Debian Codegraph @

Lib/test/test_pyexpat.py

raw ◀ ▶
Source reconstructed at patch state 0 of 1 — not the installed (fully patched) source.
1 Debian patch touch this file
1 # XXX TypeErrors on calling handlers, or on bad return values from a
2 # handler, are obscure and unhelpful.
3
4 import abc
5 import functools
6 import os
7 import re
8 import sys
9 import sysconfig
10 import textwrap
11 import unittest
12 import traceback
13 from io import BytesIO
14 from test import support
15 from test.support import import_helper, os_helper
16
17 from xml.parsers import expat
18 from xml.parsers.expat import errors
19
20 from test.support import sortdict
21
22
23 class SetAttributeTest(unittest.TestCase):
24 def setUp(self):
25 self.parser = expat.ParserCreate(namespace_separator='!')
26
27 def test_buffer_text(self):
28 self.assertIs(self.parser.buffer_text, False)
29 for x in 0, 1, 2, 0:
30 self.parser.buffer_text = x
31 self.assertIs(self.parser.buffer_text, bool(x))
32
33 def test_namespace_prefixes(self):
34 self.assertIs(self.parser.namespace_prefixes, False)
35 for x in 0, 1, 2, 0:
36 self.parser.namespace_prefixes = x
37 self.assertIs(self.parser.namespace_prefixes, bool(x))
38
39 def test_ordered_attributes(self):
40 self.assertIs(self.parser.ordered_attributes, False)
41 for x in 0, 1, 2, 0:
42 self.parser.ordered_attributes = x
43 self.assertIs(self.parser.ordered_attributes, bool(x))
44
45 def test_specified_attributes(self):
46 self.assertIs(self.parser.specified_attributes, False)
47 for x in 0, 1, 2, 0:
48 self.parser.specified_attributes = x
49 self.assertIs(self.parser.specified_attributes, bool(x))
50
51 def test_invalid_attributes(self):
52 with self.assertRaises(AttributeError):
53 self.parser.returns_unicode = 1
54 with self.assertRaises(AttributeError):
55 self.parser.returns_unicode
56
57 # Issue #25019
58 self.assertRaises(TypeError, setattr, self.parser, range(0xF), 0)
59 self.assertRaises(TypeError, self.parser.__setattr__, range(0xF), 0)
60 self.assertRaises(TypeError, getattr, self.parser, range(0xF))
61
62
63 data = b'''\
64 <?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
65 <?xml-stylesheet href="stylesheet.css"?>
66 <!-- comment data -->
67 <!DOCTYPE quotations SYSTEM "quotations.dtd" [
68 <!ELEMENT root ANY>
69 <!ATTLIST root attr1 CDATA #REQUIRED attr2 CDATA #IMPLIED>
70 <!NOTATION notation SYSTEM "notation.jpeg">
71 <!ENTITY acirc "&#226;">
72 <!ENTITY external_entity SYSTEM "entity.file">
73 <!ENTITY unparsed_entity SYSTEM "entity.file" NDATA notation>
74 %unparsed_entity;
75 ]>
76
77 <root attr1="value1" attr2="value2&#8000;">
78 <myns:subelement xmlns:myns="http://www.python.org/namespace">
79 Contents of subelements
80 </myns:subelement>
81 <sub2><![CDATA[contents of CDATA section]]></sub2>
82 &external_entity;
83 &skipped_entity;
84 \xb5
85 </root>
86 '''
87
88
89 # Produce UTF-8 output
90 class ParseTest(unittest.TestCase):
91 class Outputter:
92 def __init__(self):
93 self.out = []
94
95 def StartElementHandler(self, name, attrs):
96 self.out.append('Start element: ' + repr(name) + ' ' +
97 sortdict(attrs))
98
99 def EndElementHandler(self, name):
100 self.out.append('End element: ' + repr(name))
101
102 def CharacterDataHandler(self, data):
103 data = data.strip()
104 if data:
105 self.out.append('Character data: ' + repr(data))
106
107 def ProcessingInstructionHandler(self, target, data):
108 self.out.append('PI: ' + repr(target) + ' ' + repr(data))
109
110 def StartNamespaceDeclHandler(self, prefix, uri):
111 self.out.append('NS decl: ' + repr(prefix) + ' ' + repr(uri))
112
113 def EndNamespaceDeclHandler(self, prefix):
114 self.out.append('End of NS decl: ' + repr(prefix))
115
116 def StartCdataSectionHandler(self):
117 self.out.append('Start of CDATA section')
118
119 def EndCdataSectionHandler(self):
120 self.out.append('End of CDATA section')
121
122 def CommentHandler(self, text):
123 self.out.append('Comment: ' + repr(text))
124
125 def NotationDeclHandler(self, *args):
126 name, base, sysid, pubid = args
127 self.out.append('Notation declared: %s' %(args,))
128
129 def UnparsedEntityDeclHandler(self, *args):
130 entityName, base, systemId, publicId, notationName = args
131 self.out.append('Unparsed entity decl: %s' %(args,))
132
133 def NotStandaloneHandler(self):
134 self.out.append('Not standalone')
135 return 1
136
137 def ExternalEntityRefHandler(self, *args):
138 context, base, sysId, pubId = args
139 self.out.append('External entity ref: %s' %(args[1:],))
140 return 1
141
142 def StartDoctypeDeclHandler(self, *args):
143 self.out.append(('Start doctype', args))
144 return 1
145
146 def EndDoctypeDeclHandler(self):
147 self.out.append("End doctype")
148 return 1
149
150 def EntityDeclHandler(self, *args):
151 self.out.append(('Entity declaration', args))
152 return 1
153
154 def XmlDeclHandler(self, *args):
155 self.out.append(('XML declaration', args))
156 return 1
157
158 def ElementDeclHandler(self, *args):
159 self.out.append(('Element declaration', args))
160 return 1
161
162 def AttlistDeclHandler(self, *args):
163 self.out.append(('Attribute list declaration', args))
164 return 1
165
166 def SkippedEntityHandler(self, *args):
167 self.out.append(("Skipped entity", args))
168 return 1
169
170 def DefaultHandler(self, userData):
171 pass
172
173 def DefaultHandlerExpand(self, userData):
174 pass
175
176 handler_names = [
177 'StartElementHandler', 'EndElementHandler', 'CharacterDataHandler',
178 'ProcessingInstructionHandler', 'UnparsedEntityDeclHandler',
179 'NotationDeclHandler', 'StartNamespaceDeclHandler',
180 'EndNamespaceDeclHandler', 'CommentHandler',
181 'StartCdataSectionHandler', 'EndCdataSectionHandler', 'DefaultHandler',
182 'DefaultHandlerExpand', 'NotStandaloneHandler',
183 'ExternalEntityRefHandler', 'StartDoctypeDeclHandler',
184 'EndDoctypeDeclHandler', 'EntityDeclHandler', 'XmlDeclHandler',
185 'ElementDeclHandler', 'AttlistDeclHandler', 'SkippedEntityHandler',
186 ]
187
188 def _hookup_callbacks(self, parser, handler):
189 """
190 Set each of the callbacks defined on handler and named in
191 self.handler_names on the given parser.
192 """
193 for name in self.handler_names:
194 setattr(parser, name, getattr(handler, name))
195
196 def _verify_parse_output(self, operations):
197 expected_operations = [
198 ('XML declaration', ('1.0', 'iso-8859-1', 0)),
199 'PI: \'xml-stylesheet\' \'href="stylesheet.css"\'',
200 "Comment: ' comment data '",
201 "Not standalone",
202 ("Start doctype", ('quotations', 'quotations.dtd', None, 1)),
203 ('Element declaration', ('root', (2, 0, None, ()))),
204 ('Attribute list declaration', ('root', 'attr1', 'CDATA', None,
205 1)),
206 ('Attribute list declaration', ('root', 'attr2', 'CDATA', None,
207 0)),
208 "Notation declared: ('notation', None, 'notation.jpeg', None)",
209 ('Entity declaration', ('acirc', 0, '\xe2', None, None, None, None)),
210 ('Entity declaration', ('external_entity', 0, None, None,
211 'entity.file', None, None)),
212 "Unparsed entity decl: ('unparsed_entity', None, 'entity.file', None, 'notation')",
213 "Not standalone",
214 "End doctype",
215 "Start element: 'root' {'attr1': 'value1', 'attr2': 'value2\u1f40'}",
216 "NS decl: 'myns' 'http://www.python.org/namespace'",
217 "Start element: 'http://www.python.org/namespace!subelement' {}",
218 "Character data: 'Contents of subelements'",
219 "End element: 'http://www.python.org/namespace!subelement'",
220 "End of NS decl: 'myns'",
221 "Start element: 'sub2' {}",
222 'Start of CDATA section',
223 "Character data: 'contents of CDATA section'",
224 'End of CDATA section',
225 "End element: 'sub2'",
226 "External entity ref: (None, 'entity.file', None)",
227 ('Skipped entity', ('skipped_entity', 0)),
228 "Character data: '\xb5'",
229 "End element: 'root'",
230 ]
231 for operation, expected_operation in zip(operations, expected_operations):
232 self.assertEqual(operation, expected_operation)
233
234 def test_parse_bytes(self):
235 out = self.Outputter()
236 parser = expat.ParserCreate(namespace_separator='!')
237 self._hookup_callbacks(parser, out)
238
239 parser.Parse(data, True)
240
241 operations = out.out
242 self._verify_parse_output(operations)
243 # Issue #6697.
244 self.assertRaises(AttributeError, getattr, parser, '\uD800')
245
246 def test_parse_str(self):
247 out = self.Outputter()
248 parser = expat.ParserCreate(namespace_separator='!')
249 self._hookup_callbacks(parser, out)
250
251 parser.Parse(data.decode('iso-8859-1'), True)
252
253 operations = out.out
254 self._verify_parse_output(operations)
255
256 def test_parse_file(self):
257 # Try parsing a file
258 out = self.Outputter()
259 parser = expat.ParserCreate(namespace_separator='!')
260 self._hookup_callbacks(parser, out)
261 file = BytesIO(data)
262
263 parser.ParseFile(file)
264
265 operations = out.out
266 self._verify_parse_output(operations)
267
268 def test_parse_again(self):
269 parser = expat.ParserCreate()
270 file = BytesIO(data)
271 parser.ParseFile(file)
272 # Issue 6676: ensure a meaningful exception is raised when attempting
273 # to parse more than one XML document per xmlparser instance,
274 # a limitation of the Expat library.
275 with self.assertRaises(expat.error) as cm:
276 parser.ParseFile(file)
277 self.assertEqual(expat.ErrorString(cm.exception.code),
278 expat.errors.XML_ERROR_FINISHED)
279
280 class NamespaceSeparatorTest(unittest.TestCase):
281 def test_legal(self):
282 # Tests that make sure we get errors when the namespace_separator value
283 # is illegal, and that we don't for good values:
284 expat.ParserCreate()
285 expat.ParserCreate(namespace_separator=None)
286 expat.ParserCreate(namespace_separator=' ')
287
288 def test_illegal(self):
289 with self.assertRaisesRegex(TypeError,
290 r"ParserCreate\(\) argument (2|'namespace_separator') "
291 r"must be str or None, not int"):
292 expat.ParserCreate(namespace_separator=42)
293
294 try:
295 expat.ParserCreate(namespace_separator='too long')
296 self.fail()
297 except ValueError as e:
298 self.assertEqual(str(e),
299 'namespace_separator must be at most one character, omitted, or None')
300
301 def test_zero_length(self):
302 # ParserCreate() needs to accept a namespace_separator of zero length
303 # to satisfy the requirements of RDF applications that are required
304 # to simply glue together the namespace URI and the localname. Though
305 # considered a wart of the RDF specifications, it needs to be supported.
306 #
307 # See XML-SIG mailing list thread starting with
308 # http://mail.python.org/pipermail/xml-sig/2001-April/005202.html
309 #
310 expat.ParserCreate(namespace_separator='') # too short
311
312
313 class InterningTest(unittest.TestCase):
314 def test(self):
315 # Test the interning machinery.
316 p = expat.ParserCreate()
317 L = []
318 def collector(name, *args):
319 L.append(name)
320 p.StartElementHandler = collector
321 p.EndElementHandler = collector
322 p.Parse(b"<e> <e/> <e></e> </e>", True)
323 tag = L[0]
324 self.assertEqual(len(L), 6)
325 for entry in L:
326 # L should have the same string repeated over and over.
327 self.assertTrue(tag is entry)
328
329 def test_issue9402(self):
330 # create an ExternalEntityParserCreate with buffer text
331 class ExternalOutputter:
332 def __init__(self, parser):
333 self.parser = parser
334 self.parser_result = None
335
336 def ExternalEntityRefHandler(self, context, base, sysId, pubId):
337 external_parser = self.parser.ExternalEntityParserCreate("")
338 self.parser_result = external_parser.Parse(b"", True)
339 return 1
340
341 parser = expat.ParserCreate(namespace_separator='!')
342 parser.buffer_text = 1
343 out = ExternalOutputter(parser)
344 parser.ExternalEntityRefHandler = out.ExternalEntityRefHandler
345 parser.Parse(data, True)
346 self.assertEqual(out.parser_result, 1)
347
348
349 class BufferTextTest(unittest.TestCase):
350 def setUp(self):
351 self.stuff = []
352 self.parser = expat.ParserCreate()
353 self.parser.buffer_text = 1
354 self.parser.CharacterDataHandler = self.CharacterDataHandler
355
356 def check(self, expected, label):
357 self.assertEqual(self.stuff, expected,
358 "%s\nstuff = %r\nexpected = %r"
359 % (label, self.stuff, map(str, expected)))
360
361 def CharacterDataHandler(self, text):
362 self.stuff.append(text)
363
364 def StartElementHandler(self, name, attrs):
365 self.stuff.append("<%s>" % name)
366 bt = attrs.get("buffer-text")
367 if bt == "yes":
368 self.parser.buffer_text = 1
369 elif bt == "no":
370 self.parser.buffer_text = 0
371
372 def EndElementHandler(self, name):
373 self.stuff.append("</%s>" % name)
374
375 def CommentHandler(self, data):
376 self.stuff.append("<!--%s-->" % data)
377
378 def setHandlers(self, handlers=[]):
379 for name in handlers:
380 setattr(self.parser, name, getattr(self, name))
381
382 def test_default_to_disabled(self):
383 parser = expat.ParserCreate()
384 self.assertFalse(parser.buffer_text)
385
386 def test_buffering_enabled(self):
387 # Make sure buffering is turned on
388 self.assertTrue(self.parser.buffer_text)
389 self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True)
390 self.assertEqual(self.stuff, ['123'],
391 "buffered text not properly collapsed")
392
393 def test1(self):
394 # XXX This test exposes more detail of Expat's text chunking than we
395 # XXX like, but it tests what we need to concisely.
396 self.setHandlers(["StartElementHandler"])
397 self.parser.Parse(b"<a>1<b buffer-text='no'/>2\n3<c buffer-text='yes'/>4\n5</a>", True)
398 self.assertEqual(self.stuff,
399 ["<a>", "1", "<b>", "2", "\n", "3", "<c>", "4\n5"],
400 "buffering control not reacting as expected")
401
402 def test2(self):
403 self.parser.Parse(b"<a>1<b/>&lt;2&gt;<c/>&#32;\n&#x20;3</a>", True)
404 self.assertEqual(self.stuff, ["1<2> \n 3"],
405 "buffered text not properly collapsed")
406
407 def test3(self):
408 self.setHandlers(["StartElementHandler"])
409 self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True)
410 self.assertEqual(self.stuff, ["<a>", "1", "<b>", "2", "<c>", "3"],
411 "buffered text not properly split")
412
413 def test4(self):
414 self.setHandlers(["StartElementHandler", "EndElementHandler"])
415 self.parser.CharacterDataHandler = None
416 self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True)
417 self.assertEqual(self.stuff,
418 ["<a>", "<b>", "</b>", "<c>", "</c>", "</a>"])
419
420 def test5(self):
421 self.setHandlers(["StartElementHandler", "EndElementHandler"])
422 self.parser.Parse(b"<a>1<b></b>2<c/>3</a>", True)
423 self.assertEqual(self.stuff,
424 ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3", "</a>"])
425
426 def test6(self):
427 self.setHandlers(["CommentHandler", "EndElementHandler",
428 "StartElementHandler"])
429 self.parser.Parse(b"<a>1<b/>2<c></c>345</a> ", True)
430 self.assertEqual(self.stuff,
431 ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "345", "</a>"],
432 "buffered text not properly split")
433
434 def test7(self):
435 self.setHandlers(["CommentHandler", "EndElementHandler",
436 "StartElementHandler"])
437 self.parser.Parse(b"<a>1<b/>2<c></c>3<!--abc-->4<!--def-->5</a> ", True)
438 self.assertEqual(self.stuff,
439 ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3",
440 "<!--abc-->", "4", "<!--def-->", "5", "</a>"],
441 "buffered text not properly split")
442
443
444 # Test handling of exception from callback:
445 class HandlerExceptionTest(unittest.TestCase):
446 def StartElementHandler(self, name, attrs):
447 raise RuntimeError(f'StartElementHandler: <{name}>')
448
449 def check_traceback_entry(self, entry, filename, funcname):
450 self.assertEqual(os.path.basename(entry.filename), filename)
451 self.assertEqual(entry.name, funcname)
452
453 @support.cpython_only
454 def test_exception(self):
455 # gh-66652: test _PyTraceback_Add() used by pyexpat.c to inject frames
456
457 # Change the current directory to the Python source code directory
458 # if it is available.
459 src_dir = sysconfig.get_config_var('abs_builddir')
460 if src_dir:
461 have_source = os.path.isdir(src_dir)
462 else:
463 have_source = False
464 if have_source:
465 with os_helper.change_cwd(src_dir):
466 self._test_exception(have_source)
467 else:
468 self._test_exception(have_source)
469
470 def _test_exception(self, have_source):
471 # Use path relative to the current directory which should be the Python
472 # source code directory (if it is available).
473 PYEXPAT_C = os.path.join('Modules', 'pyexpat.c')
474
475 parser = expat.ParserCreate()
476 parser.StartElementHandler = self.StartElementHandler
477 try:
478 parser.Parse(b"<a><b><c/></b></a>", True)
479
480 self.fail("the parser did not raise RuntimeError")
481 except RuntimeError as exc:
482 self.assertEqual(exc.args[0], 'StartElementHandler: <a>', exc)
483 entries = traceback.extract_tb(exc.__traceback__)
484
485 self.assertEqual(len(entries), 3, entries)
486 self.check_traceback_entry(entries[0],
487 "test_pyexpat.py", "_test_exception")
488 self.check_traceback_entry(entries[1],
489 os.path.basename(PYEXPAT_C),
490 "StartElement")
491 self.check_traceback_entry(entries[2],
492 "test_pyexpat.py", "StartElementHandler")
493
494 # Check that the traceback contains the relevant line in
495 # Modules/pyexpat.c. Skip the test if Modules/pyexpat.c is not
496 # available.
497 if have_source and os.path.exists(PYEXPAT_C):
498 self.assertIn('call_with_frame("StartElement"',
499 entries[1].line)
500
501
502 # Test Current* members:
503 class PositionTest(unittest.TestCase):
504 def StartElementHandler(self, name, attrs):
505 self.check_pos('s')
506
507 def EndElementHandler(self, name):
508 self.check_pos('e')
509
510 def check_pos(self, event):
511 pos = (event,
512 self.parser.CurrentByteIndex,
513 self.parser.CurrentLineNumber,
514 self.parser.CurrentColumnNumber)
515 self.assertTrue(self.upto < len(self.expected_list),
516 'too many parser events')
517 expected = self.expected_list[self.upto]
518 self.assertEqual(pos, expected,
519 'Expected position %s, got position %s' %(pos, expected))
520 self.upto += 1
521
522 def test(self):
523 self.parser = expat.ParserCreate()
524 self.parser.StartElementHandler = self.StartElementHandler
525 self.parser.EndElementHandler = self.EndElementHandler
526 self.upto = 0
527 self.expected_list = [('s', 0, 1, 0), ('s', 5, 2, 1), ('s', 11, 3, 2),
528 ('e', 15, 3, 6), ('e', 17, 4, 1), ('e', 22, 5, 0)]
529
530 xml = b'<a>\n <b>\n <c/>\n </b>\n</a>'
531 self.parser.Parse(xml, True)
532
533
534 class sf1296433Test(unittest.TestCase):
535 def test_parse_only_xml_data(self):
536 # https://bugs.python.org/issue1296433
537 #
538 xml = "<?xml version='1.0' encoding='iso8859'?><s>%s</s>" % ('a' * 1025)
539 # this one doesn't crash
540 #xml = "<?xml version='1.0'?><s>%s</s>" % ('a' * 10000)
541
542 class SpecificException(Exception):
543 pass
544
545 def handler(text):
546 raise SpecificException
547
548 parser = expat.ParserCreate()
549 parser.CharacterDataHandler = handler
550
551 self.assertRaises(SpecificException, parser.Parse, xml.encode('iso8859'))
552
553 class ChardataBufferTest(unittest.TestCase):
554 """
555 test setting of chardata buffer size
556 """
557
558 def test_1025_bytes(self):
559 self.assertEqual(self.small_buffer_test(1025), 2)
560
561 def test_1000_bytes(self):
562 self.assertEqual(self.small_buffer_test(1000), 1)
563
564 def test_wrong_size(self):
565 parser = expat.ParserCreate()
566 parser.buffer_text = 1
567 with self.assertRaises(ValueError):
568 parser.buffer_size = -1
569 with self.assertRaises(ValueError):
570 parser.buffer_size = 0
571 with self.assertRaises((ValueError, OverflowError)):
572 parser.buffer_size = sys.maxsize + 1
573 with self.assertRaises(TypeError):
574 parser.buffer_size = 512.0
575
576 def test_unchanged_size(self):
577 xml1 = b"<?xml version='1.0' encoding='iso8859'?><s>" + b'a' * 512
578 xml2 = b'a'*512 + b'</s>'
579 parser = expat.ParserCreate()
580 parser.CharacterDataHandler = self.counting_handler
581 parser.buffer_size = 512
582 parser.buffer_text = 1
583
584 # Feed 512 bytes of character data: the handler should be called
585 # once.
586 self.n = 0
587 parser.Parse(xml1)
588 self.assertEqual(self.n, 1)
589
590 # Reassign to buffer_size, but assign the same size.
591 parser.buffer_size = parser.buffer_size
592 self.assertEqual(self.n, 1)
593
594 # Try parsing rest of the document
595 parser.Parse(xml2)
596 self.assertEqual(self.n, 2)
597
598
599 def test_disabling_buffer(self):
600 xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>" + b'a' * 512
601 xml2 = b'b' * 1024
602 xml3 = b'c' * 1024 + b'</a>';
603 parser = expat.ParserCreate()
604 parser.CharacterDataHandler = self.counting_handler
605 parser.buffer_text = 1
606 parser.buffer_size = 1024
607 self.assertEqual(parser.buffer_size, 1024)
608
609 # Parse one chunk of XML
610 self.n = 0
611 parser.Parse(xml1, False)
612 self.assertEqual(parser.buffer_size, 1024)
613 self.assertEqual(self.n, 1)
614
615 # Turn off buffering and parse the next chunk.
616 parser.buffer_text = 0
617 self.assertFalse(parser.buffer_text)
618 self.assertEqual(parser.buffer_size, 1024)
619 for i in range(10):
620 parser.Parse(xml2, False)
621 self.assertEqual(self.n, 11)
622
623 parser.buffer_text = 1
624 self.assertTrue(parser.buffer_text)
625 self.assertEqual(parser.buffer_size, 1024)
626 parser.Parse(xml3, True)
627 self.assertEqual(self.n, 12)
628
629 def counting_handler(self, text):
630 self.n += 1
631
632 def small_buffer_test(self, buffer_len):
633 xml = b"<?xml version='1.0' encoding='iso8859'?><s>" + b'a' * buffer_len + b'</s>'
634 parser = expat.ParserCreate()
635 parser.CharacterDataHandler = self.counting_handler
636 parser.buffer_size = 1024
637 parser.buffer_text = 1
638
639 self.n = 0
640 parser.Parse(xml)
641 return self.n
642
643 def test_change_size_1(self):
644 xml1 = b"<?xml version='1.0' encoding='iso8859'?><a><s>" + b'a' * 1024
645 xml2 = b'aaa</s><s>' + b'a' * 1025 + b'</s></a>'
646 parser = expat.ParserCreate()
647 parser.CharacterDataHandler = self.counting_handler
648 parser.buffer_text = 1
649 parser.buffer_size = 1024
650 self.assertEqual(parser.buffer_size, 1024)
651
652 self.n = 0
653 parser.Parse(xml1, False)
654 parser.buffer_size *= 2
655 self.assertEqual(parser.buffer_size, 2048)
656 parser.Parse(xml2, True)
657 self.assertEqual(self.n, 2)
658
659 def test_change_size_2(self):
660 xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>a<s>" + b'a' * 1023
661 xml2 = b'aaa</s><s>' + b'a' * 1025 + b'</s></a>'
662 parser = expat.ParserCreate()
663 parser.CharacterDataHandler = self.counting_handler
664 parser.buffer_text = 1
665 parser.buffer_size = 2048
666 self.assertEqual(parser.buffer_size, 2048)
667
668 self.n=0
669 parser.Parse(xml1, False)
670 parser.buffer_size = parser.buffer_size // 2
671 self.assertEqual(parser.buffer_size, 1024)
672 parser.Parse(xml2, True)
673 self.assertEqual(self.n, 4)
674
675 class ElementDeclHandlerTest(unittest.TestCase):
676 def test_trigger_leak(self):
677 # Unfixed, this test would leak the memory of the so-called
678 # "content model" in function ``my_ElementDeclHandler`` of pyexpat.
679 # See https://github.com/python/cpython/issues/140593.
680 data = textwrap.dedent('''\
681 <!DOCTYPE quotations SYSTEM "quotations.dtd" [
682 <!ELEMENT root ANY>
683 ]>
684 <root/>
685 ''').encode('UTF-8')
686
687 parser = expat.ParserCreate()
688 parser.NotStandaloneHandler = lambda: 1.234 # arbitrary float
689 parser.ElementDeclHandler = lambda _1, _2: None
690 self.assertRaises(TypeError, parser.Parse, data, True)
691
692 @support.skip_if_unlimited_stack_size
693 @support.skip_emscripten_stack_overflow()
694 @support.skip_wasi_stack_overflow()
695 def test_deeply_nested_content_model(self):
696 # This should raise a RecursionError and not crash.
697 # See https://github.com/python/cpython/issues/145986.
698 N = 500_000
699 data = (
700 b'<!DOCTYPE root [\n<!ELEMENT root '
701 + b'(a, ' * N + b'a' + b')' * N
702 + b'>\n]>\n<root/>\n'
703 )
704
705 parser = expat.ParserCreate()
706 parser.ElementDeclHandler = lambda _1, _2: None
707 with support.infinite_recursion():
708 with self.assertRaises(RecursionError):
709 parser.Parse(data)
710
711 class MalformedInputTest(unittest.TestCase):
712 def test1(self):
713 xml = b"\0\r\n"
714 parser = expat.ParserCreate()
715 try:
716 parser.Parse(xml, True)
717 self.fail()
718 except expat.ExpatError as e:
719 self.assertEqual(str(e), 'unclosed token: line 2, column 0')
720
721 def test2(self):
722 # \xc2\x85 is UTF-8 encoded U+0085 (NEXT LINE)
723 xml = b"<?xml version\xc2\x85='1.0'?>\r\n"
724 parser = expat.ParserCreate()
725 err_pattern = r'XML declaration not well-formed: line 1, column \d+'
726 with self.assertRaisesRegex(expat.ExpatError, err_pattern):
727 parser.Parse(xml, True)
728
729 class ErrorMessageTest(unittest.TestCase):
730 def test_codes(self):
731 # verify mapping of errors.codes and errors.messages
732 self.assertEqual(errors.XML_ERROR_SYNTAX,
733 errors.messages[errors.codes[errors.XML_ERROR_SYNTAX]])
734
735 def test_expaterror(self):
736 xml = b'<'
737 parser = expat.ParserCreate()
738 try:
739 parser.Parse(xml, True)
740 self.fail()
741 except expat.ExpatError as e:
742 self.assertEqual(e.code,
743 errors.codes[errors.XML_ERROR_UNCLOSED_TOKEN])
744
745
746 class ForeignDTDTests(unittest.TestCase):
747 """
748 Tests for the UseForeignDTD method of expat parser objects.
749 """
750 def test_use_foreign_dtd(self):
751 """
752 If UseForeignDTD is passed True and a document without an external
753 entity reference is parsed, ExternalEntityRefHandler is first called
754 with None for the public and system ids.
755 """
756 handler_call_args = []
757 def resolve_entity(context, base, system_id, public_id):
758 handler_call_args.append((public_id, system_id))
759 return 1
760
761 parser = expat.ParserCreate()
762 parser.UseForeignDTD(True)
763 parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS)
764 parser.ExternalEntityRefHandler = resolve_entity
765 parser.Parse(b"<?xml version='1.0'?><element/>")
766 self.assertEqual(handler_call_args, [(None, None)])
767
768 # test UseForeignDTD() is equal to UseForeignDTD(True)
769 handler_call_args[:] = []
770
771 parser = expat.ParserCreate()
772 parser.UseForeignDTD()
773 parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS)
774 parser.ExternalEntityRefHandler = resolve_entity
775 parser.Parse(b"<?xml version='1.0'?><element/>")
776 self.assertEqual(handler_call_args, [(None, None)])
777
778 def test_ignore_use_foreign_dtd(self):
779 """
780 If UseForeignDTD is passed True and a document with an external
781 entity reference is parsed, ExternalEntityRefHandler is called with
782 the public and system ids from the document.
783 """
784 handler_call_args = []
785 def resolve_entity(context, base, system_id, public_id):
786 handler_call_args.append((public_id, system_id))
787 return 1
788
789 parser = expat.ParserCreate()
790 parser.UseForeignDTD(True)
791 parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS)
792 parser.ExternalEntityRefHandler = resolve_entity
793 parser.Parse(
794 b"<?xml version='1.0'?><!DOCTYPE foo PUBLIC 'bar' 'baz'><element/>")
795 self.assertEqual(handler_call_args, [("bar", "baz")])
796
797
798 class ParentParserLifetimeTest(unittest.TestCase):
799 """
800 Subparsers make use of their parent XML_Parser inside of Expat.
801 As a result, parent parsers need to outlive subparsers.
802
803 See https://github.com/python/cpython/issues/139400.
804 """
805
806 def test_parent_parser_outlives_its_subparsers__single(self):
807 parser = expat.ParserCreate()
808 subparser = parser.ExternalEntityParserCreate(None)
809
810 # Now try to cause garbage collection of the parent parser
811 # while it's still being referenced by a related subparser.
812 del parser
813
814 def test_parent_parser_outlives_its_subparsers__multiple(self):
815 parser = expat.ParserCreate()
816 subparser_one = parser.ExternalEntityParserCreate(None)
817 subparser_two = parser.ExternalEntityParserCreate(None)
818
819 # Now try to cause garbage collection of the parent parser
820 # while it's still being referenced by a related subparser.
821 del parser
822
823 def test_parent_parser_outlives_its_subparsers__chain(self):
824 parser = expat.ParserCreate()
825 subparser = parser.ExternalEntityParserCreate(None)
826 subsubparser = subparser.ExternalEntityParserCreate(None)
827
828 # Now try to cause garbage collection of the parent parsers
829 # while they are still being referenced by a related subparser.
830 del parser
831 del subparser
832
833
834 class ExternalEntityParserCreateErrorTest(unittest.TestCase):
835 """ExternalEntityParserCreate error paths should not crash or leak
836 refcounts on the parent parser.
837
838 See https://github.com/python/cpython/issues/144984.
839 """
840
841 @classmethod
842 def setUpClass(cls):
843 cls.testcapi = import_helper.import_module('_testcapi')
844
845 @unittest.skipIf(support.Py_TRACE_REFS,
846 'Py_TRACE_REFS conflicts with testcapi.set_nomemory')
847 def test_error_path_no_crash(self):
848 # When an allocation inside ExternalEntityParserCreate fails,
849 # the partially-initialized subparser is deallocated. This
850 # must not dereference NULL handlers or double-decrement the
851 # parent parser's refcount.
852 parser = expat.ParserCreate()
853 parser.buffer_text = True
854 rc_before = sys.getrefcount(parser)
855
856 # We avoid self.assertRaises(MemoryError) here because the
857 # context manager itself needs memory allocations that fail
858 # while the nomemory hook is active.
859 self.testcapi.set_nomemory(1, 10)
860 raised = False
861 try:
862 parser.ExternalEntityParserCreate(None)
863 except MemoryError:
864 raised = True
865 finally:
866 self.testcapi.remove_mem_hooks()
867 self.assertTrue(raised, "MemoryError not raised")
868
869 rc_after = sys.getrefcount(parser)
870 self.assertEqual(rc_after, rc_before)
871
872
873 class ReparseDeferralTest(unittest.TestCase):
874 def test_getter_setter_round_trip(self):
875 parser = expat.ParserCreate()
876 enabled = (expat.version_info >= (2, 6, 0))
877
878 self.assertIs(parser.GetReparseDeferralEnabled(), enabled)
879 parser.SetReparseDeferralEnabled(False)
880 self.assertIs(parser.GetReparseDeferralEnabled(), False)
881 parser.SetReparseDeferralEnabled(True)
882 self.assertIs(parser.GetReparseDeferralEnabled(), enabled)
883
884 def test_reparse_deferral_enabled(self):
885 if expat.version_info < (2, 6, 0):
886 self.skipTest(f'Expat {expat.version_info} does not '
887 'support reparse deferral')
888
889 started = []
890
891 def start_element(name, _):
892 started.append(name)
893
894 parser = expat.ParserCreate()
895 parser.StartElementHandler = start_element
896 self.assertTrue(parser.GetReparseDeferralEnabled())
897
898 for chunk in (b'<doc', b'/>'):
899 parser.Parse(chunk, False)
900
901 # The key test: Have handlers already fired? Expecting: no.
902 self.assertEqual(started, [])
903
904 parser.Parse(b'', True)
905
906 self.assertEqual(started, ['doc'])
907
908 def test_reparse_deferral_disabled(self):
909 started = []
910
911 def start_element(name, _):
912 started.append(name)
913
914 parser = expat.ParserCreate()
915 parser.StartElementHandler = start_element
916 if expat.version_info >= (2, 6, 0):
917 parser.SetReparseDeferralEnabled(False)
918 self.assertFalse(parser.GetReparseDeferralEnabled())
919
920 for chunk in (b'<doc', b'/>'):
921 parser.Parse(chunk, False)
922
923 # The key test: Have handlers already fired? Expecting: yes.
924 self.assertEqual(started, ['doc'])
925
926
927 class AttackProtectionTestBase(abc.ABC):
928 """
929 Base class for testing protections against XML payloads with
930 disproportionate amplification.
931
932 The protections being tested should detect and prevent attacks
933 that leverage disproportionate amplification from small inputs.
934 """
935
936 @staticmethod
937 def exponential_expansion_payload(*, nrows, ncols, text='.'):
938 """Create a billion laughs attack payload.
939
940 Be careful: the number of total items is pow(n, k), thereby
941 requiring at least pow(ncols, nrows) * sizeof(text) memory!
942 """
943 template = textwrap.dedent(f"""\
944 <?xml version="1.0"?>
945 <!DOCTYPE doc [
946 <!ENTITY row0 "{text}">
947 <!ELEMENT doc (#PCDATA)>
948 {{body}}
949 ]>
950 <doc>&row{nrows};</doc>
951 """).rstrip()
952
953 body = '\n'.join(
954 f'<!ENTITY row{i + 1} "{f"&row{i};" * ncols}">'
955 for i in range(nrows)
956 )
957 body = textwrap.indent(body, ' ' * 4)
958 return template.format(body=body)
959
960 def test_payload_generation(self):
961 # self-test for exponential_expansion_payload()
962 payload = self.exponential_expansion_payload(nrows=2, ncols=3)
963 self.assertEqual(payload, textwrap.dedent("""\
964 <?xml version="1.0"?>
965 <!DOCTYPE doc [
966 <!ENTITY row0 ".">
967 <!ELEMENT doc (#PCDATA)>
968 <!ENTITY row1 "&row0;&row0;&row0;">
969 <!ENTITY row2 "&row1;&row1;&row1;">
970 ]>
971 <doc>&row2;</doc>
972 """).rstrip())
973
974 def assert_root_parser_failure(self, func, /, *args, **kwargs):
975 """Check that func(*args, **kwargs) is invalid for a sub-parser."""
976 msg = "parser must be a root parser"
977 self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs)
978
979 @abc.abstractmethod
980 def assert_rejected(self, func, /, *args, **kwargs):
981 """Assert that func(*args, **kwargs) triggers the attack protection.
982
983 Note: this method must ensure that the attack protection being tested
984 is the one that is actually triggered at runtime, e.g., by matching
985 the exact error message.
986 """
987
988 @abc.abstractmethod
989 def set_activation_threshold(self, parser, threshold):
990 """Set the activation threshold for the tested protection."""
991
992 @abc.abstractmethod
993 def set_maximum_amplification(self, parser, max_factor):
994 """Set the maximum amplification factor for the tested protection."""
995
996 @abc.abstractmethod
997 def test_set_activation_threshold__threshold_reached(self):
998 """Test when the activation threshold is exceeded."""
999
1000 @abc.abstractmethod
1001 def test_set_activation_threshold__threshold_not_reached(self):
1002 """Test when the activation threshold is not exceeded."""
1003
1004 def test_set_activation_threshold__invalid_threshold_type(self):
1005 parser = expat.ParserCreate()
1006 setter = functools.partial(self.set_activation_threshold, parser)
1007
1008 self.assertRaises(TypeError, setter, 1.0)
1009 self.assertRaises(TypeError, setter, -1.5)
1010 self.assertRaises(ValueError, setter, -5)
1011
1012 def test_set_activation_threshold__invalid_threshold_range(self):
1013 _testcapi = import_helper.import_module("_testcapi")
1014 parser = expat.ParserCreate()
1015 setter = functools.partial(self.set_activation_threshold, parser)
1016
1017 self.assertRaises(OverflowError, setter, _testcapi.ULLONG_MAX + 1)
1018
1019 def test_set_activation_threshold__fail_for_subparser(self):
1020 parser = expat.ParserCreate()
1021 subparser = parser.ExternalEntityParserCreate(None)
1022 setter = functools.partial(self.set_activation_threshold, subparser)
1023 self.assert_root_parser_failure(setter, 12345)
1024
1025 @abc.abstractmethod
1026 def test_set_maximum_amplification__amplification_exceeded(self):
1027 """Test when the amplification factor is exceeded."""
1028
1029 @abc.abstractmethod
1030 def test_set_maximum_amplification__amplification_not_exceeded(self):
1031 """Test when the amplification factor is not exceeded."""
1032
1033 def test_set_maximum_amplification__infinity(self):
1034 inf = float('inf') # an 'inf' threshold is allowed by Expat
1035 parser = expat.ParserCreate()
1036 self.assertIsNone(self.set_maximum_amplification(parser, inf))
1037
1038 def test_set_maximum_amplification__invalid_max_factor_type(self):
1039 parser = expat.ParserCreate()
1040 setter = functools.partial(self.set_maximum_amplification, parser)
1041
1042 self.assertRaises(TypeError, setter, None)
1043 self.assertRaises(TypeError, setter, 'abc')
1044
1045 def test_set_maximum_amplification__invalid_max_factor_range(self):
1046 parser = expat.ParserCreate()
1047 setter = functools.partial(self.set_maximum_amplification, parser)
1048
1049 msg = re.escape("'max_factor' must be at least 1.0")
1050 self.assertRaisesRegex(expat.ExpatError, msg, setter, float('nan'))
1051 self.assertRaisesRegex(expat.ExpatError, msg, setter, 0.99)
1052
1053 def test_set_maximum_amplification__fail_for_subparser(self):
1054 parser = expat.ParserCreate()
1055 subparser = parser.ExternalEntityParserCreate(None)
1056 setter = functools.partial(self.set_maximum_amplification, subparser)
1057 self.assert_root_parser_failure(setter, 123.45)
1058
1059
1060 @unittest.skipIf(not hasattr(expat.XMLParserType,
1061 "SetAllocTrackerMaximumAmplification"),
1062 "requires Python compiled with Expat >= 2.7.2")
1063 class MemoryProtectionTest(AttackProtectionTestBase, unittest.TestCase):
1064
1065 # NOTE: with the default Expat configuration, the billion laughs protection
1066 # may hit before the allocation limiter if exponential_expansion_payload()
1067 # is not carefully parametrized. As such, the payloads should be chosen so
1068 # that either the allocation limiter is hit before other protections are
1069 # triggered or no protection at all is triggered.
1070
1071 def assert_rejected(self, func, /, *args, **kwargs):
1072 """Check that func(*args, **kwargs) hits the allocation limit."""
1073 msg = r"out of memory: line \d+, column \d+"
1074 self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs)
1075
1076 def set_activation_threshold(self, parser, threshold):
1077 return parser.SetAllocTrackerActivationThreshold(threshold)
1078
1079 def set_maximum_amplification(self, parser, max_factor):
1080 return parser.SetAllocTrackerMaximumAmplification(max_factor)
1081
1082 def test_set_activation_threshold__threshold_reached(self):
1083 parser = expat.ParserCreate()
1084 # Choose a threshold expected to be always reached.
1085 self.set_activation_threshold(parser, 3)
1086 # Check that the threshold is reached by choosing a small factor
1087 # and a payload whose peak amplification factor exceeds it.
1088 self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1089 payload = self.exponential_expansion_payload(ncols=10, nrows=4)
1090 self.assert_rejected(parser.Parse, payload, True)
1091
1092 def test_set_activation_threshold__threshold_not_reached(self):
1093 parser = expat.ParserCreate()
1094 # Choose a threshold expected to be never reached.
1095 self.set_activation_threshold(parser, pow(10, 5))
1096 # Check that the threshold is reached by choosing a small factor
1097 # and a payload whose peak amplification factor exceeds it.
1098 self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1099 payload = self.exponential_expansion_payload(ncols=10, nrows=4)
1100 self.assertIsNotNone(parser.Parse(payload, True))
1101
1102 def test_set_maximum_amplification__amplification_exceeded(self):
1103 parser = expat.ParserCreate()
1104 # Unconditionally enable maximum activation factor.
1105 self.set_activation_threshold(parser, 0)
1106 # Choose a max amplification factor expected to always be exceeded.
1107 self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1108 # Craft a payload for which the peak amplification factor is > 1.0.
1109 payload = self.exponential_expansion_payload(ncols=1, nrows=2)
1110 self.assert_rejected(parser.Parse, payload, True)
1111
1112 def test_set_maximum_amplification__amplification_not_exceeded(self):
1113 parser = expat.ParserCreate()
1114 # Unconditionally enable maximum activation factor.
1115 self.set_activation_threshold(parser, 0)
1116 # Choose a max amplification factor expected to never be exceeded.
1117 self.assertIsNone(self.set_maximum_amplification(parser, 1e4))
1118 # Craft a payload for which the peak amplification factor is < 1e4.
1119 payload = self.exponential_expansion_payload(ncols=1, nrows=2)
1120 self.assertIsNotNone(parser.Parse(payload, True))
1121
1122
1123 if __name__ == "__main__":
1124 unittest.main()