Debian Codegraph @

Lib/test/test_pyexpat.py

raw ◀ ▶
1 Debian patch touch this file
1 # XXX TypeErrors on calling handlers, or on bad return values from a
2 # handler, are obscure and unhelpful.
3
4 import abc
5 import functools
6 import os
7 import re
8 import sys
9 import sysconfig
10 import textwrap
11 import unittest
12 import traceback
13 from io import BytesIO
14 from test import support
15 from test.support import import_helper, os_helper
16
17 from xml.parsers import expat
18 from xml.parsers.expat import errors
19
20 from test.support import sortdict
21
22
23 class SetAttributeTest(unittest.TestCase):
24 def setUp(self):
25 self.parser = expat.ParserCreate(namespace_separator='!')
26
27 def test_buffer_text(self):
28 self.assertIs(self.parser.buffer_text, False)
29 for x in 0, 1, 2, 0:
30 self.parser.buffer_text = x
31 self.assertIs(self.parser.buffer_text, bool(x))
32
33 def test_namespace_prefixes(self):
34 self.assertIs(self.parser.namespace_prefixes, False)
35 for x in 0, 1, 2, 0:
36 self.parser.namespace_prefixes = x
37 self.assertIs(self.parser.namespace_prefixes, bool(x))
38
39 def test_ordered_attributes(self):
40 self.assertIs(self.parser.ordered_attributes, False)
41 for x in 0, 1, 2, 0:
42 self.parser.ordered_attributes = x
43 self.assertIs(self.parser.ordered_attributes, bool(x))
44
45 def test_specified_attributes(self):
46 self.assertIs(self.parser.specified_attributes, False)
47 for x in 0, 1, 2, 0:
48 self.parser.specified_attributes = x
49 self.assertIs(self.parser.specified_attributes, bool(x))
50
51 def test_invalid_attributes(self):
52 with self.assertRaises(AttributeError):
53 self.parser.returns_unicode = 1
54 with self.assertRaises(AttributeError):
55 self.parser.returns_unicode
56
57 # Issue #25019
58 self.assertRaises(TypeError, setattr, self.parser, range(0xF), 0)
59 self.assertRaises(TypeError, self.parser.__setattr__, range(0xF), 0)
60 self.assertRaises(TypeError, getattr, self.parser, range(0xF))
61
62
63 data = b'''\
64 <?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
65 <?xml-stylesheet href="stylesheet.css"?>
66 <!-- comment data -->
67 <!DOCTYPE quotations SYSTEM "quotations.dtd" [
68 <!ELEMENT root ANY>
69 <!ATTLIST root attr1 CDATA #REQUIRED attr2 CDATA #IMPLIED>
70 <!NOTATION notation SYSTEM "notation.jpeg">
71 <!ENTITY acirc "&#226;">
72 <!ENTITY external_entity SYSTEM "entity.file">
73 <!ENTITY unparsed_entity SYSTEM "entity.file" NDATA notation>
74 %unparsed_entity;
75 ]>
76
77 <root attr1="value1" attr2="value2&#8000;">
78 <myns:subelement xmlns:myns="http://www.python.org/namespace">
79 Contents of subelements
80 </myns:subelement>
81 <sub2><![CDATA[contents of CDATA section]]></sub2>
82 &external_entity;
83 &skipped_entity;
84 \xb5
85 </root>
86 '''
87
88
89 # Produce UTF-8 output
90 class ParseTest(unittest.TestCase):
91 class Outputter:
92 def __init__(self):
93 self.out = []
94
95 def StartElementHandler(self, name, attrs):
96 self.out.append('Start element: ' + repr(name) + ' ' +
97 sortdict(attrs))
98
99 def EndElementHandler(self, name):
100 self.out.append('End element: ' + repr(name))
101
102 def CharacterDataHandler(self, data):
103 data = data.strip()
104 if data:
105 self.out.append('Character data: ' + repr(data))
106
107 def ProcessingInstructionHandler(self, target, data):
108 self.out.append('PI: ' + repr(target) + ' ' + repr(data))
109
110 def StartNamespaceDeclHandler(self, prefix, uri):
111 self.out.append('NS decl: ' + repr(prefix) + ' ' + repr(uri))
112
113 def EndNamespaceDeclHandler(self, prefix):
114 self.out.append('End of NS decl: ' + repr(prefix))
115
116 def StartCdataSectionHandler(self):
117 self.out.append('Start of CDATA section')
118
119 def EndCdataSectionHandler(self):
120 self.out.append('End of CDATA section')
121
122 def CommentHandler(self, text):
123 self.out.append('Comment: ' + repr(text))
124
125 def NotationDeclHandler(self, *args):
126 name, base, sysid, pubid = args
127 self.out.append('Notation declared: %s' %(args,))
128
129 def UnparsedEntityDeclHandler(self, *args):
130 entityName, base, systemId, publicId, notationName = args
131 self.out.append('Unparsed entity decl: %s' %(args,))
132
133 def NotStandaloneHandler(self):
134 self.out.append('Not standalone')
135 return 1
136
137 def ExternalEntityRefHandler(self, *args):
138 context, base, sysId, pubId = args
139 self.out.append('External entity ref: %s' %(args[1:],))
140 return 1
141
142 def StartDoctypeDeclHandler(self, *args):
143 self.out.append(('Start doctype', args))
144 return 1
145
146 def EndDoctypeDeclHandler(self):
147 self.out.append("End doctype")
148 return 1
149
150 def EntityDeclHandler(self, *args):
151 self.out.append(('Entity declaration', args))
152 return 1
153
154 def XmlDeclHandler(self, *args):
155 self.out.append(('XML declaration', args))
156 return 1
157
158 def ElementDeclHandler(self, *args):
159 self.out.append(('Element declaration', args))
160 return 1
161
162 def AttlistDeclHandler(self, *args):
163 self.out.append(('Attribute list declaration', args))
164 return 1
165
166 def SkippedEntityHandler(self, *args):
167 self.out.append(("Skipped entity", args))
168 return 1
169
170 def DefaultHandler(self, userData):
171 pass
172
173 def DefaultHandlerExpand(self, userData):
174 pass
175
176 handler_names = [
177 'StartElementHandler', 'EndElementHandler', 'CharacterDataHandler',
178 'ProcessingInstructionHandler', 'UnparsedEntityDeclHandler',
179 'NotationDeclHandler', 'StartNamespaceDeclHandler',
180 'EndNamespaceDeclHandler', 'CommentHandler',
181 'StartCdataSectionHandler', 'EndCdataSectionHandler', 'DefaultHandler',
182 'DefaultHandlerExpand', 'NotStandaloneHandler',
183 'ExternalEntityRefHandler', 'StartDoctypeDeclHandler',
184 'EndDoctypeDeclHandler', 'EntityDeclHandler', 'XmlDeclHandler',
185 'ElementDeclHandler', 'AttlistDeclHandler', 'SkippedEntityHandler',
186 ]
187
188 def _hookup_callbacks(self, parser, handler):
189 """
190 Set each of the callbacks defined on handler and named in
191 self.handler_names on the given parser.
192 """
193 for name in self.handler_names:
194 setattr(parser, name, getattr(handler, name))
195
196 def _verify_parse_output(self, operations):
197 expected_operations = [
198 ('XML declaration', ('1.0', 'iso-8859-1', 0)),
199 'PI: \'xml-stylesheet\' \'href="stylesheet.css"\'',
200 "Comment: ' comment data '",
201 "Not standalone",
202 ("Start doctype", ('quotations', 'quotations.dtd', None, 1)),
203 ('Element declaration', ('root', (2, 0, None, ()))),
204 ('Attribute list declaration', ('root', 'attr1', 'CDATA', None,
205 1)),
206 ('Attribute list declaration', ('root', 'attr2', 'CDATA', None,
207 0)),
208 "Notation declared: ('notation', None, 'notation.jpeg', None)",
209 ('Entity declaration', ('acirc', 0, '\xe2', None, None, None, None)),
210 ('Entity declaration', ('external_entity', 0, None, None,
211 'entity.file', None, None)),
212 "Unparsed entity decl: ('unparsed_entity', None, 'entity.file', None, 'notation')",
213 "Not standalone",
214 "End doctype",
215 "Start element: 'root' {'attr1': 'value1', 'attr2': 'value2\u1f40'}",
216 "NS decl: 'myns' 'http://www.python.org/namespace'",
217 "Start element: 'http://www.python.org/namespace!subelement' {}",
218 "Character data: 'Contents of subelements'",
219 "End element: 'http://www.python.org/namespace!subelement'",
220 "End of NS decl: 'myns'",
221 "Start element: 'sub2' {}",
222 'Start of CDATA section',
223 "Character data: 'contents of CDATA section'",
224 'End of CDATA section',
225 "End element: 'sub2'",
226 "External entity ref: (None, 'entity.file', None)",
227 ('Skipped entity', ('skipped_entity', 0)),
228 "Character data: '\xb5'",
229 "End element: 'root'",
230 ]
231 self.assertEqual(operations, expected_operations)
232
233 def test_parse_bytes(self):
234 out = self.Outputter()
235 parser = expat.ParserCreate(namespace_separator='!')
236 self._hookup_callbacks(parser, out)
237
238 parser.Parse(data, True)
239
240 operations = out.out
241 self._verify_parse_output(operations)
242 # Issue #6697.
243 self.assertRaises(AttributeError, getattr, parser, '\uD800')
244
245 def test_parse_str(self):
246 out = self.Outputter()
247 parser = expat.ParserCreate(namespace_separator='!')
248 self._hookup_callbacks(parser, out)
249
250 parser.Parse(data.decode('iso-8859-1'), True)
251
252 operations = out.out
253 self._verify_parse_output(operations)
254
255 def test_parse_file(self):
256 # Try parsing a file
257 out = self.Outputter()
258 parser = expat.ParserCreate(namespace_separator='!')
259 self._hookup_callbacks(parser, out)
260 file = BytesIO(data)
261
262 parser.ParseFile(file)
263
264 operations = out.out
265 self._verify_parse_output(operations)
266
267 def test_parse_again(self):
268 parser = expat.ParserCreate()
269 file = BytesIO(data)
270 parser.ParseFile(file)
271 # Issue 6676: ensure a meaningful exception is raised when attempting
272 # to parse more than one XML document per xmlparser instance,
273 # a limitation of the Expat library.
274 with self.assertRaises(expat.error) as cm:
275 parser.ParseFile(file)
276 self.assertEqual(expat.ErrorString(cm.exception.code),
277 expat.errors.XML_ERROR_FINISHED)
278
279 @support.subTests('encoding', [
280 'utf-8', 'utf-16', 'utf-16be', 'utf-16le',
281 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5',
282 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10',
283 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16',
284 'cp437', 'cp720', 'cp737', 'cp775', 'cp850', 'cp852',
285 'cp855', 'cp856', 'cp857', 'cp858', 'cp860', 'cp861', 'cp862',
286 'cp863', 'cp865', 'cp866', 'cp869', 'cp874', 'cp1006', 'cp1125',
287 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
288 'cp1256', 'cp1257', 'cp1258',
289 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2',
290 'mac-roman', 'mac-turkish',
291 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154',
292 ])
293 def test_supported_encodings(self, encoding):
294 out = self.Outputter()
295 parser = expat.ParserCreate()
296 self._hookup_callbacks(parser, out)
297 c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0]
298 data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
299 f'<root>{c}</root>').encode(encoding)
300 parser.Parse(data, True)
301 self.assertEqual(out.out, [
302 ('XML declaration', ('1.0', encoding, -1)),
303 "Start element: 'root' {}",
304 f'Character data: {c!r}',
305 "End element: 'root'",
306 ])
307
308 @support.subTests('encoding', [
309 'UTF-8', 'utf-8', 'utf-16', 'utf-16le', 'utf-16be',
310 'koi8-u', 'cp1125', 'cp1251', 'iso8859-5', 'mac-cyrillic',
311 ])
312 def test_supported_encodings2(self, encoding):
313 out = self.Outputter()
314 parser = expat.ParserCreate()
315 self._hookup_callbacks(parser, out)
316 data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
317 '<!-- коментар -->'
318 '<корінь атрибут="значення">зміст</корінь>').encode(encoding)
319 parser.Parse(data, True)
320 self.assertEqual(out.out, [
321 ('XML declaration', ('1.0', encoding, -1)),
322 "Comment: ' коментар '",
323 "Start element: 'корінь' {'атрибут': 'значення'}",
324 "Character data: 'зміст'",
325 "End element: 'корінь'",
326 ])
327
328 @support.subTests('encoding', [
329 'UTF-7',
330 "Big5-HKSCS", "Big5",
331 "cp932", "cp949", "cp950",
332 "EUC_JIS-2004", "EUC_JISX0213", "EUC-JP", "EUC-KR",
333 "GB18030", "GB2312", "GBK",
334 "ISO-2022-KR",
335 "johab",
336 "Shift_JIS", "Shift_JIS-2004", "Shift_JISX0213",
337 ])
338 def test_unsupported_encodings(self, encoding):
339 parser = expat.ParserCreate()
340 data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
341 '<root></root>').encode(encoding)
342 with self.assertRaises(ValueError):
343 parser.Parse(data, True)
344
345 parser = expat.ParserCreate()
346 data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
347 '<root></root>').encode()
348 with self.assertRaises(ValueError):
349 parser.Parse(data, True)
350
351 @support.subTests('encoding', [
352 'cp037', 'cp273', 'cp424', 'cp500', 'cp864', 'cp875',
353 'cp1026', 'cp1140',
354 'mac_arabic', 'mac_farsi',
355 ])
356 def test_incompatible_encodings(self, encoding):
357 parser = expat.ParserCreate()
358 data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
359 '<root></root>').encode(encoding)
360 with self.assertRaises(expat.ExpatError):
361 parser.Parse(data, True)
362
363 parser = expat.ParserCreate()
364 data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
365 '<root></root>').encode()
366 with self.assertRaisesRegex(expat.ExpatError, 'unknown encoding'):
367 parser.Parse(data, True)
368
369 @support.subTests('encoding', [
370 'hex_codec', 'rot_13',
371 ])
372 def test_non_text_encodings(self, encoding):
373 parser = expat.ParserCreate()
374 data = (f'<?xml version="1.0" encoding="{encoding}"?>\n'
375 '<root></root>').encode()
376 with self.assertRaises(LookupError):
377 parser.Parse(data, True)
378
379 def test_undefined_encoding(self):
380 parser = expat.ParserCreate()
381 data = b'<?xml version="1.0" encoding="undefined"?>\n<root></root>'
382 with self.assertRaises(UnicodeError):
383 parser.Parse(data, True)
384
385 def test_unknown_encoding(self):
386 parser = expat.ParserCreate()
387 data = b'<?xml version="1.0" encoding="xyz"?>\n<root></root>'
388 with self.assertRaises(LookupError):
389 parser.Parse(data, True)
390
391
392 class NamespaceSeparatorTest(unittest.TestCase):
393 def test_legal(self):
394 # Tests that make sure we get errors when the namespace_separator value
395 # is illegal, and that we don't for good values:
396 expat.ParserCreate()
397 expat.ParserCreate(namespace_separator=None)
398 expat.ParserCreate(namespace_separator=' ')
399
400 def test_illegal(self):
401 with self.assertRaisesRegex(TypeError,
402 r"ParserCreate\(\) argument (2|'namespace_separator') "
403 r"must be str or None, not int"):
404 expat.ParserCreate(namespace_separator=42)
405
406 try:
407 expat.ParserCreate(namespace_separator='too long')
408 self.fail()
409 except ValueError as e:
410 self.assertEqual(str(e),
411 'namespace_separator must be at most one character, omitted, or None')
412
413 def test_zero_length(self):
414 # ParserCreate() needs to accept a namespace_separator of zero length
415 # to satisfy the requirements of RDF applications that are required
416 # to simply glue together the namespace URI and the localname. Though
417 # considered a wart of the RDF specifications, it needs to be supported.
418 #
419 # See XML-SIG mailing list thread starting with
420 # http://mail.python.org/pipermail/xml-sig/2001-April/005202.html
421 #
422 expat.ParserCreate(namespace_separator='') # too short
423
424
425 class InterningTest(unittest.TestCase):
426 def test(self):
427 # Test the interning machinery.
428 p = expat.ParserCreate()
429 L = []
430 def collector(name, *args):
431 L.append(name)
432 p.StartElementHandler = collector
433 p.EndElementHandler = collector
434 p.Parse(b"<e> <e/> <e></e> </e>", True)
435 tag = L[0]
436 self.assertEqual(len(L), 6)
437 for entry in L:
438 # L should have the same string repeated over and over.
439 self.assertTrue(tag is entry)
440
441 def test_issue9402(self):
442 # create an ExternalEntityParserCreate with buffer text
443 class ExternalOutputter:
444 def __init__(self, parser):
445 self.parser = parser
446 self.parser_result = None
447
448 def ExternalEntityRefHandler(self, context, base, sysId, pubId):
449 external_parser = self.parser.ExternalEntityParserCreate("")
450 self.parser_result = external_parser.Parse(b"", True)
451 return 1
452
453 parser = expat.ParserCreate(namespace_separator='!')
454 parser.buffer_text = 1
455 out = ExternalOutputter(parser)
456 parser.ExternalEntityRefHandler = out.ExternalEntityRefHandler
457 parser.Parse(data, True)
458 self.assertEqual(out.parser_result, 1)
459
460
461 class BufferTextTest(unittest.TestCase):
462 def setUp(self):
463 self.stuff = []
464 self.parser = expat.ParserCreate()
465 self.parser.buffer_text = 1
466 self.parser.CharacterDataHandler = self.CharacterDataHandler
467
468 def check(self, expected, label):
469 self.assertEqual(self.stuff, expected,
470 "%s\nstuff = %r\nexpected = %r"
471 % (label, self.stuff, map(str, expected)))
472
473 def CharacterDataHandler(self, text):
474 self.stuff.append(text)
475
476 def StartElementHandler(self, name, attrs):
477 self.stuff.append("<%s>" % name)
478 bt = attrs.get("buffer-text")
479 if bt == "yes":
480 self.parser.buffer_text = 1
481 elif bt == "no":
482 self.parser.buffer_text = 0
483
484 def EndElementHandler(self, name):
485 self.stuff.append("</%s>" % name)
486
487 def CommentHandler(self, data):
488 self.stuff.append("<!--%s-->" % data)
489
490 def setHandlers(self, handlers=[]):
491 for name in handlers:
492 setattr(self.parser, name, getattr(self, name))
493
494 def test_default_to_disabled(self):
495 parser = expat.ParserCreate()
496 self.assertFalse(parser.buffer_text)
497
498 def test_buffering_enabled(self):
499 # Make sure buffering is turned on
500 self.assertTrue(self.parser.buffer_text)
501 self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True)
502 self.assertEqual(self.stuff, ['123'],
503 "buffered text not properly collapsed")
504
505 def test1(self):
506 # XXX This test exposes more detail of Expat's text chunking than we
507 # XXX like, but it tests what we need to concisely.
508 self.setHandlers(["StartElementHandler"])
509 self.parser.Parse(b"<a>1<b buffer-text='no'/>2\n3<c buffer-text='yes'/>4\n5</a>", True)
510 self.assertEqual(self.stuff,
511 ["<a>", "1", "<b>", "2", "\n", "3", "<c>", "4\n5"],
512 "buffering control not reacting as expected")
513
514 def test2(self):
515 self.parser.Parse(b"<a>1<b/>&lt;2&gt;<c/>&#32;\n&#x20;3</a>", True)
516 self.assertEqual(self.stuff, ["1<2> \n 3"],
517 "buffered text not properly collapsed")
518
519 def test3(self):
520 self.setHandlers(["StartElementHandler"])
521 self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True)
522 self.assertEqual(self.stuff, ["<a>", "1", "<b>", "2", "<c>", "3"],
523 "buffered text not properly split")
524
525 def test4(self):
526 self.setHandlers(["StartElementHandler", "EndElementHandler"])
527 self.parser.CharacterDataHandler = None
528 self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True)
529 self.assertEqual(self.stuff,
530 ["<a>", "<b>", "</b>", "<c>", "</c>", "</a>"])
531
532 def test5(self):
533 self.setHandlers(["StartElementHandler", "EndElementHandler"])
534 self.parser.Parse(b"<a>1<b></b>2<c/>3</a>", True)
535 self.assertEqual(self.stuff,
536 ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3", "</a>"])
537
538 def test6(self):
539 self.setHandlers(["CommentHandler", "EndElementHandler",
540 "StartElementHandler"])
541 self.parser.Parse(b"<a>1<b/>2<c></c>345</a> ", True)
542 self.assertEqual(self.stuff,
543 ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "345", "</a>"],
544 "buffered text not properly split")
545
546 def test7(self):
547 self.setHandlers(["CommentHandler", "EndElementHandler",
548 "StartElementHandler"])
549 self.parser.Parse(b"<a>1<b/>2<c></c>3<!--abc-->4<!--def-->5</a> ", True)
550 self.assertEqual(self.stuff,
551 ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3",
552 "<!--abc-->", "4", "<!--def-->", "5", "</a>"],
553 "buffered text not properly split")
554
555
556 # Test handling of exception from callback:
557 class HandlerExceptionTest(unittest.TestCase):
558 def StartElementHandler(self, name, attrs):
559 raise RuntimeError(f'StartElementHandler: <{name}>')
560
561 def check_traceback_entry(self, entry, filename, funcname):
562 self.assertEqual(os.path.basename(entry.filename), filename)
563 self.assertEqual(entry.name, funcname)
564
565 @support.cpython_only
566 def test_exception(self):
567 # gh-66652: test _PyTraceback_Add() used by pyexpat.c to inject frames
568
569 # Change the current directory to the Python source code directory
570 # if it is available.
571 src_dir = sysconfig.get_config_var('abs_builddir')
572 if src_dir:
573 have_source = os.path.isdir(src_dir)
574 else:
575 have_source = False
576 if have_source:
577 with os_helper.change_cwd(src_dir):
578 self._test_exception(have_source)
579 else:
580 self._test_exception(have_source)
581
582 def _test_exception(self, have_source):
583 # Use path relative to the current directory which should be the Python
584 # source code directory (if it is available).
585 PYEXPAT_C = os.path.join('Modules', 'pyexpat.c')
586
587 parser = expat.ParserCreate()
588 parser.StartElementHandler = self.StartElementHandler
589 try:
590 parser.Parse(b"<a><b><c/></b></a>", True)
591
592 self.fail("the parser did not raise RuntimeError")
593 except RuntimeError as exc:
594 self.assertEqual(exc.args[0], 'StartElementHandler: <a>', exc)
595 entries = traceback.extract_tb(exc.__traceback__)
596
597 self.assertEqual(len(entries), 3, entries)
598 self.check_traceback_entry(entries[0],
599 "test_pyexpat.py", "_test_exception")
600 self.check_traceback_entry(entries[1],
601 os.path.basename(PYEXPAT_C),
602 "StartElement")
603 self.check_traceback_entry(entries[2],
604 "test_pyexpat.py", "StartElementHandler")
605
606 # Check that the traceback contains the relevant line in
607 # Modules/pyexpat.c. Skip the test if Modules/pyexpat.c is not
608 # available.
609 if have_source and os.path.exists(PYEXPAT_C):
610 self.assertIn('call_with_frame("StartElement"',
611 entries[1].line)
612
613
614 # Test Current* members:
615 class PositionTest(unittest.TestCase):
616 def StartElementHandler(self, name, attrs):
617 self.check_pos('s')
618
619 def EndElementHandler(self, name):
620 self.check_pos('e')
621
622 def check_pos(self, event):
623 pos = (event,
624 self.parser.CurrentByteIndex,
625 self.parser.CurrentLineNumber,
626 self.parser.CurrentColumnNumber)
627 self.assertTrue(self.upto < len(self.expected_list),
628 'too many parser events')
629 expected = self.expected_list[self.upto]
630 self.assertEqual(pos, expected,
631 'Expected position %s, got position %s' %(pos, expected))
632 self.upto += 1
633
634 def test(self):
635 self.parser = expat.ParserCreate()
636 self.parser.StartElementHandler = self.StartElementHandler
637 self.parser.EndElementHandler = self.EndElementHandler
638 self.upto = 0
639 self.expected_list = [('s', 0, 1, 0), ('s', 5, 2, 1), ('s', 11, 3, 2),
640 ('e', 15, 3, 6), ('e', 17, 4, 1), ('e', 22, 5, 0)]
641
642 xml = b'<a>\n <b>\n <c/>\n </b>\n</a>'
643 self.parser.Parse(xml, True)
644
645
646 class sf1296433Test(unittest.TestCase):
647 def test_parse_only_xml_data(self):
648 # https://bugs.python.org/issue1296433
649 #
650 xml = "<?xml version='1.0' encoding='iso8859'?><s>%s</s>" % ('a' * 1025)
651 # this one doesn't crash
652 #xml = "<?xml version='1.0'?><s>%s</s>" % ('a' * 10000)
653
654 class SpecificException(Exception):
655 pass
656
657 def handler(text):
658 raise SpecificException
659
660 parser = expat.ParserCreate()
661 parser.CharacterDataHandler = handler
662
663 self.assertRaises(SpecificException, parser.Parse, xml.encode('iso8859'))
664
665 class ChardataBufferTest(unittest.TestCase):
666 """
667 test setting of chardata buffer size
668 """
669
670 def test_1025_bytes(self):
671 self.assertEqual(self.small_buffer_test(1025), 2)
672
673 def test_1000_bytes(self):
674 self.assertEqual(self.small_buffer_test(1000), 1)
675
676 def test_wrong_size(self):
677 parser = expat.ParserCreate()
678 parser.buffer_text = 1
679 with self.assertRaises(ValueError):
680 parser.buffer_size = -1
681 with self.assertRaises(ValueError):
682 parser.buffer_size = 0
683 with self.assertRaises((ValueError, OverflowError)):
684 parser.buffer_size = sys.maxsize + 1
685 with self.assertRaises(TypeError):
686 parser.buffer_size = 512.0
687
688 def test_unchanged_size(self):
689 xml1 = b"<?xml version='1.0' encoding='iso8859'?><s>" + b'a' * 512
690 xml2 = b'a'*512 + b'</s>'
691 parser = expat.ParserCreate()
692 parser.CharacterDataHandler = self.counting_handler
693 parser.buffer_size = 512
694 parser.buffer_text = 1
695
696 # Feed 512 bytes of character data: the handler should be called
697 # once.
698 self.n = 0
699 parser.Parse(xml1)
700 self.assertEqual(self.n, 1)
701
702 # Reassign to buffer_size, but assign the same size.
703 parser.buffer_size = parser.buffer_size
704 self.assertEqual(self.n, 1)
705
706 # Try parsing rest of the document
707 parser.Parse(xml2)
708 self.assertEqual(self.n, 2)
709
710
711 def test_disabling_buffer(self):
712 xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>" + b'a' * 512
713 xml2 = b'b' * 1024
714 xml3 = b'c' * 1024 + b'</a>';
715 parser = expat.ParserCreate()
716 parser.CharacterDataHandler = self.counting_handler
717 parser.buffer_text = 1
718 parser.buffer_size = 1024
719 self.assertEqual(parser.buffer_size, 1024)
720
721 # Parse one chunk of XML
722 self.n = 0
723 parser.Parse(xml1, False)
724 self.assertEqual(parser.buffer_size, 1024)
725 self.assertEqual(self.n, 1)
726
727 # Turn off buffering and parse the next chunk.
728 parser.buffer_text = 0
729 self.assertFalse(parser.buffer_text)
730 self.assertEqual(parser.buffer_size, 1024)
731 for i in range(10):
732 parser.Parse(xml2, False)
733 self.assertEqual(self.n, 11)
734
735 parser.buffer_text = 1
736 self.assertTrue(parser.buffer_text)
737 self.assertEqual(parser.buffer_size, 1024)
738 parser.Parse(xml3, True)
739 self.assertEqual(self.n, 12)
740
741 def counting_handler(self, text):
742 self.n += 1
743
744 def small_buffer_test(self, buffer_len):
745 xml = b"<?xml version='1.0' encoding='iso8859'?><s>" + b'a' * buffer_len + b'</s>'
746 parser = expat.ParserCreate()
747 parser.CharacterDataHandler = self.counting_handler
748 parser.buffer_size = 1024
749 parser.buffer_text = 1
750
751 self.n = 0
752 parser.Parse(xml)
753 return self.n
754
755 def test_change_size_1(self):
756 xml1 = b"<?xml version='1.0' encoding='iso8859'?><a><s>" + b'a' * 1024
757 xml2 = b'aaa</s><s>' + b'a' * 1025 + b'</s></a>'
758 parser = expat.ParserCreate()
759 parser.CharacterDataHandler = self.counting_handler
760 parser.buffer_text = 1
761 parser.buffer_size = 1024
762 self.assertEqual(parser.buffer_size, 1024)
763
764 self.n = 0
765 parser.Parse(xml1, False)
766 parser.buffer_size *= 2
767 self.assertEqual(parser.buffer_size, 2048)
768 parser.Parse(xml2, True)
769 self.assertEqual(self.n, 2)
770
771 def test_change_size_2(self):
772 xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>a<s>" + b'a' * 1023
773 xml2 = b'aaa</s><s>' + b'a' * 1025 + b'</s></a>'
774 parser = expat.ParserCreate()
775 parser.CharacterDataHandler = self.counting_handler
776 parser.buffer_text = 1
777 parser.buffer_size = 2048
778 self.assertEqual(parser.buffer_size, 2048)
779
780 self.n=0
781 parser.Parse(xml1, False)
782 parser.buffer_size = parser.buffer_size // 2
783 self.assertEqual(parser.buffer_size, 1024)
784 parser.Parse(xml2, True)
785 self.assertEqual(self.n, 4)
786
787 @support.requires_resource('cpu')
788 @support.requires_resource('walltime')
789 @support.bigmemtest(size=2**31, memuse=4, dry_run=False)
790 def test_large_character_data_does_not_crash(self):
791 # See https://github.com/python/cpython/issues/148441
792 parser = expat.ParserCreate()
793 parser.buffer_text = True
794 parser.buffer_size = 2**31 - 1 # INT_MAX
795 N = 2049 * (1 << 20) - 3 # Character data greater than INT_MAX
796 self.assertGreater(N, parser.buffer_size)
797 parser.CharacterDataHandler = lambda text: None
798 xml_data = b"<r>" + b"A" * N + b"</r>"
799 self.assertEqual(parser.Parse(xml_data, True), 1)
800
801 class ElementDeclHandlerTest(unittest.TestCase):
802 def test_trigger_leak(self):
803 # Unfixed, this test would leak the memory of the so-called
804 # "content model" in function ``my_ElementDeclHandler`` of pyexpat.
805 # See https://github.com/python/cpython/issues/140593.
806 data = textwrap.dedent('''\
807 <!DOCTYPE quotations SYSTEM "quotations.dtd" [
808 <!ELEMENT root ANY>
809 ]>
810 <root/>
811 ''').encode('UTF-8')
812
813 parser = expat.ParserCreate()
814 parser.NotStandaloneHandler = lambda: 1.234 # arbitrary float
815 parser.ElementDeclHandler = lambda _1, _2: None
816 self.assertRaises(TypeError, parser.Parse, data, True)
817
818 @support.skip_if_unlimited_stack_size
819 @support.skip_emscripten_stack_overflow()
820 @support.skip_wasi_stack_overflow()
821 def test_deeply_nested_content_model(self):
822 # This should raise a RecursionError and not crash.
823 # See https://github.com/python/cpython/issues/145986.
•824 N = 800_000
825 data = (
826 b'<!DOCTYPE root [\n<!ELEMENT root '
827 + b'(a, ' * N + b'a' + b')' * N
828 + b'>\n]>\n<root/>\n'
829 )
830
831 parser = expat.ParserCreate()
832 parser.ElementDeclHandler = lambda _1, _2: None
833 with support.infinite_recursion():
834 with self.assertRaises(RecursionError):
835 parser.Parse(data)
836
837 class MalformedInputTest(unittest.TestCase):
838 def test1(self):
839 xml = b"\0\r\n"
840 parser = expat.ParserCreate()
841 try:
842 parser.Parse(xml, True)
843 self.fail()
844 except expat.ExpatError as e:
845 self.assertEqual(str(e), 'unclosed token: line 2, column 0')
846
847 def test2(self):
848 # \xc2\x85 is UTF-8 encoded U+0085 (NEXT LINE)
849 xml = b"<?xml version\xc2\x85='1.0'?>\r\n"
850 parser = expat.ParserCreate()
851 err_pattern = r'XML declaration not well-formed: line 1, column \d+'
852 with self.assertRaisesRegex(expat.ExpatError, err_pattern):
853 parser.Parse(xml, True)
854
855 class ErrorMessageTest(unittest.TestCase):
856 def test_codes(self):
857 # verify mapping of errors.codes and errors.messages
858 self.assertEqual(errors.XML_ERROR_SYNTAX,
859 errors.messages[errors.codes[errors.XML_ERROR_SYNTAX]])
860
861 def test_expaterror(self):
862 xml = b'<'
863 parser = expat.ParserCreate()
864 try:
865 parser.Parse(xml, True)
866 self.fail()
867 except expat.ExpatError as e:
868 self.assertEqual(e.code,
869 errors.codes[errors.XML_ERROR_UNCLOSED_TOKEN])
870
871
872 class ForeignDTDTests(unittest.TestCase):
873 """
874 Tests for the UseForeignDTD method of expat parser objects.
875 """
876 def test_use_foreign_dtd(self):
877 """
878 If UseForeignDTD is passed True and a document without an external
879 entity reference is parsed, ExternalEntityRefHandler is first called
880 with None for the public and system ids.
881 """
882 handler_call_args = []
883 def resolve_entity(context, base, system_id, public_id):
884 handler_call_args.append((public_id, system_id))
885 return 1
886
887 parser = expat.ParserCreate()
888 parser.UseForeignDTD(True)
889 parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS)
890 parser.ExternalEntityRefHandler = resolve_entity
891 parser.Parse(b"<?xml version='1.0'?><element/>")
892 self.assertEqual(handler_call_args, [(None, None)])
893
894 # test UseForeignDTD() is equal to UseForeignDTD(True)
895 handler_call_args[:] = []
896
897 parser = expat.ParserCreate()
898 parser.UseForeignDTD()
899 parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS)
900 parser.ExternalEntityRefHandler = resolve_entity
901 parser.Parse(b"<?xml version='1.0'?><element/>")
902 self.assertEqual(handler_call_args, [(None, None)])
903
904 def test_ignore_use_foreign_dtd(self):
905 """
906 If UseForeignDTD is passed True and a document with an external
907 entity reference is parsed, ExternalEntityRefHandler is called with
908 the public and system ids from the document.
909 """
910 handler_call_args = []
911 def resolve_entity(context, base, system_id, public_id):
912 handler_call_args.append((public_id, system_id))
913 return 1
914
915 parser = expat.ParserCreate()
916 parser.UseForeignDTD(True)
917 parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS)
918 parser.ExternalEntityRefHandler = resolve_entity
919 parser.Parse(
920 b"<?xml version='1.0'?><!DOCTYPE foo PUBLIC 'bar' 'baz'><element/>")
921 self.assertEqual(handler_call_args, [("bar", "baz")])
922
923
924 class ParentParserLifetimeTest(unittest.TestCase):
925 """
926 Subparsers make use of their parent XML_Parser inside of Expat.
927 As a result, parent parsers need to outlive subparsers.
928
929 See https://github.com/python/cpython/issues/139400.
930 """
931
932 def test_parent_parser_outlives_its_subparsers__single(self):
933 parser = expat.ParserCreate()
934 subparser = parser.ExternalEntityParserCreate(None)
935
936 # Now try to cause garbage collection of the parent parser
937 # while it's still being referenced by a related subparser.
938 del parser
939
940 def test_parent_parser_outlives_its_subparsers__multiple(self):
941 parser = expat.ParserCreate()
942 subparser_one = parser.ExternalEntityParserCreate(None)
943 subparser_two = parser.ExternalEntityParserCreate(None)
944
945 # Now try to cause garbage collection of the parent parser
946 # while it's still being referenced by a related subparser.
947 del parser
948
949 def test_parent_parser_outlives_its_subparsers__chain(self):
950 parser = expat.ParserCreate()
951 subparser = parser.ExternalEntityParserCreate(None)
952 subsubparser = subparser.ExternalEntityParserCreate(None)
953
954 # Now try to cause garbage collection of the parent parsers
955 # while they are still being referenced by a related subparser.
956 del parser
957 del subparser
958
959
960 class ExternalEntityParserCreateErrorTest(unittest.TestCase):
961 """ExternalEntityParserCreate error paths should not crash or leak
962 refcounts on the parent parser.
963
964 See https://github.com/python/cpython/issues/144984.
965 """
966
967 @classmethod
968 def setUpClass(cls):
969 cls.testcapi = import_helper.import_module('_testcapi')
970
971 @unittest.skipIf(support.Py_TRACE_REFS,
972 'Py_TRACE_REFS conflicts with testcapi.set_nomemory')
973 def test_error_path_no_crash(self):
974 # When an allocation inside ExternalEntityParserCreate fails,
975 # the partially-initialized subparser is deallocated. This
976 # must not dereference NULL handlers or double-decrement the
977 # parent parser's refcount.
978 parser = expat.ParserCreate()
979 parser.buffer_text = True
980 rc_before = sys.getrefcount(parser)
981
982 # We avoid self.assertRaises(MemoryError) here because the
983 # context manager itself needs memory allocations that fail
984 # while the nomemory hook is active.
985 self.testcapi.set_nomemory(1, 10)
986 raised = False
987 try:
988 parser.ExternalEntityParserCreate(None)
989 except MemoryError:
990 raised = True
991 finally:
992 self.testcapi.remove_mem_hooks()
993 self.assertTrue(raised, "MemoryError not raised")
994
995 rc_after = sys.getrefcount(parser)
996 self.assertEqual(rc_after, rc_before)
997
998
999 class ReparseDeferralTest(unittest.TestCase):
1000 def test_getter_setter_round_trip(self):
1001 parser = expat.ParserCreate()
1002 enabled = (expat.version_info >= (2, 6, 0))
1003
1004 self.assertIs(parser.GetReparseDeferralEnabled(), enabled)
1005 parser.SetReparseDeferralEnabled(False)
1006 self.assertIs(parser.GetReparseDeferralEnabled(), False)
1007 parser.SetReparseDeferralEnabled(True)
1008 self.assertIs(parser.GetReparseDeferralEnabled(), enabled)
1009
1010 def test_reparse_deferral_enabled(self):
1011 if expat.version_info < (2, 6, 0):
1012 self.skipTest(f'Expat {expat.version_info} does not '
1013 'support reparse deferral')
1014
1015 started = []
1016
1017 def start_element(name, _):
1018 started.append(name)
1019
1020 parser = expat.ParserCreate()
1021 parser.StartElementHandler = start_element
1022 self.assertTrue(parser.GetReparseDeferralEnabled())
1023
1024 for chunk in (b'<doc', b'/>'):
1025 parser.Parse(chunk, False)
1026
1027 # The key test: Have handlers already fired? Expecting: no.
1028 self.assertEqual(started, [])
1029
1030 parser.Parse(b'', True)
1031
1032 self.assertEqual(started, ['doc'])
1033
1034 def test_reparse_deferral_disabled(self):
1035 started = []
1036
1037 def start_element(name, _):
1038 started.append(name)
1039
1040 parser = expat.ParserCreate()
1041 parser.StartElementHandler = start_element
1042 if expat.version_info >= (2, 6, 0):
1043 parser.SetReparseDeferralEnabled(False)
1044 self.assertFalse(parser.GetReparseDeferralEnabled())
1045
1046 for chunk in (b'<doc', b'/>'):
1047 parser.Parse(chunk, False)
1048
1049 # The key test: Have handlers already fired? Expecting: yes.
1050 self.assertEqual(started, ['doc'])
1051
1052
1053 class AttackProtectionTestBase(abc.ABC):
1054 """
1055 Base class for testing protections against XML payloads with
1056 disproportionate amplification.
1057
1058 The protections being tested should detect and prevent attacks
1059 that leverage disproportionate amplification from small inputs.
1060 """
1061
1062 @staticmethod
1063 def exponential_expansion_payload(*, nrows, ncols, text='.'):
1064 """Create a billion laughs attack payload.
1065
1066 Be careful: the number of total items is pow(n, k), thereby
1067 requiring at least pow(ncols, nrows) * sizeof(text) memory!
1068 """
1069 template = textwrap.dedent(f"""\
1070 <?xml version="1.0"?>
1071 <!DOCTYPE doc [
1072 <!ENTITY row0 "{text}">
1073 <!ELEMENT doc (#PCDATA)>
1074 {{body}}
1075 ]>
1076 <doc>&row{nrows};</doc>
1077 """).rstrip()
1078
1079 body = '\n'.join(
1080 f'<!ENTITY row{i + 1} "{f"&row{i};" * ncols}">'
1081 for i in range(nrows)
1082 )
1083 body = textwrap.indent(body, ' ' * 4)
1084 return template.format(body=body)
1085
1086 def test_payload_generation(self):
1087 # self-test for exponential_expansion_payload()
1088 payload = self.exponential_expansion_payload(nrows=2, ncols=3)
1089 self.assertEqual(payload, textwrap.dedent("""\
1090 <?xml version="1.0"?>
1091 <!DOCTYPE doc [
1092 <!ENTITY row0 ".">
1093 <!ELEMENT doc (#PCDATA)>
1094 <!ENTITY row1 "&row0;&row0;&row0;">
1095 <!ENTITY row2 "&row1;&row1;&row1;">
1096 ]>
1097 <doc>&row2;</doc>
1098 """).rstrip())
1099
1100 def assert_root_parser_failure(self, func, /, *args, **kwargs):
1101 """Check that func(*args, **kwargs) is invalid for a sub-parser."""
1102 msg = "parser must be a root parser"
1103 self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs)
1104
1105 @abc.abstractmethod
1106 def assert_rejected(self, func, /, *args, **kwargs):
1107 """Assert that func(*args, **kwargs) triggers the attack protection.
1108
1109 Note: this method must ensure that the attack protection being tested
1110 is the one that is actually triggered at runtime, e.g., by matching
1111 the exact error message.
1112 """
1113
1114 @abc.abstractmethod
1115 def set_activation_threshold(self, parser, threshold):
1116 """Set the activation threshold for the tested protection."""
1117
1118 @abc.abstractmethod
1119 def set_maximum_amplification(self, parser, max_factor):
1120 """Set the maximum amplification factor for the tested protection."""
1121
1122 @abc.abstractmethod
1123 def test_set_activation_threshold__threshold_reached(self):
1124 """Test when the activation threshold is exceeded."""
1125
1126 @abc.abstractmethod
1127 def test_set_activation_threshold__threshold_not_reached(self):
1128 """Test when the activation threshold is not exceeded."""
1129
1130 def test_set_activation_threshold__invalid_threshold_type(self):
1131 parser = expat.ParserCreate()
1132 setter = functools.partial(self.set_activation_threshold, parser)
1133
1134 self.assertRaises(TypeError, setter, 1.0)
1135 self.assertRaises(TypeError, setter, -1.5)
1136 self.assertRaises(ValueError, setter, -5)
1137
1138 def test_set_activation_threshold__invalid_threshold_range(self):
1139 _testcapi = import_helper.import_module("_testcapi")
1140 parser = expat.ParserCreate()
1141 setter = functools.partial(self.set_activation_threshold, parser)
1142
1143 self.assertRaises(OverflowError, setter, _testcapi.ULLONG_MAX + 1)
1144
1145 def test_set_activation_threshold__fail_for_subparser(self):
1146 parser = expat.ParserCreate()
1147 subparser = parser.ExternalEntityParserCreate(None)
1148 setter = functools.partial(self.set_activation_threshold, subparser)
1149 self.assert_root_parser_failure(setter, 12345)
1150
1151 @abc.abstractmethod
1152 def test_set_maximum_amplification__amplification_exceeded(self):
1153 """Test when the amplification factor is exceeded."""
1154
1155 @abc.abstractmethod
1156 def test_set_maximum_amplification__amplification_not_exceeded(self):
1157 """Test when the amplification factor is not exceeded."""
1158
1159 def test_set_maximum_amplification__infinity(self):
1160 inf = float('inf') # an 'inf' threshold is allowed by Expat
1161 parser = expat.ParserCreate()
1162 self.assertIsNone(self.set_maximum_amplification(parser, inf))
1163
1164 def test_set_maximum_amplification__invalid_max_factor_type(self):
1165 parser = expat.ParserCreate()
1166 setter = functools.partial(self.set_maximum_amplification, parser)
1167
1168 self.assertRaises(TypeError, setter, None)
1169 self.assertRaises(TypeError, setter, 'abc')
1170
1171 def test_set_maximum_amplification__invalid_max_factor_range(self):
1172 parser = expat.ParserCreate()
1173 setter = functools.partial(self.set_maximum_amplification, parser)
1174
1175 msg = re.escape("'max_factor' must be at least 1.0")
1176 self.assertRaisesRegex(expat.ExpatError, msg, setter, float('nan'))
1177 self.assertRaisesRegex(expat.ExpatError, msg, setter, 0.99)
1178
1179 def test_set_maximum_amplification__fail_for_subparser(self):
1180 parser = expat.ParserCreate()
1181 subparser = parser.ExternalEntityParserCreate(None)
1182 setter = functools.partial(self.set_maximum_amplification, subparser)
1183 self.assert_root_parser_failure(setter, 123.45)
1184
1185
1186 @unittest.skipIf(expat.version_info < (2, 4, 0), "requires Expat >= 2.4.0")
1187 class ExpansionProtectionTest(AttackProtectionTestBase, unittest.TestCase):
1188
1189 def assert_rejected(self, func, /, *args, **kwargs):
1190 """Check that func(*args, **kwargs) hits the allocation limit."""
1191 msg = (
1192 r"limit on input amplification factor \(from DTD and entities\) "
1193 r"breached: line \d+, column \d+"
1194 )
1195 self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs)
1196
1197 def set_activation_threshold(self, parser, threshold):
1198 return parser.SetBillionLaughsAttackProtectionActivationThreshold(threshold)
1199
1200 def set_maximum_amplification(self, parser, max_factor):
1201 return parser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor)
1202
1203 def test_set_activation_threshold__threshold_reached(self):
1204 parser = expat.ParserCreate()
1205 # Choose a threshold expected to be always reached.
1206 self.set_activation_threshold(parser, 3)
1207 # Check that the threshold is reached by choosing a small factor
1208 # and a payload whose peak amplification factor exceeds it.
1209 self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1210 payload = self.exponential_expansion_payload(ncols=10, nrows=4)
1211 self.assert_rejected(parser.Parse, payload, True)
1212
1213 def test_set_activation_threshold__threshold_not_reached(self):
1214 parser = expat.ParserCreate()
1215 # Choose a threshold expected to be never reached.
1216 self.set_activation_threshold(parser, pow(10, 5))
1217 # Check that the threshold is reached by choosing a small factor
1218 # and a payload whose peak amplification factor exceeds it.
1219 self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1220 payload = self.exponential_expansion_payload(ncols=10, nrows=4)
1221 self.assertIsNotNone(parser.Parse(payload, True))
1222
1223 def test_set_maximum_amplification__amplification_exceeded(self):
1224 parser = expat.ParserCreate()
1225 # Unconditionally enable maximum activation factor.
1226 self.set_activation_threshold(parser, 0)
1227 # Choose a max amplification factor expected to always be exceeded.
1228 self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1229 # Craft a payload for which the peak amplification factor is > 1.0.
1230 payload = self.exponential_expansion_payload(ncols=1, nrows=2)
1231 self.assert_rejected(parser.Parse, payload, True)
1232
1233 def test_set_maximum_amplification__amplification_not_exceeded(self):
1234 parser = expat.ParserCreate()
1235 # Unconditionally enable maximum activation factor.
1236 self.set_activation_threshold(parser, 0)
1237 # Choose a max amplification factor expected to never be exceeded.
1238 self.assertIsNone(self.set_maximum_amplification(parser, 1e4))
1239 # Craft a payload for which the peak amplification factor is < 1e4.
1240 payload = self.exponential_expansion_payload(ncols=1, nrows=2)
1241 self.assertIsNotNone(parser.Parse(payload, True))
1242
1243
1244 @unittest.skipIf(not hasattr(expat.XMLParserType,
1245 "SetAllocTrackerMaximumAmplification"),
1246 "requires Python compiled with Expat >= 2.7.2")
1247 class MemoryProtectionTest(AttackProtectionTestBase, unittest.TestCase):
1248
1249 # NOTE: with the default Expat configuration, the billion laughs protection
1250 # may hit before the allocation limiter if exponential_expansion_payload()
1251 # is not carefully parametrized. As such, the payloads should be chosen so
1252 # that either the allocation limiter is hit before other protections are
1253 # triggered or no protection at all is triggered.
1254
1255 def assert_rejected(self, func, /, *args, **kwargs):
1256 """Check that func(*args, **kwargs) hits the allocation limit."""
1257 msg = r"out of memory: line \d+, column \d+"
1258 self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs)
1259
1260 def set_activation_threshold(self, parser, threshold):
1261 return parser.SetAllocTrackerActivationThreshold(threshold)
1262
1263 def set_maximum_amplification(self, parser, max_factor):
1264 return parser.SetAllocTrackerMaximumAmplification(max_factor)
1265
1266 def test_set_activation_threshold__threshold_reached(self):
1267 parser = expat.ParserCreate()
1268 # Choose a threshold expected to be always reached.
1269 self.set_activation_threshold(parser, 3)
1270 # Check that the threshold is reached by choosing a small factor
1271 # and a payload whose peak amplification factor exceeds it.
1272 self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1273 payload = self.exponential_expansion_payload(ncols=10, nrows=4)
1274 self.assert_rejected(parser.Parse, payload, True)
1275
1276 def test_set_activation_threshold__threshold_not_reached(self):
1277 parser = expat.ParserCreate()
1278 # Choose a threshold expected to be never reached.
1279 self.set_activation_threshold(parser, pow(10, 5))
1280 # Check that the threshold is reached by choosing a small factor
1281 # and a payload whose peak amplification factor exceeds it.
1282 self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1283 payload = self.exponential_expansion_payload(ncols=10, nrows=4)
1284 self.assertIsNotNone(parser.Parse(payload, True))
1285
1286 def test_set_maximum_amplification__amplification_exceeded(self):
1287 parser = expat.ParserCreate()
1288 # Unconditionally enable maximum activation factor.
1289 self.set_activation_threshold(parser, 0)
1290 # Choose a max amplification factor expected to always be exceeded.
1291 self.assertIsNone(self.set_maximum_amplification(parser, 1.0))
1292 # Craft a payload for which the peak amplification factor is > 1.0.
1293 payload = self.exponential_expansion_payload(ncols=1, nrows=2)
1294 self.assert_rejected(parser.Parse, payload, True)
1295
1296 def test_set_maximum_amplification__amplification_not_exceeded(self):
1297 parser = expat.ParserCreate()
1298 # Unconditionally enable maximum activation factor.
1299 self.set_activation_threshold(parser, 0)
1300 # Choose a max amplification factor expected to never be exceeded.
1301 self.assertIsNone(self.set_maximum_amplification(parser, 1e4))
1302 # Craft a payload for which the peak amplification factor is < 1e4.
1303 payload = self.exponential_expansion_payload(ncols=1, nrows=2)
1304 self.assertIsNotNone(parser.Parse(payload, True))
1305
1306
1307 if __name__ == "__main__":
1308 unittest.main()