| 1 |
# XXX TypeErrors on calling handlers, or on bad return values from a |
| 2 |
# handler, are obscure and unhelpful. |
| 3 |
|
| 4 |
import abc |
| 5 |
import functools |
| 6 |
import os |
| 7 |
import re |
| 8 |
import sys |
| 9 |
import sysconfig |
| 10 |
import textwrap |
| 11 |
import unittest |
| 12 |
import traceback |
| 13 |
from io import BytesIO |
| 14 |
from test import support |
| 15 |
from test.support import import_helper, os_helper |
| 16 |
|
| 17 |
from xml.parsers import expat |
| 18 |
from xml.parsers.expat import errors |
| 19 |
|
| 20 |
from test.support import sortdict |
| 21 |
|
| 22 |
|
| 23 |
class SetAttributeTest(unittest.TestCase): |
| 24 |
def setUp(self): |
| 25 |
self.parser = expat.ParserCreate(namespace_separator='!') |
| 26 |
|
| 27 |
def test_buffer_text(self): |
| 28 |
self.assertIs(self.parser.buffer_text, False) |
| 29 |
for x in 0, 1, 2, 0: |
| 30 |
self.parser.buffer_text = x |
| 31 |
self.assertIs(self.parser.buffer_text, bool(x)) |
| 32 |
|
| 33 |
def test_namespace_prefixes(self): |
| 34 |
self.assertIs(self.parser.namespace_prefixes, False) |
| 35 |
for x in 0, 1, 2, 0: |
| 36 |
self.parser.namespace_prefixes = x |
| 37 |
self.assertIs(self.parser.namespace_prefixes, bool(x)) |
| 38 |
|
| 39 |
def test_ordered_attributes(self): |
| 40 |
self.assertIs(self.parser.ordered_attributes, False) |
| 41 |
for x in 0, 1, 2, 0: |
| 42 |
self.parser.ordered_attributes = x |
| 43 |
self.assertIs(self.parser.ordered_attributes, bool(x)) |
| 44 |
|
| 45 |
def test_specified_attributes(self): |
| 46 |
self.assertIs(self.parser.specified_attributes, False) |
| 47 |
for x in 0, 1, 2, 0: |
| 48 |
self.parser.specified_attributes = x |
| 49 |
self.assertIs(self.parser.specified_attributes, bool(x)) |
| 50 |
|
| 51 |
def test_invalid_attributes(self): |
| 52 |
with self.assertRaises(AttributeError): |
| 53 |
self.parser.returns_unicode = 1 |
| 54 |
with self.assertRaises(AttributeError): |
| 55 |
self.parser.returns_unicode |
| 56 |
|
| 57 |
# Issue #25019 |
| 58 |
self.assertRaises(TypeError, setattr, self.parser, range(0xF), 0) |
| 59 |
self.assertRaises(TypeError, self.parser.__setattr__, range(0xF), 0) |
| 60 |
self.assertRaises(TypeError, getattr, self.parser, range(0xF)) |
| 61 |
|
| 62 |
|
| 63 |
data = b'''\ |
| 64 |
<?xml version="1.0" encoding="iso-8859-1" standalone="no"?> |
| 65 |
<?xml-stylesheet href="stylesheet.css"?> |
| 66 |
<!-- comment data --> |
| 67 |
<!DOCTYPE quotations SYSTEM "quotations.dtd" [ |
| 68 |
<!ELEMENT root ANY> |
| 69 |
<!ATTLIST root attr1 CDATA #REQUIRED attr2 CDATA #IMPLIED> |
| 70 |
<!NOTATION notation SYSTEM "notation.jpeg"> |
| 71 |
<!ENTITY acirc "â"> |
| 72 |
<!ENTITY external_entity SYSTEM "entity.file"> |
| 73 |
<!ENTITY unparsed_entity SYSTEM "entity.file" NDATA notation> |
| 74 |
%unparsed_entity; |
| 75 |
]> |
| 76 |
|
| 77 |
<root attr1="value1" attr2="value2ὀ"> |
| 78 |
<myns:subelement xmlns:myns="http://www.python.org/namespace"> |
| 79 |
Contents of subelements |
| 80 |
</myns:subelement> |
| 81 |
<sub2><![CDATA[contents of CDATA section]]></sub2> |
| 82 |
&external_entity; |
| 83 |
&skipped_entity; |
| 84 |
\xb5 |
| 85 |
</root> |
| 86 |
''' |
| 87 |
|
| 88 |
|
| 89 |
# Produce UTF-8 output |
| 90 |
class ParseTest(unittest.TestCase): |
| 91 |
class Outputter: |
| 92 |
def __init__(self): |
| 93 |
self.out = [] |
| 94 |
|
| 95 |
def StartElementHandler(self, name, attrs): |
| 96 |
self.out.append('Start element: ' + repr(name) + ' ' + |
| 97 |
sortdict(attrs)) |
| 98 |
|
| 99 |
def EndElementHandler(self, name): |
| 100 |
self.out.append('End element: ' + repr(name)) |
| 101 |
|
| 102 |
def CharacterDataHandler(self, data): |
| 103 |
data = data.strip() |
| 104 |
if data: |
| 105 |
self.out.append('Character data: ' + repr(data)) |
| 106 |
|
| 107 |
def ProcessingInstructionHandler(self, target, data): |
| 108 |
self.out.append('PI: ' + repr(target) + ' ' + repr(data)) |
| 109 |
|
| 110 |
def StartNamespaceDeclHandler(self, prefix, uri): |
| 111 |
self.out.append('NS decl: ' + repr(prefix) + ' ' + repr(uri)) |
| 112 |
|
| 113 |
def EndNamespaceDeclHandler(self, prefix): |
| 114 |
self.out.append('End of NS decl: ' + repr(prefix)) |
| 115 |
|
| 116 |
def StartCdataSectionHandler(self): |
| 117 |
self.out.append('Start of CDATA section') |
| 118 |
|
| 119 |
def EndCdataSectionHandler(self): |
| 120 |
self.out.append('End of CDATA section') |
| 121 |
|
| 122 |
def CommentHandler(self, text): |
| 123 |
self.out.append('Comment: ' + repr(text)) |
| 124 |
|
| 125 |
def NotationDeclHandler(self, *args): |
| 126 |
name, base, sysid, pubid = args |
| 127 |
self.out.append('Notation declared: %s' %(args,)) |
| 128 |
|
| 129 |
def UnparsedEntityDeclHandler(self, *args): |
| 130 |
entityName, base, systemId, publicId, notationName = args |
| 131 |
self.out.append('Unparsed entity decl: %s' %(args,)) |
| 132 |
|
| 133 |
def NotStandaloneHandler(self): |
| 134 |
self.out.append('Not standalone') |
| 135 |
return 1 |
| 136 |
|
| 137 |
def ExternalEntityRefHandler(self, *args): |
| 138 |
context, base, sysId, pubId = args |
| 139 |
self.out.append('External entity ref: %s' %(args[1:],)) |
| 140 |
return 1 |
| 141 |
|
| 142 |
def StartDoctypeDeclHandler(self, *args): |
| 143 |
self.out.append(('Start doctype', args)) |
| 144 |
return 1 |
| 145 |
|
| 146 |
def EndDoctypeDeclHandler(self): |
| 147 |
self.out.append("End doctype") |
| 148 |
return 1 |
| 149 |
|
| 150 |
def EntityDeclHandler(self, *args): |
| 151 |
self.out.append(('Entity declaration', args)) |
| 152 |
return 1 |
| 153 |
|
| 154 |
def XmlDeclHandler(self, *args): |
| 155 |
self.out.append(('XML declaration', args)) |
| 156 |
return 1 |
| 157 |
|
| 158 |
def ElementDeclHandler(self, *args): |
| 159 |
self.out.append(('Element declaration', args)) |
| 160 |
return 1 |
| 161 |
|
| 162 |
def AttlistDeclHandler(self, *args): |
| 163 |
self.out.append(('Attribute list declaration', args)) |
| 164 |
return 1 |
| 165 |
|
| 166 |
def SkippedEntityHandler(self, *args): |
| 167 |
self.out.append(("Skipped entity", args)) |
| 168 |
return 1 |
| 169 |
|
| 170 |
def DefaultHandler(self, userData): |
| 171 |
pass |
| 172 |
|
| 173 |
def DefaultHandlerExpand(self, userData): |
| 174 |
pass |
| 175 |
|
| 176 |
handler_names = [ |
| 177 |
'StartElementHandler', 'EndElementHandler', 'CharacterDataHandler', |
| 178 |
'ProcessingInstructionHandler', 'UnparsedEntityDeclHandler', |
| 179 |
'NotationDeclHandler', 'StartNamespaceDeclHandler', |
| 180 |
'EndNamespaceDeclHandler', 'CommentHandler', |
| 181 |
'StartCdataSectionHandler', 'EndCdataSectionHandler', 'DefaultHandler', |
| 182 |
'DefaultHandlerExpand', 'NotStandaloneHandler', |
| 183 |
'ExternalEntityRefHandler', 'StartDoctypeDeclHandler', |
| 184 |
'EndDoctypeDeclHandler', 'EntityDeclHandler', 'XmlDeclHandler', |
| 185 |
'ElementDeclHandler', 'AttlistDeclHandler', 'SkippedEntityHandler', |
| 186 |
] |
| 187 |
|
| 188 |
def _hookup_callbacks(self, parser, handler): |
| 189 |
""" |
| 190 |
Set each of the callbacks defined on handler and named in |
| 191 |
self.handler_names on the given parser. |
| 192 |
""" |
| 193 |
for name in self.handler_names: |
| 194 |
setattr(parser, name, getattr(handler, name)) |
| 195 |
|
| 196 |
def _verify_parse_output(self, operations): |
| 197 |
expected_operations = [ |
| 198 |
('XML declaration', ('1.0', 'iso-8859-1', 0)), |
| 199 |
'PI: \'xml-stylesheet\' \'href="stylesheet.css"\'', |
| 200 |
"Comment: ' comment data '", |
| 201 |
"Not standalone", |
| 202 |
("Start doctype", ('quotations', 'quotations.dtd', None, 1)), |
| 203 |
('Element declaration', ('root', (2, 0, None, ()))), |
| 204 |
('Attribute list declaration', ('root', 'attr1', 'CDATA', None, |
| 205 |
1)), |
| 206 |
('Attribute list declaration', ('root', 'attr2', 'CDATA', None, |
| 207 |
0)), |
| 208 |
"Notation declared: ('notation', None, 'notation.jpeg', None)", |
| 209 |
('Entity declaration', ('acirc', 0, '\xe2', None, None, None, None)), |
| 210 |
('Entity declaration', ('external_entity', 0, None, None, |
| 211 |
'entity.file', None, None)), |
| 212 |
"Unparsed entity decl: ('unparsed_entity', None, 'entity.file', None, 'notation')", |
| 213 |
"Not standalone", |
| 214 |
"End doctype", |
| 215 |
"Start element: 'root' {'attr1': 'value1', 'attr2': 'value2\u1f40'}", |
| 216 |
"NS decl: 'myns' 'http://www.python.org/namespace'", |
| 217 |
"Start element: 'http://www.python.org/namespace!subelement' {}", |
| 218 |
"Character data: 'Contents of subelements'", |
| 219 |
"End element: 'http://www.python.org/namespace!subelement'", |
| 220 |
"End of NS decl: 'myns'", |
| 221 |
"Start element: 'sub2' {}", |
| 222 |
'Start of CDATA section', |
| 223 |
"Character data: 'contents of CDATA section'", |
| 224 |
'End of CDATA section', |
| 225 |
"End element: 'sub2'", |
| 226 |
"External entity ref: (None, 'entity.file', None)", |
| 227 |
('Skipped entity', ('skipped_entity', 0)), |
| 228 |
"Character data: '\xb5'", |
| 229 |
"End element: 'root'", |
| 230 |
] |
| 231 |
for operation, expected_operation in zip(operations, expected_operations): |
| 232 |
self.assertEqual(operation, expected_operation) |
| 233 |
|
| 234 |
def test_parse_bytes(self): |
| 235 |
out = self.Outputter() |
| 236 |
parser = expat.ParserCreate(namespace_separator='!') |
| 237 |
self._hookup_callbacks(parser, out) |
| 238 |
|
| 239 |
parser.Parse(data, True) |
| 240 |
|
| 241 |
operations = out.out |
| 242 |
self._verify_parse_output(operations) |
| 243 |
# Issue #6697. |
| 244 |
self.assertRaises(AttributeError, getattr, parser, '\uD800') |
| 245 |
|
| 246 |
def test_parse_str(self): |
| 247 |
out = self.Outputter() |
| 248 |
parser = expat.ParserCreate(namespace_separator='!') |
| 249 |
self._hookup_callbacks(parser, out) |
| 250 |
|
| 251 |
parser.Parse(data.decode('iso-8859-1'), True) |
| 252 |
|
| 253 |
operations = out.out |
| 254 |
self._verify_parse_output(operations) |
| 255 |
|
| 256 |
def test_parse_file(self): |
| 257 |
# Try parsing a file |
| 258 |
out = self.Outputter() |
| 259 |
parser = expat.ParserCreate(namespace_separator='!') |
| 260 |
self._hookup_callbacks(parser, out) |
| 261 |
file = BytesIO(data) |
| 262 |
|
| 263 |
parser.ParseFile(file) |
| 264 |
|
| 265 |
operations = out.out |
| 266 |
self._verify_parse_output(operations) |
| 267 |
|
| 268 |
def test_parse_again(self): |
| 269 |
parser = expat.ParserCreate() |
| 270 |
file = BytesIO(data) |
| 271 |
parser.ParseFile(file) |
| 272 |
# Issue 6676: ensure a meaningful exception is raised when attempting |
| 273 |
# to parse more than one XML document per xmlparser instance, |
| 274 |
# a limitation of the Expat library. |
| 275 |
with self.assertRaises(expat.error) as cm: |
| 276 |
parser.ParseFile(file) |
| 277 |
self.assertEqual(expat.ErrorString(cm.exception.code), |
| 278 |
expat.errors.XML_ERROR_FINISHED) |
| 279 |
|
| 280 |
class NamespaceSeparatorTest(unittest.TestCase): |
| 281 |
def test_legal(self): |
| 282 |
# Tests that make sure we get errors when the namespace_separator value |
| 283 |
# is illegal, and that we don't for good values: |
| 284 |
expat.ParserCreate() |
| 285 |
expat.ParserCreate(namespace_separator=None) |
| 286 |
expat.ParserCreate(namespace_separator=' ') |
| 287 |
|
| 288 |
def test_illegal(self): |
| 289 |
with self.assertRaisesRegex(TypeError, |
| 290 |
r"ParserCreate\(\) argument (2|'namespace_separator') " |
| 291 |
r"must be str or None, not int"): |
| 292 |
expat.ParserCreate(namespace_separator=42) |
| 293 |
|
| 294 |
try: |
| 295 |
expat.ParserCreate(namespace_separator='too long') |
| 296 |
self.fail() |
| 297 |
except ValueError as e: |
| 298 |
self.assertEqual(str(e), |
| 299 |
'namespace_separator must be at most one character, omitted, or None') |
| 300 |
|
| 301 |
def test_zero_length(self): |
| 302 |
# ParserCreate() needs to accept a namespace_separator of zero length |
| 303 |
# to satisfy the requirements of RDF applications that are required |
| 304 |
# to simply glue together the namespace URI and the localname. Though |
| 305 |
# considered a wart of the RDF specifications, it needs to be supported. |
| 306 |
# |
| 307 |
# See XML-SIG mailing list thread starting with |
| 308 |
# http://mail.python.org/pipermail/xml-sig/2001-April/005202.html |
| 309 |
# |
| 310 |
expat.ParserCreate(namespace_separator='') # too short |
| 311 |
|
| 312 |
|
| 313 |
class InterningTest(unittest.TestCase): |
| 314 |
def test(self): |
| 315 |
# Test the interning machinery. |
| 316 |
p = expat.ParserCreate() |
| 317 |
L = [] |
| 318 |
def collector(name, *args): |
| 319 |
L.append(name) |
| 320 |
p.StartElementHandler = collector |
| 321 |
p.EndElementHandler = collector |
| 322 |
p.Parse(b"<e> <e/> <e></e> </e>", True) |
| 323 |
tag = L[0] |
| 324 |
self.assertEqual(len(L), 6) |
| 325 |
for entry in L: |
| 326 |
# L should have the same string repeated over and over. |
| 327 |
self.assertTrue(tag is entry) |
| 328 |
|
| 329 |
def test_issue9402(self): |
| 330 |
# create an ExternalEntityParserCreate with buffer text |
| 331 |
class ExternalOutputter: |
| 332 |
def __init__(self, parser): |
| 333 |
self.parser = parser |
| 334 |
self.parser_result = None |
| 335 |
|
| 336 |
def ExternalEntityRefHandler(self, context, base, sysId, pubId): |
| 337 |
external_parser = self.parser.ExternalEntityParserCreate("") |
| 338 |
self.parser_result = external_parser.Parse(b"", True) |
| 339 |
return 1 |
| 340 |
|
| 341 |
parser = expat.ParserCreate(namespace_separator='!') |
| 342 |
parser.buffer_text = 1 |
| 343 |
out = ExternalOutputter(parser) |
| 344 |
parser.ExternalEntityRefHandler = out.ExternalEntityRefHandler |
| 345 |
parser.Parse(data, True) |
| 346 |
self.assertEqual(out.parser_result, 1) |
| 347 |
|
| 348 |
|
| 349 |
class BufferTextTest(unittest.TestCase): |
| 350 |
def setUp(self): |
| 351 |
self.stuff = [] |
| 352 |
self.parser = expat.ParserCreate() |
| 353 |
self.parser.buffer_text = 1 |
| 354 |
self.parser.CharacterDataHandler = self.CharacterDataHandler |
| 355 |
|
| 356 |
def check(self, expected, label): |
| 357 |
self.assertEqual(self.stuff, expected, |
| 358 |
"%s\nstuff = %r\nexpected = %r" |
| 359 |
% (label, self.stuff, map(str, expected))) |
| 360 |
|
| 361 |
def CharacterDataHandler(self, text): |
| 362 |
self.stuff.append(text) |
| 363 |
|
| 364 |
def StartElementHandler(self, name, attrs): |
| 365 |
self.stuff.append("<%s>" % name) |
| 366 |
bt = attrs.get("buffer-text") |
| 367 |
if bt == "yes": |
| 368 |
self.parser.buffer_text = 1 |
| 369 |
elif bt == "no": |
| 370 |
self.parser.buffer_text = 0 |
| 371 |
|
| 372 |
def EndElementHandler(self, name): |
| 373 |
self.stuff.append("</%s>" % name) |
| 374 |
|
| 375 |
def CommentHandler(self, data): |
| 376 |
self.stuff.append("<!--%s-->" % data) |
| 377 |
|
| 378 |
def setHandlers(self, handlers=[]): |
| 379 |
for name in handlers: |
| 380 |
setattr(self.parser, name, getattr(self, name)) |
| 381 |
|
| 382 |
def test_default_to_disabled(self): |
| 383 |
parser = expat.ParserCreate() |
| 384 |
self.assertFalse(parser.buffer_text) |
| 385 |
|
| 386 |
def test_buffering_enabled(self): |
| 387 |
# Make sure buffering is turned on |
| 388 |
self.assertTrue(self.parser.buffer_text) |
| 389 |
self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True) |
| 390 |
self.assertEqual(self.stuff, ['123'], |
| 391 |
"buffered text not properly collapsed") |
| 392 |
|
| 393 |
def test1(self): |
| 394 |
# XXX This test exposes more detail of Expat's text chunking than we |
| 395 |
# XXX like, but it tests what we need to concisely. |
| 396 |
self.setHandlers(["StartElementHandler"]) |
| 397 |
self.parser.Parse(b"<a>1<b buffer-text='no'/>2\n3<c buffer-text='yes'/>4\n5</a>", True) |
| 398 |
self.assertEqual(self.stuff, |
| 399 |
["<a>", "1", "<b>", "2", "\n", "3", "<c>", "4\n5"], |
| 400 |
"buffering control not reacting as expected") |
| 401 |
|
| 402 |
def test2(self): |
| 403 |
self.parser.Parse(b"<a>1<b/><2><c/> \n 3</a>", True) |
| 404 |
self.assertEqual(self.stuff, ["1<2> \n 3"], |
| 405 |
"buffered text not properly collapsed") |
| 406 |
|
| 407 |
def test3(self): |
| 408 |
self.setHandlers(["StartElementHandler"]) |
| 409 |
self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True) |
| 410 |
self.assertEqual(self.stuff, ["<a>", "1", "<b>", "2", "<c>", "3"], |
| 411 |
"buffered text not properly split") |
| 412 |
|
| 413 |
def test4(self): |
| 414 |
self.setHandlers(["StartElementHandler", "EndElementHandler"]) |
| 415 |
self.parser.CharacterDataHandler = None |
| 416 |
self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True) |
| 417 |
self.assertEqual(self.stuff, |
| 418 |
["<a>", "<b>", "</b>", "<c>", "</c>", "</a>"]) |
| 419 |
|
| 420 |
def test5(self): |
| 421 |
self.setHandlers(["StartElementHandler", "EndElementHandler"]) |
| 422 |
self.parser.Parse(b"<a>1<b></b>2<c/>3</a>", True) |
| 423 |
self.assertEqual(self.stuff, |
| 424 |
["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3", "</a>"]) |
| 425 |
|
| 426 |
def test6(self): |
| 427 |
self.setHandlers(["CommentHandler", "EndElementHandler", |
| 428 |
"StartElementHandler"]) |
| 429 |
self.parser.Parse(b"<a>1<b/>2<c></c>345</a> ", True) |
| 430 |
self.assertEqual(self.stuff, |
| 431 |
["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "345", "</a>"], |
| 432 |
"buffered text not properly split") |
| 433 |
|
| 434 |
def test7(self): |
| 435 |
self.setHandlers(["CommentHandler", "EndElementHandler", |
| 436 |
"StartElementHandler"]) |
| 437 |
self.parser.Parse(b"<a>1<b/>2<c></c>3<!--abc-->4<!--def-->5</a> ", True) |
| 438 |
self.assertEqual(self.stuff, |
| 439 |
["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3", |
| 440 |
"<!--abc-->", "4", "<!--def-->", "5", "</a>"], |
| 441 |
"buffered text not properly split") |
| 442 |
|
| 443 |
|
| 444 |
# Test handling of exception from callback: |
| 445 |
class HandlerExceptionTest(unittest.TestCase): |
| 446 |
def StartElementHandler(self, name, attrs): |
| 447 |
raise RuntimeError(f'StartElementHandler: <{name}>') |
| 448 |
|
| 449 |
def check_traceback_entry(self, entry, filename, funcname): |
| 450 |
self.assertEqual(os.path.basename(entry.filename), filename) |
| 451 |
self.assertEqual(entry.name, funcname) |
| 452 |
|
| 453 |
@support.cpython_only |
| 454 |
def test_exception(self): |
| 455 |
# gh-66652: test _PyTraceback_Add() used by pyexpat.c to inject frames |
| 456 |
|
| 457 |
# Change the current directory to the Python source code directory |
| 458 |
# if it is available. |
| 459 |
src_dir = sysconfig.get_config_var('abs_builddir') |
| 460 |
if src_dir: |
| 461 |
have_source = os.path.isdir(src_dir) |
| 462 |
else: |
| 463 |
have_source = False |
| 464 |
if have_source: |
| 465 |
with os_helper.change_cwd(src_dir): |
| 466 |
self._test_exception(have_source) |
| 467 |
else: |
| 468 |
self._test_exception(have_source) |
| 469 |
|
| 470 |
def _test_exception(self, have_source): |
| 471 |
# Use path relative to the current directory which should be the Python |
| 472 |
# source code directory (if it is available). |
| 473 |
PYEXPAT_C = os.path.join('Modules', 'pyexpat.c') |
| 474 |
|
| 475 |
parser = expat.ParserCreate() |
| 476 |
parser.StartElementHandler = self.StartElementHandler |
| 477 |
try: |
| 478 |
parser.Parse(b"<a><b><c/></b></a>", True) |
| 479 |
|
| 480 |
self.fail("the parser did not raise RuntimeError") |
| 481 |
except RuntimeError as exc: |
| 482 |
self.assertEqual(exc.args[0], 'StartElementHandler: <a>', exc) |
| 483 |
entries = traceback.extract_tb(exc.__traceback__) |
| 484 |
|
| 485 |
self.assertEqual(len(entries), 3, entries) |
| 486 |
self.check_traceback_entry(entries[0], |
| 487 |
"test_pyexpat.py", "_test_exception") |
| 488 |
self.check_traceback_entry(entries[1], |
| 489 |
os.path.basename(PYEXPAT_C), |
| 490 |
"StartElement") |
| 491 |
self.check_traceback_entry(entries[2], |
| 492 |
"test_pyexpat.py", "StartElementHandler") |
| 493 |
|
| 494 |
# Check that the traceback contains the relevant line in |
| 495 |
# Modules/pyexpat.c. Skip the test if Modules/pyexpat.c is not |
| 496 |
# available. |
| 497 |
if have_source and os.path.exists(PYEXPAT_C): |
| 498 |
self.assertIn('call_with_frame("StartElement"', |
| 499 |
entries[1].line) |
| 500 |
|
| 501 |
|
| 502 |
# Test Current* members: |
| 503 |
class PositionTest(unittest.TestCase): |
| 504 |
def StartElementHandler(self, name, attrs): |
| 505 |
self.check_pos('s') |
| 506 |
|
| 507 |
def EndElementHandler(self, name): |
| 508 |
self.check_pos('e') |
| 509 |
|
| 510 |
def check_pos(self, event): |
| 511 |
pos = (event, |
| 512 |
self.parser.CurrentByteIndex, |
| 513 |
self.parser.CurrentLineNumber, |
| 514 |
self.parser.CurrentColumnNumber) |
| 515 |
self.assertTrue(self.upto < len(self.expected_list), |
| 516 |
'too many parser events') |
| 517 |
expected = self.expected_list[self.upto] |
| 518 |
self.assertEqual(pos, expected, |
| 519 |
'Expected position %s, got position %s' %(pos, expected)) |
| 520 |
self.upto += 1 |
| 521 |
|
| 522 |
def test(self): |
| 523 |
self.parser = expat.ParserCreate() |
| 524 |
self.parser.StartElementHandler = self.StartElementHandler |
| 525 |
self.parser.EndElementHandler = self.EndElementHandler |
| 526 |
self.upto = 0 |
| 527 |
self.expected_list = [('s', 0, 1, 0), ('s', 5, 2, 1), ('s', 11, 3, 2), |
| 528 |
('e', 15, 3, 6), ('e', 17, 4, 1), ('e', 22, 5, 0)] |
| 529 |
|
| 530 |
xml = b'<a>\n <b>\n <c/>\n </b>\n</a>' |
| 531 |
self.parser.Parse(xml, True) |
| 532 |
|
| 533 |
|
| 534 |
class sf1296433Test(unittest.TestCase): |
| 535 |
def test_parse_only_xml_data(self): |
| 536 |
# https://bugs.python.org/issue1296433 |
| 537 |
# |
| 538 |
xml = "<?xml version='1.0' encoding='iso8859'?><s>%s</s>" % ('a' * 1025) |
| 539 |
# this one doesn't crash |
| 540 |
#xml = "<?xml version='1.0'?><s>%s</s>" % ('a' * 10000) |
| 541 |
|
| 542 |
class SpecificException(Exception): |
| 543 |
pass |
| 544 |
|
| 545 |
def handler(text): |
| 546 |
raise SpecificException |
| 547 |
|
| 548 |
parser = expat.ParserCreate() |
| 549 |
parser.CharacterDataHandler = handler |
| 550 |
|
| 551 |
self.assertRaises(SpecificException, parser.Parse, xml.encode('iso8859')) |
| 552 |
|
| 553 |
class ChardataBufferTest(unittest.TestCase): |
| 554 |
""" |
| 555 |
test setting of chardata buffer size |
| 556 |
""" |
| 557 |
|
| 558 |
def test_1025_bytes(self): |
| 559 |
self.assertEqual(self.small_buffer_test(1025), 2) |
| 560 |
|
| 561 |
def test_1000_bytes(self): |
| 562 |
self.assertEqual(self.small_buffer_test(1000), 1) |
| 563 |
|
| 564 |
def test_wrong_size(self): |
| 565 |
parser = expat.ParserCreate() |
| 566 |
parser.buffer_text = 1 |
| 567 |
with self.assertRaises(ValueError): |
| 568 |
parser.buffer_size = -1 |
| 569 |
with self.assertRaises(ValueError): |
| 570 |
parser.buffer_size = 0 |
| 571 |
with self.assertRaises((ValueError, OverflowError)): |
| 572 |
parser.buffer_size = sys.maxsize + 1 |
| 573 |
with self.assertRaises(TypeError): |
| 574 |
parser.buffer_size = 512.0 |
| 575 |
|
| 576 |
def test_unchanged_size(self): |
| 577 |
xml1 = b"<?xml version='1.0' encoding='iso8859'?><s>" + b'a' * 512 |
| 578 |
xml2 = b'a'*512 + b'</s>' |
| 579 |
parser = expat.ParserCreate() |
| 580 |
parser.CharacterDataHandler = self.counting_handler |
| 581 |
parser.buffer_size = 512 |
| 582 |
parser.buffer_text = 1 |
| 583 |
|
| 584 |
# Feed 512 bytes of character data: the handler should be called |
| 585 |
# once. |
| 586 |
self.n = 0 |
| 587 |
parser.Parse(xml1) |
| 588 |
self.assertEqual(self.n, 1) |
| 589 |
|
| 590 |
# Reassign to buffer_size, but assign the same size. |
| 591 |
parser.buffer_size = parser.buffer_size |
| 592 |
self.assertEqual(self.n, 1) |
| 593 |
|
| 594 |
# Try parsing rest of the document |
| 595 |
parser.Parse(xml2) |
| 596 |
self.assertEqual(self.n, 2) |
| 597 |
|
| 598 |
|
| 599 |
def test_disabling_buffer(self): |
| 600 |
xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>" + b'a' * 512 |
| 601 |
xml2 = b'b' * 1024 |
| 602 |
xml3 = b'c' * 1024 + b'</a>'; |
| 603 |
parser = expat.ParserCreate() |
| 604 |
parser.CharacterDataHandler = self.counting_handler |
| 605 |
parser.buffer_text = 1 |
| 606 |
parser.buffer_size = 1024 |
| 607 |
self.assertEqual(parser.buffer_size, 1024) |
| 608 |
|
| 609 |
# Parse one chunk of XML |
| 610 |
self.n = 0 |
| 611 |
parser.Parse(xml1, False) |
| 612 |
self.assertEqual(parser.buffer_size, 1024) |
| 613 |
self.assertEqual(self.n, 1) |
| 614 |
|
| 615 |
# Turn off buffering and parse the next chunk. |
| 616 |
parser.buffer_text = 0 |
| 617 |
self.assertFalse(parser.buffer_text) |
| 618 |
self.assertEqual(parser.buffer_size, 1024) |
| 619 |
for i in range(10): |
| 620 |
parser.Parse(xml2, False) |
| 621 |
self.assertEqual(self.n, 11) |
| 622 |
|
| 623 |
parser.buffer_text = 1 |
| 624 |
self.assertTrue(parser.buffer_text) |
| 625 |
self.assertEqual(parser.buffer_size, 1024) |
| 626 |
parser.Parse(xml3, True) |
| 627 |
self.assertEqual(self.n, 12) |
| 628 |
|
| 629 |
def counting_handler(self, text): |
| 630 |
self.n += 1 |
| 631 |
|
| 632 |
def small_buffer_test(self, buffer_len): |
| 633 |
xml = b"<?xml version='1.0' encoding='iso8859'?><s>" + b'a' * buffer_len + b'</s>' |
| 634 |
parser = expat.ParserCreate() |
| 635 |
parser.CharacterDataHandler = self.counting_handler |
| 636 |
parser.buffer_size = 1024 |
| 637 |
parser.buffer_text = 1 |
| 638 |
|
| 639 |
self.n = 0 |
| 640 |
parser.Parse(xml) |
| 641 |
return self.n |
| 642 |
|
| 643 |
def test_change_size_1(self): |
| 644 |
xml1 = b"<?xml version='1.0' encoding='iso8859'?><a><s>" + b'a' * 1024 |
| 645 |
xml2 = b'aaa</s><s>' + b'a' * 1025 + b'</s></a>' |
| 646 |
parser = expat.ParserCreate() |
| 647 |
parser.CharacterDataHandler = self.counting_handler |
| 648 |
parser.buffer_text = 1 |
| 649 |
parser.buffer_size = 1024 |
| 650 |
self.assertEqual(parser.buffer_size, 1024) |
| 651 |
|
| 652 |
self.n = 0 |
| 653 |
parser.Parse(xml1, False) |
| 654 |
parser.buffer_size *= 2 |
| 655 |
self.assertEqual(parser.buffer_size, 2048) |
| 656 |
parser.Parse(xml2, True) |
| 657 |
self.assertEqual(self.n, 2) |
| 658 |
|
| 659 |
def test_change_size_2(self): |
| 660 |
xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>a<s>" + b'a' * 1023 |
| 661 |
xml2 = b'aaa</s><s>' + b'a' * 1025 + b'</s></a>' |
| 662 |
parser = expat.ParserCreate() |
| 663 |
parser.CharacterDataHandler = self.counting_handler |
| 664 |
parser.buffer_text = 1 |
| 665 |
parser.buffer_size = 2048 |
| 666 |
self.assertEqual(parser.buffer_size, 2048) |
| 667 |
|
| 668 |
self.n=0 |
| 669 |
parser.Parse(xml1, False) |
| 670 |
parser.buffer_size = parser.buffer_size // 2 |
| 671 |
self.assertEqual(parser.buffer_size, 1024) |
| 672 |
parser.Parse(xml2, True) |
| 673 |
self.assertEqual(self.n, 4) |
| 674 |
|
| 675 |
class ElementDeclHandlerTest(unittest.TestCase): |
| 676 |
def test_trigger_leak(self): |
| 677 |
# Unfixed, this test would leak the memory of the so-called |
| 678 |
# "content model" in function ``my_ElementDeclHandler`` of pyexpat. |
| 679 |
# See https://github.com/python/cpython/issues/140593. |
| 680 |
data = textwrap.dedent('''\ |
| 681 |
<!DOCTYPE quotations SYSTEM "quotations.dtd" [ |
| 682 |
<!ELEMENT root ANY> |
| 683 |
]> |
| 684 |
<root/> |
| 685 |
''').encode('UTF-8') |
| 686 |
|
| 687 |
parser = expat.ParserCreate() |
| 688 |
parser.NotStandaloneHandler = lambda: 1.234 # arbitrary float |
| 689 |
parser.ElementDeclHandler = lambda _1, _2: None |
| 690 |
self.assertRaises(TypeError, parser.Parse, data, True) |
| 691 |
|
| 692 |
@support.skip_if_unlimited_stack_size |
| 693 |
@support.skip_emscripten_stack_overflow() |
| 694 |
@support.skip_wasi_stack_overflow() |
| 695 |
def test_deeply_nested_content_model(self): |
| 696 |
# This should raise a RecursionError and not crash. |
| 697 |
# See https://github.com/python/cpython/issues/145986. |
| •698 |
N = 800_000 |
| 699 |
data = ( |
| 700 |
b'<!DOCTYPE root [\n<!ELEMENT root ' |
| 701 |
+ b'(a, ' * N + b'a' + b')' * N |
| 702 |
+ b'>\n]>\n<root/>\n' |
| 703 |
) |
| 704 |
|
| 705 |
parser = expat.ParserCreate() |
| 706 |
parser.ElementDeclHandler = lambda _1, _2: None |
| 707 |
with support.infinite_recursion(): |
| 708 |
with self.assertRaises(RecursionError): |
| 709 |
parser.Parse(data) |
| 710 |
|
| 711 |
class MalformedInputTest(unittest.TestCase): |
| 712 |
def test1(self): |
| 713 |
xml = b"\0\r\n" |
| 714 |
parser = expat.ParserCreate() |
| 715 |
try: |
| 716 |
parser.Parse(xml, True) |
| 717 |
self.fail() |
| 718 |
except expat.ExpatError as e: |
| 719 |
self.assertEqual(str(e), 'unclosed token: line 2, column 0') |
| 720 |
|
| 721 |
def test2(self): |
| 722 |
# \xc2\x85 is UTF-8 encoded U+0085 (NEXT LINE) |
| 723 |
xml = b"<?xml version\xc2\x85='1.0'?>\r\n" |
| 724 |
parser = expat.ParserCreate() |
| 725 |
err_pattern = r'XML declaration not well-formed: line 1, column \d+' |
| 726 |
with self.assertRaisesRegex(expat.ExpatError, err_pattern): |
| 727 |
parser.Parse(xml, True) |
| 728 |
|
| 729 |
class ErrorMessageTest(unittest.TestCase): |
| 730 |
def test_codes(self): |
| 731 |
# verify mapping of errors.codes and errors.messages |
| 732 |
self.assertEqual(errors.XML_ERROR_SYNTAX, |
| 733 |
errors.messages[errors.codes[errors.XML_ERROR_SYNTAX]]) |
| 734 |
|
| 735 |
def test_expaterror(self): |
| 736 |
xml = b'<' |
| 737 |
parser = expat.ParserCreate() |
| 738 |
try: |
| 739 |
parser.Parse(xml, True) |
| 740 |
self.fail() |
| 741 |
except expat.ExpatError as e: |
| 742 |
self.assertEqual(e.code, |
| 743 |
errors.codes[errors.XML_ERROR_UNCLOSED_TOKEN]) |
| 744 |
|
| 745 |
|
| 746 |
class ForeignDTDTests(unittest.TestCase): |
| 747 |
""" |
| 748 |
Tests for the UseForeignDTD method of expat parser objects. |
| 749 |
""" |
| 750 |
def test_use_foreign_dtd(self): |
| 751 |
""" |
| 752 |
If UseForeignDTD is passed True and a document without an external |
| 753 |
entity reference is parsed, ExternalEntityRefHandler is first called |
| 754 |
with None for the public and system ids. |
| 755 |
""" |
| 756 |
handler_call_args = [] |
| 757 |
def resolve_entity(context, base, system_id, public_id): |
| 758 |
handler_call_args.append((public_id, system_id)) |
| 759 |
return 1 |
| 760 |
|
| 761 |
parser = expat.ParserCreate() |
| 762 |
parser.UseForeignDTD(True) |
| 763 |
parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS) |
| 764 |
parser.ExternalEntityRefHandler = resolve_entity |
| 765 |
parser.Parse(b"<?xml version='1.0'?><element/>") |
| 766 |
self.assertEqual(handler_call_args, [(None, None)]) |
| 767 |
|
| 768 |
# test UseForeignDTD() is equal to UseForeignDTD(True) |
| 769 |
handler_call_args[:] = [] |
| 770 |
|
| 771 |
parser = expat.ParserCreate() |
| 772 |
parser.UseForeignDTD() |
| 773 |
parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS) |
| 774 |
parser.ExternalEntityRefHandler = resolve_entity |
| 775 |
parser.Parse(b"<?xml version='1.0'?><element/>") |
| 776 |
self.assertEqual(handler_call_args, [(None, None)]) |
| 777 |
|
| 778 |
def test_ignore_use_foreign_dtd(self): |
| 779 |
""" |
| 780 |
If UseForeignDTD is passed True and a document with an external |
| 781 |
entity reference is parsed, ExternalEntityRefHandler is called with |
| 782 |
the public and system ids from the document. |
| 783 |
""" |
| 784 |
handler_call_args = [] |
| 785 |
def resolve_entity(context, base, system_id, public_id): |
| 786 |
handler_call_args.append((public_id, system_id)) |
| 787 |
return 1 |
| 788 |
|
| 789 |
parser = expat.ParserCreate() |
| 790 |
parser.UseForeignDTD(True) |
| 791 |
parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_ALWAYS) |
| 792 |
parser.ExternalEntityRefHandler = resolve_entity |
| 793 |
parser.Parse( |
| 794 |
b"<?xml version='1.0'?><!DOCTYPE foo PUBLIC 'bar' 'baz'><element/>") |
| 795 |
self.assertEqual(handler_call_args, [("bar", "baz")]) |
| 796 |
|
| 797 |
|
| 798 |
class ParentParserLifetimeTest(unittest.TestCase): |
| 799 |
""" |
| 800 |
Subparsers make use of their parent XML_Parser inside of Expat. |
| 801 |
As a result, parent parsers need to outlive subparsers. |
| 802 |
|
| 803 |
See https://github.com/python/cpython/issues/139400. |
| 804 |
""" |
| 805 |
|
| 806 |
def test_parent_parser_outlives_its_subparsers__single(self): |
| 807 |
parser = expat.ParserCreate() |
| 808 |
subparser = parser.ExternalEntityParserCreate(None) |
| 809 |
|
| 810 |
# Now try to cause garbage collection of the parent parser |
| 811 |
# while it's still being referenced by a related subparser. |
| 812 |
del parser |
| 813 |
|
| 814 |
def test_parent_parser_outlives_its_subparsers__multiple(self): |
| 815 |
parser = expat.ParserCreate() |
| 816 |
subparser_one = parser.ExternalEntityParserCreate(None) |
| 817 |
subparser_two = parser.ExternalEntityParserCreate(None) |
| 818 |
|
| 819 |
# Now try to cause garbage collection of the parent parser |
| 820 |
# while it's still being referenced by a related subparser. |
| 821 |
del parser |
| 822 |
|
| 823 |
def test_parent_parser_outlives_its_subparsers__chain(self): |
| 824 |
parser = expat.ParserCreate() |
| 825 |
subparser = parser.ExternalEntityParserCreate(None) |
| 826 |
subsubparser = subparser.ExternalEntityParserCreate(None) |
| 827 |
|
| 828 |
# Now try to cause garbage collection of the parent parsers |
| 829 |
# while they are still being referenced by a related subparser. |
| 830 |
del parser |
| 831 |
del subparser |
| 832 |
|
| 833 |
|
| 834 |
class ExternalEntityParserCreateErrorTest(unittest.TestCase): |
| 835 |
"""ExternalEntityParserCreate error paths should not crash or leak |
| 836 |
refcounts on the parent parser. |
| 837 |
|
| 838 |
See https://github.com/python/cpython/issues/144984. |
| 839 |
""" |
| 840 |
|
| 841 |
@classmethod |
| 842 |
def setUpClass(cls): |
| 843 |
cls.testcapi = import_helper.import_module('_testcapi') |
| 844 |
|
| 845 |
@unittest.skipIf(support.Py_TRACE_REFS, |
| 846 |
'Py_TRACE_REFS conflicts with testcapi.set_nomemory') |
| 847 |
def test_error_path_no_crash(self): |
| 848 |
# When an allocation inside ExternalEntityParserCreate fails, |
| 849 |
# the partially-initialized subparser is deallocated. This |
| 850 |
# must not dereference NULL handlers or double-decrement the |
| 851 |
# parent parser's refcount. |
| 852 |
parser = expat.ParserCreate() |
| 853 |
parser.buffer_text = True |
| 854 |
rc_before = sys.getrefcount(parser) |
| 855 |
|
| 856 |
# We avoid self.assertRaises(MemoryError) here because the |
| 857 |
# context manager itself needs memory allocations that fail |
| 858 |
# while the nomemory hook is active. |
| 859 |
self.testcapi.set_nomemory(1, 10) |
| 860 |
raised = False |
| 861 |
try: |
| 862 |
parser.ExternalEntityParserCreate(None) |
| 863 |
except MemoryError: |
| 864 |
raised = True |
| 865 |
finally: |
| 866 |
self.testcapi.remove_mem_hooks() |
| 867 |
self.assertTrue(raised, "MemoryError not raised") |
| 868 |
|
| 869 |
rc_after = sys.getrefcount(parser) |
| 870 |
self.assertEqual(rc_after, rc_before) |
| 871 |
|
| 872 |
|
| 873 |
class ReparseDeferralTest(unittest.TestCase): |
| 874 |
def test_getter_setter_round_trip(self): |
| 875 |
parser = expat.ParserCreate() |
| 876 |
enabled = (expat.version_info >= (2, 6, 0)) |
| 877 |
|
| 878 |
self.assertIs(parser.GetReparseDeferralEnabled(), enabled) |
| 879 |
parser.SetReparseDeferralEnabled(False) |
| 880 |
self.assertIs(parser.GetReparseDeferralEnabled(), False) |
| 881 |
parser.SetReparseDeferralEnabled(True) |
| 882 |
self.assertIs(parser.GetReparseDeferralEnabled(), enabled) |
| 883 |
|
| 884 |
def test_reparse_deferral_enabled(self): |
| 885 |
if expat.version_info < (2, 6, 0): |
| 886 |
self.skipTest(f'Expat {expat.version_info} does not ' |
| 887 |
'support reparse deferral') |
| 888 |
|
| 889 |
started = [] |
| 890 |
|
| 891 |
def start_element(name, _): |
| 892 |
started.append(name) |
| 893 |
|
| 894 |
parser = expat.ParserCreate() |
| 895 |
parser.StartElementHandler = start_element |
| 896 |
self.assertTrue(parser.GetReparseDeferralEnabled()) |
| 897 |
|
| 898 |
for chunk in (b'<doc', b'/>'): |
| 899 |
parser.Parse(chunk, False) |
| 900 |
|
| 901 |
# The key test: Have handlers already fired? Expecting: no. |
| 902 |
self.assertEqual(started, []) |
| 903 |
|
| 904 |
parser.Parse(b'', True) |
| 905 |
|
| 906 |
self.assertEqual(started, ['doc']) |
| 907 |
|
| 908 |
def test_reparse_deferral_disabled(self): |
| 909 |
started = [] |
| 910 |
|
| 911 |
def start_element(name, _): |
| 912 |
started.append(name) |
| 913 |
|
| 914 |
parser = expat.ParserCreate() |
| 915 |
parser.StartElementHandler = start_element |
| 916 |
if expat.version_info >= (2, 6, 0): |
| 917 |
parser.SetReparseDeferralEnabled(False) |
| 918 |
self.assertFalse(parser.GetReparseDeferralEnabled()) |
| 919 |
|
| 920 |
for chunk in (b'<doc', b'/>'): |
| 921 |
parser.Parse(chunk, False) |
| 922 |
|
| 923 |
# The key test: Have handlers already fired? Expecting: yes. |
| 924 |
self.assertEqual(started, ['doc']) |
| 925 |
|
| 926 |
|
| 927 |
class AttackProtectionTestBase(abc.ABC): |
| 928 |
""" |
| 929 |
Base class for testing protections against XML payloads with |
| 930 |
disproportionate amplification. |
| 931 |
|
| 932 |
The protections being tested should detect and prevent attacks |
| 933 |
that leverage disproportionate amplification from small inputs. |
| 934 |
""" |
| 935 |
|
| 936 |
@staticmethod |
| 937 |
def exponential_expansion_payload(*, nrows, ncols, text='.'): |
| 938 |
"""Create a billion laughs attack payload. |
| 939 |
|
| 940 |
Be careful: the number of total items is pow(n, k), thereby |
| 941 |
requiring at least pow(ncols, nrows) * sizeof(text) memory! |
| 942 |
""" |
| 943 |
template = textwrap.dedent(f"""\ |
| 944 |
<?xml version="1.0"?> |
| 945 |
<!DOCTYPE doc [ |
| 946 |
<!ENTITY row0 "{text}"> |
| 947 |
<!ELEMENT doc (#PCDATA)> |
| 948 |
{{body}} |
| 949 |
]> |
| 950 |
<doc>&row{nrows};</doc> |
| 951 |
""").rstrip() |
| 952 |
|
| 953 |
body = '\n'.join( |
| 954 |
f'<!ENTITY row{i + 1} "{f"&row{i};" * ncols}">' |
| 955 |
for i in range(nrows) |
| 956 |
) |
| 957 |
body = textwrap.indent(body, ' ' * 4) |
| 958 |
return template.format(body=body) |
| 959 |
|
| 960 |
def test_payload_generation(self): |
| 961 |
# self-test for exponential_expansion_payload() |
| 962 |
payload = self.exponential_expansion_payload(nrows=2, ncols=3) |
| 963 |
self.assertEqual(payload, textwrap.dedent("""\ |
| 964 |
<?xml version="1.0"?> |
| 965 |
<!DOCTYPE doc [ |
| 966 |
<!ENTITY row0 "."> |
| 967 |
<!ELEMENT doc (#PCDATA)> |
| 968 |
<!ENTITY row1 "&row0;&row0;&row0;"> |
| 969 |
<!ENTITY row2 "&row1;&row1;&row1;"> |
| 970 |
]> |
| 971 |
<doc>&row2;</doc> |
| 972 |
""").rstrip()) |
| 973 |
|
| 974 |
def assert_root_parser_failure(self, func, /, *args, **kwargs): |
| 975 |
"""Check that func(*args, **kwargs) is invalid for a sub-parser.""" |
| 976 |
msg = "parser must be a root parser" |
| 977 |
self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) |
| 978 |
|
| 979 |
@abc.abstractmethod |
| 980 |
def assert_rejected(self, func, /, *args, **kwargs): |
| 981 |
"""Assert that func(*args, **kwargs) triggers the attack protection. |
| 982 |
|
| 983 |
Note: this method must ensure that the attack protection being tested |
| 984 |
is the one that is actually triggered at runtime, e.g., by matching |
| 985 |
the exact error message. |
| 986 |
""" |
| 987 |
|
| 988 |
@abc.abstractmethod |
| 989 |
def set_activation_threshold(self, parser, threshold): |
| 990 |
"""Set the activation threshold for the tested protection.""" |
| 991 |
|
| 992 |
@abc.abstractmethod |
| 993 |
def set_maximum_amplification(self, parser, max_factor): |
| 994 |
"""Set the maximum amplification factor for the tested protection.""" |
| 995 |
|
| 996 |
@abc.abstractmethod |
| 997 |
def test_set_activation_threshold__threshold_reached(self): |
| 998 |
"""Test when the activation threshold is exceeded.""" |
| 999 |
|
| 1000 |
@abc.abstractmethod |
| 1001 |
def test_set_activation_threshold__threshold_not_reached(self): |
| 1002 |
"""Test when the activation threshold is not exceeded.""" |
| 1003 |
|
| 1004 |
def test_set_activation_threshold__invalid_threshold_type(self): |
| 1005 |
parser = expat.ParserCreate() |
| 1006 |
setter = functools.partial(self.set_activation_threshold, parser) |
| 1007 |
|
| 1008 |
self.assertRaises(TypeError, setter, 1.0) |
| 1009 |
self.assertRaises(TypeError, setter, -1.5) |
| 1010 |
self.assertRaises(ValueError, setter, -5) |
| 1011 |
|
| 1012 |
def test_set_activation_threshold__invalid_threshold_range(self): |
| 1013 |
_testcapi = import_helper.import_module("_testcapi") |
| 1014 |
parser = expat.ParserCreate() |
| 1015 |
setter = functools.partial(self.set_activation_threshold, parser) |
| 1016 |
|
| 1017 |
self.assertRaises(OverflowError, setter, _testcapi.ULLONG_MAX + 1) |
| 1018 |
|
| 1019 |
def test_set_activation_threshold__fail_for_subparser(self): |
| 1020 |
parser = expat.ParserCreate() |
| 1021 |
subparser = parser.ExternalEntityParserCreate(None) |
| 1022 |
setter = functools.partial(self.set_activation_threshold, subparser) |
| 1023 |
self.assert_root_parser_failure(setter, 12345) |
| 1024 |
|
| 1025 |
@abc.abstractmethod |
| 1026 |
def test_set_maximum_amplification__amplification_exceeded(self): |
| 1027 |
"""Test when the amplification factor is exceeded.""" |
| 1028 |
|
| 1029 |
@abc.abstractmethod |
| 1030 |
def test_set_maximum_amplification__amplification_not_exceeded(self): |
| 1031 |
"""Test when the amplification factor is not exceeded.""" |
| 1032 |
|
| 1033 |
def test_set_maximum_amplification__infinity(self): |
| 1034 |
inf = float('inf') # an 'inf' threshold is allowed by Expat |
| 1035 |
parser = expat.ParserCreate() |
| 1036 |
self.assertIsNone(self.set_maximum_amplification(parser, inf)) |
| 1037 |
|
| 1038 |
def test_set_maximum_amplification__invalid_max_factor_type(self): |
| 1039 |
parser = expat.ParserCreate() |
| 1040 |
setter = functools.partial(self.set_maximum_amplification, parser) |
| 1041 |
|
| 1042 |
self.assertRaises(TypeError, setter, None) |
| 1043 |
self.assertRaises(TypeError, setter, 'abc') |
| 1044 |
|
| 1045 |
def test_set_maximum_amplification__invalid_max_factor_range(self): |
| 1046 |
parser = expat.ParserCreate() |
| 1047 |
setter = functools.partial(self.set_maximum_amplification, parser) |
| 1048 |
|
| 1049 |
msg = re.escape("'max_factor' must be at least 1.0") |
| 1050 |
self.assertRaisesRegex(expat.ExpatError, msg, setter, float('nan')) |
| 1051 |
self.assertRaisesRegex(expat.ExpatError, msg, setter, 0.99) |
| 1052 |
|
| 1053 |
def test_set_maximum_amplification__fail_for_subparser(self): |
| 1054 |
parser = expat.ParserCreate() |
| 1055 |
subparser = parser.ExternalEntityParserCreate(None) |
| 1056 |
setter = functools.partial(self.set_maximum_amplification, subparser) |
| 1057 |
self.assert_root_parser_failure(setter, 123.45) |
| 1058 |
|
| 1059 |
|
| 1060 |
@unittest.skipIf(not hasattr(expat.XMLParserType, |
| 1061 |
"SetAllocTrackerMaximumAmplification"), |
| 1062 |
"requires Python compiled with Expat >= 2.7.2") |
| 1063 |
class MemoryProtectionTest(AttackProtectionTestBase, unittest.TestCase): |
| 1064 |
|
| 1065 |
# NOTE: with the default Expat configuration, the billion laughs protection |
| 1066 |
# may hit before the allocation limiter if exponential_expansion_payload() |
| 1067 |
# is not carefully parametrized. As such, the payloads should be chosen so |
| 1068 |
# that either the allocation limiter is hit before other protections are |
| 1069 |
# triggered or no protection at all is triggered. |
| 1070 |
|
| 1071 |
def assert_rejected(self, func, /, *args, **kwargs): |
| 1072 |
"""Check that func(*args, **kwargs) hits the allocation limit.""" |
| 1073 |
msg = r"out of memory: line \d+, column \d+" |
| 1074 |
self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) |
| 1075 |
|
| 1076 |
def set_activation_threshold(self, parser, threshold): |
| 1077 |
return parser.SetAllocTrackerActivationThreshold(threshold) |
| 1078 |
|
| 1079 |
def set_maximum_amplification(self, parser, max_factor): |
| 1080 |
return parser.SetAllocTrackerMaximumAmplification(max_factor) |
| 1081 |
|
| 1082 |
def test_set_activation_threshold__threshold_reached(self): |
| 1083 |
parser = expat.ParserCreate() |
| 1084 |
# Choose a threshold expected to be always reached. |
| 1085 |
self.set_activation_threshold(parser, 3) |
| 1086 |
# Check that the threshold is reached by choosing a small factor |
| 1087 |
# and a payload whose peak amplification factor exceeds it. |
| 1088 |
self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) |
| 1089 |
payload = self.exponential_expansion_payload(ncols=10, nrows=4) |
| 1090 |
self.assert_rejected(parser.Parse, payload, True) |
| 1091 |
|
| 1092 |
def test_set_activation_threshold__threshold_not_reached(self): |
| 1093 |
parser = expat.ParserCreate() |
| 1094 |
# Choose a threshold expected to be never reached. |
| 1095 |
self.set_activation_threshold(parser, pow(10, 5)) |
| 1096 |
# Check that the threshold is reached by choosing a small factor |
| 1097 |
# and a payload whose peak amplification factor exceeds it. |
| 1098 |
self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) |
| 1099 |
payload = self.exponential_expansion_payload(ncols=10, nrows=4) |
| 1100 |
self.assertIsNotNone(parser.Parse(payload, True)) |
| 1101 |
|
| 1102 |
def test_set_maximum_amplification__amplification_exceeded(self): |
| 1103 |
parser = expat.ParserCreate() |
| 1104 |
# Unconditionally enable maximum activation factor. |
| 1105 |
self.set_activation_threshold(parser, 0) |
| 1106 |
# Choose a max amplification factor expected to always be exceeded. |
| 1107 |
self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) |
| 1108 |
# Craft a payload for which the peak amplification factor is > 1.0. |
| 1109 |
payload = self.exponential_expansion_payload(ncols=1, nrows=2) |
| 1110 |
self.assert_rejected(parser.Parse, payload, True) |
| 1111 |
|
| 1112 |
def test_set_maximum_amplification__amplification_not_exceeded(self): |
| 1113 |
parser = expat.ParserCreate() |
| 1114 |
# Unconditionally enable maximum activation factor. |
| 1115 |
self.set_activation_threshold(parser, 0) |
| 1116 |
# Choose a max amplification factor expected to never be exceeded. |
| 1117 |
self.assertIsNone(self.set_maximum_amplification(parser, 1e4)) |
| 1118 |
# Craft a payload for which the peak amplification factor is < 1e4. |
| 1119 |
payload = self.exponential_expansion_payload(ncols=1, nrows=2) |
| 1120 |
self.assertIsNotNone(parser.Parse(payload, True)) |
| 1121 |
|
| 1122 |
|
| 1123 |
if __name__ == "__main__": |
| 1124 |
unittest.main() |