| 1 |
from test import support |
| 2 |
from test.support import is_apple_mobile, os_helper, requires_debug_ranges, is_emscripten |
| 3 |
from test.support.script_helper import assert_python_ok |
| 4 |
import array |
| 5 |
import io |
| 6 |
import marshal |
| 7 |
import sys |
| 8 |
import unittest |
| 9 |
import os |
| 10 |
import types |
| 11 |
import textwrap |
| 12 |
|
| 13 |
try: |
| 14 |
import _testcapi |
| 15 |
except ImportError: |
| 16 |
_testcapi = None |
| 17 |
|
| 18 |
class HelperMixin: |
| 19 |
def helper(self, sample, *extra): |
| 20 |
new = marshal.loads(marshal.dumps(sample, *extra)) |
| 21 |
self.assertEqual(sample, new) |
| 22 |
try: |
| 23 |
with open(os_helper.TESTFN, "wb") as f: |
| 24 |
marshal.dump(sample, f, *extra) |
| 25 |
with open(os_helper.TESTFN, "rb") as f: |
| 26 |
new = marshal.load(f) |
| 27 |
self.assertEqual(sample, new) |
| 28 |
finally: |
| 29 |
os_helper.unlink(os_helper.TESTFN) |
| 30 |
|
| 31 |
def omit_last_byte(data): |
| 32 |
"""return data[:-1]""" |
| 33 |
# This file's code is used in CompatibilityTestCase, |
| 34 |
# but slices need marshal version 5. |
| 35 |
# Avoid the slice literal. |
| 36 |
return data[slice(0, -1)] |
| 37 |
|
| 38 |
class IntTestCase(unittest.TestCase, HelperMixin): |
| 39 |
def test_ints(self): |
| 40 |
# Test a range of Python ints larger than the machine word size. |
| 41 |
n = sys.maxsize ** 2 |
| 42 |
while n: |
| 43 |
for expected in (-n, n): |
| 44 |
self.helper(expected) |
| 45 |
n = n >> 1 |
| 46 |
n = 1 << 100 |
| 47 |
while n: |
| 48 |
for expected in (-n, -n+1, n-1, n): |
| 49 |
self.helper(expected) |
| 50 |
n = n >> 1 |
| 51 |
|
| 52 |
def test_int64(self): |
| 53 |
# Simulate int marshaling with TYPE_INT64. |
| 54 |
maxint64 = (1 << 63) - 1 |
| 55 |
minint64 = -maxint64-1 |
| 56 |
for base in maxint64, minint64, -maxint64, -(minint64 >> 1): |
| 57 |
while base: |
| 58 |
s = b'I' + int.to_bytes(base, 8, 'little', signed=True) |
| 59 |
got = marshal.loads(s) |
| 60 |
self.assertEqual(base, got) |
| 61 |
if base == -1: # a fixed-point for shifting right 1 |
| 62 |
base = 0 |
| 63 |
else: |
| 64 |
base >>= 1 |
| 65 |
|
| 66 |
got = marshal.loads(b'I\xfe\xdc\xba\x98\x76\x54\x32\x10') |
| 67 |
self.assertEqual(got, 0x1032547698badcfe) |
| 68 |
got = marshal.loads(b'I\x01\x23\x45\x67\x89\xab\xcd\xef') |
| 69 |
self.assertEqual(got, -0x1032547698badcff) |
| 70 |
got = marshal.loads(b'I\x08\x19\x2a\x3b\x4c\x5d\x6e\x7f') |
| 71 |
self.assertEqual(got, 0x7f6e5d4c3b2a1908) |
| 72 |
got = marshal.loads(b'I\xf7\xe6\xd5\xc4\xb3\xa2\x91\x80') |
| 73 |
self.assertEqual(got, -0x7f6e5d4c3b2a1909) |
| 74 |
|
| 75 |
def test_bool(self): |
| 76 |
for b in (True, False): |
| 77 |
self.helper(b) |
| 78 |
|
| 79 |
class FloatTestCase(unittest.TestCase, HelperMixin): |
| 80 |
def test_floats(self): |
| 81 |
# Test a few floats |
| 82 |
small = 1e-25 |
| 83 |
n = sys.maxsize * 3.7e250 |
| 84 |
while n > small: |
| 85 |
for expected in (-n, n): |
| 86 |
self.helper(float(expected)) |
| 87 |
n /= 123.4567 |
| 88 |
|
| 89 |
f = 0.0 |
| 90 |
s = marshal.dumps(f, 2) |
| 91 |
got = marshal.loads(s) |
| 92 |
self.assertEqual(f, got) |
| 93 |
# and with version <= 1 (floats marshalled differently then) |
| 94 |
s = marshal.dumps(f, 1) |
| 95 |
got = marshal.loads(s) |
| 96 |
self.assertEqual(f, got) |
| 97 |
|
| 98 |
n = sys.maxsize * 3.7e-250 |
| 99 |
while n < small: |
| 100 |
for expected in (-n, n): |
| 101 |
f = float(expected) |
| 102 |
self.helper(f) |
| 103 |
self.helper(f, 1) |
| 104 |
n *= 123.4567 |
| 105 |
|
| 106 |
class StringTestCase(unittest.TestCase, HelperMixin): |
| 107 |
def test_unicode(self): |
| 108 |
for s in ["", "Andr\xe8 Previn", "abc", " "*10000]: |
| 109 |
self.helper(marshal.loads(marshal.dumps(s))) |
| 110 |
|
| 111 |
def test_string(self): |
| 112 |
for s in ["", "Andr\xe8 Previn", "abc", " "*10000]: |
| 113 |
self.helper(s) |
| 114 |
|
| 115 |
def test_bytes(self): |
| 116 |
for s in [b"", b"Andr\xe8 Previn", b"abc", b" "*10000]: |
| 117 |
self.helper(s) |
| 118 |
|
| 119 |
class ExceptionTestCase(unittest.TestCase): |
| 120 |
def test_exceptions(self): |
| 121 |
new = marshal.loads(marshal.dumps(StopIteration)) |
| 122 |
self.assertEqual(StopIteration, new) |
| 123 |
|
| 124 |
class CodeTestCase(unittest.TestCase): |
| 125 |
def test_code(self): |
| 126 |
co = ExceptionTestCase.test_exceptions.__code__ |
| 127 |
new = marshal.loads(marshal.dumps(co)) |
| 128 |
self.assertEqual(co, new) |
| 129 |
|
| 130 |
def test_many_codeobjects(self): |
| 131 |
# Issue2957: bad recursion count on code objects |
| 132 |
# more than MAX_MARSHAL_STACK_DEPTH |
| 133 |
codes = (ExceptionTestCase.test_exceptions.__code__,) * 10_000 |
| 134 |
marshal.loads(marshal.dumps(codes)) |
| 135 |
|
| 136 |
def test_different_filenames(self): |
| 137 |
co1 = compile("x", "f1", "exec") |
| 138 |
co2 = compile("y", "f2", "exec") |
| 139 |
co1, co2 = marshal.loads(marshal.dumps((co1, co2))) |
| 140 |
self.assertEqual(co1.co_filename, "f1") |
| 141 |
self.assertEqual(co2.co_filename, "f2") |
| 142 |
|
| 143 |
def test_no_allow_code(self): |
| 144 |
data = {'a': [({0},)]} |
| 145 |
dump = marshal.dumps(data, allow_code=False) |
| 146 |
self.assertEqual(marshal.loads(dump, allow_code=False), data) |
| 147 |
|
| 148 |
f = io.BytesIO() |
| 149 |
marshal.dump(data, f, allow_code=False) |
| 150 |
f.seek(0) |
| 151 |
self.assertEqual(marshal.load(f, allow_code=False), data) |
| 152 |
|
| 153 |
co = ExceptionTestCase.test_exceptions.__code__ |
| 154 |
data = {'a': [({co, 0},)]} |
| 155 |
dump = marshal.dumps(data, allow_code=True) |
| 156 |
self.assertEqual(marshal.loads(dump, allow_code=True), data) |
| 157 |
with self.assertRaises(ValueError): |
| 158 |
marshal.dumps(data, allow_code=False) |
| 159 |
with self.assertRaises(ValueError): |
| 160 |
marshal.loads(dump, allow_code=False) |
| 161 |
|
| 162 |
marshal.dump(data, io.BytesIO(), allow_code=True) |
| 163 |
self.assertEqual(marshal.load(io.BytesIO(dump), allow_code=True), data) |
| 164 |
with self.assertRaises(ValueError): |
| 165 |
marshal.dump(data, io.BytesIO(), allow_code=False) |
| 166 |
with self.assertRaises(ValueError): |
| 167 |
marshal.load(io.BytesIO(dump), allow_code=False) |
| 168 |
|
| 169 |
@requires_debug_ranges() |
| 170 |
def test_minimal_linetable_with_no_debug_ranges(self): |
| 171 |
# Make sure when demarshalling objects with `-X no_debug_ranges` |
| 172 |
# that the columns are None. |
| 173 |
co = ExceptionTestCase.test_exceptions.__code__ |
| 174 |
code = textwrap.dedent(""" |
| 175 |
import sys |
| 176 |
import marshal |
| 177 |
with open(sys.argv[1], 'rb') as f: |
| 178 |
co = marshal.load(f) |
| 179 |
positions = list(co.co_positions()) |
| 180 |
assert positions[0][2] is None |
| 181 |
assert positions[0][3] is None |
| 182 |
""") |
| 183 |
|
| 184 |
try: |
| 185 |
with open(os_helper.TESTFN, 'wb') as f: |
| 186 |
marshal.dump(co, f) |
| 187 |
|
| 188 |
assert_python_ok('-X', 'no_debug_ranges', |
| 189 |
'-c', code, os_helper.TESTFN) |
| 190 |
finally: |
| 191 |
os_helper.unlink(os_helper.TESTFN) |
| 192 |
|
| 193 |
@support.cpython_only |
| 194 |
def test_same_filename_used(self): |
| 195 |
s = """def f(): pass\ndef g(): pass""" |
| 196 |
co = compile(s, "myfile", "exec") |
| 197 |
co = marshal.loads(marshal.dumps(co)) |
| 198 |
for obj in co.co_consts: |
| 199 |
if isinstance(obj, types.CodeType): |
| 200 |
self.assertIs(co.co_filename, obj.co_filename) |
| 201 |
|
| 202 |
class ContainerTestCase(unittest.TestCase, HelperMixin): |
| 203 |
d = {'astring': '[email protected]', |
| 204 |
'afloat': 7283.43, |
| 205 |
'anint': 2**20, |
| 206 |
'ashortlong': 2, |
| 207 |
'alist': ['.zyx.41'], |
| 208 |
'atuple': ('.zyx.41',)*10, |
| 209 |
'aboolean': False, |
| 210 |
'aunicode': "Andr\xe8 Previn" |
| 211 |
} |
| 212 |
|
| 213 |
def test_dict(self): |
| 214 |
self.helper(self.d) |
| 215 |
|
| 216 |
def test_list(self): |
| 217 |
self.helper(list(self.d.items())) |
| 218 |
|
| 219 |
def test_tuple(self): |
| 220 |
self.helper(tuple(self.d.keys())) |
| 221 |
|
| 222 |
def test_sets(self): |
| 223 |
for constructor in (set, frozenset): |
| 224 |
self.helper(constructor(self.d.keys())) |
| 225 |
|
| 226 |
|
| 227 |
class BufferTestCase(unittest.TestCase, HelperMixin): |
| 228 |
|
| 229 |
def test_bytearray(self): |
| 230 |
b = bytearray(b"abc") |
| 231 |
self.helper(b) |
| 232 |
new = marshal.loads(marshal.dumps(b)) |
| 233 |
self.assertEqual(type(new), bytes) |
| 234 |
|
| 235 |
def test_memoryview(self): |
| 236 |
b = memoryview(b"abc") |
| 237 |
self.helper(b) |
| 238 |
new = marshal.loads(marshal.dumps(b)) |
| 239 |
self.assertEqual(type(new), bytes) |
| 240 |
|
| 241 |
def test_array(self): |
| 242 |
a = array.array('B', b"abc") |
| 243 |
new = marshal.loads(marshal.dumps(a)) |
| 244 |
self.assertEqual(new, b"abc") |
| 245 |
|
| 246 |
|
| 247 |
class BugsTestCase(unittest.TestCase): |
| 248 |
def test_bug_5888452(self): |
| 249 |
# Simple-minded check for SF 588452: Debug build crashes |
| 250 |
marshal.dumps([128] * 1000) |
| 251 |
|
| 252 |
def test_patch_873224(self): |
| 253 |
self.assertRaises(Exception, marshal.loads, b'0') |
| 254 |
self.assertRaises(Exception, marshal.loads, b'f') |
| 255 |
self.assertRaises(Exception, marshal.loads, |
| 256 |
omit_last_byte(marshal.dumps(2**65))) |
| 257 |
|
| 258 |
def test_version_argument(self): |
| 259 |
# Python 2.4.0 crashes for any call to marshal.dumps(x, y) |
| 260 |
self.assertEqual(marshal.loads(marshal.dumps(5, 0)), 5) |
| 261 |
self.assertEqual(marshal.loads(marshal.dumps(5, 1)), 5) |
| 262 |
|
| 263 |
def test_fuzz(self): |
| 264 |
# simple test that it's at least not *totally* trivial to |
| 265 |
# crash from bad marshal data |
| 266 |
for i in range(256): |
| 267 |
c = bytes([i]) |
| 268 |
try: |
| 269 |
marshal.loads(c) |
| 270 |
except Exception: |
| 271 |
pass |
| 272 |
|
| 273 |
def test_loads_recursion(self): |
| 274 |
def run_tests(N, check): |
| 275 |
# (((...None...),),) |
| 276 |
check(b')\x01' * N + b'N') |
| 277 |
check(b'(\x01\x00\x00\x00' * N + b'N') |
| 278 |
# [[[...None...]]] |
| 279 |
check(b'[\x01\x00\x00\x00' * N + b'N') |
| 280 |
# {None: {None: {None: ...None...}}} |
| 281 |
check(b'{N' * N + b'N' + b'0' * N) |
| 282 |
# frozenset([frozenset([frozenset([...None...])])]) |
| 283 |
check(b'>\x01\x00\x00\x00' * N + b'N') |
| 284 |
# Check that the generated marshal data is valid and marshal.loads() |
| 285 |
# works for moderately deep nesting |
| 286 |
run_tests(100, marshal.loads) |
| 287 |
# Very deeply nested structure shouldn't blow the stack |
| 288 |
def check(s): |
| 289 |
self.assertRaises(ValueError, marshal.loads, s) |
| 290 |
run_tests(2**20, check) |
| 291 |
|
| 292 |
def test_recursion_limit(self): |
| 293 |
# Create a deeply nested structure. |
| 294 |
head = last = [] |
| 295 |
# The max stack depth should match the value in Python/marshal.c. |
| 296 |
# BUG: https://bugs.python.org/issue33720 |
| 297 |
# Windows always limits the maximum depth on release and debug builds |
| 298 |
#if os.name == 'nt' and support.Py_DEBUG: |
| 299 |
if os.name == 'nt': |
| 300 |
MAX_MARSHAL_STACK_DEPTH = 1000 |
| 301 |
elif sys.platform == 'wasi' or is_emscripten or is_apple_mobile: |
| 302 |
MAX_MARSHAL_STACK_DEPTH = 1500 |
| 303 |
else: |
| 304 |
MAX_MARSHAL_STACK_DEPTH = 2000 |
| 305 |
for i in range(MAX_MARSHAL_STACK_DEPTH - 2): |
| 306 |
last.append([0]) |
| 307 |
last = last[-1] |
| 308 |
|
| 309 |
# Verify we don't blow out the stack with dumps/load. |
| 310 |
data = marshal.dumps(head) |
| 311 |
new_head = marshal.loads(data) |
| 312 |
# Don't use == to compare objects, it can exceed the recursion limit. |
| 313 |
self.assertEqual(len(new_head), len(head)) |
| 314 |
self.assertEqual(len(new_head[0]), len(head[0])) |
| 315 |
self.assertEqual(len(new_head[-1]), len(head[-1])) |
| 316 |
|
| 317 |
last.append([0]) |
| 318 |
self.assertRaises(ValueError, marshal.dumps, head) |
| 319 |
|
| 320 |
def test_reference_loop_list(self): |
| 321 |
a = [] |
| 322 |
a.append(a) |
| 323 |
for v in range(3): |
| 324 |
self.assertRaises(ValueError, marshal.dumps, a, v) |
| 325 |
for v in range(3, marshal.version + 1): |
| 326 |
d = marshal.dumps(a, v) |
| 327 |
b = marshal.loads(d) |
| 328 |
self.assertIsInstance(b, list) |
| 329 |
self.assertIs(b[0], b) |
| 330 |
|
| 331 |
def test_reference_loop_dict(self): |
| 332 |
a = {} |
| 333 |
a[None] = a |
| 334 |
for v in range(3): |
| 335 |
self.assertRaises(ValueError, marshal.dumps, a, v) |
| 336 |
for v in range(3, marshal.version + 1): |
| 337 |
d = marshal.dumps(a, v) |
| 338 |
b = marshal.loads(d) |
| 339 |
self.assertIsInstance(b, dict) |
| 340 |
self.assertIs(b[None], b) |
| 341 |
|
| 342 |
def test_reference_loop_tuple(self): |
| 343 |
a = ([],) |
| 344 |
a[0].append(a) |
| 345 |
for v in range(3): |
| 346 |
self.assertRaises(ValueError, marshal.dumps, a, v) |
| 347 |
for v in range(3, marshal.version + 1): |
| 348 |
d = marshal.dumps(a, v) |
| 349 |
b = marshal.loads(d) |
| 350 |
self.assertIsInstance(b, tuple) |
| 351 |
self.assertIsInstance(b[0], list) |
| 352 |
self.assertIs(b[0][0], b) |
| 353 |
|
| 354 |
def test_reference_loop_code(self): |
| 355 |
def f(): |
| 356 |
return 1234.5 |
| 357 |
code = f.__code__ |
| 358 |
a = [] |
| 359 |
code = code.replace(co_consts=code.co_consts + (a,)) |
| 360 |
# This test creates a reference loop which leads to reference leaks, |
| 361 |
# so we need to break the loop manually. See gh-148722. |
| 362 |
self.addCleanup(a.clear) |
| 363 |
a.append(code) |
| 364 |
for v in range(marshal.version + 1): |
| 365 |
self.assertRaises(ValueError, marshal.dumps, code, v) |
| 366 |
|
| 367 |
def test_reference_loop_slice(self): |
| 368 |
a = slice([], None) |
| 369 |
a.start.append(a) |
| 370 |
for v in range(marshal.version + 1): |
| 371 |
self.assertRaises(ValueError, marshal.dumps, a, v) |
| 372 |
|
| 373 |
a = slice(None, []) |
| 374 |
a.stop.append(a) |
| 375 |
for v in range(marshal.version + 1): |
| 376 |
self.assertRaises(ValueError, marshal.dumps, a, v) |
| 377 |
|
| 378 |
a = slice(None, None, []) |
| 379 |
a.step.append(a) |
| 380 |
for v in range(marshal.version + 1): |
| 381 |
self.assertRaises(ValueError, marshal.dumps, a, v) |
| 382 |
|
| 383 |
def test_loads_reference_loop_list(self): |
| 384 |
data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [<R>] |
| 385 |
a = marshal.loads(data) |
| 386 |
self.assertIsInstance(a, list) |
| 387 |
self.assertIs(a[0], a) |
| 388 |
|
| 389 |
def test_loads_reference_loop_dict(self): |
| 390 |
data = b'\xfbNr\x00\x00\x00\x000' # {None: <R>} |
| 391 |
a = marshal.loads(data) |
| 392 |
self.assertIsInstance(a, dict) |
| 393 |
self.assertIs(a[None], a) |
| 394 |
|
| 395 |
def test_loads_abnormal_reference_loops(self): |
| 396 |
# Indirect self-references of tuples. |
| 397 |
data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([<R>],) |
| 398 |
a = marshal.loads(data) |
| 399 |
self.assertIsInstance(a, tuple) |
| 400 |
self.assertIsInstance(a[0], list) |
| 401 |
self.assertIs(a[0][0], a) |
| 402 |
|
| 403 |
data = b'\xa8\x01\x00\x00\x00{Nr\x00\x00\x00\x000' # ({None: <R>},) |
| 404 |
a = marshal.loads(data) |
| 405 |
self.assertIsInstance(a, tuple) |
| 406 |
self.assertIsInstance(a[0], dict) |
| 407 |
self.assertIs(a[0][None], a) |
| 408 |
|
| 409 |
# Direct self-reference which cannot be created in Python. |
| 410 |
# This creates a reference loop which cannot be collected. |
| 411 |
if False: |
| 412 |
data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (<R>,) |
| 413 |
a = marshal.loads(data) |
| 414 |
self.assertIsInstance(a, tuple) |
| 415 |
self.assertIs(a[0], a) |
| 416 |
|
| 417 |
# Direct self-references which cannot be created in Python |
| 418 |
# because of unhashability. |
| 419 |
data = b'\xfbr\x00\x00\x00\x00N0' # {<R>: None} |
| 420 |
self.assertRaises(TypeError, marshal.loads, data) |
| 421 |
data = b'\xbc\x01\x00\x00\x00r\x00\x00\x00\x00' # {<R>} |
| 422 |
self.assertRaises(TypeError, marshal.loads, data) |
| 423 |
|
| 424 |
for data in [ |
| 425 |
# Indirect self-references of immutable objects. |
| 426 |
b'\xba[\x01\x00\x00\x00r\x00\x00\x00\x00NN', # slice([<R>], None) |
| 427 |
b'\xbaN[\x01\x00\x00\x00r\x00\x00\x00\x00N', # slice(None, [<R>]) |
| 428 |
b'\xbaNN[\x01\x00\x00\x00r\x00\x00\x00\x00', # slice(None, None, [<R>]) |
| 429 |
b'\xba{Nr\x00\x00\x00\x000NN', # slice({None: <R>}, None) |
| 430 |
b'\xbaN{Nr\x00\x00\x00\x000N', # slice(None, {None: <R>}) |
| 431 |
b'\xbaNN{Nr\x00\x00\x00\x000', # slice(None, None, {None: <R>}) |
| 432 |
|
| 433 |
# Direct self-references which cannot be created in Python. |
| 434 |
b'\xbe\x01\x00\x00\x00r\x00\x00\x00\x00', # frozenset({<R>}) |
| 435 |
b'\xbar\x00\x00\x00\x00NN', # slice(<R>, None) |
| 436 |
b'\xbaNr\x00\x00\x00\x00N', # slice(None, <R>) |
| 437 |
b'\xbaNNr\x00\x00\x00\x00', # slice(None, None, <R>) |
| 438 |
]: |
| 439 |
with self.subTest(data=data): |
| 440 |
self.assertRaises(ValueError, marshal.loads, data) |
| 441 |
|
| 442 |
def test_exact_type_match(self): |
| 443 |
# Former bug: |
| 444 |
# >>> class Int(int): pass |
| 445 |
# >>> type(loads(dumps(Int()))) |
| 446 |
# <type 'int'> |
| 447 |
for typ in (int, float, complex, tuple, list, dict, set, frozenset): |
| 448 |
# Note: str subclasses are not tested because they get handled |
| 449 |
# by marshal's routines for objects supporting the buffer API. |
| 450 |
subtyp = type('subtyp', (typ,), {}) |
| 451 |
self.assertRaises(ValueError, marshal.dumps, subtyp()) |
| 452 |
|
| 453 |
# Issue #1792 introduced a change in how marshal increases the size of its |
| 454 |
# internal buffer; this test ensures that the new code is exercised. |
| 455 |
def test_large_marshal(self): |
| 456 |
size = int(1e6) |
| 457 |
testString = 'abc' * size |
| 458 |
marshal.dumps(testString) |
| 459 |
|
| 460 |
def test_invalid_longs(self): |
| 461 |
# Issue #7019: marshal.loads shouldn't produce unnormalized PyLongs |
| 462 |
invalid_string = b'l\x02\x00\x00\x00\x00\x00\x00\x00' |
| 463 |
self.assertRaises(ValueError, marshal.loads, invalid_string) |
| 464 |
|
| 465 |
def test_multiple_dumps_and_loads(self): |
| 466 |
# Issue 12291: marshal.load() should be callable multiple times |
| 467 |
# with interleaved data written by non-marshal code |
| 468 |
# Adapted from a patch by Engelbert Gruber. |
| 469 |
data = (1, 'abc', b'def', 1.0, (2, 'a', ['b', b'c'])) |
| 470 |
for interleaved in (b'', b'0123'): |
| 471 |
ilen = len(interleaved) |
| 472 |
positions = [] |
| 473 |
try: |
| 474 |
with open(os_helper.TESTFN, 'wb') as f: |
| 475 |
for d in data: |
| 476 |
marshal.dump(d, f) |
| 477 |
if ilen: |
| 478 |
f.write(interleaved) |
| 479 |
positions.append(f.tell()) |
| 480 |
with open(os_helper.TESTFN, 'rb') as f: |
| 481 |
for i, d in enumerate(data): |
| 482 |
self.assertEqual(d, marshal.load(f)) |
| 483 |
if ilen: |
| 484 |
f.read(ilen) |
| 485 |
self.assertEqual(positions[i], f.tell()) |
| 486 |
finally: |
| 487 |
os_helper.unlink(os_helper.TESTFN) |
| 488 |
|
| 489 |
def test_loads_reject_unicode_strings(self): |
| 490 |
# Issue #14177: marshal.loads() should not accept unicode strings |
| 491 |
unicode_string = 'T' |
| 492 |
self.assertRaises(TypeError, marshal.loads, unicode_string) |
| 493 |
|
| 494 |
def test_bad_reader(self): |
| 495 |
class BadReader(io.BytesIO): |
| 496 |
def readinto(self, buf): |
| 497 |
n = super().readinto(buf) |
| 498 |
if n is not None and n > 4: |
| 499 |
n += 10**6 |
| 500 |
return n |
| 501 |
for value in (1.0, 1j, b'0123456789', '0123456789'): |
| 502 |
self.assertRaises(ValueError, marshal.load, |
| 503 |
BadReader(marshal.dumps(value))) |
| 504 |
|
| 505 |
def test_eof(self): |
| 506 |
data = marshal.dumps(("hello", "dolly", None)) |
| 507 |
for i in range(len(data)): |
| 508 |
self.assertRaises(EOFError, marshal.loads, data[0: i]) |
| 509 |
|
| 510 |
def test_deterministic_sets(self): |
| 511 |
# bpo-37596: To support reproducible builds, sets and frozensets need to |
| 512 |
# have their elements serialized in a consistent order (even when they |
| 513 |
# have been scrambled by hash randomization): |
| 514 |
for kind in ("set", "frozenset"): |
| 515 |
for elements in ( |
| 516 |
"float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'", |
| 517 |
# Also test for bad interactions with backreferencing: |
| 518 |
"('Spam', 0), ('Spam', 1), ('Spam', 2), ('Spam', 3), ('Spam', 4), ('Spam', 5)", |
| 519 |
): |
| 520 |
s = f"{kind}([{elements}])" |
| 521 |
with self.subTest(s): |
| 522 |
# First, make sure that our test case still has different |
| 523 |
# orders under hash seeds 0 and 1. If this check fails, we |
| 524 |
# need to update this test with different elements. Skip |
| 525 |
# this part if we are configured to use any other hash |
| 526 |
# algorithm (for example, using Py_HASH_EXTERNAL): |
| 527 |
if sys.hash_info.algorithm in {"fnv", "siphash24"}: |
| 528 |
args = ["-c", f"print({s})"] |
| 529 |
_, repr_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0") |
| 530 |
_, repr_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1") |
| 531 |
self.assertNotEqual(repr_0, repr_1) |
| 532 |
# Then, perform the actual test: |
| 533 |
args = ["-c", f"import marshal; print(marshal.dumps({s}))"] |
| 534 |
_, dump_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0") |
| 535 |
_, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1") |
| 536 |
self.assertEqual(dump_0, dump_1) |
| 537 |
|
| 538 |
def test_unmarshallable(self): |
| 539 |
# Check no crash after encountering unmarshallable objects. |
| 540 |
# See https://github.com/python/cpython/issues/106287. |
| 541 |
fset = frozenset([int]) |
| 542 |
code = compile("a = 1", "<string>", "exec") |
| 543 |
code = code.replace(co_consts=(1, fset, None)) |
| 544 |
cases = (('tuple', (fset,)), |
| 545 |
('list', [fset]), |
| 546 |
('set', fset), |
| 547 |
('dict key', {fset: 'x'}), |
| 548 |
('dict value', {'x': fset}), |
| 549 |
('dict key & value', {fset: fset}), |
| 550 |
('slice', slice(fset, fset)), |
| 551 |
('code', code)) |
| 552 |
for name, arg in cases: |
| 553 |
with self.subTest(name, arg=arg): |
| 554 |
with self.assertRaisesRegex(ValueError, "unmarshallable object"): |
| 555 |
marshal.dumps((arg, memoryview(b''))) |
| 556 |
|
| 557 |
|
| 558 |
LARGE_SIZE = 2**31 |
| 559 |
pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4 |
| 560 |
|
| 561 |
class NullWriter: |
| 562 |
def write(self, s): |
| 563 |
pass |
| 564 |
|
| 565 |
@unittest.skipIf(LARGE_SIZE > sys.maxsize, "test cannot run on 32-bit systems") |
| 566 |
class LargeValuesTestCase(unittest.TestCase): |
| 567 |
def check_unmarshallable(self, data): |
| 568 |
self.assertRaises(ValueError, marshal.dump, data, NullWriter()) |
| 569 |
|
| 570 |
@support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) |
| 571 |
def test_bytes(self, size): |
| 572 |
self.check_unmarshallable(b'x' * size) |
| 573 |
|
| 574 |
@support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) |
| 575 |
def test_str(self, size): |
| 576 |
self.check_unmarshallable('x' * size) |
| 577 |
|
| 578 |
@support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False) |
| 579 |
def test_tuple(self, size): |
| 580 |
self.check_unmarshallable((None,) * size) |
| 581 |
|
| 582 |
@support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False) |
| 583 |
def test_list(self, size): |
| 584 |
self.check_unmarshallable([None] * size) |
| 585 |
|
| 586 |
@support.bigmemtest(size=LARGE_SIZE, |
| 587 |
memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1), |
| 588 |
dry_run=False) |
| 589 |
def test_set(self, size): |
| 590 |
self.check_unmarshallable(set(range(size))) |
| 591 |
|
| 592 |
@support.bigmemtest(size=LARGE_SIZE, |
| 593 |
memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1), |
| 594 |
dry_run=False) |
| 595 |
def test_frozenset(self, size): |
| 596 |
self.check_unmarshallable(frozenset(range(size))) |
| 597 |
|
| 598 |
@support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) |
| 599 |
def test_bytearray(self, size): |
| 600 |
self.check_unmarshallable(bytearray(size)) |
| 601 |
|
| 602 |
def CollectObjectIDs(ids, obj): |
| 603 |
"""Collect object ids seen in a structure""" |
| 604 |
if id(obj) in ids: |
| 605 |
return |
| 606 |
ids.add(id(obj)) |
| 607 |
if isinstance(obj, (list, tuple, set, frozenset)): |
| 608 |
for e in obj: |
| 609 |
CollectObjectIDs(ids, e) |
| 610 |
elif isinstance(obj, dict): |
| 611 |
for k, v in obj.items(): |
| 612 |
CollectObjectIDs(ids, k) |
| 613 |
CollectObjectIDs(ids, v) |
| 614 |
return len(ids) |
| 615 |
|
| 616 |
class InstancingTestCase(unittest.TestCase, HelperMixin): |
| 617 |
keys = (123, 1.2345, 'abc', (123, 'abc'), frozenset({123, 'abc'})) |
| 618 |
|
| 619 |
def helper3(self, rsample, recursive=False, simple=False): |
| 620 |
#we have two instances |
| 621 |
sample = (rsample, rsample) |
| 622 |
|
| 623 |
n0 = CollectObjectIDs(set(), sample) |
| 624 |
|
| 625 |
for v in range(3, marshal.version + 1): |
| 626 |
s3 = marshal.dumps(sample, v) |
| 627 |
n3 = CollectObjectIDs(set(), marshal.loads(s3)) |
| 628 |
|
| 629 |
#same number of instances generated |
| 630 |
self.assertEqual(n3, n0) |
| 631 |
|
| 632 |
if not recursive: |
| 633 |
#can compare with version 2 |
| 634 |
s2 = marshal.dumps(sample, 2) |
| 635 |
n2 = CollectObjectIDs(set(), marshal.loads(s2)) |
| 636 |
#old format generated more instances |
| 637 |
self.assertGreater(n2, n0) |
| 638 |
|
| 639 |
#if complex objects are in there, old format is larger |
| 640 |
if not simple: |
| 641 |
self.assertGreater(len(s2), len(s3)) |
| 642 |
else: |
| 643 |
self.assertGreaterEqual(len(s2), len(s3)) |
| 644 |
|
| 645 |
def testInt(self): |
| 646 |
intobj = 123321 |
| 647 |
self.helper(intobj) |
| 648 |
self.helper3(intobj, simple=True) |
| 649 |
|
| 650 |
def testFloat(self): |
| 651 |
floatobj = 1.2345 |
| 652 |
self.helper(floatobj) |
| 653 |
self.helper3(floatobj) |
| 654 |
|
| 655 |
def testStr(self): |
| 656 |
strobj = "abcde"*3 |
| 657 |
self.helper(strobj) |
| 658 |
self.helper3(strobj) |
| 659 |
|
| 660 |
def testBytes(self): |
| 661 |
bytesobj = b"abcde"*3 |
| 662 |
self.helper(bytesobj) |
| 663 |
self.helper3(bytesobj) |
| 664 |
|
| 665 |
def testList(self): |
| 666 |
for obj in self.keys: |
| 667 |
listobj = [obj, obj] |
| 668 |
self.helper(listobj) |
| 669 |
self.helper3(listobj) |
| 670 |
|
| 671 |
def testTuple(self): |
| 672 |
for obj in self.keys: |
| 673 |
tupleobj = (obj, obj) |
| 674 |
self.helper(tupleobj) |
| 675 |
self.helper3(tupleobj) |
| 676 |
|
| 677 |
def testSet(self): |
| 678 |
for obj in self.keys: |
| 679 |
setobj = {(obj, 1), (obj, 2)} |
| 680 |
self.helper(setobj) |
| 681 |
self.helper3(setobj) |
| 682 |
|
| 683 |
def testFrozenSet(self): |
| 684 |
for obj in self.keys: |
| 685 |
frozensetobj = frozenset({(obj, 1), (obj, 2)}) |
| 686 |
self.helper(frozensetobj) |
| 687 |
self.helper3(frozensetobj) |
| 688 |
|
| 689 |
def testDict(self): |
| 690 |
for obj in self.keys: |
| 691 |
dictobj = {"hello": obj, "goodbye": obj, obj: "hello"} |
| 692 |
self.helper(dictobj) |
| 693 |
self.helper3(dictobj) |
| 694 |
|
| 695 |
def testModule(self): |
| 696 |
with open(__file__, "rb") as f: |
| 697 |
code = f.read() |
| 698 |
if __file__.endswith(".py"): |
| 699 |
code = compile(code, __file__, "exec") |
| 700 |
self.helper(code) |
| 701 |
self.helper3(code) |
| 702 |
|
| 703 |
def testRecursion(self): |
| 704 |
obj = 1.2345 |
| 705 |
d = {"hello": obj, "goodbye": obj, obj: "hello"} |
| 706 |
d["self"] = d |
| 707 |
self.helper3(d, recursive=True) |
| 708 |
l = [obj, obj] |
| 709 |
l.append(l) |
| 710 |
self.helper3(l, recursive=True) |
| 711 |
|
| 712 |
class CompatibilityTestCase(unittest.TestCase): |
| 713 |
def _test(self, version): |
| 714 |
with open(__file__, "rb") as f: |
| 715 |
code = f.read() |
| 716 |
if __file__.endswith(".py"): |
| 717 |
code = compile(code, __file__, "exec") |
| 718 |
data = marshal.dumps(code, version) |
| 719 |
marshal.loads(data) |
| 720 |
|
| 721 |
def test0To3(self): |
| 722 |
self._test(0) |
| 723 |
|
| 724 |
def test1To3(self): |
| 725 |
self._test(1) |
| 726 |
|
| 727 |
def test2To3(self): |
| 728 |
self._test(2) |
| 729 |
|
| 730 |
def test3To3(self): |
| 731 |
self._test(3) |
| 732 |
|
| 733 |
class InterningTestCase(unittest.TestCase, HelperMixin): |
| 734 |
strobj = "this is an interned string" |
| 735 |
strobj = sys.intern(strobj) |
| 736 |
|
| 737 |
def testIntern(self): |
| 738 |
s = marshal.loads(marshal.dumps(self.strobj)) |
| 739 |
self.assertEqual(s, self.strobj) |
| 740 |
self.assertEqual(id(s), id(self.strobj)) |
| 741 |
s2 = sys.intern(s) |
| 742 |
self.assertEqual(id(s2), id(s)) |
| 743 |
|
| 744 |
def testNoIntern(self): |
| 745 |
s = marshal.loads(marshal.dumps(self.strobj, 2)) |
| 746 |
self.assertEqual(s, self.strobj) |
| 747 |
self.assertNotEqual(id(s), id(self.strobj)) |
| 748 |
s2 = sys.intern(s) |
| 749 |
self.assertNotEqual(id(s2), id(s)) |
| 750 |
|
| 751 |
class SliceTestCase(unittest.TestCase, HelperMixin): |
| 752 |
def test_slice(self): |
| 753 |
for obj in ( |
| 754 |
slice(None), slice(1), slice(1, 2), slice(1, 2, 3), |
| 755 |
slice({'set'}, ('tuple', {'with': 'dict'}, ), self.helper.__code__) |
| 756 |
): |
| 757 |
with self.subTest(obj=str(obj)): |
| 758 |
self.helper(obj) |
| 759 |
|
| 760 |
for version in range(4): |
| 761 |
with self.assertRaises(ValueError): |
| 762 |
marshal.dumps(obj, version) |
| 763 |
|
| 764 |
@support.cpython_only |
| 765 |
@unittest.skipUnless(_testcapi, 'requires _testcapi') |
| 766 |
class CAPI_TestCase(unittest.TestCase, HelperMixin): |
| 767 |
|
| 768 |
def test_write_long_to_file(self): |
| 769 |
for v in range(marshal.version + 1): |
| 770 |
_testcapi.pymarshal_write_long_to_file(0x12345678, os_helper.TESTFN, v) |
| 771 |
with open(os_helper.TESTFN, 'rb') as f: |
| 772 |
data = f.read() |
| 773 |
os_helper.unlink(os_helper.TESTFN) |
| 774 |
self.assertEqual(data, b'\x78\x56\x34\x12') |
| 775 |
|
| 776 |
def test_write_object_to_file(self): |
| 777 |
obj = ('\u20ac', b'abc', 123, 45.6, 7+8j, 'long line '*1000) |
| 778 |
for v in range(marshal.version + 1): |
| 779 |
_testcapi.pymarshal_write_object_to_file(obj, os_helper.TESTFN, v) |
| 780 |
with open(os_helper.TESTFN, 'rb') as f: |
| 781 |
data = f.read() |
| 782 |
os_helper.unlink(os_helper.TESTFN) |
| 783 |
self.assertEqual(marshal.loads(data), obj) |
| 784 |
|
| 785 |
def test_read_short_from_file(self): |
| 786 |
with open(os_helper.TESTFN, 'wb') as f: |
| 787 |
f.write(b'\x34\x12xxxx') |
| 788 |
r, p = _testcapi.pymarshal_read_short_from_file(os_helper.TESTFN) |
| 789 |
os_helper.unlink(os_helper.TESTFN) |
| 790 |
self.assertEqual(r, 0x1234) |
| 791 |
self.assertEqual(p, 2) |
| 792 |
|
| 793 |
with open(os_helper.TESTFN, 'wb') as f: |
| 794 |
f.write(b'\x12') |
| 795 |
with self.assertRaises(EOFError): |
| 796 |
_testcapi.pymarshal_read_short_from_file(os_helper.TESTFN) |
| 797 |
os_helper.unlink(os_helper.TESTFN) |
| 798 |
|
| 799 |
def test_read_long_from_file(self): |
| 800 |
with open(os_helper.TESTFN, 'wb') as f: |
| 801 |
f.write(b'\x78\x56\x34\x12xxxx') |
| 802 |
r, p = _testcapi.pymarshal_read_long_from_file(os_helper.TESTFN) |
| 803 |
os_helper.unlink(os_helper.TESTFN) |
| 804 |
self.assertEqual(r, 0x12345678) |
| 805 |
self.assertEqual(p, 4) |
| 806 |
|
| 807 |
with open(os_helper.TESTFN, 'wb') as f: |
| 808 |
f.write(b'\x56\x34\x12') |
| 809 |
with self.assertRaises(EOFError): |
| 810 |
_testcapi.pymarshal_read_long_from_file(os_helper.TESTFN) |
| 811 |
os_helper.unlink(os_helper.TESTFN) |
| 812 |
|
| 813 |
def test_read_last_object_from_file(self): |
| 814 |
obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) |
| 815 |
for v in range(marshal.version + 1): |
| 816 |
data = marshal.dumps(obj, v) |
| 817 |
with open(os_helper.TESTFN, 'wb') as f: |
| 818 |
f.write(data + b'xxxx') |
| 819 |
r, p = _testcapi.pymarshal_read_last_object_from_file(os_helper.TESTFN) |
| 820 |
os_helper.unlink(os_helper.TESTFN) |
| 821 |
self.assertEqual(r, obj) |
| 822 |
|
| 823 |
with open(os_helper.TESTFN, 'wb') as f: |
| 824 |
f.write(omit_last_byte(data)) |
| 825 |
with self.assertRaises(EOFError): |
| 826 |
_testcapi.pymarshal_read_last_object_from_file(os_helper.TESTFN) |
| 827 |
os_helper.unlink(os_helper.TESTFN) |
| 828 |
|
| 829 |
def test_read_object_from_file(self): |
| 830 |
obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) |
| 831 |
for v in range(marshal.version + 1): |
| 832 |
data = marshal.dumps(obj, v) |
| 833 |
with open(os_helper.TESTFN, 'wb') as f: |
| 834 |
f.write(data + b'xxxx') |
| 835 |
r, p = _testcapi.pymarshal_read_object_from_file(os_helper.TESTFN) |
| 836 |
os_helper.unlink(os_helper.TESTFN) |
| 837 |
self.assertEqual(r, obj) |
| 838 |
self.assertEqual(p, len(data)) |
| 839 |
|
| 840 |
with open(os_helper.TESTFN, 'wb') as f: |
| 841 |
f.write(omit_last_byte(data)) |
| 842 |
with self.assertRaises(EOFError): |
| 843 |
_testcapi.pymarshal_read_object_from_file(os_helper.TESTFN) |
| 844 |
os_helper.unlink(os_helper.TESTFN) |
| 845 |
|
| 846 |
|
| 847 |
if __name__ == "__main__": |
| 848 |
unittest.main() |