Debian Codegraph @

Lib/test/test_marshal.py

raw
1 from test import support
2 from test.support import is_apple_mobile, os_helper, requires_debug_ranges, is_emscripten
3 from test.support.script_helper import assert_python_ok
4 import array
5 import io
6 import marshal
7 import sys
8 import unittest
9 import os
10 import types
11 import textwrap
12
13 try:
14 import _testcapi
15 except ImportError:
16 _testcapi = None
17
18 class HelperMixin:
19 def helper(self, sample, *extra):
20 new = marshal.loads(marshal.dumps(sample, *extra))
21 self.assertEqual(sample, new)
22 try:
23 with open(os_helper.TESTFN, "wb") as f:
24 marshal.dump(sample, f, *extra)
25 with open(os_helper.TESTFN, "rb") as f:
26 new = marshal.load(f)
27 self.assertEqual(sample, new)
28 finally:
29 os_helper.unlink(os_helper.TESTFN)
30
31 def omit_last_byte(data):
32 """return data[:-1]"""
33 # This file's code is used in CompatibilityTestCase,
34 # but slices need marshal version 5.
35 # Avoid the slice literal.
36 return data[slice(0, -1)]
37
38 class IntTestCase(unittest.TestCase, HelperMixin):
39 def test_ints(self):
40 # Test a range of Python ints larger than the machine word size.
41 n = sys.maxsize ** 2
42 while n:
43 for expected in (-n, n):
44 self.helper(expected)
45 n = n >> 1
46 n = 1 << 100
47 while n:
48 for expected in (-n, -n+1, n-1, n):
49 self.helper(expected)
50 n = n >> 1
51
52 def test_int64(self):
53 # Simulate int marshaling with TYPE_INT64.
54 maxint64 = (1 << 63) - 1
55 minint64 = -maxint64-1
56 for base in maxint64, minint64, -maxint64, -(minint64 >> 1):
57 while base:
58 s = b'I' + int.to_bytes(base, 8, 'little', signed=True)
59 got = marshal.loads(s)
60 self.assertEqual(base, got)
61 if base == -1: # a fixed-point for shifting right 1
62 base = 0
63 else:
64 base >>= 1
65
66 got = marshal.loads(b'I\xfe\xdc\xba\x98\x76\x54\x32\x10')
67 self.assertEqual(got, 0x1032547698badcfe)
68 got = marshal.loads(b'I\x01\x23\x45\x67\x89\xab\xcd\xef')
69 self.assertEqual(got, -0x1032547698badcff)
70 got = marshal.loads(b'I\x08\x19\x2a\x3b\x4c\x5d\x6e\x7f')
71 self.assertEqual(got, 0x7f6e5d4c3b2a1908)
72 got = marshal.loads(b'I\xf7\xe6\xd5\xc4\xb3\xa2\x91\x80')
73 self.assertEqual(got, -0x7f6e5d4c3b2a1909)
74
75 def test_bool(self):
76 for b in (True, False):
77 self.helper(b)
78
79 class FloatTestCase(unittest.TestCase, HelperMixin):
80 def test_floats(self):
81 # Test a few floats
82 small = 1e-25
83 n = sys.maxsize * 3.7e250
84 while n > small:
85 for expected in (-n, n):
86 self.helper(float(expected))
87 n /= 123.4567
88
89 f = 0.0
90 s = marshal.dumps(f, 2)
91 got = marshal.loads(s)
92 self.assertEqual(f, got)
93 # and with version <= 1 (floats marshalled differently then)
94 s = marshal.dumps(f, 1)
95 got = marshal.loads(s)
96 self.assertEqual(f, got)
97
98 n = sys.maxsize * 3.7e-250
99 while n < small:
100 for expected in (-n, n):
101 f = float(expected)
102 self.helper(f)
103 self.helper(f, 1)
104 n *= 123.4567
105
106 class StringTestCase(unittest.TestCase, HelperMixin):
107 def test_unicode(self):
108 for s in ["", "Andr\xe8 Previn", "abc", " "*10000]:
109 self.helper(marshal.loads(marshal.dumps(s)))
110
111 def test_string(self):
112 for s in ["", "Andr\xe8 Previn", "abc", " "*10000]:
113 self.helper(s)
114
115 def test_bytes(self):
116 for s in [b"", b"Andr\xe8 Previn", b"abc", b" "*10000]:
117 self.helper(s)
118
119 class ExceptionTestCase(unittest.TestCase):
120 def test_exceptions(self):
121 new = marshal.loads(marshal.dumps(StopIteration))
122 self.assertEqual(StopIteration, new)
123
124 class CodeTestCase(unittest.TestCase):
125 def test_code(self):
126 co = ExceptionTestCase.test_exceptions.__code__
127 new = marshal.loads(marshal.dumps(co))
128 self.assertEqual(co, new)
129
130 def test_many_codeobjects(self):
131 # Issue2957: bad recursion count on code objects
132 # more than MAX_MARSHAL_STACK_DEPTH
133 codes = (ExceptionTestCase.test_exceptions.__code__,) * 10_000
134 marshal.loads(marshal.dumps(codes))
135
136 def test_different_filenames(self):
137 co1 = compile("x", "f1", "exec")
138 co2 = compile("y", "f2", "exec")
139 co1, co2 = marshal.loads(marshal.dumps((co1, co2)))
140 self.assertEqual(co1.co_filename, "f1")
141 self.assertEqual(co2.co_filename, "f2")
142
143 def test_no_allow_code(self):
144 data = {'a': [({0},)]}
145 dump = marshal.dumps(data, allow_code=False)
146 self.assertEqual(marshal.loads(dump, allow_code=False), data)
147
148 f = io.BytesIO()
149 marshal.dump(data, f, allow_code=False)
150 f.seek(0)
151 self.assertEqual(marshal.load(f, allow_code=False), data)
152
153 co = ExceptionTestCase.test_exceptions.__code__
154 data = {'a': [({co, 0},)]}
155 dump = marshal.dumps(data, allow_code=True)
156 self.assertEqual(marshal.loads(dump, allow_code=True), data)
157 with self.assertRaises(ValueError):
158 marshal.dumps(data, allow_code=False)
159 with self.assertRaises(ValueError):
160 marshal.loads(dump, allow_code=False)
161
162 marshal.dump(data, io.BytesIO(), allow_code=True)
163 self.assertEqual(marshal.load(io.BytesIO(dump), allow_code=True), data)
164 with self.assertRaises(ValueError):
165 marshal.dump(data, io.BytesIO(), allow_code=False)
166 with self.assertRaises(ValueError):
167 marshal.load(io.BytesIO(dump), allow_code=False)
168
169 @requires_debug_ranges()
170 def test_minimal_linetable_with_no_debug_ranges(self):
171 # Make sure when demarshalling objects with `-X no_debug_ranges`
172 # that the columns are None.
173 co = ExceptionTestCase.test_exceptions.__code__
174 code = textwrap.dedent("""
175 import sys
176 import marshal
177 with open(sys.argv[1], 'rb') as f:
178 co = marshal.load(f)
179 positions = list(co.co_positions())
180 assert positions[0][2] is None
181 assert positions[0][3] is None
182 """)
183
184 try:
185 with open(os_helper.TESTFN, 'wb') as f:
186 marshal.dump(co, f)
187
188 assert_python_ok('-X', 'no_debug_ranges',
189 '-c', code, os_helper.TESTFN)
190 finally:
191 os_helper.unlink(os_helper.TESTFN)
192
193 @support.cpython_only
194 def test_same_filename_used(self):
195 s = """def f(): pass\ndef g(): pass"""
196 co = compile(s, "myfile", "exec")
197 co = marshal.loads(marshal.dumps(co))
198 for obj in co.co_consts:
199 if isinstance(obj, types.CodeType):
200 self.assertIs(co.co_filename, obj.co_filename)
201
202 class ContainerTestCase(unittest.TestCase, HelperMixin):
203 d = {'astring': '[email protected]',
204 'afloat': 7283.43,
205 'anint': 2**20,
206 'ashortlong': 2,
207 'alist': ['.zyx.41'],
208 'atuple': ('.zyx.41',)*10,
209 'aboolean': False,
210 'aunicode': "Andr\xe8 Previn"
211 }
212
213 def test_dict(self):
214 self.helper(self.d)
215
216 def test_list(self):
217 self.helper(list(self.d.items()))
218
219 def test_tuple(self):
220 self.helper(tuple(self.d.keys()))
221
222 def test_sets(self):
223 for constructor in (set, frozenset):
224 self.helper(constructor(self.d.keys()))
225
226
227 class BufferTestCase(unittest.TestCase, HelperMixin):
228
229 def test_bytearray(self):
230 b = bytearray(b"abc")
231 self.helper(b)
232 new = marshal.loads(marshal.dumps(b))
233 self.assertEqual(type(new), bytes)
234
235 def test_memoryview(self):
236 b = memoryview(b"abc")
237 self.helper(b)
238 new = marshal.loads(marshal.dumps(b))
239 self.assertEqual(type(new), bytes)
240
241 def test_array(self):
242 a = array.array('B', b"abc")
243 new = marshal.loads(marshal.dumps(a))
244 self.assertEqual(new, b"abc")
245
246
247 class BugsTestCase(unittest.TestCase):
248 def test_bug_5888452(self):
249 # Simple-minded check for SF 588452: Debug build crashes
250 marshal.dumps([128] * 1000)
251
252 def test_patch_873224(self):
253 self.assertRaises(Exception, marshal.loads, b'0')
254 self.assertRaises(Exception, marshal.loads, b'f')
255 self.assertRaises(Exception, marshal.loads,
256 omit_last_byte(marshal.dumps(2**65)))
257
258 def test_version_argument(self):
259 # Python 2.4.0 crashes for any call to marshal.dumps(x, y)
260 self.assertEqual(marshal.loads(marshal.dumps(5, 0)), 5)
261 self.assertEqual(marshal.loads(marshal.dumps(5, 1)), 5)
262
263 def test_fuzz(self):
264 # simple test that it's at least not *totally* trivial to
265 # crash from bad marshal data
266 for i in range(256):
267 c = bytes([i])
268 try:
269 marshal.loads(c)
270 except Exception:
271 pass
272
273 def test_loads_recursion(self):
274 def run_tests(N, check):
275 # (((...None...),),)
276 check(b')\x01' * N + b'N')
277 check(b'(\x01\x00\x00\x00' * N + b'N')
278 # [[[...None...]]]
279 check(b'[\x01\x00\x00\x00' * N + b'N')
280 # {None: {None: {None: ...None...}}}
281 check(b'{N' * N + b'N' + b'0' * N)
282 # frozenset([frozenset([frozenset([...None...])])])
283 check(b'>\x01\x00\x00\x00' * N + b'N')
284 # Check that the generated marshal data is valid and marshal.loads()
285 # works for moderately deep nesting
286 run_tests(100, marshal.loads)
287 # Very deeply nested structure shouldn't blow the stack
288 def check(s):
289 self.assertRaises(ValueError, marshal.loads, s)
290 run_tests(2**20, check)
291
292 def test_recursion_limit(self):
293 # Create a deeply nested structure.
294 head = last = []
295 # The max stack depth should match the value in Python/marshal.c.
296 # BUG: https://bugs.python.org/issue33720
297 # Windows always limits the maximum depth on release and debug builds
298 #if os.name == 'nt' and support.Py_DEBUG:
299 if os.name == 'nt':
300 MAX_MARSHAL_STACK_DEPTH = 1000
301 elif sys.platform == 'wasi' or is_emscripten or is_apple_mobile:
302 MAX_MARSHAL_STACK_DEPTH = 1500
303 else:
304 MAX_MARSHAL_STACK_DEPTH = 2000
305 for i in range(MAX_MARSHAL_STACK_DEPTH - 2):
306 last.append([0])
307 last = last[-1]
308
309 # Verify we don't blow out the stack with dumps/load.
310 data = marshal.dumps(head)
311 new_head = marshal.loads(data)
312 # Don't use == to compare objects, it can exceed the recursion limit.
313 self.assertEqual(len(new_head), len(head))
314 self.assertEqual(len(new_head[0]), len(head[0]))
315 self.assertEqual(len(new_head[-1]), len(head[-1]))
316
317 last.append([0])
318 self.assertRaises(ValueError, marshal.dumps, head)
319
320 def test_reference_loop_list(self):
321 a = []
322 a.append(a)
323 for v in range(3):
324 self.assertRaises(ValueError, marshal.dumps, a, v)
325 for v in range(3, marshal.version + 1):
326 d = marshal.dumps(a, v)
327 b = marshal.loads(d)
328 self.assertIsInstance(b, list)
329 self.assertIs(b[0], b)
330
331 def test_reference_loop_dict(self):
332 a = {}
333 a[None] = a
334 for v in range(3):
335 self.assertRaises(ValueError, marshal.dumps, a, v)
336 for v in range(3, marshal.version + 1):
337 d = marshal.dumps(a, v)
338 b = marshal.loads(d)
339 self.assertIsInstance(b, dict)
340 self.assertIs(b[None], b)
341
342 def test_reference_loop_tuple(self):
343 a = ([],)
344 a[0].append(a)
345 for v in range(3):
346 self.assertRaises(ValueError, marshal.dumps, a, v)
347 for v in range(3, marshal.version + 1):
348 d = marshal.dumps(a, v)
349 b = marshal.loads(d)
350 self.assertIsInstance(b, tuple)
351 self.assertIsInstance(b[0], list)
352 self.assertIs(b[0][0], b)
353
354 def test_reference_loop_code(self):
355 def f():
356 return 1234.5
357 code = f.__code__
358 a = []
359 code = code.replace(co_consts=code.co_consts + (a,))
360 # This test creates a reference loop which leads to reference leaks,
361 # so we need to break the loop manually. See gh-148722.
362 self.addCleanup(a.clear)
363 a.append(code)
364 for v in range(marshal.version + 1):
365 self.assertRaises(ValueError, marshal.dumps, code, v)
366
367 def test_reference_loop_slice(self):
368 a = slice([], None)
369 a.start.append(a)
370 for v in range(marshal.version + 1):
371 self.assertRaises(ValueError, marshal.dumps, a, v)
372
373 a = slice(None, [])
374 a.stop.append(a)
375 for v in range(marshal.version + 1):
376 self.assertRaises(ValueError, marshal.dumps, a, v)
377
378 a = slice(None, None, [])
379 a.step.append(a)
380 for v in range(marshal.version + 1):
381 self.assertRaises(ValueError, marshal.dumps, a, v)
382
383 def test_loads_reference_loop_list(self):
384 data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [<R>]
385 a = marshal.loads(data)
386 self.assertIsInstance(a, list)
387 self.assertIs(a[0], a)
388
389 def test_loads_reference_loop_dict(self):
390 data = b'\xfbNr\x00\x00\x00\x000' # {None: <R>}
391 a = marshal.loads(data)
392 self.assertIsInstance(a, dict)
393 self.assertIs(a[None], a)
394
395 def test_loads_abnormal_reference_loops(self):
396 # Indirect self-references of tuples.
397 data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([<R>],)
398 a = marshal.loads(data)
399 self.assertIsInstance(a, tuple)
400 self.assertIsInstance(a[0], list)
401 self.assertIs(a[0][0], a)
402
403 data = b'\xa8\x01\x00\x00\x00{Nr\x00\x00\x00\x000' # ({None: <R>},)
404 a = marshal.loads(data)
405 self.assertIsInstance(a, tuple)
406 self.assertIsInstance(a[0], dict)
407 self.assertIs(a[0][None], a)
408
409 # Direct self-reference which cannot be created in Python.
410 # This creates a reference loop which cannot be collected.
411 if False:
412 data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (<R>,)
413 a = marshal.loads(data)
414 self.assertIsInstance(a, tuple)
415 self.assertIs(a[0], a)
416
417 # Direct self-references which cannot be created in Python
418 # because of unhashability.
419 data = b'\xfbr\x00\x00\x00\x00N0' # {<R>: None}
420 self.assertRaises(TypeError, marshal.loads, data)
421 data = b'\xbc\x01\x00\x00\x00r\x00\x00\x00\x00' # {<R>}
422 self.assertRaises(TypeError, marshal.loads, data)
423
424 for data in [
425 # Indirect self-references of immutable objects.
426 b'\xba[\x01\x00\x00\x00r\x00\x00\x00\x00NN', # slice([<R>], None)
427 b'\xbaN[\x01\x00\x00\x00r\x00\x00\x00\x00N', # slice(None, [<R>])
428 b'\xbaNN[\x01\x00\x00\x00r\x00\x00\x00\x00', # slice(None, None, [<R>])
429 b'\xba{Nr\x00\x00\x00\x000NN', # slice({None: <R>}, None)
430 b'\xbaN{Nr\x00\x00\x00\x000N', # slice(None, {None: <R>})
431 b'\xbaNN{Nr\x00\x00\x00\x000', # slice(None, None, {None: <R>})
432
433 # Direct self-references which cannot be created in Python.
434 b'\xbe\x01\x00\x00\x00r\x00\x00\x00\x00', # frozenset({<R>})
435 b'\xbar\x00\x00\x00\x00NN', # slice(<R>, None)
436 b'\xbaNr\x00\x00\x00\x00N', # slice(None, <R>)
437 b'\xbaNNr\x00\x00\x00\x00', # slice(None, None, <R>)
438 ]:
439 with self.subTest(data=data):
440 self.assertRaises(ValueError, marshal.loads, data)
441
442 def test_exact_type_match(self):
443 # Former bug:
444 # >>> class Int(int): pass
445 # >>> type(loads(dumps(Int())))
446 # <type 'int'>
447 for typ in (int, float, complex, tuple, list, dict, set, frozenset):
448 # Note: str subclasses are not tested because they get handled
449 # by marshal's routines for objects supporting the buffer API.
450 subtyp = type('subtyp', (typ,), {})
451 self.assertRaises(ValueError, marshal.dumps, subtyp())
452
453 # Issue #1792 introduced a change in how marshal increases the size of its
454 # internal buffer; this test ensures that the new code is exercised.
455 def test_large_marshal(self):
456 size = int(1e6)
457 testString = 'abc' * size
458 marshal.dumps(testString)
459
460 def test_invalid_longs(self):
461 # Issue #7019: marshal.loads shouldn't produce unnormalized PyLongs
462 invalid_string = b'l\x02\x00\x00\x00\x00\x00\x00\x00'
463 self.assertRaises(ValueError, marshal.loads, invalid_string)
464
465 def test_multiple_dumps_and_loads(self):
466 # Issue 12291: marshal.load() should be callable multiple times
467 # with interleaved data written by non-marshal code
468 # Adapted from a patch by Engelbert Gruber.
469 data = (1, 'abc', b'def', 1.0, (2, 'a', ['b', b'c']))
470 for interleaved in (b'', b'0123'):
471 ilen = len(interleaved)
472 positions = []
473 try:
474 with open(os_helper.TESTFN, 'wb') as f:
475 for d in data:
476 marshal.dump(d, f)
477 if ilen:
478 f.write(interleaved)
479 positions.append(f.tell())
480 with open(os_helper.TESTFN, 'rb') as f:
481 for i, d in enumerate(data):
482 self.assertEqual(d, marshal.load(f))
483 if ilen:
484 f.read(ilen)
485 self.assertEqual(positions[i], f.tell())
486 finally:
487 os_helper.unlink(os_helper.TESTFN)
488
489 def test_loads_reject_unicode_strings(self):
490 # Issue #14177: marshal.loads() should not accept unicode strings
491 unicode_string = 'T'
492 self.assertRaises(TypeError, marshal.loads, unicode_string)
493
494 def test_bad_reader(self):
495 class BadReader(io.BytesIO):
496 def readinto(self, buf):
497 n = super().readinto(buf)
498 if n is not None and n > 4:
499 n += 10**6
500 return n
501 for value in (1.0, 1j, b'0123456789', '0123456789'):
502 self.assertRaises(ValueError, marshal.load,
503 BadReader(marshal.dumps(value)))
504
505 def test_eof(self):
506 data = marshal.dumps(("hello", "dolly", None))
507 for i in range(len(data)):
508 self.assertRaises(EOFError, marshal.loads, data[0: i])
509
510 def test_deterministic_sets(self):
511 # bpo-37596: To support reproducible builds, sets and frozensets need to
512 # have their elements serialized in a consistent order (even when they
513 # have been scrambled by hash randomization):
514 for kind in ("set", "frozenset"):
515 for elements in (
516 "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'",
517 # Also test for bad interactions with backreferencing:
518 "('Spam', 0), ('Spam', 1), ('Spam', 2), ('Spam', 3), ('Spam', 4), ('Spam', 5)",
519 ):
520 s = f"{kind}([{elements}])"
521 with self.subTest(s):
522 # First, make sure that our test case still has different
523 # orders under hash seeds 0 and 1. If this check fails, we
524 # need to update this test with different elements. Skip
525 # this part if we are configured to use any other hash
526 # algorithm (for example, using Py_HASH_EXTERNAL):
527 if sys.hash_info.algorithm in {"fnv", "siphash24"}:
528 args = ["-c", f"print({s})"]
529 _, repr_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0")
530 _, repr_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
531 self.assertNotEqual(repr_0, repr_1)
532 # Then, perform the actual test:
533 args = ["-c", f"import marshal; print(marshal.dumps({s}))"]
534 _, dump_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0")
535 _, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1")
536 self.assertEqual(dump_0, dump_1)
537
538 def test_unmarshallable(self):
539 # Check no crash after encountering unmarshallable objects.
540 # See https://github.com/python/cpython/issues/106287.
541 fset = frozenset([int])
542 code = compile("a = 1", "<string>", "exec")
543 code = code.replace(co_consts=(1, fset, None))
544 cases = (('tuple', (fset,)),
545 ('list', [fset]),
546 ('set', fset),
547 ('dict key', {fset: 'x'}),
548 ('dict value', {'x': fset}),
549 ('dict key & value', {fset: fset}),
550 ('slice', slice(fset, fset)),
551 ('code', code))
552 for name, arg in cases:
553 with self.subTest(name, arg=arg):
554 with self.assertRaisesRegex(ValueError, "unmarshallable object"):
555 marshal.dumps((arg, memoryview(b'')))
556
557
558 LARGE_SIZE = 2**31
559 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
560
561 class NullWriter:
562 def write(self, s):
563 pass
564
565 @unittest.skipIf(LARGE_SIZE > sys.maxsize, "test cannot run on 32-bit systems")
566 class LargeValuesTestCase(unittest.TestCase):
567 def check_unmarshallable(self, data):
568 self.assertRaises(ValueError, marshal.dump, data, NullWriter())
569
570 @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
571 def test_bytes(self, size):
572 self.check_unmarshallable(b'x' * size)
573
574 @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
575 def test_str(self, size):
576 self.check_unmarshallable('x' * size)
577
578 @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False)
579 def test_tuple(self, size):
580 self.check_unmarshallable((None,) * size)
581
582 @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False)
583 def test_list(self, size):
584 self.check_unmarshallable([None] * size)
585
586 @support.bigmemtest(size=LARGE_SIZE,
587 memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
588 dry_run=False)
589 def test_set(self, size):
590 self.check_unmarshallable(set(range(size)))
591
592 @support.bigmemtest(size=LARGE_SIZE,
593 memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
594 dry_run=False)
595 def test_frozenset(self, size):
596 self.check_unmarshallable(frozenset(range(size)))
597
598 @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False)
599 def test_bytearray(self, size):
600 self.check_unmarshallable(bytearray(size))
601
602 def CollectObjectIDs(ids, obj):
603 """Collect object ids seen in a structure"""
604 if id(obj) in ids:
605 return
606 ids.add(id(obj))
607 if isinstance(obj, (list, tuple, set, frozenset)):
608 for e in obj:
609 CollectObjectIDs(ids, e)
610 elif isinstance(obj, dict):
611 for k, v in obj.items():
612 CollectObjectIDs(ids, k)
613 CollectObjectIDs(ids, v)
614 return len(ids)
615
616 class InstancingTestCase(unittest.TestCase, HelperMixin):
617 keys = (123, 1.2345, 'abc', (123, 'abc'), frozenset({123, 'abc'}))
618
619 def helper3(self, rsample, recursive=False, simple=False):
620 #we have two instances
621 sample = (rsample, rsample)
622
623 n0 = CollectObjectIDs(set(), sample)
624
625 for v in range(3, marshal.version + 1):
626 s3 = marshal.dumps(sample, v)
627 n3 = CollectObjectIDs(set(), marshal.loads(s3))
628
629 #same number of instances generated
630 self.assertEqual(n3, n0)
631
632 if not recursive:
633 #can compare with version 2
634 s2 = marshal.dumps(sample, 2)
635 n2 = CollectObjectIDs(set(), marshal.loads(s2))
636 #old format generated more instances
637 self.assertGreater(n2, n0)
638
639 #if complex objects are in there, old format is larger
640 if not simple:
641 self.assertGreater(len(s2), len(s3))
642 else:
643 self.assertGreaterEqual(len(s2), len(s3))
644
645 def testInt(self):
646 intobj = 123321
647 self.helper(intobj)
648 self.helper3(intobj, simple=True)
649
650 def testFloat(self):
651 floatobj = 1.2345
652 self.helper(floatobj)
653 self.helper3(floatobj)
654
655 def testStr(self):
656 strobj = "abcde"*3
657 self.helper(strobj)
658 self.helper3(strobj)
659
660 def testBytes(self):
661 bytesobj = b"abcde"*3
662 self.helper(bytesobj)
663 self.helper3(bytesobj)
664
665 def testList(self):
666 for obj in self.keys:
667 listobj = [obj, obj]
668 self.helper(listobj)
669 self.helper3(listobj)
670
671 def testTuple(self):
672 for obj in self.keys:
673 tupleobj = (obj, obj)
674 self.helper(tupleobj)
675 self.helper3(tupleobj)
676
677 def testSet(self):
678 for obj in self.keys:
679 setobj = {(obj, 1), (obj, 2)}
680 self.helper(setobj)
681 self.helper3(setobj)
682
683 def testFrozenSet(self):
684 for obj in self.keys:
685 frozensetobj = frozenset({(obj, 1), (obj, 2)})
686 self.helper(frozensetobj)
687 self.helper3(frozensetobj)
688
689 def testDict(self):
690 for obj in self.keys:
691 dictobj = {"hello": obj, "goodbye": obj, obj: "hello"}
692 self.helper(dictobj)
693 self.helper3(dictobj)
694
695 def testModule(self):
696 with open(__file__, "rb") as f:
697 code = f.read()
698 if __file__.endswith(".py"):
699 code = compile(code, __file__, "exec")
700 self.helper(code)
701 self.helper3(code)
702
703 def testRecursion(self):
704 obj = 1.2345
705 d = {"hello": obj, "goodbye": obj, obj: "hello"}
706 d["self"] = d
707 self.helper3(d, recursive=True)
708 l = [obj, obj]
709 l.append(l)
710 self.helper3(l, recursive=True)
711
712 class CompatibilityTestCase(unittest.TestCase):
713 def _test(self, version):
714 with open(__file__, "rb") as f:
715 code = f.read()
716 if __file__.endswith(".py"):
717 code = compile(code, __file__, "exec")
718 data = marshal.dumps(code, version)
719 marshal.loads(data)
720
721 def test0To3(self):
722 self._test(0)
723
724 def test1To3(self):
725 self._test(1)
726
727 def test2To3(self):
728 self._test(2)
729
730 def test3To3(self):
731 self._test(3)
732
733 class InterningTestCase(unittest.TestCase, HelperMixin):
734 strobj = "this is an interned string"
735 strobj = sys.intern(strobj)
736
737 def testIntern(self):
738 s = marshal.loads(marshal.dumps(self.strobj))
739 self.assertEqual(s, self.strobj)
740 self.assertEqual(id(s), id(self.strobj))
741 s2 = sys.intern(s)
742 self.assertEqual(id(s2), id(s))
743
744 def testNoIntern(self):
745 s = marshal.loads(marshal.dumps(self.strobj, 2))
746 self.assertEqual(s, self.strobj)
747 self.assertNotEqual(id(s), id(self.strobj))
748 s2 = sys.intern(s)
749 self.assertNotEqual(id(s2), id(s))
750
751 class SliceTestCase(unittest.TestCase, HelperMixin):
752 def test_slice(self):
753 for obj in (
754 slice(None), slice(1), slice(1, 2), slice(1, 2, 3),
755 slice({'set'}, ('tuple', {'with': 'dict'}, ), self.helper.__code__)
756 ):
757 with self.subTest(obj=str(obj)):
758 self.helper(obj)
759
760 for version in range(4):
761 with self.assertRaises(ValueError):
762 marshal.dumps(obj, version)
763
764 @support.cpython_only
765 @unittest.skipUnless(_testcapi, 'requires _testcapi')
766 class CAPI_TestCase(unittest.TestCase, HelperMixin):
767
768 def test_write_long_to_file(self):
769 for v in range(marshal.version + 1):
770 _testcapi.pymarshal_write_long_to_file(0x12345678, os_helper.TESTFN, v)
771 with open(os_helper.TESTFN, 'rb') as f:
772 data = f.read()
773 os_helper.unlink(os_helper.TESTFN)
774 self.assertEqual(data, b'\x78\x56\x34\x12')
775
776 def test_write_object_to_file(self):
777 obj = ('\u20ac', b'abc', 123, 45.6, 7+8j, 'long line '*1000)
778 for v in range(marshal.version + 1):
779 _testcapi.pymarshal_write_object_to_file(obj, os_helper.TESTFN, v)
780 with open(os_helper.TESTFN, 'rb') as f:
781 data = f.read()
782 os_helper.unlink(os_helper.TESTFN)
783 self.assertEqual(marshal.loads(data), obj)
784
785 def test_read_short_from_file(self):
786 with open(os_helper.TESTFN, 'wb') as f:
787 f.write(b'\x34\x12xxxx')
788 r, p = _testcapi.pymarshal_read_short_from_file(os_helper.TESTFN)
789 os_helper.unlink(os_helper.TESTFN)
790 self.assertEqual(r, 0x1234)
791 self.assertEqual(p, 2)
792
793 with open(os_helper.TESTFN, 'wb') as f:
794 f.write(b'\x12')
795 with self.assertRaises(EOFError):
796 _testcapi.pymarshal_read_short_from_file(os_helper.TESTFN)
797 os_helper.unlink(os_helper.TESTFN)
798
799 def test_read_long_from_file(self):
800 with open(os_helper.TESTFN, 'wb') as f:
801 f.write(b'\x78\x56\x34\x12xxxx')
802 r, p = _testcapi.pymarshal_read_long_from_file(os_helper.TESTFN)
803 os_helper.unlink(os_helper.TESTFN)
804 self.assertEqual(r, 0x12345678)
805 self.assertEqual(p, 4)
806
807 with open(os_helper.TESTFN, 'wb') as f:
808 f.write(b'\x56\x34\x12')
809 with self.assertRaises(EOFError):
810 _testcapi.pymarshal_read_long_from_file(os_helper.TESTFN)
811 os_helper.unlink(os_helper.TESTFN)
812
813 def test_read_last_object_from_file(self):
814 obj = ('\u20ac', b'abc', 123, 45.6, 7+8j)
815 for v in range(marshal.version + 1):
816 data = marshal.dumps(obj, v)
817 with open(os_helper.TESTFN, 'wb') as f:
818 f.write(data + b'xxxx')
819 r, p = _testcapi.pymarshal_read_last_object_from_file(os_helper.TESTFN)
820 os_helper.unlink(os_helper.TESTFN)
821 self.assertEqual(r, obj)
822
823 with open(os_helper.TESTFN, 'wb') as f:
824 f.write(omit_last_byte(data))
825 with self.assertRaises(EOFError):
826 _testcapi.pymarshal_read_last_object_from_file(os_helper.TESTFN)
827 os_helper.unlink(os_helper.TESTFN)
828
829 def test_read_object_from_file(self):
830 obj = ('\u20ac', b'abc', 123, 45.6, 7+8j)
831 for v in range(marshal.version + 1):
832 data = marshal.dumps(obj, v)
833 with open(os_helper.TESTFN, 'wb') as f:
834 f.write(data + b'xxxx')
835 r, p = _testcapi.pymarshal_read_object_from_file(os_helper.TESTFN)
836 os_helper.unlink(os_helper.TESTFN)
837 self.assertEqual(r, obj)
838 self.assertEqual(p, len(data))
839
840 with open(os_helper.TESTFN, 'wb') as f:
841 f.write(omit_last_byte(data))
842 with self.assertRaises(EOFError):
843 _testcapi.pymarshal_read_object_from_file(os_helper.TESTFN)
844 os_helper.unlink(os_helper.TESTFN)
845
846
847 if __name__ == "__main__":
848 unittest.main()