Debian Codegraph @

Lib/test/test_pickletools.py

raw
1 import io
2 import pickle
3 import pickletools
4 from test import support
5 from test.pickletester import AbstractPickleTests
6 import doctest
7 import unittest
8
9 class OptimizedPickleTests(AbstractPickleTests, unittest.TestCase):
10
11 def dumps(self, arg, proto=None, **kwargs):
12 return pickletools.optimize(pickle.dumps(arg, proto, **kwargs))
13
14 def loads(self, buf, **kwds):
15 return pickle.loads(buf, **kwds)
16
17 # Test relies on precise output of dumps()
18 test_pickle_to_2x = None
19
20 # Test relies on writing by chunks into a file object.
21 test_framed_write_sizes_with_delayed_writer = None
22
23 def test_optimize_long_binget(self):
24 data = [str(i) for i in range(257)]
25 data.append(data[-1])
26 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
27 pickled = pickle.dumps(data, proto)
28 unpickled = pickle.loads(pickled)
29 self.assertEqual(unpickled, data)
30 self.assertIs(unpickled[-1], unpickled[-2])
31
32 pickled2 = pickletools.optimize(pickled)
33 unpickled2 = pickle.loads(pickled2)
34 self.assertEqual(unpickled2, data)
35 self.assertIs(unpickled2[-1], unpickled2[-2])
36 self.assertNotIn(pickle.LONG_BINGET, pickled2)
37 self.assertNotIn(pickle.LONG_BINPUT, pickled2)
38
39 def test_optimize_binput_and_memoize(self):
40 pickled = (b'\x80\x04\x95\x15\x00\x00\x00\x00\x00\x00\x00'
41 b']\x94(\x8c\x04spamq\x01\x8c\x03ham\x94h\x02e.')
42 # 0: \x80 PROTO 4
43 # 2: \x95 FRAME 21
44 # 11: ] EMPTY_LIST
45 # 12: \x94 MEMOIZE
46 # 13: ( MARK
47 # 14: \x8c SHORT_BINUNICODE 'spam'
48 # 20: q BINPUT 1
49 # 22: \x8c SHORT_BINUNICODE 'ham'
50 # 27: \x94 MEMOIZE
51 # 28: h BINGET 2
52 # 30: e APPENDS (MARK at 13)
53 # 31: . STOP
54 self.assertIn(pickle.BINPUT, pickled)
55 unpickled = pickle.loads(pickled)
56 self.assertEqual(unpickled, ['spam', 'ham', 'ham'])
57 self.assertIs(unpickled[1], unpickled[2])
58
59 pickled2 = pickletools.optimize(pickled)
60 unpickled2 = pickle.loads(pickled2)
61 self.assertEqual(unpickled2, ['spam', 'ham', 'ham'])
62 self.assertIs(unpickled2[1], unpickled2[2])
63 self.assertNotIn(pickle.BINPUT, pickled2)
64
65
66 class SimpleReader:
67 def __init__(self, data):
68 self.data = data
69 self.pos = 0
70
71 def read(self, n):
72 data = self.data[self.pos: self.pos + n]
73 self.pos += n
74 return data
75
76 def readline(self):
77 nl = self.data.find(b'\n', self.pos) + 1
78 if not nl:
79 nl = len(self.data)
80 data = self.data[self.pos: nl]
81 self.pos = nl
82 return data
83
84
85 class GenopsTests(unittest.TestCase):
86 def test_genops(self):
87 it = pickletools.genops(b'(I123\nK\x12J\x12\x34\x56\x78t.')
88 self.assertEqual([(item[0].name,) + item[1:] for item in it], [
89 ('MARK', None, 0),
90 ('INT', 123, 1),
91 ('BININT1', 0x12, 6),
92 ('BININT', 0x78563412, 8),
93 ('TUPLE', None, 13),
94 ('STOP', None, 14),
95 ])
96
97 def test_from_file(self):
98 f = io.BytesIO(b'prefix(I123\nK\x12J\x12\x34\x56\x78t.suffix')
99 self.assertEqual(f.read(6), b'prefix')
100 it = pickletools.genops(f)
101 self.assertEqual([(item[0].name,) + item[1:] for item in it], [
102 ('MARK', None, 6),
103 ('INT', 123, 7),
104 ('BININT1', 0x12, 12),
105 ('BININT', 0x78563412, 14),
106 ('TUPLE', None, 19),
107 ('STOP', None, 20),
108 ])
109 self.assertEqual(f.read(), b'suffix')
110
111 def test_without_pos(self):
112 f = SimpleReader(b'(I123\nK\x12J\x12\x34\x56\x78t.')
113 it = pickletools.genops(f)
114 self.assertEqual([(item[0].name,) + item[1:] for item in it], [
115 ('MARK', None, None),
116 ('INT', 123, None),
117 ('BININT1', 0x12, None),
118 ('BININT', 0x78563412, None),
119 ('TUPLE', None, None),
120 ('STOP', None, None),
121 ])
122
123 def test_no_stop(self):
124 it = pickletools.genops(b'N')
125 item = next(it)
126 self.assertEqual(item[0].name, 'NONE')
127 with self.assertRaisesRegex(ValueError,
128 'pickle exhausted before seeing STOP'):
129 next(it)
130
131 def test_truncated_data(self):
132 it = pickletools.genops(b'I123')
133 with self.assertRaisesRegex(ValueError,
134 'no newline found when trying to read stringnl'):
135 next(it)
136 it = pickletools.genops(b'J\x12\x34')
137 with self.assertRaisesRegex(ValueError,
138 'not enough data in stream to read int4'):
139 next(it)
140
141 def test_unknown_opcode(self):
142 it = pickletools.genops(b'N\xff')
143 item = next(it)
144 self.assertEqual(item[0].name, 'NONE')
145 with self.assertRaisesRegex(ValueError,
146 r"at position 1, opcode b'\\xff' unknown"):
147 next(it)
148
149 def test_unknown_opcode_without_pos(self):
150 f = SimpleReader(b'N\xff')
151 it = pickletools.genops(f)
152 item = next(it)
153 self.assertEqual(item[0].name, 'NONE')
154 with self.assertRaisesRegex(ValueError,
155 r"at position <unknown>, opcode b'\\xff' unknown"):
156 next(it)
157
158
159 class DisTests(unittest.TestCase):
160 maxDiff = None
161
162 def check_dis(self, data, expected, **kwargs):
163 out = io.StringIO()
164 pickletools.dis(data, out=out, **kwargs)
165 self.assertEqual(out.getvalue(), expected)
166
167 def check_dis_error(self, data, expected, expected_error, **kwargs):
168 out = io.StringIO()
169 with self.assertRaisesRegex(ValueError, expected_error):
170 pickletools.dis(data, out=out, **kwargs)
171 self.assertEqual(out.getvalue(), expected)
172
173 def test_mark(self):
174 self.check_dis(b'(N(tl.', '''\
175 0: ( MARK
176 1: N NONE
177 2: ( MARK
178 3: t TUPLE (MARK at 2)
179 4: l LIST (MARK at 0)
180 5: . STOP
181 highest protocol among opcodes = 0
182 ''')
183
184 def test_indentlevel(self):
185 self.check_dis(b'(N(tl.', '''\
186 0: ( MARK
187 1: N NONE
188 2: ( MARK
189 3: t TUPLE (MARK at 2)
190 4: l LIST (MARK at 0)
191 5: . STOP
192 highest protocol among opcodes = 0
193 ''', indentlevel=2)
194
195 def test_mark_without_pos(self):
196 self.check_dis(SimpleReader(b'(N(tl.'), '''\
197 ( MARK
198 N NONE
199 ( MARK
200 t TUPLE (MARK at unknown opcode offset)
201 l LIST (MARK at unknown opcode offset)
202 . STOP
203 highest protocol among opcodes = 0
204 ''')
205
206 def test_no_mark(self):
207 self.check_dis_error(b'Nt.', '''\
208 0: N NONE
209 1: t TUPLE
210 ''', 'no MARK exists on stack')
211
212 def test_put(self):
213 self.check_dis(b'Np0\nq\x01r\x02\x00\x00\x00\x94.', '''\
214 0: N NONE
215 1: p PUT 0
216 4: q BINPUT 1
217 6: r LONG_BINPUT 2
218 11: \\x94 MEMOIZE (as 3)
219 12: . STOP
220 highest protocol among opcodes = 4
221 ''')
222
223 def test_put_redefined(self):
224 self.check_dis(b'Np1\np1\nq\x01r\x01\x00\x00\x00\x94.', '''\
225 0: N NONE
226 1: p PUT 1
227 4: p PUT 1
228 7: q BINPUT 1
229 9: r LONG_BINPUT 1
230 14: \\x94 MEMOIZE (as 1)
231 15: . STOP
232 highest protocol among opcodes = 4
233 ''')
234
235 def test_put_empty_stack(self):
236 self.check_dis_error(b'p0\n', '''\
237 0: p PUT 0
238 ''', "stack is empty -- can't store into memo")
239
240 def test_put_markobject(self):
241 self.check_dis_error(b'(p0\n', '''\
242 0: ( MARK
243 1: p PUT 0
244 ''', "can't store markobject in the memo")
245
246 def test_get(self):
247 self.check_dis(b'(Np1\ng1\nh\x01j\x01\x00\x00\x00t.', '''\
248 0: ( MARK
249 1: N NONE
250 2: p PUT 1
251 5: g GET 1
252 8: h BINGET 1
253 10: j LONG_BINGET 1
254 15: t TUPLE (MARK at 0)
255 16: . STOP
256 highest protocol among opcodes = 1
257 ''')
258
259 def test_get_without_put(self):
260 self.check_dis_error(b'g1\n.', '''\
261 0: g GET 1
262 ''', 'memo key 1 has never been stored into')
263 self.check_dis_error(b'h\x01.', '''\
264 0: h BINGET 1
265 ''', 'memo key 1 has never been stored into')
266 self.check_dis_error(b'j\x01\x00\x00\x00.', '''\
267 0: j LONG_BINGET 1
268 ''', 'memo key 1 has never been stored into')
269
270 def test_memo(self):
271 memo = {}
272 self.check_dis(b'Np1\n.', '''\
273 0: N NONE
274 1: p PUT 1
275 4: . STOP
276 highest protocol among opcodes = 0
277 ''', memo=memo)
278 self.check_dis(b'g1\n.', '''\
279 0: g GET 1
280 3: . STOP
281 highest protocol among opcodes = 0
282 ''', memo=memo)
283
284 def test_mark_pop(self):
285 self.check_dis(b'(N00N.', '''\
286 0: ( MARK
287 1: N NONE
288 2: 0 POP
289 3: 0 POP (MARK at 0)
290 4: N NONE
291 5: . STOP
292 highest protocol among opcodes = 0
293 ''')
294
295 def test_too_small_stack(self):
296 self.check_dis_error(b'a', '''\
297 0: a APPEND
298 ''', 'tries to pop 2 items from stack with only 0 items')
299 self.check_dis_error(b']a', '''\
300 0: ] EMPTY_LIST
301 1: a APPEND
302 ''', 'tries to pop 2 items from stack with only 1 items')
303
304 def test_no_stop(self):
305 self.check_dis_error(b'N', '''\
306 0: N NONE
307 ''', 'pickle exhausted before seeing STOP')
308
309 def test_truncated_data(self):
310 self.check_dis_error(b'NI123', '''\
311 0: N NONE
312 ''', 'no newline found when trying to read stringnl')
313 self.check_dis_error(b'NJ\x12\x34', '''\
314 0: N NONE
315 ''', 'not enough data in stream to read int4')
316
317 def test_unknown_opcode(self):
318 self.check_dis_error(b'N\xff', '''\
319 0: N NONE
320 ''', r"at position 1, opcode b'\\xff' unknown")
321
322 def test_stop_not_empty_stack(self):
323 self.check_dis_error(b']N.', '''\
324 0: ] EMPTY_LIST
325 1: N NONE
326 2: . STOP
327 highest protocol among opcodes = 1
328 ''', r'stack not empty after STOP: \[list\]')
329
330 def test_annotate(self):
331 self.check_dis(b'(Nt.', '''\
332 0: ( MARK Push markobject onto the stack.
333 1: N NONE Push None on the stack.
334 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject.
335 3: . STOP Stop the unpickling machine.
336 highest protocol among opcodes = 0
337 ''', annotate=1)
338 self.check_dis(b'(Nt.', '''\
339 0: ( MARK Push markobject onto the stack.
340 1: N NONE Push None on the stack.
341 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject.
342 3: . STOP Stop the unpickling machine.
343 highest protocol among opcodes = 0
344 ''', annotate=20)
345 self.check_dis(b'(((((((ttttttt.', '''\
346 0: ( MARK Push markobject onto the stack.
347 1: ( MARK Push markobject onto the stack.
348 2: ( MARK Push markobject onto the stack.
349 3: ( MARK Push markobject onto the stack.
350 4: ( MARK Push markobject onto the stack.
351 5: ( MARK Push markobject onto the stack.
352 6: ( MARK Push markobject onto the stack.
353 7: t TUPLE (MARK at 6) Build a tuple out of the topmost stack slice, after markobject.
354 8: t TUPLE (MARK at 5) Build a tuple out of the topmost stack slice, after markobject.
355 9: t TUPLE (MARK at 4) Build a tuple out of the topmost stack slice, after markobject.
356 10: t TUPLE (MARK at 3) Build a tuple out of the topmost stack slice, after markobject.
357 11: t TUPLE (MARK at 2) Build a tuple out of the topmost stack slice, after markobject.
358 12: t TUPLE (MARK at 1) Build a tuple out of the topmost stack slice, after markobject.
359 13: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject.
360 14: . STOP Stop the unpickling machine.
361 highest protocol among opcodes = 0
362 ''', annotate=20)
363
364 def test_string(self):
365 self.check_dis(b"S'abc'\n.", '''\
366 0: S STRING 'abc'
367 7: . STOP
368 highest protocol among opcodes = 0
369 ''')
370 self.check_dis(b'S"abc"\n.', '''\
371 0: S STRING 'abc'
372 7: . STOP
373 highest protocol among opcodes = 0
374 ''')
375 self.check_dis(b"S'\xc3\xb5'\n.", '''\
376 0: S STRING '\\xc3\\xb5'
377 6: . STOP
378 highest protocol among opcodes = 0
379 ''')
380
381 def test_string_without_quotes(self):
382 self.check_dis_error(b"Sabc'\n.", '',
383 'no string quotes around b"abc\'"')
384 self.check_dis_error(b'Sabc"\n.', '',
385 "no string quotes around b'abc\"'")
386 self.check_dis_error(b"S'abc\n.", '',
387 '''string quote b"'" not found at both ends of b"'abc"''')
388 self.check_dis_error(b'S"abc\n.', '',
389 r"""string quote b'"' not found at both ends of b'"abc'""")
390 self.check_dis_error(b"S'abc\"\n.", '',
391 r"""string quote b"'" not found at both ends of b'\\'abc"'""")
392 self.check_dis_error(b"S\"abc'\n.", '',
393 r"""string quote b'"' not found at both ends of b'"abc\\''""")
394
395 def test_binstring(self):
396 self.check_dis(b"T\x03\x00\x00\x00abc.", '''\
397 0: T BINSTRING 'abc'
398 8: . STOP
399 highest protocol among opcodes = 1
400 ''')
401 self.check_dis(b"T\x02\x00\x00\x00\xc3\xb5.", '''\
402 0: T BINSTRING '\\xc3\\xb5'
403 7: . STOP
404 highest protocol among opcodes = 1
405 ''')
406
407 def test_short_binstring(self):
408 self.check_dis(b"U\x03abc.", '''\
409 0: U SHORT_BINSTRING 'abc'
410 5: . STOP
411 highest protocol among opcodes = 1
412 ''')
413 self.check_dis(b"U\x02\xc3\xb5.", '''\
414 0: U SHORT_BINSTRING '\\xc3\\xb5'
415 4: . STOP
416 highest protocol among opcodes = 1
417 ''')
418
419 def test_global(self):
420 self.check_dis(b"cmodule\nname\n.", '''\
421 0: c GLOBAL 'module name'
422 13: . STOP
423 highest protocol among opcodes = 0
424 ''')
425 self.check_dis(b"cm\xc3\xb6dule\nn\xc3\xa4me\n.", '''\
426 0: c GLOBAL 'm\xf6dule n\xe4me'
427 15: . STOP
428 highest protocol among opcodes = 0
429 ''')
430
431 def test_inst(self):
432 self.check_dis(b"(imodule\nname\n.", '''\
433 0: ( MARK
434 1: i INST 'module name' (MARK at 0)
435 14: . STOP
436 highest protocol among opcodes = 0
437 ''')
438
439 def test_persid(self):
440 self.check_dis(b"Pabc\n.", '''\
441 0: P PERSID 'abc'
442 5: . STOP
443 highest protocol among opcodes = 0
444 ''')
445
446 def test_constants(self):
447 self.check_dis(b"(NI00\nI01\n\x89\x88t.", '''\
448 0: ( MARK
449 1: N NONE
450 2: I INT False
451 6: I INT True
452 10: \\x89 NEWFALSE
453 11: \\x88 NEWTRUE
454 12: t TUPLE (MARK at 0)
455 13: . STOP
456 highest protocol among opcodes = 2
457 ''')
458
459 def test_integers(self):
460 self.check_dis(b"(I0\nI1\nI10\nI011\nL12\nL13L\nL014\nL015L\nt.", '''\
461 0: ( MARK
462 1: I INT 0
463 4: I INT 1
464 7: I INT 10
465 11: I INT 11
466 16: L LONG 12
467 20: L LONG 13
468 25: L LONG 14
469 30: L LONG 15
470 36: t TUPLE (MARK at 0)
471 37: . STOP
472 highest protocol among opcodes = 0
473 ''')
474
475 def test_nondecimal_integers(self):
476 self.check_dis_error(b'I0b10\n.', '', 'invalid literal for int')
477 self.check_dis_error(b'I0o10\n.', '', 'invalid literal for int')
478 self.check_dis_error(b'I0x10\n.', '', 'invalid literal for int')
479 self.check_dis_error(b'L0b10L\n.', '', 'invalid literal for int')
480 self.check_dis_error(b'L0o10L\n.', '', 'invalid literal for int')
481 self.check_dis_error(b'L0x10L\n.', '', 'invalid literal for int')
482
483
484 class MiscTestCase(unittest.TestCase):
485 def test__all__(self):
486 not_exported = {
487 'bytes_types',
488 'UP_TO_NEWLINE', 'TAKEN_FROM_ARGUMENT1',
489 'TAKEN_FROM_ARGUMENT4', 'TAKEN_FROM_ARGUMENT4U',
490 'TAKEN_FROM_ARGUMENT8U', 'ArgumentDescriptor',
491 'read_uint1', 'read_uint2', 'read_int4', 'read_uint4',
492 'read_uint8', 'read_stringnl', 'read_stringnl_noescape',
493 'read_stringnl_noescape_pair', 'read_string1',
494 'read_string4', 'read_bytes1', 'read_bytes4',
495 'read_bytes8', 'read_bytearray8', 'read_unicodestringnl',
496 'read_unicodestring1', 'read_unicodestring4',
497 'read_unicodestring8', 'read_decimalnl_short',
498 'read_decimalnl_long', 'read_floatnl', 'read_float8',
499 'read_long1', 'read_long4',
500 'uint1', 'uint2', 'int4', 'uint4', 'uint8', 'stringnl',
501 'stringnl_noescape', 'stringnl_noescape_pair', 'string1',
502 'string4', 'bytes1', 'bytes4', 'bytes8', 'bytearray8',
503 'unicodestringnl', 'unicodestring1', 'unicodestring4',
504 'unicodestring8', 'decimalnl_short', 'decimalnl_long',
505 'floatnl', 'float8', 'long1', 'long4',
506 'StackObject',
507 'pyint', 'pylong', 'pyinteger_or_bool', 'pybool', 'pyfloat',
508 'pybytes_or_str', 'pystring', 'pybytes', 'pybytearray',
509 'pyunicode', 'pynone', 'pytuple', 'pylist', 'pydict',
510 'pyset', 'pyfrozenset', 'pybuffer', 'anyobject',
511 'markobject', 'stackslice', 'OpcodeInfo', 'opcodes',
512 'code2op',
513 }
514 support.check__all__(self, pickletools, not_exported=not_exported)
515
516
517 def load_tests(loader, tests, pattern):
518 tests.addTest(doctest.DocTestSuite(pickletools))
519 return tests
520
521
522 if __name__ == "__main__":
523 unittest.main()