Debian Codegraph @

Lib/test/pickletester.py

raw
1 import builtins
2 import collections
3 import copyreg
4 import dbm
5 import io
6 import functools
7 import os
8 import math
9 import pickle
10 import pickletools
11 import shutil
12 import struct
13 import sys
14 import threading
15 import types
16 import unittest
17 import weakref
18 import __main__
19 from textwrap import dedent
20 from http.cookies import SimpleCookie
21
22 try:
23 import _testbuffer
24 except ImportError:
25 _testbuffer = None
26
27 from test import support
28 from test.support import os_helper
29 from test.support import (
30 run_with_locales, no_tracing,
31 _2G, _4G, bigmemtest
32 )
33 from test.support.import_helper import forget
34 from test.support.os_helper import TESTFN
35 from test.support import threading_helper
36 from test.support.warnings_helper import save_restore_warnings_filters
37 from test import picklecommon
38 from test.picklecommon import *
39
40 from pickle import bytes_types
41
42
43 # bpo-41003: Save/restore warnings filters to leave them unchanged.
44 # Ignore filters installed by numpy.
45 try:
46 with save_restore_warnings_filters():
47 import numpy as np
48 except ImportError:
49 np = None
50
51
52 requires_32b = unittest.skipUnless(sys.maxsize < 2**32,
53 "test is only meaningful on 32-bit builds")
54
55 # Tests that try a number of pickle protocols should have a
56 # for proto in protocols:
57 # kind of outer loop.
58 protocols = range(pickle.HIGHEST_PROTOCOL + 1)
59
60 FAST_NESTING_LIMIT = 50
61
62
63 # Return True if opcode code appears in the pickle, else False.
64 def opcode_in_pickle(code, pickle):
65 for op, dummy, dummy in pickletools.genops(pickle):
66 if op.code == code.decode("latin-1"):
67 return True
68 return False
69
70 # Return the number of times opcode code appears in pickle.
71 def count_opcode(code, pickle):
72 n = 0
73 for op, dummy, dummy in pickletools.genops(pickle):
74 if op.code == code.decode("latin-1"):
75 n += 1
76 return n
77
78
79 def identity(x):
80 return x
81
82
83 class UnseekableIO(io.BytesIO):
84 def peek(self, *args):
85 raise NotImplementedError
86
87 def seekable(self):
88 return False
89
90 def seek(self, *args):
91 raise io.UnsupportedOperation
92
93 def tell(self):
94 raise io.UnsupportedOperation
95
96
97 class MinimalIO(object):
98 """
99 A file-like object that doesn't support readinto().
100 """
101 def __init__(self, *args):
102 self._bio = io.BytesIO(*args)
103 self.getvalue = self._bio.getvalue
104 self.read = self._bio.read
105 self.readline = self._bio.readline
106 self.write = self._bio.write
107
108
109 # We can't very well test the extension registry without putting known stuff
110 # in it, but we have to be careful to restore its original state. Code
111 # should do this:
112 #
113 # e = ExtensionSaver(extension_code)
114 # try:
115 # fiddle w/ the extension registry's stuff for extension_code
116 # finally:
117 # e.restore()
118
119 class ExtensionSaver:
120 # Remember current registration for code (if any), and remove it (if
121 # there is one).
122 def __init__(self, code):
123 self.code = code
124 if code in copyreg._inverted_registry:
125 self.pair = copyreg._inverted_registry[code]
126 copyreg.remove_extension(self.pair[0], self.pair[1], code)
127 else:
128 self.pair = None
129
130 # Restore previous registration for code.
131 def restore(self):
132 code = self.code
133 curpair = copyreg._inverted_registry.get(code)
134 if curpair is not None:
135 copyreg.remove_extension(curpair[0], curpair[1], code)
136 pair = self.pair
137 if pair is not None:
138 copyreg.add_extension(pair[0], pair[1], code)
139
140 class pickling_metaclass(type):
141 def __eq__(self, other):
142 return (type(self) == type(other) and
143 self.reduce_args == other.reduce_args)
144
145 def __reduce__(self):
146 return (create_dynamic_class, self.reduce_args)
147
148 def create_dynamic_class(name, bases):
149 result = pickling_metaclass(name, bases, dict())
150 result.reduce_args = (name, bases)
151 return result
152
153
154 if _testbuffer is not None:
155
156 class PicklableNDArray:
157 # A not-really-zero-copy picklable ndarray, as the ndarray()
158 # constructor doesn't allow for it
159
160 zero_copy_reconstruct = False
161
162 def __init__(self, *args, **kwargs):
163 self.array = _testbuffer.ndarray(*args, **kwargs)
164
165 def __getitem__(self, idx):
166 cls = type(self)
167 new = cls.__new__(cls)
168 new.array = self.array[idx]
169 return new
170
171 @property
172 def readonly(self):
173 return self.array.readonly
174
175 @property
176 def c_contiguous(self):
177 return self.array.c_contiguous
178
179 @property
180 def f_contiguous(self):
181 return self.array.f_contiguous
182
183 def __eq__(self, other):
184 if not isinstance(other, PicklableNDArray):
185 return NotImplemented
186 return (other.array.format == self.array.format and
187 other.array.shape == self.array.shape and
188 other.array.strides == self.array.strides and
189 other.array.readonly == self.array.readonly and
190 other.array.tobytes() == self.array.tobytes())
191
192 def __ne__(self, other):
193 if not isinstance(other, PicklableNDArray):
194 return NotImplemented
195 return not (self == other)
196
197 def __repr__(self):
198 return ("{name}(shape={array.shape},"
199 "strides={array.strides}, "
200 "bytes={array.tobytes()})").format(
201 name=type(self).__name__, array=self.array.shape)
202
203 def __reduce_ex__(self, protocol):
204 if not self.array.contiguous:
205 raise NotImplementedError("Reconstructing a non-contiguous "
206 "ndarray does not seem possible")
207 ndarray_kwargs = {"shape": self.array.shape,
208 "strides": self.array.strides,
209 "format": self.array.format,
210 "flags": (0 if self.readonly
211 else _testbuffer.ND_WRITABLE)}
212 pb = pickle.PickleBuffer(self.array)
213 if protocol >= 5:
214 return (type(self)._reconstruct,
215 (pb, ndarray_kwargs))
216 else:
217 # Need to serialize the bytes in physical order
218 with pb.raw() as m:
219 return (type(self)._reconstruct,
220 (m.tobytes(), ndarray_kwargs))
221
222 @classmethod
223 def _reconstruct(cls, obj, kwargs):
224 with memoryview(obj) as m:
225 # For some reason, ndarray() wants a list of integers...
226 # XXX This only works if format == 'B'
227 items = list(m.tobytes())
228 return cls(items, **kwargs)
229
230
231 # DATA0 .. DATA4 are the pickles we expect under the various protocols, for
232 # the object returned by create_data().
233
234 DATA0 = (
235 b'(lp0\nL0L\naL1L\naF2.0\n'
236 b'ac__builtin__\ncomple'
237 b'x\np1\n(F3.0\nF0.0\ntp2\n'
238 b'Rp3\naL1L\naL-1L\naL255'
239 b'L\naL-255L\naL-256L\naL'
240 b'65535L\naL-65535L\naL-'
241 b'65536L\naL2147483647L'
242 b'\naL-2147483647L\naL-2'
243 b'147483648L\na(Vabc\np4'
244 b'\ng4\nccopy_reg\n_recon'
245 b'structor\np5\n(c__main'
246 b'__\nC\np6\nc__builtin__'
247 b'\nobject\np7\nNtp8\nRp9\n'
248 b'(dp10\nVfoo\np11\nL1L\ns'
249 b'Vbar\np12\nL2L\nsbg9\ntp'
250 b'13\nag13\naL5L\na.'
251 )
252
253 # Disassembly of DATA0
254 DATA0_DIS = """\
255 0: ( MARK
256 1: l LIST (MARK at 0)
257 2: p PUT 0
258 5: L LONG 0
259 9: a APPEND
260 10: L LONG 1
261 14: a APPEND
262 15: F FLOAT 2.0
263 20: a APPEND
264 21: c GLOBAL '__builtin__ complex'
265 42: p PUT 1
266 45: ( MARK
267 46: F FLOAT 3.0
268 51: F FLOAT 0.0
269 56: t TUPLE (MARK at 45)
270 57: p PUT 2
271 60: R REDUCE
272 61: p PUT 3
273 64: a APPEND
274 65: L LONG 1
275 69: a APPEND
276 70: L LONG -1
277 75: a APPEND
278 76: L LONG 255
279 82: a APPEND
280 83: L LONG -255
281 90: a APPEND
282 91: L LONG -256
283 98: a APPEND
284 99: L LONG 65535
285 107: a APPEND
286 108: L LONG -65535
287 117: a APPEND
288 118: L LONG -65536
289 127: a APPEND
290 128: L LONG 2147483647
291 141: a APPEND
292 142: L LONG -2147483647
293 156: a APPEND
294 157: L LONG -2147483648
295 171: a APPEND
296 172: ( MARK
297 173: V UNICODE 'abc'
298 178: p PUT 4
299 181: g GET 4
300 184: c GLOBAL 'copy_reg _reconstructor'
301 209: p PUT 5
302 212: ( MARK
303 213: c GLOBAL '__main__ C'
304 225: p PUT 6
305 228: c GLOBAL '__builtin__ object'
306 248: p PUT 7
307 251: N NONE
308 252: t TUPLE (MARK at 212)
309 253: p PUT 8
310 256: R REDUCE
311 257: p PUT 9
312 260: ( MARK
313 261: d DICT (MARK at 260)
314 262: p PUT 10
315 266: V UNICODE 'foo'
316 271: p PUT 11
317 275: L LONG 1
318 279: s SETITEM
319 280: V UNICODE 'bar'
320 285: p PUT 12
321 289: L LONG 2
322 293: s SETITEM
323 294: b BUILD
324 295: g GET 9
325 298: t TUPLE (MARK at 172)
326 299: p PUT 13
327 303: a APPEND
328 304: g GET 13
329 308: a APPEND
330 309: L LONG 5
331 313: a APPEND
332 314: . STOP
333 highest protocol among opcodes = 0
334 """
335
336 DATA1 = (
337 b']q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c__'
338 b'builtin__\ncomplex\nq\x01'
339 b'(G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00t'
340 b'q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ'
341 b'\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff'
342 b'\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00ab'
343 b'cq\x04h\x04ccopy_reg\n_reco'
344 b'nstructor\nq\x05(c__main'
345 b'__\nC\nq\x06c__builtin__\n'
346 b'object\nq\x07Ntq\x08Rq\t}q\n('
347 b'X\x03\x00\x00\x00fooq\x0bK\x01X\x03\x00\x00\x00bar'
348 b'q\x0cK\x02ubh\ttq\rh\rK\x05e.'
349 )
350
351 # Disassembly of DATA1
352 DATA1_DIS = """\
353 0: ] EMPTY_LIST
354 1: q BINPUT 0
355 3: ( MARK
356 4: K BININT1 0
357 6: K BININT1 1
358 8: G BINFLOAT 2.0
359 17: c GLOBAL '__builtin__ complex'
360 38: q BINPUT 1
361 40: ( MARK
362 41: G BINFLOAT 3.0
363 50: G BINFLOAT 0.0
364 59: t TUPLE (MARK at 40)
365 60: q BINPUT 2
366 62: R REDUCE
367 63: q BINPUT 3
368 65: K BININT1 1
369 67: J BININT -1
370 72: K BININT1 255
371 74: J BININT -255
372 79: J BININT -256
373 84: M BININT2 65535
374 87: J BININT -65535
375 92: J BININT -65536
376 97: J BININT 2147483647
377 102: J BININT -2147483647
378 107: J BININT -2147483648
379 112: ( MARK
380 113: X BINUNICODE 'abc'
381 121: q BINPUT 4
382 123: h BINGET 4
383 125: c GLOBAL 'copy_reg _reconstructor'
384 150: q BINPUT 5
385 152: ( MARK
386 153: c GLOBAL '__main__ C'
387 165: q BINPUT 6
388 167: c GLOBAL '__builtin__ object'
389 187: q BINPUT 7
390 189: N NONE
391 190: t TUPLE (MARK at 152)
392 191: q BINPUT 8
393 193: R REDUCE
394 194: q BINPUT 9
395 196: } EMPTY_DICT
396 197: q BINPUT 10
397 199: ( MARK
398 200: X BINUNICODE 'foo'
399 208: q BINPUT 11
400 210: K BININT1 1
401 212: X BINUNICODE 'bar'
402 220: q BINPUT 12
403 222: K BININT1 2
404 224: u SETITEMS (MARK at 199)
405 225: b BUILD
406 226: h BINGET 9
407 228: t TUPLE (MARK at 112)
408 229: q BINPUT 13
409 231: h BINGET 13
410 233: K BININT1 5
411 235: e APPENDS (MARK at 3)
412 236: . STOP
413 highest protocol among opcodes = 1
414 """
415
416 DATA2 = (
417 b'\x80\x02]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
418 b'__builtin__\ncomplex\n'
419 b'q\x01G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00'
420 b'\x86q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xff'
421 b'J\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff'
422 b'\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00a'
423 b'bcq\x04h\x04c__main__\nC\nq\x05'
424 b')\x81q\x06}q\x07(X\x03\x00\x00\x00fooq\x08K\x01'
425 b'X\x03\x00\x00\x00barq\tK\x02ubh\x06tq\nh'
426 b'\nK\x05e.'
427 )
428
429 # Disassembly of DATA2
430 DATA2_DIS = """\
431 0: \x80 PROTO 2
432 2: ] EMPTY_LIST
433 3: q BINPUT 0
434 5: ( MARK
435 6: K BININT1 0
436 8: K BININT1 1
437 10: G BINFLOAT 2.0
438 19: c GLOBAL '__builtin__ complex'
439 40: q BINPUT 1
440 42: G BINFLOAT 3.0
441 51: G BINFLOAT 0.0
442 60: \x86 TUPLE2
443 61: q BINPUT 2
444 63: R REDUCE
445 64: q BINPUT 3
446 66: K BININT1 1
447 68: J BININT -1
448 73: K BININT1 255
449 75: J BININT -255
450 80: J BININT -256
451 85: M BININT2 65535
452 88: J BININT -65535
453 93: J BININT -65536
454 98: J BININT 2147483647
455 103: J BININT -2147483647
456 108: J BININT -2147483648
457 113: ( MARK
458 114: X BINUNICODE 'abc'
459 122: q BINPUT 4
460 124: h BINGET 4
461 126: c GLOBAL '__main__ C'
462 138: q BINPUT 5
463 140: ) EMPTY_TUPLE
464 141: \x81 NEWOBJ
465 142: q BINPUT 6
466 144: } EMPTY_DICT
467 145: q BINPUT 7
468 147: ( MARK
469 148: X BINUNICODE 'foo'
470 156: q BINPUT 8
471 158: K BININT1 1
472 160: X BINUNICODE 'bar'
473 168: q BINPUT 9
474 170: K BININT1 2
475 172: u SETITEMS (MARK at 147)
476 173: b BUILD
477 174: h BINGET 6
478 176: t TUPLE (MARK at 113)
479 177: q BINPUT 10
480 179: h BINGET 10
481 181: K BININT1 5
482 183: e APPENDS (MARK at 5)
483 184: . STOP
484 highest protocol among opcodes = 2
485 """
486
487 DATA3 = (
488 b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
489 b'builtins\ncomplex\nq\x01G'
490 b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02'
491 b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff'
492 b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f'
493 b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq'
494 b'\x04h\x04c__main__\nC\nq\x05)\x81q'
495 b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00'
496 b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05'
497 b'e.'
498 )
499
500 # Disassembly of DATA3
501 DATA3_DIS = """\
502 0: \x80 PROTO 3
503 2: ] EMPTY_LIST
504 3: q BINPUT 0
505 5: ( MARK
506 6: K BININT1 0
507 8: K BININT1 1
508 10: G BINFLOAT 2.0
509 19: c GLOBAL 'builtins complex'
510 37: q BINPUT 1
511 39: G BINFLOAT 3.0
512 48: G BINFLOAT 0.0
513 57: \x86 TUPLE2
514 58: q BINPUT 2
515 60: R REDUCE
516 61: q BINPUT 3
517 63: K BININT1 1
518 65: J BININT -1
519 70: K BININT1 255
520 72: J BININT -255
521 77: J BININT -256
522 82: M BININT2 65535
523 85: J BININT -65535
524 90: J BININT -65536
525 95: J BININT 2147483647
526 100: J BININT -2147483647
527 105: J BININT -2147483648
528 110: ( MARK
529 111: X BINUNICODE 'abc'
530 119: q BINPUT 4
531 121: h BINGET 4
532 123: c GLOBAL '__main__ C'
533 135: q BINPUT 5
534 137: ) EMPTY_TUPLE
535 138: \x81 NEWOBJ
536 139: q BINPUT 6
537 141: } EMPTY_DICT
538 142: q BINPUT 7
539 144: ( MARK
540 145: X BINUNICODE 'bar'
541 153: q BINPUT 8
542 155: K BININT1 2
543 157: X BINUNICODE 'foo'
544 165: q BINPUT 9
545 167: K BININT1 1
546 169: u SETITEMS (MARK at 144)
547 170: b BUILD
548 171: h BINGET 6
549 173: t TUPLE (MARK at 110)
550 174: q BINPUT 10
551 176: h BINGET 10
552 178: K BININT1 5
553 180: e APPENDS (MARK at 5)
554 181: . STOP
555 highest protocol among opcodes = 2
556 """
557
558 DATA4 = (
559 b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@'
560 b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07'
561 b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G'
562 b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK'
563 b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ'
564 b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80('
565 b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c'
566 b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c'
567 b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.'
568 )
569
570 # Disassembly of DATA4
571 DATA4_DIS = """\
572 0: \x80 PROTO 4
573 2: \x95 FRAME 168
574 11: ] EMPTY_LIST
575 12: \x94 MEMOIZE
576 13: ( MARK
577 14: K BININT1 0
578 16: K BININT1 1
579 18: G BINFLOAT 2.0
580 27: \x8c SHORT_BINUNICODE 'builtins'
581 37: \x94 MEMOIZE
582 38: \x8c SHORT_BINUNICODE 'complex'
583 47: \x94 MEMOIZE
584 48: \x93 STACK_GLOBAL
585 49: \x94 MEMOIZE
586 50: G BINFLOAT 3.0
587 59: G BINFLOAT 0.0
588 68: \x86 TUPLE2
589 69: \x94 MEMOIZE
590 70: R REDUCE
591 71: \x94 MEMOIZE
592 72: K BININT1 1
593 74: J BININT -1
594 79: K BININT1 255
595 81: J BININT -255
596 86: J BININT -256
597 91: M BININT2 65535
598 94: J BININT -65535
599 99: J BININT -65536
600 104: J BININT 2147483647
601 109: J BININT -2147483647
602 114: J BININT -2147483648
603 119: ( MARK
604 120: \x8c SHORT_BINUNICODE 'abc'
605 125: \x94 MEMOIZE
606 126: h BINGET 6
607 128: \x8c SHORT_BINUNICODE '__main__'
608 138: \x94 MEMOIZE
609 139: \x8c SHORT_BINUNICODE 'C'
610 142: \x94 MEMOIZE
611 143: \x93 STACK_GLOBAL
612 144: \x94 MEMOIZE
613 145: ) EMPTY_TUPLE
614 146: \x81 NEWOBJ
615 147: \x94 MEMOIZE
616 148: } EMPTY_DICT
617 149: \x94 MEMOIZE
618 150: ( MARK
619 151: \x8c SHORT_BINUNICODE 'bar'
620 156: \x94 MEMOIZE
621 157: K BININT1 2
622 159: \x8c SHORT_BINUNICODE 'foo'
623 164: \x94 MEMOIZE
624 165: K BININT1 1
625 167: u SETITEMS (MARK at 150)
626 168: b BUILD
627 169: h BINGET 10
628 171: t TUPLE (MARK at 119)
629 172: \x94 MEMOIZE
630 173: h BINGET 14
631 175: K BININT1 5
632 177: e APPENDS (MARK at 13)
633 178: . STOP
634 highest protocol among opcodes = 4
635 """
636
637 # set([1,2]) pickled from 2.x with protocol 2
638 DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.'
639
640 # xrange(5) pickled from 2.x with protocol 2
641 DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.'
642
643 # a SimpleCookie() object pickled from 2.x with protocol 2
644 DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
645 b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U'
646 b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07'
647 b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U'
648 b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b'
649 b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.')
650
651 # set([3]) pickled from 2.x with protocol 2
652 DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
653
654 python2_exceptions_without_args = (
655 ArithmeticError,
656 AssertionError,
657 AttributeError,
658 BaseException,
659 BufferError,
660 BytesWarning,
661 DeprecationWarning,
662 EOFError,
663 EnvironmentError,
664 Exception,
665 FloatingPointError,
666 FutureWarning,
667 GeneratorExit,
668 IOError,
669 ImportError,
670 ImportWarning,
671 IndentationError,
672 IndexError,
673 KeyError,
674 KeyboardInterrupt,
675 LookupError,
676 MemoryError,
677 NameError,
678 NotImplementedError,
679 OSError,
680 OverflowError,
681 PendingDeprecationWarning,
682 ReferenceError,
683 RuntimeError,
684 RuntimeWarning,
685 # StandardError is gone in Python 3, we map it to Exception
686 StopIteration,
687 SyntaxError,
688 SyntaxWarning,
689 SystemError,
690 SystemExit,
691 TabError,
692 TypeError,
693 UnboundLocalError,
694 UnicodeError,
695 UnicodeWarning,
696 UserWarning,
697 ValueError,
698 Warning,
699 ZeroDivisionError,
700 )
701
702 exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.'
703
704 # UnicodeEncodeError object pickled from 2.x with protocol 2
705 DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n'
706 b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01'
707 b'U\x03badq\x03tq\x04Rq\x05.')
708
709
710 def create_data():
711 c = C()
712 c.foo = 1
713 c.bar = 2
714 x = [0, 1, 2.0, 3.0+0j]
715 # Append some integer test cases at cPickle.c's internal size
716 # cutoffs.
717 uint1max = 0xff
718 uint2max = 0xffff
719 int4max = 0x7fffffff
720 x.extend([1, -1,
721 uint1max, -uint1max, -uint1max-1,
722 uint2max, -uint2max, -uint2max-1,
723 int4max, -int4max, -int4max-1])
724 y = ('abc', 'abc', c, c)
725 x.append(y)
726 x.append(y)
727 x.append(5)
728 return x
729
730
731 class AbstractUnpickleTests:
732 # Subclass must define self.loads.
733
734 _testdata = create_data()
735
736 def assert_is_copy(self, obj, objcopy, msg=None):
737 """Utility method to verify if two objects are copies of each others.
738 """
739 if msg is None:
740 msg = "{!r} is not a copy of {!r}".format(obj, objcopy)
741 self.assertEqual(obj, objcopy, msg=msg)
742 self.assertIs(type(obj), type(objcopy), msg=msg)
743 if hasattr(obj, '__dict__'):
744 self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg)
745 self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg)
746 if hasattr(obj, '__slots__'):
747 self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg)
748 for slot in obj.__slots__:
749 self.assertEqual(
750 hasattr(obj, slot), hasattr(objcopy, slot), msg=msg)
751 self.assertEqual(getattr(obj, slot, None),
752 getattr(objcopy, slot, None), msg=msg)
753
754 def check_unpickling_error(self, errors, data):
755 with self.subTest(data=data), \
756 self.assertRaises(errors):
757 try:
758 self.loads(data)
759 except BaseException as exc:
760 if support.verbose > 1:
761 print('%-32r - %s: %s' %
762 (data, exc.__class__.__name__, exc))
763 raise
764
765 def test_load_from_data0(self):
766 self.assert_is_copy(self._testdata, self.loads(DATA0))
767
768 def test_load_from_data1(self):
769 self.assert_is_copy(self._testdata, self.loads(DATA1))
770
771 def test_load_from_data2(self):
772 self.assert_is_copy(self._testdata, self.loads(DATA2))
773
774 def test_load_from_data3(self):
775 self.assert_is_copy(self._testdata, self.loads(DATA3))
776
777 def test_load_from_data4(self):
778 self.assert_is_copy(self._testdata, self.loads(DATA4))
779
780 def test_load_classic_instance(self):
781 # See issue5180. Test loading 2.x pickles that
782 # contain an instance of old style class.
783 for X, args in [(C, ()), (D, ('x',)), (E, ())]:
784 xname = X.__name__.encode('ascii')
785 # Protocol 0 (text mode pickle):
786 """
787 0: ( MARK
788 1: i INST '__main__ X' (MARK at 0)
789 13: p PUT 0
790 16: ( MARK
791 17: d DICT (MARK at 16)
792 18: p PUT 1
793 21: b BUILD
794 22: . STOP
795 """
796 pickle0 = (b"(i__main__\n"
797 b"X\n"
798 b"p0\n"
799 b"(dp1\nb.").replace(b'X', xname)
800 self.assert_is_copy(X(*args), self.loads(pickle0))
801
802 # Protocol 1 (binary mode pickle)
803 """
804 0: ( MARK
805 1: c GLOBAL '__main__ X'
806 13: q BINPUT 0
807 15: o OBJ (MARK at 0)
808 16: q BINPUT 1
809 18: } EMPTY_DICT
810 19: q BINPUT 2
811 21: b BUILD
812 22: . STOP
813 """
814 pickle1 = (b'(c__main__\n'
815 b'X\n'
816 b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
817 self.assert_is_copy(X(*args), self.loads(pickle1))
818
819 # Protocol 2 (pickle2 = b'\x80\x02' + pickle1)
820 """
821 0: \x80 PROTO 2
822 2: ( MARK
823 3: c GLOBAL '__main__ X'
824 15: q BINPUT 0
825 17: o OBJ (MARK at 2)
826 18: q BINPUT 1
827 20: } EMPTY_DICT
828 21: q BINPUT 2
829 23: b BUILD
830 24: . STOP
831 """
832 pickle2 = (b'\x80\x02(c__main__\n'
833 b'X\n'
834 b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
835 self.assert_is_copy(X(*args), self.loads(pickle2))
836
837 def test_maxint64(self):
838 maxint64 = (1 << 63) - 1
839 data = b'I' + str(maxint64).encode("ascii") + b'\n.'
840 got = self.loads(data)
841 self.assert_is_copy(maxint64, got)
842
843 # Try too with a bogus literal.
844 data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.'
845 self.check_unpickling_error(ValueError, data)
846
847 def test_unpickle_from_2x(self):
848 # Unpickle non-trivial data from Python 2.x.
849 loaded = self.loads(DATA_SET)
850 self.assertEqual(loaded, set([1, 2]))
851 loaded = self.loads(DATA_XRANGE)
852 self.assertEqual(type(loaded), type(range(0)))
853 self.assertEqual(list(loaded), list(range(5)))
854 loaded = self.loads(DATA_COOKIE)
855 self.assertEqual(type(loaded), SimpleCookie)
856 self.assertEqual(list(loaded.keys()), ["key"])
857 self.assertEqual(loaded["key"].value, "value")
858
859 # Exception objects without arguments pickled from 2.x with protocol 2
860 for exc in python2_exceptions_without_args:
861 data = exception_pickle.replace(b'?', exc.__name__.encode("ascii"))
862 loaded = self.loads(data)
863 self.assertIs(type(loaded), exc)
864
865 # StandardError is mapped to Exception, test that separately
866 loaded = self.loads(exception_pickle.replace(b'?', b'StandardError'))
867 self.assertIs(type(loaded), Exception)
868
869 loaded = self.loads(DATA_UEERR)
870 self.assertIs(type(loaded), UnicodeEncodeError)
871 self.assertEqual(loaded.object, "foo")
872 self.assertEqual(loaded.encoding, "ascii")
873 self.assertEqual(loaded.start, 0)
874 self.assertEqual(loaded.end, 1)
875 self.assertEqual(loaded.reason, "bad")
876
877 def test_load_python2_str_as_bytes(self):
878 # From Python 2: pickle.dumps('a\x00\xa0', protocol=0)
879 self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.",
880 encoding="bytes"), b'a\x00\xa0')
881 # From Python 2: pickle.dumps('a\x00\xa0', protocol=1)
882 self.assertEqual(self.loads(b'U\x03a\x00\xa0.',
883 encoding="bytes"), b'a\x00\xa0')
884 # From Python 2: pickle.dumps('a\x00\xa0', protocol=2)
885 self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.',
886 encoding="bytes"), b'a\x00\xa0')
887
888 def test_load_python2_unicode_as_str(self):
889 # From Python 2: pickle.dumps(u'Ï€', protocol=0)
890 self.assertEqual(self.loads(b'V\\u03c0\n.',
891 encoding='bytes'), 'Ï€')
892 # From Python 2: pickle.dumps(u'Ï€', protocol=1)
893 self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.',
894 encoding="bytes"), 'Ï€')
895 # From Python 2: pickle.dumps(u'Ï€', protocol=2)
896 self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.',
897 encoding="bytes"), 'Ï€')
898
899 def test_load_long_python2_str_as_bytes(self):
900 # From Python 2: pickle.dumps('x' * 300, protocol=1)
901 self.assertEqual(self.loads(pickle.BINSTRING +
902 struct.pack("<I", 300) +
903 b'x' * 300 + pickle.STOP,
904 encoding='bytes'), b'x' * 300)
905
906 def test_constants(self):
907 self.assertIsNone(self.loads(b'N.'))
908 self.assertIs(self.loads(b'\x88.'), True)
909 self.assertIs(self.loads(b'\x89.'), False)
910 self.assertIs(self.loads(b'I01\n.'), True)
911 self.assertIs(self.loads(b'I00\n.'), False)
912
913 def test_zero_padded_integers(self):
914 self.assertEqual(self.loads(b'I010\n.'), 10)
915 self.assertEqual(self.loads(b'I-010\n.'), -10)
916 self.assertEqual(self.loads(b'I0010\n.'), 10)
917 self.assertEqual(self.loads(b'I-0010\n.'), -10)
918 self.assertEqual(self.loads(b'L010\n.'), 10)
919 self.assertEqual(self.loads(b'L-010\n.'), -10)
920 self.assertEqual(self.loads(b'L0010\n.'), 10)
921 self.assertEqual(self.loads(b'L-0010\n.'), -10)
922 self.assertEqual(self.loads(b'L010L\n.'), 10)
923 self.assertEqual(self.loads(b'L-010L\n.'), -10)
924
925 def test_nondecimal_integers(self):
926 self.assertRaises(ValueError, self.loads, b'I0b10\n.')
927 self.assertRaises(ValueError, self.loads, b'I0o10\n.')
928 self.assertRaises(ValueError, self.loads, b'I0x10\n.')
929 self.assertRaises(ValueError, self.loads, b'L0b10L\n.')
930 self.assertRaises(ValueError, self.loads, b'L0o10L\n.')
931 self.assertRaises(ValueError, self.loads, b'L0x10L\n.')
932
933 def test_empty_bytestring(self):
934 # issue 11286
935 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r')
936 self.assertEqual(empty, '')
937
938 def test_short_binbytes(self):
939 dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.'
940 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
941
942 def test_binbytes(self):
943 dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.'
944 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
945
946 @requires_32b
947 def test_negative_32b_binbytes(self):
948 # On 32-bit builds, a BINBYTES of 2**31 or more is refused
949 dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.'
950 self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
951 dumped)
952
953 @requires_32b
954 def test_negative_32b_binunicode(self):
955 # On 32-bit builds, a BINUNICODE of 2**31 or more is refused
956 dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.'
957 self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
958 dumped)
959
960 def test_short_binunicode(self):
961 dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.'
962 self.assertEqual(self.loads(dumped), '\u20ac\x00')
963
964 def test_misc_get(self):
965 self.check_unpickling_error(pickle.UnpicklingError, b'g0\np0')
966 self.check_unpickling_error(pickle.UnpicklingError, b'jens:')
967 self.check_unpickling_error(pickle.UnpicklingError, b'hens:')
968 self.assert_is_copy([(100,), (100,)],
969 self.loads(b'((Kdtp0\nh\x00l.))'))
970
971 def test_binbytes8(self):
972 dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
973 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
974
975 def test_binunicode8(self):
976 dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
977 self.assertEqual(self.loads(dumped), '\u20ac\x00')
978
979 def test_bytearray8(self):
980 dumped = b'\x80\x05\x96\x03\x00\x00\x00\x00\x00\x00\x00xxx.'
981 self.assertEqual(self.loads(dumped), bytearray(b'xxx'))
982
983 @requires_32b
984 def test_large_32b_binbytes8(self):
985 dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
986 self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
987 dumped)
988
989 @requires_32b
990 def test_large_32b_bytearray8(self):
991 dumped = b'\x80\x05\x96\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
992 self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
993 dumped)
994
995 @requires_32b
996 def test_large_32b_binunicode8(self):
997 dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
998 self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
999 dumped)
1000
1001 def test_large_binstring(self):
1002 errmsg = 'BINSTRING pickle has negative byte count'
1003 with self.assertRaisesRegex(pickle.UnpicklingError, errmsg):
1004 self.loads(b'T\0\0\0\x80')
1005
1006 def test_get(self):
1007 pickled = b'((lp100000\ng100000\nt.'
1008 unpickled = self.loads(pickled)
1009 self.assertEqual(unpickled, ([],)*2)
1010 self.assertIs(unpickled[0], unpickled[1])
1011
1012 def test_binget(self):
1013 pickled = b'(]q\xffh\xfft.'
1014 unpickled = self.loads(pickled)
1015 self.assertEqual(unpickled, ([],)*2)
1016 self.assertIs(unpickled[0], unpickled[1])
1017
1018 def test_long_binget(self):
1019 pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.'
1020 unpickled = self.loads(pickled)
1021 self.assertEqual(unpickled, ([],)*2)
1022 self.assertIs(unpickled[0], unpickled[1])
1023
1024 def test_dup(self):
1025 pickled = b'((l2t.'
1026 unpickled = self.loads(pickled)
1027 self.assertEqual(unpickled, ([],)*2)
1028 self.assertIs(unpickled[0], unpickled[1])
1029
1030 def test_negative_put(self):
1031 # Issue #12847
1032 dumped = b'Va\np-1\n.'
1033 self.check_unpickling_error(ValueError, dumped)
1034
1035 @requires_32b
1036 def test_negative_32b_binput(self):
1037 # Issue #12847
1038 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.'
1039 self.check_unpickling_error(ValueError, dumped)
1040
1041 def test_badly_escaped_string(self):
1042 self.check_unpickling_error(ValueError, b"S'\\'\n.")
1043
1044 def test_badly_quoted_string(self):
1045 # Issue #17710
1046 badpickles = [b"S'\n.",
1047 b'S"\n.',
1048 b'S\' \n.',
1049 b'S" \n.',
1050 b'S\'"\n.',
1051 b'S"\'\n.',
1052 b"S' ' \n.",
1053 b'S" " \n.',
1054 b"S ''\n.",
1055 b'S ""\n.',
1056 b'S \n.',
1057 b'S\n.',
1058 b'S.']
1059 for p in badpickles:
1060 self.check_unpickling_error(pickle.UnpicklingError, p)
1061
1062 def test_correctly_quoted_string(self):
1063 goodpickles = [(b"S''\n.", ''),
1064 (b'S""\n.', ''),
1065 (b'S"\\n"\n.', '\n'),
1066 (b"S'\\n'\n.", '\n')]
1067 for p, expected in goodpickles:
1068 self.assertEqual(self.loads(p), expected)
1069
1070 def test_frame_readline(self):
1071 pickled = b'\x80\x04\x95\x05\x00\x00\x00\x00\x00\x00\x00I42\n.'
1072 # 0: \x80 PROTO 4
1073 # 2: \x95 FRAME 5
1074 # 11: I INT 42
1075 # 15: . STOP
1076 self.assertEqual(self.loads(pickled), 42)
1077
1078 def test_compat_unpickle(self):
1079 # xrange(1, 7)
1080 pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.'
1081 unpickled = self.loads(pickled)
1082 self.assertIs(type(unpickled), range)
1083 self.assertEqual(unpickled, range(1, 7))
1084 self.assertEqual(list(unpickled), [1, 2, 3, 4, 5, 6])
1085 # reduce
1086 pickled = b'\x80\x02c__builtin__\nreduce\n.'
1087 self.assertIs(self.loads(pickled), functools.reduce)
1088 # whichdb.whichdb
1089 pickled = b'\x80\x02cwhichdb\nwhichdb\n.'
1090 self.assertIs(self.loads(pickled), dbm.whichdb)
1091 # Exception(), StandardError()
1092 for name in (b'Exception', b'StandardError'):
1093 pickled = (b'\x80\x02cexceptions\n' + name + b'\nU\x03ugh\x85R.')
1094 unpickled = self.loads(pickled)
1095 self.assertIs(type(unpickled), Exception)
1096 self.assertEqual(str(unpickled), 'ugh')
1097 # UserDict.UserDict({1: 2}), UserDict.IterableUserDict({1: 2})
1098 for name in (b'UserDict', b'IterableUserDict'):
1099 pickled = (b'\x80\x02(cUserDict\n' + name +
1100 b'\no}U\x04data}K\x01K\x02ssb.')
1101 unpickled = self.loads(pickled)
1102 self.assertIs(type(unpickled), collections.UserDict)
1103 self.assertEqual(unpickled, collections.UserDict({1: 2}))
1104
1105 def test_load_global(self):
1106 self.assertIs(self.loads(b'cbuiltins\nstr\n.'), str)
1107 self.assertIs(self.loads(b'cmath\nlog\n.'), math.log)
1108 self.assertIs(self.loads(b'cos.path\njoin\n.'), os.path.join)
1109 self.assertIs(self.loads(b'\x80\x04cbuiltins\nstr.upper\n.'), str.upper)
1110 with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)):
1111 self.assertEqual(self.loads(b'\x80\x04cm\xc3\xb6dule\ngl\xc3\xb6bal\n.'), 42)
1112
1113 self.assertRaises(UnicodeDecodeError, self.loads, b'c\xff\nlog\n.')
1114 self.assertRaises(UnicodeDecodeError, self.loads, b'cmath\n\xff\n.')
1115 self.assertRaises(self.truncated_errors, self.loads, b'c\nlog\n.')
1116 self.assertRaises(self.truncated_errors, self.loads, b'cmath\n\n.')
1117 self.assertRaises(self.truncated_errors, self.loads, b'\x80\x04cmath\n\n.')
1118
1119 def test_load_stack_global(self):
1120 self.assertIs(self.loads(b'\x8c\x08builtins\x8c\x03str\x93.'), str)
1121 self.assertIs(self.loads(b'\x8c\x04math\x8c\x03log\x93.'), math.log)
1122 self.assertIs(self.loads(b'\x8c\x07os.path\x8c\x04join\x93.'),
1123 os.path.join)
1124 self.assertIs(self.loads(b'\x80\x04\x8c\x08builtins\x8c\x09str.upper\x93.'),
1125 str.upper)
1126 with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)):
1127 self.assertEqual(self.loads(b'\x80\x04\x8c\x07m\xc3\xb6dule\x8c\x07gl\xc3\xb6bal\x93.'), 42)
1128
1129 self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x01\xff\x8c\x03log\x93.')
1130 self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x04math\x8c\x01\xff\x93.')
1131 self.assertRaises(ValueError, self.loads, b'\x8c\x00\x8c\x03log\x93.')
1132 self.assertRaises(AttributeError, self.loads, b'\x8c\x04math\x8c\x00\x93.')
1133 self.assertRaises(AttributeError, self.loads, b'\x80\x04\x8c\x04math\x8c\x00\x93.')
1134
1135 self.assertRaises(pickle.UnpicklingError, self.loads, b'N\x8c\x03log\x93.')
1136 self.assertRaises(pickle.UnpicklingError, self.loads, b'\x8c\x04mathN\x93.')
1137 self.assertRaises(pickle.UnpicklingError, self.loads, b'\x80\x04\x8c\x04mathN\x93.')
1138
1139 def test_find_class(self):
1140 unpickler = self.unpickler(io.BytesIO())
1141 unpickler_nofix = self.unpickler(io.BytesIO(), fix_imports=False)
1142 unpickler4 = self.unpickler(io.BytesIO(b'\x80\x04N.'))
1143 unpickler4.load()
1144
1145 self.assertIs(unpickler.find_class('__builtin__', 'str'), str)
1146 self.assertRaises(ModuleNotFoundError,
1147 unpickler_nofix.find_class, '__builtin__', 'str')
1148 self.assertIs(unpickler.find_class('builtins', 'str'), str)
1149 self.assertIs(unpickler_nofix.find_class('builtins', 'str'), str)
1150 self.assertIs(unpickler.find_class('math', 'log'), math.log)
1151 self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join)
1152 self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join)
1153
1154 self.assertIs(unpickler4.find_class('builtins', 'str.upper'), str.upper)
1155 with self.assertRaisesRegex(AttributeError,
1156 r"module 'builtins' has no attribute 'str\.upper'"):
1157 unpickler.find_class('builtins', 'str.upper')
1158
1159 with self.assertRaisesRegex(AttributeError,
1160 "module 'math' has no attribute 'spam'"):
1161 unpickler.find_class('math', 'spam')
1162 with self.assertRaisesRegex(AttributeError,
1163 "module 'math' has no attribute 'spam'"):
1164 unpickler4.find_class('math', 'spam')
1165 with self.assertRaisesRegex(AttributeError,
1166 r"module 'math' has no attribute 'log\.spam'"):
1167 unpickler.find_class('math', 'log.spam')
1168 with self.assertRaisesRegex(AttributeError,
1169 r"Can't resolve path 'log\.spam' on module 'math'") as cm:
1170 unpickler4.find_class('math', 'log.spam')
1171 self.assertEqual(str(cm.exception.__context__),
1172 "'builtin_function_or_method' object has no attribute 'spam'")
1173 with self.assertRaisesRegex(AttributeError,
1174 r"module 'math' has no attribute 'log\.<locals>\.spam'"):
1175 unpickler.find_class('math', 'log.<locals>.spam')
1176 with self.assertRaisesRegex(AttributeError,
1177 r"Can't resolve path 'log\.<locals>\.spam' on module 'math'") as cm:
1178 unpickler4.find_class('math', 'log.<locals>.spam')
1179 self.assertEqual(str(cm.exception.__context__),
1180 "'builtin_function_or_method' object has no attribute '<locals>'")
1181 with self.assertRaisesRegex(AttributeError,
1182 "module 'math' has no attribute ''"):
1183 unpickler.find_class('math', '')
1184 with self.assertRaisesRegex(AttributeError,
1185 "module 'math' has no attribute ''"):
1186 unpickler4.find_class('math', '')
1187 self.assertRaises(ModuleNotFoundError, unpickler.find_class, 'spam', 'log')
1188 self.assertRaises(ValueError, unpickler.find_class, '', 'log')
1189
1190 self.assertRaises(TypeError, unpickler.find_class, None, 'log')
1191 self.assertRaises(TypeError, unpickler.find_class, 'math', None)
1192 self.assertRaises((TypeError, AttributeError), unpickler4.find_class, 'math', None)
1193
1194 def test_custom_find_class(self):
1195 def loads(data):
1196 class Unpickler(self.unpickler):
1197 def find_class(self, module_name, global_name):
1198 return (module_name, global_name)
1199 return Unpickler(io.BytesIO(data)).load()
1200
1201 self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log'))
1202 self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log'))
1203
1204 def loads(data):
1205 class Unpickler(self.unpickler):
1206 @staticmethod
1207 def find_class(module_name, global_name):
1208 return (module_name, global_name)
1209 return Unpickler(io.BytesIO(data)).load()
1210
1211 self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log'))
1212 self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log'))
1213
1214 def loads(data):
1215 class Unpickler(self.unpickler):
1216 @classmethod
1217 def find_class(cls, module_name, global_name):
1218 return (module_name, global_name)
1219 return Unpickler(io.BytesIO(data)).load()
1220
1221 self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log'))
1222 self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log'))
1223
1224 def loads(data):
1225 class Unpickler(self.unpickler):
1226 pass
1227 def find_class(module_name, global_name):
1228 return (module_name, global_name)
1229 unpickler = Unpickler(io.BytesIO(data))
1230 unpickler.find_class = find_class
1231 return unpickler.load()
1232
1233 self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log'))
1234 self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log'))
1235
1236 def test_bad_ext_code(self):
1237 # unregistered extension code
1238 self.check_unpickling_error(ValueError, b'\x82\x01.')
1239 self.check_unpickling_error(ValueError, b'\x82\xff.')
1240 self.check_unpickling_error(ValueError, b'\x83\x01\x00.')
1241 self.check_unpickling_error(ValueError, b'\x83\xff\xff.')
1242 self.check_unpickling_error(ValueError, b'\x84\x01\x00\x00\x00.')
1243 self.check_unpickling_error(ValueError, b'\x84\xff\xff\xff\x7f.')
1244 # EXT specifies code <= 0
1245 self.check_unpickling_error(pickle.UnpicklingError, b'\x82\x00.')
1246 self.check_unpickling_error(pickle.UnpicklingError, b'\x83\x00\x00.')
1247 self.check_unpickling_error(pickle.UnpicklingError, b'\x84\x00\x00\x00\x00.')
1248 self.check_unpickling_error(pickle.UnpicklingError, b'\x84\x00\x00\x00\x80.')
1249 self.check_unpickling_error(pickle.UnpicklingError, b'\x84\xff\xff\xff\xff.')
1250
1251 @support.cpython_only
1252 def test_bad_ext_inverted_registry(self):
1253 code = 1
1254 def check(key, exc):
1255 with support.swap_item(copyreg._inverted_registry, code, key):
1256 with self.assertRaises(exc):
1257 self.loads(b'\x82\x01.')
1258 check(None, ValueError)
1259 check((), ValueError)
1260 check((MyList.__module__,), (TypeError, ValueError))
1261 check((MyList.__module__, "MyList", "x"), (TypeError, ValueError))
1262 check((MyList.__module__, None), (TypeError, ValueError))
1263 check((None, "MyList"), (TypeError, ValueError))
1264
1265 def test_bad_reduce(self):
1266 self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0)
1267 self.check_unpickling_error(TypeError, b'N)R.')
1268 self.check_unpickling_error(TypeError, b'cbuiltins\nint\nNR.')
1269
1270 def test_bad_newobj(self):
1271 error = (pickle.UnpicklingError, TypeError)
1272 self.assertEqual(self.loads(b'cbuiltins\nint\n)\x81.'), 0)
1273 self.check_unpickling_error(error, b'cbuiltins\nlen\n)\x81.')
1274 self.check_unpickling_error(error, b'cbuiltins\nint\nN\x81.')
1275
1276 def test_bad_newobj_ex(self):
1277 error = (pickle.UnpicklingError, TypeError)
1278 self.assertEqual(self.loads(b'cbuiltins\nint\n)}\x92.'), 0)
1279 self.check_unpickling_error(error, b'cbuiltins\nlen\n)}\x92.')
1280 self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.')
1281 self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.')
1282
1283 def test_bad_state(self):
1284 c = C()
1285 c.x = None
1286 base = b'c__main__\nC\n)\x81'
1287 self.assertEqual(self.loads(base + b'}X\x01\x00\x00\x00xNsb.'), c)
1288 self.assertEqual(self.loads(base + b'N}X\x01\x00\x00\x00xNs\x86b.'), c)
1289 # non-hashable dict key
1290 self.check_unpickling_error(TypeError, base + b'}]Nsb.')
1291 # state = list
1292 error = (pickle.UnpicklingError, AttributeError)
1293 self.check_unpickling_error(error, base + b'](}}eb.')
1294 # state = 1-tuple
1295 self.check_unpickling_error(error, base + b'}\x85b.')
1296 # state = 3-tuple
1297 self.check_unpickling_error(error, base + b'}}}\x87b.')
1298 # non-hashable slot name
1299 self.check_unpickling_error(TypeError, base + b'}}]Ns\x86b.')
1300 # non-string slot name
1301 self.check_unpickling_error(TypeError, base + b'}}NNs\x86b.')
1302 # dict = True
1303 self.check_unpickling_error(error, base + b'\x88}\x86b.')
1304 # slots dict = True
1305 self.check_unpickling_error(error, base + b'}\x88\x86b.')
1306
1307 class BadKey1:
1308 count = 1
1309 def __hash__(self):
1310 if not self.count:
1311 raise CustomError
1312 self.count -= 1
1313 return 42
1314 __main__.BadKey1 = BadKey1
1315 # bad hashable dict key
1316 self.check_unpickling_error(CustomError, base + b'}c__main__\nBadKey1\n)\x81Nsb.')
1317
1318 def test_bad_stack(self):
1319 badpickles = [
1320 b'.', # STOP
1321 b'0', # POP
1322 b'1', # POP_MARK
1323 b'2', # DUP
1324 b'(2',
1325 b'R', # REDUCE
1326 b')R',
1327 b'a', # APPEND
1328 b'Na',
1329 b'b', # BUILD
1330 b'Nb',
1331 b'd', # DICT
1332 b'e', # APPENDS
1333 b'(e',
1334 b'ibuiltins\nlist\n', # INST
1335 b'l', # LIST
1336 b'o', # OBJ
1337 b'(o',
1338 b'p1\n', # PUT
1339 b'q\x00', # BINPUT
1340 b'r\x00\x00\x00\x00', # LONG_BINPUT
1341 b's', # SETITEM
1342 b'Ns',
1343 b'NNs',
1344 b't', # TUPLE
1345 b'u', # SETITEMS
1346 b'(u',
1347 b'}(Nu',
1348 b'\x81', # NEWOBJ
1349 b')\x81',
1350 b'\x85', # TUPLE1
1351 b'\x86', # TUPLE2
1352 b'N\x86',
1353 b'\x87', # TUPLE3
1354 b'N\x87',
1355 b'NN\x87',
1356 b'\x90', # ADDITEMS
1357 b'(\x90',
1358 b'\x91', # FROZENSET
1359 b'\x92', # NEWOBJ_EX
1360 b')}\x92',
1361 b'\x93', # STACK_GLOBAL
1362 b'Vlist\n\x93',
1363 b'\x94', # MEMOIZE
1364 ]
1365 for p in badpickles:
1366 self.check_unpickling_error(self.bad_stack_errors, p)
1367
1368 def test_bad_mark(self):
1369 badpickles = [
1370 b'N(.', # STOP
1371 b'N(2', # DUP
1372 b'cbuiltins\nlist\n)(R', # REDUCE
1373 b'cbuiltins\nlist\n()R',
1374 b']N(a', # APPEND
1375 # BUILD
1376 b'cbuiltins\nValueError\n)R}(b',
1377 b'cbuiltins\nValueError\n)R(}b',
1378 b'(Nd', # DICT
1379 b'N(p1\n', # PUT
1380 b'N(q\x00', # BINPUT
1381 b'N(r\x00\x00\x00\x00', # LONG_BINPUT
1382 b'}NN(s', # SETITEM
1383 b'}N(Ns',
1384 b'}(NNs',
1385 b'}((u', # SETITEMS
1386 b'cbuiltins\nlist\n)(\x81', # NEWOBJ
1387 b'cbuiltins\nlist\n()\x81',
1388 b'N(\x85', # TUPLE1
1389 b'NN(\x86', # TUPLE2
1390 b'N(N\x86',
1391 b'NNN(\x87', # TUPLE3
1392 b'NN(N\x87',
1393 b'N(NN\x87',
1394 b']((\x90', # ADDITEMS
1395 # NEWOBJ_EX
1396 b'cbuiltins\nlist\n)}(\x92',
1397 b'cbuiltins\nlist\n)(}\x92',
1398 b'cbuiltins\nlist\n()}\x92',
1399 # STACK_GLOBAL
1400 b'Vbuiltins\n(Vlist\n\x93',
1401 b'Vbuiltins\nVlist\n(\x93',
1402 b'N(\x94', # MEMOIZE
1403 ]
1404 for p in badpickles:
1405 self.check_unpickling_error(self.bad_stack_errors, p)
1406
1407 def test_truncated_data(self):
1408 self.check_unpickling_error(EOFError, b'')
1409 self.check_unpickling_error(EOFError, b'N')
1410 badpickles = [
1411 b'B', # BINBYTES
1412 b'B\x03\x00\x00',
1413 b'B\x03\x00\x00\x00',
1414 b'B\x03\x00\x00\x00ab',
1415 b'C', # SHORT_BINBYTES
1416 b'C\x03',
1417 b'C\x03ab',
1418 b'F', # FLOAT
1419 b'F0.0',
1420 b'F0.00',
1421 b'G', # BINFLOAT
1422 b'G\x00\x00\x00\x00\x00\x00\x00',
1423 b'I', # INT
1424 b'I0',
1425 b'J', # BININT
1426 b'J\x00\x00\x00',
1427 b'K', # BININT1
1428 b'L', # LONG
1429 b'L0',
1430 b'L10',
1431 b'L0L',
1432 b'L10L',
1433 b'M', # BININT2
1434 b'M\x00',
1435 # b'P', # PERSID
1436 # b'Pabc',
1437 b'S', # STRING
1438 b"S'abc'",
1439 b'T', # BINSTRING
1440 b'T\x03\x00\x00',
1441 b'T\x03\x00\x00\x00',
1442 b'T\x03\x00\x00\x00ab',
1443 b'U', # SHORT_BINSTRING
1444 b'U\x03',
1445 b'U\x03ab',
1446 b'V', # UNICODE
1447 b'Vabc',
1448 b'X', # BINUNICODE
1449 b'X\x03\x00\x00',
1450 b'X\x03\x00\x00\x00',
1451 b'X\x03\x00\x00\x00ab',
1452 b'(c', # GLOBAL
1453 b'(cbuiltins',
1454 b'(cbuiltins\n',
1455 b'(cbuiltins\nlist',
1456 b'Ng', # GET
1457 b'Ng0',
1458 b'(i', # INST
1459 b'(ibuiltins',
1460 b'(ibuiltins\n',
1461 b'(ibuiltins\nlist',
1462 b'Nh', # BINGET
1463 b'Nj', # LONG_BINGET
1464 b'Nj\x00\x00\x00',
1465 b'Np', # PUT
1466 b'Np0',
1467 b'Nq', # BINPUT
1468 b'Nr', # LONG_BINPUT
1469 b'Nr\x00\x00\x00',
1470 b'\x80', # PROTO
1471 b'\x82', # EXT1
1472 b'\x83', # EXT2
1473 b'\x84\x01',
1474 b'\x84', # EXT4
1475 b'\x84\x01\x00\x00',
1476 b'\x8a', # LONG1
1477 b'\x8b', # LONG4
1478 b'\x8b\x00\x00\x00',
1479 b'\x8c', # SHORT_BINUNICODE
1480 b'\x8c\x03',
1481 b'\x8c\x03ab',
1482 b'\x8d', # BINUNICODE8
1483 b'\x8d\x03\x00\x00\x00\x00\x00\x00',
1484 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00',
1485 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab',
1486 b'\x8e', # BINBYTES8
1487 b'\x8e\x03\x00\x00\x00\x00\x00\x00',
1488 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00',
1489 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab',
1490 b'\x96', # BYTEARRAY8
1491 b'\x96\x03\x00\x00\x00\x00\x00\x00',
1492 b'\x96\x03\x00\x00\x00\x00\x00\x00\x00',
1493 b'\x96\x03\x00\x00\x00\x00\x00\x00\x00ab',
1494 b'\x95', # FRAME
1495 b'\x95\x02\x00\x00\x00\x00\x00\x00',
1496 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00',
1497 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N',
1498 ]
1499 for p in badpickles:
1500 self.check_unpickling_error(self.truncated_errors, p)
1501
1502 @threading_helper.reap_threads
1503 @threading_helper.requires_working_threading()
1504 def test_unpickle_module_race(self):
1505 # https://bugs.python.org/issue34572
1506 locker_module = dedent("""
1507 import threading
1508 barrier = threading.Barrier(2)
1509 """)
1510 locking_import_module = dedent("""
1511 import locker
1512 locker.barrier.wait()
1513 class ToBeUnpickled(object):
1514 pass
1515 """)
1516
1517 os.mkdir(TESTFN)
1518 self.addCleanup(shutil.rmtree, TESTFN)
1519 sys.path.insert(0, TESTFN)
1520 self.addCleanup(sys.path.remove, TESTFN)
1521 with open(os.path.join(TESTFN, "locker.py"), "wb") as f:
1522 f.write(locker_module.encode('utf-8'))
1523 with open(os.path.join(TESTFN, "locking_import.py"), "wb") as f:
1524 f.write(locking_import_module.encode('utf-8'))
1525 self.addCleanup(forget, "locker")
1526 self.addCleanup(forget, "locking_import")
1527
1528 import locker
1529
1530 pickle_bytes = (
1531 b'\x80\x03clocking_import\nToBeUnpickled\nq\x00)\x81q\x01.')
1532
1533 # Then try to unpickle two of these simultaneously
1534 # One of them will cause the module import, and we want it to block
1535 # until the other one either:
1536 # - fails (before the patch for this issue)
1537 # - blocks on the import lock for the module, as it should
1538 results = []
1539 barrier = threading.Barrier(3)
1540 def t():
1541 # This ensures the threads have all started
1542 # presumably barrier release is faster than thread startup
1543 barrier.wait()
1544 results.append(pickle.loads(pickle_bytes))
1545
1546 t1 = threading.Thread(target=t)
1547 t2 = threading.Thread(target=t)
1548 t1.start()
1549 t2.start()
1550
1551 barrier.wait()
1552 # could have delay here
1553 locker.barrier.wait()
1554
1555 t1.join()
1556 t2.join()
1557
1558 from locking_import import ToBeUnpickled
1559 self.assertEqual(
1560 [type(x) for x in results],
1561 [ToBeUnpickled] * 2)
1562
1563
1564 class AbstractPicklingErrorTests:
1565 # Subclass must define self.dumps, self.pickler.
1566
1567 def test_bad_reduce_result(self):
1568 obj = REX([print, ()])
1569 for proto in protocols:
1570 with self.subTest(proto=proto):
1571 with self.assertRaises(pickle.PicklingError) as cm:
1572 self.dumps(obj, proto)
1573 self.assertEqual(str(cm.exception),
1574 '__reduce__ must return a string or tuple, not list')
1575 self.assertEqual(cm.exception.__notes__, [
1576 f'when serializing {REX.__module__}.REX object'])
1577
1578 obj = REX((print,))
1579 for proto in protocols:
1580 with self.subTest(proto=proto):
1581 with self.assertRaises(pickle.PicklingError) as cm:
1582 self.dumps(obj, proto)
1583 self.assertEqual(str(cm.exception),
1584 'tuple returned by __reduce__ must contain 2 through 6 elements')
1585 self.assertEqual(cm.exception.__notes__, [
1586 f'when serializing {REX.__module__}.REX object'])
1587
1588 obj = REX((print, (), None, None, None, None, None))
1589 for proto in protocols:
1590 with self.subTest(proto=proto):
1591 with self.assertRaises(pickle.PicklingError) as cm:
1592 self.dumps(obj, proto)
1593 self.assertEqual(str(cm.exception),
1594 'tuple returned by __reduce__ must contain 2 through 6 elements')
1595 self.assertEqual(cm.exception.__notes__, [
1596 f'when serializing {REX.__module__}.REX object'])
1597
1598 def test_bad_reconstructor(self):
1599 obj = REX((42, ()))
1600 for proto in protocols:
1601 with self.subTest(proto=proto):
1602 with self.assertRaises(pickle.PicklingError) as cm:
1603 self.dumps(obj, proto)
1604 self.assertEqual(str(cm.exception),
1605 'first item of the tuple returned by __reduce__ '
1606 'must be callable, not int')
1607 self.assertEqual(cm.exception.__notes__, [
1608 f'when serializing {REX.__module__}.REX object'])
1609
1610 def test_unpickleable_reconstructor(self):
1611 obj = REX((UnpickleableCallable(), ()))
1612 for proto in protocols:
1613 with self.subTest(proto=proto):
1614 with self.assertRaises(CustomError) as cm:
1615 self.dumps(obj, proto)
1616 self.assertEqual(cm.exception.__notes__, [
1617 f'when serializing {REX.__module__}.REX reconstructor',
1618 f'when serializing {REX.__module__}.REX object'])
1619
1620 def test_bad_reconstructor_args(self):
1621 obj = REX((print, []))
1622 for proto in protocols:
1623 with self.subTest(proto=proto):
1624 with self.assertRaises(pickle.PicklingError) as cm:
1625 self.dumps(obj, proto)
1626 self.assertEqual(str(cm.exception),
1627 'second item of the tuple returned by __reduce__ '
1628 'must be a tuple, not list')
1629 self.assertEqual(cm.exception.__notes__, [
1630 f'when serializing {REX.__module__}.REX object'])
1631
1632 def test_unpickleable_reconstructor_args(self):
1633 obj = REX((print, (1, 2, UNPICKLEABLE)))
1634 for proto in protocols:
1635 with self.subTest(proto=proto):
1636 with self.assertRaises(CustomError) as cm:
1637 self.dumps(obj, proto)
1638 self.assertEqual(cm.exception.__notes__, [
1639 'when serializing tuple item 2',
1640 f'when serializing {REX.__module__}.REX reconstructor arguments',
1641 f'when serializing {REX.__module__}.REX object'])
1642
1643 def test_bad_newobj_args(self):
1644 obj = REX((copyreg.__newobj__, ()))
1645 for proto in protocols[2:]:
1646 with self.subTest(proto=proto):
1647 with self.assertRaises((IndexError, pickle.PicklingError)) as cm:
1648 self.dumps(obj, proto)
1649 self.assertIn(str(cm.exception), {
1650 'tuple index out of range',
1651 '__newobj__ expected at least 1 argument, got 0'})
1652 self.assertEqual(cm.exception.__notes__, [
1653 f'when serializing {REX.__module__}.REX object'])
1654
1655 obj = REX((copyreg.__newobj__, [REX]))
1656 for proto in protocols[2:]:
1657 with self.subTest(proto=proto):
1658 with self.assertRaises(pickle.PicklingError) as cm:
1659 self.dumps(obj, proto)
1660 self.assertEqual(str(cm.exception),
1661 'second item of the tuple returned by __reduce__ '
1662 'must be a tuple, not list')
1663 self.assertEqual(cm.exception.__notes__, [
1664 f'when serializing {REX.__module__}.REX object'])
1665
1666 def test_bad_newobj_class(self):
1667 obj = REX((copyreg.__newobj__, (NoNew(),)))
1668 for proto in protocols[2:]:
1669 with self.subTest(proto=proto):
1670 with self.assertRaises(pickle.PicklingError) as cm:
1671 self.dumps(obj, proto)
1672 self.assertIn(str(cm.exception), {
1673 'first argument to __newobj__() has no __new__',
1674 f'first argument to __newobj__() must be a class, not {NoNew.__module__}.NoNew'})
1675 self.assertEqual(cm.exception.__notes__, [
1676 f'when serializing {REX.__module__}.REX object'])
1677
1678 def test_wrong_newobj_class(self):
1679 obj = REX((copyreg.__newobj__, (str,)))
1680 for proto in protocols[2:]:
1681 with self.subTest(proto=proto):
1682 with self.assertRaises(pickle.PicklingError) as cm:
1683 self.dumps(obj, proto)
1684 self.assertEqual(str(cm.exception),
1685 f'first argument to __newobj__() must be {REX!r}, not {str!r}')
1686 self.assertEqual(cm.exception.__notes__, [
1687 f'when serializing {REX.__module__}.REX object'])
1688
1689 def test_unpickleable_newobj_class(self):
1690 class LocalREX(REX): pass
1691 obj = LocalREX((copyreg.__newobj__, (LocalREX,)))
1692 for proto in protocols:
1693 with self.subTest(proto=proto):
1694 with self.assertRaises(pickle.PicklingError) as cm:
1695 self.dumps(obj, proto)
1696 if proto >= 2:
1697 self.assertEqual(cm.exception.__notes__, [
1698 f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class',
1699 f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object'])
1700 else:
1701 self.assertEqual(cm.exception.__notes__, [
1702 'when serializing tuple item 0',
1703 f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments',
1704 f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object'])
1705
1706 def test_unpickleable_newobj_args(self):
1707 obj = REX((copyreg.__newobj__, (REX, 1, 2, UNPICKLEABLE)))
1708 for proto in protocols:
1709 with self.subTest(proto=proto):
1710 with self.assertRaises(CustomError) as cm:
1711 self.dumps(obj, proto)
1712 if proto >= 2:
1713 self.assertEqual(cm.exception.__notes__, [
1714 'when serializing tuple item 2',
1715 f'when serializing {REX.__module__}.REX __new__ arguments',
1716 f'when serializing {REX.__module__}.REX object'])
1717 else:
1718 self.assertEqual(cm.exception.__notes__, [
1719 'when serializing tuple item 3',
1720 f'when serializing {REX.__module__}.REX reconstructor arguments',
1721 f'when serializing {REX.__module__}.REX object'])
1722
1723 def test_bad_newobj_ex_args(self):
1724 obj = REX((copyreg.__newobj_ex__, ()))
1725 for proto in protocols[2:]:
1726 with self.subTest(proto=proto):
1727 with self.assertRaises((ValueError, pickle.PicklingError)) as cm:
1728 self.dumps(obj, proto)
1729 self.assertIn(str(cm.exception), {
1730 'not enough values to unpack (expected 3, got 0)',
1731 '__newobj_ex__ expected 3 arguments, got 0'})
1732 self.assertEqual(cm.exception.__notes__, [
1733 f'when serializing {REX.__module__}.REX object'])
1734
1735 obj = REX((copyreg.__newobj_ex__, 42))
1736 for proto in protocols[2:]:
1737 with self.subTest(proto=proto):
1738 with self.assertRaises(pickle.PicklingError) as cm:
1739 self.dumps(obj, proto)
1740 self.assertEqual(str(cm.exception),
1741 'second item of the tuple returned by __reduce__ '
1742 'must be a tuple, not int')
1743 self.assertEqual(cm.exception.__notes__, [
1744 f'when serializing {REX.__module__}.REX object'])
1745
1746 obj = REX((copyreg.__newobj_ex__, (REX, 42, {})))
1747 if self.pickler is pickle._Pickler:
1748 for proto in protocols[2:4]:
1749 with self.subTest(proto=proto):
1750 with self.assertRaises(TypeError) as cm:
1751 self.dumps(obj, proto)
1752 self.assertEqual(str(cm.exception),
1753 'Value after * must be an iterable, not int')
1754 self.assertEqual(cm.exception.__notes__, [
1755 f'when serializing {REX.__module__}.REX object'])
1756 else:
1757 for proto in protocols[2:]:
1758 with self.subTest(proto=proto):
1759 with self.assertRaises(pickle.PicklingError) as cm:
1760 self.dumps(obj, proto)
1761 self.assertEqual(str(cm.exception),
1762 'second argument to __newobj_ex__() must be a tuple, not int')
1763 self.assertEqual(cm.exception.__notes__, [
1764 f'when serializing {REX.__module__}.REX object'])
1765
1766 obj = REX((copyreg.__newobj_ex__, (REX, (), [])))
1767 if self.pickler is pickle._Pickler:
1768 for proto in protocols[2:4]:
1769 with self.subTest(proto=proto):
1770 with self.assertRaises(TypeError) as cm:
1771 self.dumps(obj, proto)
1772 self.assertEqual(str(cm.exception),
1773 'functools.partial() argument after ** must be a mapping, not list')
1774 self.assertEqual(cm.exception.__notes__, [
1775 f'when serializing {REX.__module__}.REX object'])
1776 else:
1777 for proto in protocols[2:]:
1778 with self.subTest(proto=proto):
1779 with self.assertRaises(pickle.PicklingError) as cm:
1780 self.dumps(obj, proto)
1781 self.assertEqual(str(cm.exception),
1782 'third argument to __newobj_ex__() must be a dict, not list')
1783 self.assertEqual(cm.exception.__notes__, [
1784 f'when serializing {REX.__module__}.REX object'])
1785
1786 def test_bad_newobj_ex__class(self):
1787 obj = REX((copyreg.__newobj_ex__, (NoNew(), (), {})))
1788 for proto in protocols[2:]:
1789 with self.subTest(proto=proto):
1790 with self.assertRaises(pickle.PicklingError) as cm:
1791 self.dumps(obj, proto)
1792 self.assertIn(str(cm.exception), {
1793 'first argument to __newobj_ex__() has no __new__',
1794 f'first argument to __newobj_ex__() must be a class, not {NoNew.__module__}.NoNew'})
1795 self.assertEqual(cm.exception.__notes__, [
1796 f'when serializing {REX.__module__}.REX object'])
1797
1798 def test_wrong_newobj_ex_class(self):
1799 if self.pickler is not pickle._Pickler:
1800 self.skipTest('only verified in the Python implementation')
1801 obj = REX((copyreg.__newobj_ex__, (str, (), {})))
1802 for proto in protocols[2:]:
1803 with self.subTest(proto=proto):
1804 with self.assertRaises(pickle.PicklingError) as cm:
1805 self.dumps(obj, proto)
1806 self.assertEqual(str(cm.exception),
1807 f'first argument to __newobj_ex__() must be {REX}, not {str}')
1808 self.assertEqual(cm.exception.__notes__, [
1809 f'when serializing {REX.__module__}.REX object'])
1810
1811 def test_unpickleable_newobj_ex_class(self):
1812 class LocalREX(REX): pass
1813 obj = LocalREX((copyreg.__newobj_ex__, (LocalREX, (), {})))
1814 for proto in protocols:
1815 with self.subTest(proto=proto):
1816 with self.assertRaises(pickle.PicklingError) as cm:
1817 self.dumps(obj, proto)
1818 if proto >= 4:
1819 self.assertEqual(cm.exception.__notes__, [
1820 f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class',
1821 f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object'])
1822 elif proto >= 2:
1823 self.assertEqual(cm.exception.__notes__, [
1824 'when serializing tuple item 0',
1825 'when serializing tuple item 1',
1826 'when serializing functools.partial state',
1827 'when serializing functools.partial object',
1828 f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor',
1829 f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object'])
1830 else:
1831 self.assertEqual(cm.exception.__notes__, [
1832 'when serializing tuple item 0',
1833 f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments',
1834 f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object'])
1835
1836 def test_unpickleable_newobj_ex_args(self):
1837 obj = REX((copyreg.__newobj_ex__, (REX, (1, 2, UNPICKLEABLE), {})))
1838 for proto in protocols:
1839 with self.subTest(proto=proto):
1840 with self.assertRaises(CustomError) as cm:
1841 self.dumps(obj, proto)
1842 if proto >= 4:
1843 self.assertEqual(cm.exception.__notes__, [
1844 'when serializing tuple item 2',
1845 f'when serializing {REX.__module__}.REX __new__ arguments',
1846 f'when serializing {REX.__module__}.REX object'])
1847 elif proto >= 2:
1848 self.assertEqual(cm.exception.__notes__, [
1849 'when serializing tuple item 3',
1850 'when serializing tuple item 1',
1851 'when serializing functools.partial state',
1852 'when serializing functools.partial object',
1853 f'when serializing {REX.__module__}.REX reconstructor',
1854 f'when serializing {REX.__module__}.REX object'])
1855 else:
1856 self.assertEqual(cm.exception.__notes__, [
1857 'when serializing tuple item 2',
1858 'when serializing tuple item 1',
1859 f'when serializing {REX.__module__}.REX reconstructor arguments',
1860 f'when serializing {REX.__module__}.REX object'])
1861
1862 def test_unpickleable_newobj_ex_kwargs(self):
1863 obj = REX((copyreg.__newobj_ex__, (REX, (), {'a': UNPICKLEABLE})))
1864 for proto in protocols:
1865 with self.subTest(proto=proto):
1866 with self.assertRaises(CustomError) as cm:
1867 self.dumps(obj, proto)
1868 if proto >= 4:
1869 self.assertEqual(cm.exception.__notes__, [
1870 "when serializing dict item 'a'",
1871 f'when serializing {REX.__module__}.REX __new__ arguments',
1872 f'when serializing {REX.__module__}.REX object'])
1873 elif proto >= 2:
1874 self.assertEqual(cm.exception.__notes__, [
1875 "when serializing dict item 'a'",
1876 'when serializing tuple item 2',
1877 'when serializing functools.partial state',
1878 'when serializing functools.partial object',
1879 f'when serializing {REX.__module__}.REX reconstructor',
1880 f'when serializing {REX.__module__}.REX object'])
1881 else:
1882 self.assertEqual(cm.exception.__notes__, [
1883 "when serializing dict item 'a'",
1884 'when serializing tuple item 2',
1885 f'when serializing {REX.__module__}.REX reconstructor arguments',
1886 f'when serializing {REX.__module__}.REX object'])
1887
1888 def test_unpickleable_state(self):
1889 obj = REX_state(UNPICKLEABLE)
1890 for proto in protocols:
1891 with self.subTest(proto=proto):
1892 with self.assertRaises(CustomError) as cm:
1893 self.dumps(obj, proto)
1894 self.assertEqual(cm.exception.__notes__, [
1895 f'when serializing {REX_state.__module__}.REX_state state',
1896 f'when serializing {REX_state.__module__}.REX_state object'])
1897
1898 def test_bad_state_setter(self):
1899 if self.pickler is pickle._Pickler:
1900 self.skipTest('only verified in the C implementation')
1901 obj = REX((print, (), 'state', None, None, 42))
1902 for proto in protocols:
1903 with self.subTest(proto=proto):
1904 with self.assertRaises(pickle.PicklingError) as cm:
1905 self.dumps(obj, proto)
1906 self.assertEqual(str(cm.exception),
1907 'sixth item of the tuple returned by __reduce__ '
1908 'must be callable, not int')
1909 self.assertEqual(cm.exception.__notes__, [
1910 f'when serializing {REX.__module__}.REX object'])
1911
1912 def test_unpickleable_state_setter(self):
1913 obj = REX((print, (), 'state', None, None, UnpickleableCallable()))
1914 for proto in protocols:
1915 with self.subTest(proto=proto):
1916 with self.assertRaises(CustomError) as cm:
1917 self.dumps(obj, proto)
1918 self.assertEqual(cm.exception.__notes__, [
1919 f'when serializing {REX.__module__}.REX state setter',
1920 f'when serializing {REX.__module__}.REX object'])
1921
1922 def test_unpickleable_state_with_state_setter(self):
1923 obj = REX((print, (), UNPICKLEABLE, None, None, print))
1924 for proto in protocols:
1925 with self.subTest(proto=proto):
1926 with self.assertRaises(CustomError) as cm:
1927 self.dumps(obj, proto)
1928 self.assertEqual(cm.exception.__notes__, [
1929 f'when serializing {REX.__module__}.REX state',
1930 f'when serializing {REX.__module__}.REX object'])
1931
1932 def test_bad_object_list_items(self):
1933 # Issue4176: crash when 4th and 5th items of __reduce__()
1934 # are not iterators
1935 obj = REX((list, (), None, 42))
1936 for proto in protocols:
1937 with self.subTest(proto=proto):
1938 with self.assertRaises((TypeError, pickle.PicklingError)) as cm:
1939 self.dumps(obj, proto)
1940 self.assertIn(str(cm.exception), {
1941 "'int' object is not iterable",
1942 'fourth item of the tuple returned by __reduce__ '
1943 'must be an iterator, not int'})
1944 self.assertEqual(cm.exception.__notes__, [
1945 f'when serializing {REX.__module__}.REX object'])
1946
1947 if self.pickler is not pickle._Pickler:
1948 # Python implementation is less strict and also accepts iterables.
1949 obj = REX((list, (), None, []))
1950 for proto in protocols:
1951 with self.subTest(proto=proto):
1952 with self.assertRaises(pickle.PicklingError):
1953 self.dumps(obj, proto)
1954 self.assertEqual(str(cm.exception),
1955 'fourth item of the tuple returned by __reduce__ '
1956 'must be an iterator, not int')
1957 self.assertEqual(cm.exception.__notes__, [
1958 f'when serializing {REX.__module__}.REX object'])
1959
1960 def test_unpickleable_object_list_items(self):
1961 obj = REX_six([1, 2, UNPICKLEABLE])
1962 for proto in protocols:
1963 with self.subTest(proto=proto):
1964 with self.assertRaises(CustomError) as cm:
1965 self.dumps(obj, proto)
1966 self.assertEqual(cm.exception.__notes__, [
1967 f'when serializing {REX_six.__module__}.REX_six item 2',
1968 f'when serializing {REX_six.__module__}.REX_six object'])
1969
1970 def test_bad_object_dict_items(self):
1971 # Issue4176: crash when 4th and 5th items of __reduce__()
1972 # are not iterators
1973 obj = REX((dict, (), None, None, 42))
1974 for proto in protocols:
1975 with self.subTest(proto=proto):
1976 with self.assertRaises((TypeError, pickle.PicklingError)) as cm:
1977 self.dumps(obj, proto)
1978 self.assertIn(str(cm.exception), {
1979 "'int' object is not iterable",
1980 'fifth item of the tuple returned by __reduce__ '
1981 'must be an iterator, not int'})
1982 self.assertEqual(cm.exception.__notes__, [
1983 f'when serializing {REX.__module__}.REX object'])
1984
1985 for proto in protocols:
1986 obj = REX((dict, (), None, None, iter([('a',)])))
1987 with self.subTest(proto=proto):
1988 with self.assertRaises((ValueError, TypeError)) as cm:
1989 self.dumps(obj, proto)
1990 self.assertIn(str(cm.exception), {
1991 'not enough values to unpack (expected 2, got 1)',
1992 'dict items iterator must return 2-tuples'})
1993 self.assertEqual(cm.exception.__notes__, [
1994 f'when serializing {REX.__module__}.REX object'])
1995
1996 if self.pickler is not pickle._Pickler:
1997 # Python implementation is less strict and also accepts iterables.
1998 obj = REX((dict, (), None, None, []))
1999 for proto in protocols:
2000 with self.subTest(proto=proto):
2001 with self.assertRaises(pickle.PicklingError):
2002 self.dumps(obj, proto)
2003 self.assertEqual(str(cm.exception),
2004 'dict items iterator must return 2-tuples')
2005 self.assertEqual(cm.exception.__notes__, [
2006 f'when serializing {REX.__module__}.REX object'])
2007
2008 def test_unpickleable_object_dict_items(self):
2009 obj = REX_seven({'a': UNPICKLEABLE})
2010 for proto in protocols:
2011 with self.subTest(proto=proto):
2012 with self.assertRaises(CustomError) as cm:
2013 self.dumps(obj, proto)
2014 self.assertEqual(cm.exception.__notes__, [
2015 f"when serializing {REX_seven.__module__}.REX_seven item 'a'",
2016 f'when serializing {REX_seven.__module__}.REX_seven object'])
2017
2018 def test_unpickleable_list_items(self):
2019 obj = [1, [2, 3, UNPICKLEABLE]]
2020 for proto in protocols:
2021 with self.subTest(proto=proto):
2022 with self.assertRaises(CustomError) as cm:
2023 self.dumps(obj, proto)
2024 self.assertEqual(cm.exception.__notes__, [
2025 'when serializing list item 2',
2026 'when serializing list item 1'])
2027 for n in [0, 1, 1000, 1005]:
2028 obj = [*range(n), UNPICKLEABLE]
2029 for proto in protocols:
2030 with self.subTest(proto=proto):
2031 with self.assertRaises(CustomError) as cm:
2032 self.dumps(obj, proto)
2033 self.assertEqual(cm.exception.__notes__, [
2034 f'when serializing list item {n}'])
2035
2036 def test_unpickleable_tuple_items(self):
2037 obj = (1, (2, 3, UNPICKLEABLE))
2038 for proto in protocols:
2039 with self.subTest(proto=proto):
2040 with self.assertRaises(CustomError) as cm:
2041 self.dumps(obj, proto)
2042 self.assertEqual(cm.exception.__notes__, [
2043 'when serializing tuple item 2',
2044 'when serializing tuple item 1'])
2045 obj = (*range(10), UNPICKLEABLE)
2046 for proto in protocols:
2047 with self.subTest(proto=proto):
2048 with self.assertRaises(CustomError) as cm:
2049 self.dumps(obj, proto)
2050 self.assertEqual(cm.exception.__notes__, [
2051 'when serializing tuple item 10'])
2052
2053 def test_unpickleable_dict_items(self):
2054 obj = {'a': {'b': UNPICKLEABLE}}
2055 for proto in protocols:
2056 with self.subTest(proto=proto):
2057 with self.assertRaises(CustomError) as cm:
2058 self.dumps(obj, proto)
2059 self.assertEqual(cm.exception.__notes__, [
2060 "when serializing dict item 'b'",
2061 "when serializing dict item 'a'"])
2062 for n in [0, 1, 1000, 1005]:
2063 obj = dict.fromkeys(range(n))
2064 obj['a'] = UNPICKLEABLE
2065 for proto in protocols:
2066 with self.subTest(proto=proto, n=n):
2067 with self.assertRaises(CustomError) as cm:
2068 self.dumps(obj, proto)
2069 self.assertEqual(cm.exception.__notes__, [
2070 "when serializing dict item 'a'"])
2071
2072 def test_unpickleable_set_items(self):
2073 obj = {UNPICKLEABLE}
2074 for proto in protocols:
2075 with self.subTest(proto=proto):
2076 with self.assertRaises(CustomError) as cm:
2077 self.dumps(obj, proto)
2078 if proto >= 4:
2079 self.assertEqual(cm.exception.__notes__, [
2080 'when serializing set element'])
2081 else:
2082 self.assertEqual(cm.exception.__notes__, [
2083 'when serializing list item 0',
2084 'when serializing tuple item 0',
2085 'when serializing set reconstructor arguments'])
2086
2087 def test_unpickleable_frozenset_items(self):
2088 obj = frozenset({frozenset({UNPICKLEABLE})})
2089 for proto in protocols:
2090 with self.subTest(proto=proto):
2091 with self.assertRaises(CustomError) as cm:
2092 self.dumps(obj, proto)
2093 if proto >= 4:
2094 self.assertEqual(cm.exception.__notes__, [
2095 'when serializing frozenset element',
2096 'when serializing frozenset element'])
2097 else:
2098 self.assertEqual(cm.exception.__notes__, [
2099 'when serializing list item 0',
2100 'when serializing tuple item 0',
2101 'when serializing frozenset reconstructor arguments',
2102 'when serializing list item 0',
2103 'when serializing tuple item 0',
2104 'when serializing frozenset reconstructor arguments'])
2105
2106 def test_global_lookup_error(self):
2107 # Global name does not exist
2108 obj = REX('spam')
2109 obj.__module__ = 'test.picklecommon'
2110 for proto in protocols:
2111 with self.subTest(proto=proto):
2112 with self.assertRaises(pickle.PicklingError) as cm:
2113 self.dumps(obj, proto)
2114 self.assertEqual(str(cm.exception),
2115 f"Can't pickle {obj!r}: it's not found as test.picklecommon.spam")
2116 self.assertEqual(str(cm.exception.__context__),
2117 "module 'test.picklecommon' has no attribute 'spam'")
2118
2119 obj.__module__ = 'nonexisting'
2120 for proto in protocols:
2121 with self.subTest(proto=proto):
2122 with self.assertRaises(pickle.PicklingError) as cm:
2123 self.dumps(obj, proto)
2124 self.assertEqual(str(cm.exception),
2125 f"Can't pickle {obj!r}: No module named 'nonexisting'")
2126 self.assertEqual(str(cm.exception.__context__),
2127 "No module named 'nonexisting'")
2128
2129 obj.__module__ = ''
2130 for proto in protocols:
2131 with self.subTest(proto=proto):
2132 with self.assertRaises(pickle.PicklingError) as cm:
2133 self.dumps(obj, proto)
2134 self.assertEqual(str(cm.exception),
2135 f"Can't pickle {obj!r}: Empty module name")
2136 self.assertEqual(str(cm.exception.__context__),
2137 "Empty module name")
2138
2139 obj.__module__ = None
2140 for proto in protocols:
2141 with self.subTest(proto=proto):
2142 with self.assertRaises(pickle.PicklingError) as cm:
2143 self.dumps(obj, proto)
2144 self.assertEqual(str(cm.exception),
2145 f"Can't pickle {obj!r}: it's not found as __main__.spam")
2146 self.assertEqual(str(cm.exception.__context__),
2147 "module '__main__' has no attribute 'spam'")
2148
2149 def test_nonencodable_global_name_error(self):
2150 for proto in protocols[:4]:
2151 with self.subTest(proto=proto):
2152 name = 'nonascii\xff' if proto < 3 else 'nonencodable\udbff'
2153 obj = REX(name)
2154 obj.__module__ = __name__
2155 with support.swap_item(globals(), name, obj):
2156 with self.assertRaises(pickle.PicklingError) as cm:
2157 self.dumps(obj, proto)
2158 self.assertEqual(str(cm.exception),
2159 f"can't pickle global identifier {name!r} using pickle protocol {proto}")
2160 self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError)
2161
2162 def test_nonencodable_module_name_error(self):
2163 for proto in protocols[:4]:
2164 with self.subTest(proto=proto):
2165 name = 'nonascii\xff' if proto < 3 else 'nonencodable\udbff'
2166 obj = REX('test')
2167 obj.__module__ = name
2168 mod = types.SimpleNamespace(test=obj)
2169 with support.swap_item(sys.modules, name, mod):
2170 with self.assertRaises(pickle.PicklingError) as cm:
2171 self.dumps(obj, proto)
2172 self.assertEqual(str(cm.exception),
2173 f"can't pickle module identifier {name!r} using pickle protocol {proto}")
2174 self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError)
2175
2176 def test_nested_lookup_error(self):
2177 # Nested name does not exist
2178 global TestGlobal
2179 class TestGlobal:
2180 class A:
2181 pass
2182 obj = REX('TestGlobal.A.B.C')
2183 obj.__module__ = __name__
2184 for proto in protocols:
2185 with self.subTest(proto=proto):
2186 with self.assertRaises(pickle.PicklingError) as cm:
2187 self.dumps(obj, proto)
2188 self.assertEqual(str(cm.exception),
2189 f"Can't pickle {obj!r}: "
2190 f"it's not found as {__name__}.TestGlobal.A.B.C")
2191 self.assertEqual(str(cm.exception.__context__),
2192 "type object 'A' has no attribute 'B'")
2193
2194 obj.__module__ = None
2195 for proto in protocols:
2196 with self.subTest(proto=proto):
2197 with self.assertRaises(pickle.PicklingError) as cm:
2198 self.dumps(obj, proto)
2199 self.assertEqual(str(cm.exception),
2200 f"Can't pickle {obj!r}: "
2201 f"it's not found as __main__.TestGlobal.A.B.C")
2202 self.assertEqual(str(cm.exception.__context__),
2203 "module '__main__' has no attribute 'TestGlobal'")
2204
2205 def test_wrong_object_lookup_error(self):
2206 # Name is bound to different object
2207 global TestGlobal
2208 class TestGlobal:
2209 pass
2210 obj = REX('TestGlobal')
2211 obj.__module__ = __name__
2212 for proto in protocols:
2213 with self.subTest(proto=proto):
2214 with self.assertRaises(pickle.PicklingError) as cm:
2215 self.dumps(obj, proto)
2216 self.assertEqual(str(cm.exception),
2217 f"Can't pickle {obj!r}: "
2218 f"it's not the same object as {__name__}.TestGlobal")
2219 self.assertIsNone(cm.exception.__context__)
2220
2221 obj.__module__ = None
2222 for proto in protocols:
2223 with self.subTest(proto=proto):
2224 with self.assertRaises(pickle.PicklingError) as cm:
2225 self.dumps(obj, proto)
2226 self.assertEqual(str(cm.exception),
2227 f"Can't pickle {obj!r}: "
2228 f"it's not found as __main__.TestGlobal")
2229 self.assertEqual(str(cm.exception.__context__),
2230 "module '__main__' has no attribute 'TestGlobal'")
2231
2232 def test_local_lookup_error(self):
2233 # Test that whichmodule() errors out cleanly when looking up
2234 # an assumed globally-reachable object fails.
2235 def f():
2236 pass
2237 # Since the function is local, lookup will fail
2238 for proto in protocols:
2239 with self.subTest(proto=proto):
2240 with self.assertRaises(pickle.PicklingError) as cm:
2241 self.dumps(f, proto)
2242 self.assertEqual(str(cm.exception),
2243 f"Can't pickle local object {f!r}")
2244 # Same without a __module__ attribute (exercises a different path
2245 # in _pickle.c).
2246 del f.__module__
2247 for proto in protocols:
2248 with self.subTest(proto=proto):
2249 with self.assertRaises(pickle.PicklingError) as cm:
2250 self.dumps(f, proto)
2251 self.assertEqual(str(cm.exception),
2252 f"Can't pickle local object {f!r}")
2253 # Yet a different path.
2254 f.__name__ = f.__qualname__
2255 for proto in protocols:
2256 with self.subTest(proto=proto):
2257 with self.assertRaises(pickle.PicklingError) as cm:
2258 self.dumps(f, proto)
2259 self.assertEqual(str(cm.exception),
2260 f"Can't pickle local object {f!r}")
2261
2262 def test_reduce_ex_None(self):
2263 c = REX_None()
2264 with self.assertRaises(TypeError):
2265 self.dumps(c)
2266
2267 def test_reduce_None(self):
2268 c = R_None()
2269 with self.assertRaises(TypeError):
2270 self.dumps(c)
2271
2272 @support.skip_if_unlimited_stack_size
2273 @no_tracing
2274 def test_bad_getattr(self):
2275 # Issue #3514: crash when there is an infinite loop in __getattr__
2276 x = BadGetattr()
2277 for proto in range(2):
2278 with support.infinite_recursion(25):
2279 self.assertRaises(RuntimeError, self.dumps, x, proto)
2280 for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
2281 s = self.dumps(x, proto)
2282
2283 def test_picklebuffer_error(self):
2284 # PickleBuffer forbidden with protocol < 5
2285 pb = pickle.PickleBuffer(b"foobar")
2286 for proto in range(0, 5):
2287 with self.subTest(proto=proto):
2288 with self.assertRaises(pickle.PickleError) as cm:
2289 self.dumps(pb, proto)
2290 self.assertEqual(str(cm.exception),
2291 'PickleBuffer can only be pickled with protocol >= 5')
2292
2293 def test_non_continuous_buffer(self):
2294 for proto in protocols[5:]:
2295 with self.subTest(proto=proto):
2296 pb = pickle.PickleBuffer(memoryview(b"foobar")[::2])
2297 with self.assertRaises((pickle.PicklingError, BufferError)):
2298 self.dumps(pb, proto)
2299
2300 def test_buffer_callback_error(self):
2301 def buffer_callback(buffers):
2302 raise CustomError
2303 pb = pickle.PickleBuffer(b"foobar")
2304 with self.assertRaises(CustomError):
2305 self.dumps(pb, 5, buffer_callback=buffer_callback)
2306
2307 def test_evil_pickler_mutating_collection(self):
2308 # https://github.com/python/cpython/issues/92930
2309 global Clearer
2310 class Clearer:
2311 pass
2312
2313 def check(collection):
2314 class EvilPickler(self.pickler):
2315 def persistent_id(self, obj):
2316 if isinstance(obj, Clearer):
2317 collection.clear()
2318 return None
2319 pickler = EvilPickler(io.BytesIO(), proto)
2320 try:
2321 pickler.dump(collection)
2322 except RuntimeError as e:
2323 expected = "changed size during iteration"
2324 self.assertIn(expected, str(e))
2325
2326 for proto in protocols:
2327 check([Clearer()])
2328 check([Clearer(), Clearer()])
2329 check({Clearer()})
2330 check({Clearer(), Clearer()})
2331 check({Clearer(): 1})
2332 check({Clearer(): 1, Clearer(): 2})
2333 check({1: Clearer(), 2: Clearer()})
2334
2335 @support.cpython_only
2336 def test_bad_ext_code(self):
2337 # This should never happen in normal circumstances, because the type
2338 # and the value of the extension code is checked in copyreg.add_extension().
2339 key = (MyList.__module__, 'MyList')
2340 def check(code, exc):
2341 assert key not in copyreg._extension_registry
2342 assert code not in copyreg._inverted_registry
2343 with (support.swap_item(copyreg._extension_registry, key, code),
2344 support.swap_item(copyreg._inverted_registry, code, key)):
2345 for proto in protocols[2:]:
2346 with self.assertRaises(exc):
2347 self.dumps(MyList, proto)
2348
2349 check(object(), TypeError)
2350 check(None, TypeError)
2351 check(-1, (RuntimeError, struct.error))
2352 check(0, RuntimeError)
2353 check(2**31, (RuntimeError, OverflowError, struct.error))
2354 check(2**1000, (OverflowError, struct.error))
2355 check(-2**1000, (OverflowError, struct.error))
2356
2357
2358 class AbstractPickleTests:
2359 # Subclass must define self.dumps, self.loads.
2360
2361 py_version = sys.version_info # for test_xpickle
2362 optimized = False
2363
2364 _testdata = AbstractUnpickleTests._testdata
2365
2366 def setUp(self):
2367 pass
2368
2369 assert_is_copy = AbstractUnpickleTests.assert_is_copy
2370
2371 def test_misc(self):
2372 # test various datatypes not tested by testdata
2373 for proto in protocols:
2374 with self.subTest('myint', proto=proto):
2375 if self.py_version < (3, 0) and proto < 2:
2376 self.skipTest('int subclasses are not interoperable with Python 2')
2377 x = myint(4)
2378 s = self.dumps(x, proto)
2379 y = self.loads(s)
2380 self.assert_is_copy(x, y)
2381
2382 with self.subTest('tuple', proto=proto):
2383 x = (1, ())
2384 s = self.dumps(x, proto)
2385 y = self.loads(s)
2386 self.assert_is_copy(x, y)
2387
2388 with self.subTest('initarg', proto=proto):
2389 if self.py_version < (3, 0):
2390 self.skipTest('"classic" classes are not interoperable with Python 2')
2391 x = initarg(1, x)
2392 s = self.dumps(x, proto)
2393 y = self.loads(s)
2394 self.assert_is_copy(x, y)
2395
2396 # XXX test __reduce__ protocol?
2397
2398 def test_roundtrip_equality(self):
2399 if self.py_version < (3, 0):
2400 self.skipTest('"classic" classes are not interoperable with Python 2')
2401 expected = self._testdata
2402 for proto in protocols:
2403 s = self.dumps(expected, proto)
2404 got = self.loads(s)
2405 self.assert_is_copy(expected, got)
2406
2407 # There are gratuitous differences between pickles produced by
2408 # pickle and cPickle, largely because cPickle starts PUT indices at
2409 # 1 and pickle starts them at 0. See XXX comment in cPickle's put2() --
2410 # there's a comment with an exclamation point there whose meaning
2411 # is a mystery. cPickle also suppresses PUT for objects with a refcount
2412 # of 1.
2413 def dont_test_disassembly(self):
2414 from io import StringIO
2415 from pickletools import dis
2416
2417 for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS):
2418 s = self.dumps(self._testdata, proto)
2419 filelike = StringIO()
2420 dis(s, out=filelike)
2421 got = filelike.getvalue()
2422 self.assertEqual(expected, got)
2423
2424 def _test_recursive_list(self, cls, aslist=identity, minprotocol=0):
2425 # List containing itself.
2426 l = cls()
2427 l.append(l)
2428 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
2429 s = self.dumps(l, proto)
2430 x = self.loads(s)
2431 self.assertIsInstance(x, cls)
2432 y = aslist(x)
2433 self.assertEqual(len(y), 1)
2434 self.assertIs(y[0], x)
2435
2436 def test_recursive_list(self):
2437 self._test_recursive_list(list)
2438
2439 def test_recursive_list_subclass(self):
2440 self._test_recursive_list(MyList, minprotocol=2)
2441
2442 def test_recursive_list_like(self):
2443 self._test_recursive_list(REX_six, aslist=lambda x: x.items)
2444
2445 def _test_recursive_tuple_and_list(self, cls, aslist=identity, minprotocol=0):
2446 # Tuple containing a list containing the original tuple.
2447 t = (cls(),)
2448 t[0].append(t)
2449 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
2450 s = self.dumps(t, proto)
2451 x = self.loads(s)
2452 self.assertIsInstance(x, tuple)
2453 self.assertEqual(len(x), 1)
2454 self.assertIsInstance(x[0], cls)
2455 y = aslist(x[0])
2456 self.assertEqual(len(y), 1)
2457 self.assertIs(y[0], x)
2458
2459 # List containing a tuple containing the original list.
2460 t, = t
2461 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
2462 s = self.dumps(t, proto)
2463 x = self.loads(s)
2464 self.assertIsInstance(x, cls)
2465 y = aslist(x)
2466 self.assertEqual(len(y), 1)
2467 self.assertIsInstance(y[0], tuple)
2468 self.assertEqual(len(y[0]), 1)
2469 self.assertIs(y[0][0], x)
2470
2471 def test_recursive_tuple_and_list(self):
2472 self._test_recursive_tuple_and_list(list)
2473
2474 def test_recursive_tuple_and_list_subclass(self):
2475 self._test_recursive_tuple_and_list(MyList, minprotocol=2)
2476
2477 def test_recursive_tuple_and_list_like(self):
2478 self._test_recursive_tuple_and_list(REX_six, aslist=lambda x: x.items)
2479
2480 def _test_recursive_dict(self, cls, asdict=identity, minprotocol=0):
2481 # Dict containing itself.
2482 d = cls()
2483 d[1] = d
2484 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
2485 s = self.dumps(d, proto)
2486 x = self.loads(s)
2487 self.assertIsInstance(x, cls)
2488 y = asdict(x)
2489 self.assertEqual(list(y.keys()), [1])
2490 self.assertIs(y[1], x)
2491
2492 def test_recursive_dict(self):
2493 self._test_recursive_dict(dict)
2494
2495 def test_recursive_dict_subclass(self):
2496 self._test_recursive_dict(MyDict, minprotocol=2)
2497
2498 def test_recursive_dict_like(self):
2499 self._test_recursive_dict(REX_seven, asdict=lambda x: x.table)
2500
2501 def _test_recursive_tuple_and_dict(self, cls, asdict=identity, minprotocol=0):
2502 # Tuple containing a dict containing the original tuple.
2503 t = (cls(),)
2504 t[0][1] = t
2505 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
2506 s = self.dumps(t, proto)
2507 x = self.loads(s)
2508 self.assertIsInstance(x, tuple)
2509 self.assertEqual(len(x), 1)
2510 self.assertIsInstance(x[0], cls)
2511 y = asdict(x[0])
2512 self.assertEqual(list(y), [1])
2513 self.assertIs(y[1], x)
2514
2515 # Dict containing a tuple containing the original dict.
2516 t, = t
2517 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
2518 s = self.dumps(t, proto)
2519 x = self.loads(s)
2520 self.assertIsInstance(x, cls)
2521 y = asdict(x)
2522 self.assertEqual(list(y), [1])
2523 self.assertIsInstance(y[1], tuple)
2524 self.assertEqual(len(y[1]), 1)
2525 self.assertIs(y[1][0], x)
2526
2527 def test_recursive_tuple_and_dict(self):
2528 self._test_recursive_tuple_and_dict(dict)
2529
2530 def test_recursive_tuple_and_dict_subclass(self):
2531 self._test_recursive_tuple_and_dict(MyDict, minprotocol=2)
2532
2533 def test_recursive_tuple_and_dict_like(self):
2534 self._test_recursive_tuple_and_dict(REX_seven, asdict=lambda x: x.table)
2535
2536 def _test_recursive_dict_key(self, cls, asdict=identity, minprotocol=0):
2537 # Dict containing an immutable object (as key) containing the original
2538 # dict.
2539 d = cls()
2540 d[K(d)] = 1
2541 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
2542 s = self.dumps(d, proto)
2543 x = self.loads(s)
2544 self.assertIsInstance(x, cls)
2545 y = asdict(x)
2546 self.assertEqual(len(y.keys()), 1)
2547 self.assertIsInstance(list(y.keys())[0], K)
2548 self.assertIs(list(y.keys())[0].value, x)
2549
2550 def test_recursive_dict_key(self):
2551 self._test_recursive_dict_key(dict)
2552
2553 def test_recursive_dict_subclass_key(self):
2554 self._test_recursive_dict_key(MyDict, minprotocol=2)
2555
2556 def test_recursive_dict_like_key(self):
2557 self._test_recursive_dict_key(REX_seven, asdict=lambda x: x.table)
2558
2559 def _test_recursive_tuple_and_dict_key(self, cls, asdict=identity, minprotocol=0):
2560 # Tuple containing a dict containing an immutable object (as key)
2561 # containing the original tuple.
2562 t = (cls(),)
2563 t[0][K(t)] = 1
2564 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
2565 s = self.dumps(t, proto)
2566 x = self.loads(s)
2567 self.assertIsInstance(x, tuple)
2568 self.assertEqual(len(x), 1)
2569 self.assertIsInstance(x[0], cls)
2570 y = asdict(x[0])
2571 self.assertEqual(len(y), 1)
2572 self.assertIsInstance(list(y.keys())[0], K)
2573 self.assertIs(list(y.keys())[0].value, x)
2574
2575 # Dict containing an immutable object (as key) containing a tuple
2576 # containing the original dict.
2577 t, = t
2578 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
2579 s = self.dumps(t, proto)
2580 x = self.loads(s)
2581 self.assertIsInstance(x, cls)
2582 y = asdict(x)
2583 self.assertEqual(len(y), 1)
2584 self.assertIsInstance(list(y.keys())[0], K)
2585 self.assertIs(list(y.keys())[0].value[0], x)
2586
2587 def test_recursive_tuple_and_dict_key(self):
2588 self._test_recursive_tuple_and_dict_key(dict)
2589
2590 def test_recursive_tuple_and_dict_subclass_key(self):
2591 self._test_recursive_tuple_and_dict_key(MyDict, minprotocol=2)
2592
2593 def test_recursive_tuple_and_dict_like_key(self):
2594 self._test_recursive_tuple_and_dict_key(REX_seven, asdict=lambda x: x.table)
2595
2596 def test_recursive_set(self):
2597 if self.py_version < (3, 4):
2598 self.skipTest('not supported in Python < 3.4')
2599 # Set containing an immutable object containing the original set.
2600 y = set()
2601 y.add(K(y))
2602 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2603 s = self.dumps(y, proto)
2604 x = self.loads(s)
2605 self.assertIsInstance(x, set)
2606 self.assertEqual(len(x), 1)
2607 self.assertIsInstance(list(x)[0], K)
2608 self.assertIs(list(x)[0].value, x)
2609
2610 # Immutable object containing a set containing the original object.
2611 y, = y
2612 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2613 s = self.dumps(y, proto)
2614 x = self.loads(s)
2615 self.assertIsInstance(x, K)
2616 self.assertIsInstance(x.value, set)
2617 self.assertEqual(len(x.value), 1)
2618 self.assertIs(list(x.value)[0], x)
2619
2620 def test_recursive_inst(self):
2621 # Mutable object containing itself.
2622 if self.py_version < (3, 0):
2623 self.skipTest('"classic" classes are not interoperable with Python 2')
2624 i = Object()
2625 i.attr = i
2626 for proto in protocols:
2627 s = self.dumps(i, proto)
2628 x = self.loads(s)
2629 self.assertIsInstance(x, Object)
2630 self.assertEqual(dir(x), dir(i))
2631 self.assertIs(x.attr, x)
2632
2633 def test_recursive_multi(self):
2634 if self.py_version < (3, 0):
2635 self.skipTest('"classic" classes are not interoperable with Python 2')
2636 l = []
2637 d = {1:l}
2638 i = Object()
2639 i.attr = d
2640 l.append(i)
2641 for proto in protocols:
2642 s = self.dumps(l, proto)
2643 x = self.loads(s)
2644 self.assertIsInstance(x, list)
2645 self.assertEqual(len(x), 1)
2646 self.assertEqual(dir(x[0]), dir(i))
2647 self.assertEqual(list(x[0].attr.keys()), [1])
2648 self.assertIs(x[0].attr[1], x)
2649
2650 def _test_recursive_collection_and_inst(self, factory, oldminproto=None):
2651 if self.py_version < (3, 0):
2652 self.skipTest('"classic" classes are not interoperable with Python 2')
2653 # Mutable object containing a collection containing the original
2654 # object.
2655 o = Object()
2656 o.attr = factory([o])
2657 t = type(o.attr)
2658 with self.subTest('obj -> {t.__name__} -> obj'):
2659 for proto in protocols:
2660 with self.subTest(proto=proto):
2661 s = self.dumps(o, proto)
2662 x = self.loads(s)
2663 self.assertIsInstance(x.attr, t)
2664 self.assertEqual(len(x.attr), 1)
2665 self.assertIsInstance(list(x.attr)[0], Object)
2666 self.assertIs(list(x.attr)[0], x)
2667
2668 # Collection containing a mutable object containing the original
2669 # collection.
2670 o = o.attr
2671 with self.subTest(f'{t.__name__} -> obj -> {t.__name__}'):
2672 if self.py_version < (3, 4) and oldminproto is None:
2673 self.skipTest('not supported in Python < 3.4')
2674 for proto in protocols:
2675 with self.subTest(proto=proto):
2676 if self.py_version < (3, 4) and proto < oldminproto:
2677 self.skipTest(f'requires protocol {oldminproto} in Python < 3.4')
2678 s = self.dumps(o, proto)
2679 x = self.loads(s)
2680 self.assertIsInstance(x, t)
2681 self.assertEqual(len(x), 1)
2682 self.assertIsInstance(list(x)[0], Object)
2683 self.assertIs(list(x)[0].attr, x)
2684
2685 def test_recursive_list_and_inst(self):
2686 self._test_recursive_collection_and_inst(list, oldminproto=0)
2687
2688 def test_recursive_tuple_and_inst(self):
2689 self._test_recursive_collection_and_inst(tuple, oldminproto=0)
2690
2691 def test_recursive_dict_and_inst(self):
2692 self._test_recursive_collection_and_inst(dict.fromkeys, oldminproto=0)
2693
2694 def test_recursive_set_and_inst(self):
2695 self._test_recursive_collection_and_inst(set)
2696
2697 def test_recursive_frozenset_and_inst(self):
2698 self._test_recursive_collection_and_inst(frozenset)
2699
2700 def test_recursive_list_subclass_and_inst(self):
2701 self._test_recursive_collection_and_inst(MyList, oldminproto=2)
2702
2703 def test_recursive_tuple_subclass_and_inst(self):
2704 self._test_recursive_collection_and_inst(MyTuple)
2705
2706 def test_recursive_dict_subclass_and_inst(self):
2707 self._test_recursive_collection_and_inst(MyDict.fromkeys, oldminproto=2)
2708
2709 def test_recursive_set_subclass_and_inst(self):
2710 self._test_recursive_collection_and_inst(MySet)
2711
2712 def test_recursive_frozenset_subclass_and_inst(self):
2713 self._test_recursive_collection_and_inst(MyFrozenSet)
2714
2715 def test_recursive_inst_state(self):
2716 # Mutable object containing itself.
2717 y = REX_state()
2718 y.state = y
2719 for proto in protocols:
2720 s = self.dumps(y, proto)
2721 x = self.loads(s)
2722 self.assertIsInstance(x, REX_state)
2723 self.assertIs(x.state, x)
2724
2725 def test_recursive_tuple_and_inst_state(self):
2726 # Tuple containing a mutable object containing the original tuple.
2727 t = (REX_state(),)
2728 t[0].state = t
2729 for proto in protocols:
2730 s = self.dumps(t, proto)
2731 x = self.loads(s)
2732 self.assertIsInstance(x, tuple)
2733 self.assertEqual(len(x), 1)
2734 self.assertIsInstance(x[0], REX_state)
2735 self.assertIs(x[0].state, x)
2736
2737 # Mutable object containing a tuple containing the object.
2738 t, = t
2739 for proto in protocols:
2740 s = self.dumps(t, proto)
2741 x = self.loads(s)
2742 self.assertIsInstance(x, REX_state)
2743 self.assertIsInstance(x.state, tuple)
2744 self.assertEqual(len(x.state), 1)
2745 self.assertIs(x.state[0], x)
2746
2747 def test_unicode(self):
2748 endcases = ['', '<\\u>', '<\\\u1234>', '<\n>',
2749 '<\\>', '<\\\U00012345>',
2750 # surrogates
2751 '<\udc80>']
2752 for proto in protocols:
2753 for u in endcases:
2754 p = self.dumps(u, proto)
2755 u2 = self.loads(p)
2756 self.assert_is_copy(u, u2)
2757
2758 def test_unicode_high_plane(self):
2759 t = '\U00012345'
2760 for proto in protocols:
2761 p = self.dumps(t, proto)
2762 t2 = self.loads(p)
2763 self.assert_is_copy(t, t2)
2764
2765 def test_unicode_memoization(self):
2766 # Repeated str is re-used (even when escapes added).
2767 if self.py_version < (3, 0):
2768 self.skipTest('not supported in Python < 3.0')
2769 for proto in protocols:
2770 for s in '', 'xyz', 'xyz\n', 'x\\yz', 'x\xa1yz\r':
2771 p = self.dumps((s, s), proto)
2772 s1, s2 = self.loads(p)
2773 self.assertIs(s1, s2)
2774
2775 def test_bytes(self):
2776 for proto in protocols:
2777 for s in b'', b'xyz', b'xyz'*100:
2778 p = self.dumps(s, proto)
2779 self.assert_is_copy(s, self.loads(p))
2780 for s in [bytes([i]) for i in range(256)]:
2781 p = self.dumps(s, proto)
2782 self.assert_is_copy(s, self.loads(p))
2783 for s in [bytes([i, i]) for i in range(256)]:
2784 p = self.dumps(s, proto)
2785 self.assert_is_copy(s, self.loads(p))
2786
2787 def test_bytes_memoization(self):
2788 array_types = [bytes]
2789 if self.py_version >= (3, 4):
2790 array_types += [ZeroCopyBytes]
2791 for proto in protocols:
2792 for array_type in array_types:
2793 for s in b'', b'xyz', b'xyz'*100:
2794 b = array_type(s)
2795 expected = (b, b) if self.py_version >= (3, 0) else (b.decode(),)*2
2796 with self.subTest(proto=proto, array_type=array_type, s=s, independent=False):
2797 p = self.dumps((b, b), proto)
2798 x, y = self.loads(p)
2799 self.assertIs(x, y)
2800 self.assert_is_copy(expected, (x, y))
2801
2802 b2 = array_type(s)
2803 with self.subTest(proto=proto, array_type=array_type, s=s, independent=True):
2804 p = self.dumps((b, b2), proto)
2805 # Note that (b, b2) = self.loads(p) might have identical
2806 # components, i.e., b is b2, but this is not always the
2807 # case if the content is large (equality still holds).
2808 self.assert_is_copy(expected, self.loads(p))
2809
2810 def test_bytearray(self):
2811 for proto in protocols:
2812 for s in b'', b'xyz', b'xyz'*100:
2813 b = bytearray(s)
2814 p = self.dumps(b, proto)
2815 bb = self.loads(p)
2816 self.assertIsNot(bb, b)
2817 self.assert_is_copy(b, bb)
2818 if proto <= 3:
2819 # bytearray is serialized using a global reference
2820 self.assertIn(b'bytearray', p)
2821 self.assertTrue(opcode_in_pickle(pickle.GLOBAL, p))
2822 elif proto == 4:
2823 self.assertIn(b'bytearray', p)
2824 self.assertTrue(opcode_in_pickle(pickle.STACK_GLOBAL, p))
2825 elif proto == 5:
2826 self.assertNotIn(b'bytearray', p)
2827 self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p))
2828
2829 def test_bytearray_memoization(self):
2830 array_types = [bytearray]
2831 if self.py_version >= (3, 4):
2832 array_types += [ZeroCopyBytearray]
2833 for proto in protocols:
2834 for array_type in array_types:
2835 for s in b'', b'xyz', b'xyz'*100:
2836 with self.subTest(proto=proto, array_type=array_type, s=s, independent=False):
2837 b = array_type(s)
2838 p = self.dumps((b, b), proto)
2839 b1, b2 = self.loads(p)
2840 self.assertIs(b1, b2)
2841
2842 with self.subTest(proto=proto, array_type=array_type, s=s, independent=True):
2843 b1a, b2a = array_type(s), array_type(s)
2844 # Unlike bytes, equal but independent bytearray objects are
2845 # never identical.
2846 self.assertIsNot(b1a, b2a)
2847
2848 p = self.dumps((b1a, b2a), proto)
2849 b1b, b2b = self.loads(p)
2850 self.assertIsNot(b1b, b2b)
2851
2852 self.assertIsNot(b1a, b1b)
2853 self.assert_is_copy(b1a, b1b)
2854
2855 self.assertIsNot(b2a, b2b)
2856 self.assert_is_copy(b2a, b2b)
2857
2858 def test_picklebuffer_memoization(self):
2859 if self.py_version < (3, 8):
2860 self.skipTest('not supported in Python < 3.8')
2861 array_types = [bytes, bytearray]
2862 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2863 for array_type in array_types:
2864 for s in b'', b'xyz', b'xyz'*100:
2865 with self.subTest(proto=proto, array_type=array_type, s=s, independent=False):
2866 b = pickle.PickleBuffer(array_type(s))
2867 p = self.dumps((b, b), proto)
2868 b1, b2 = self.loads(p)
2869 self.assertIs(b1, b2)
2870
2871 with self.subTest(proto=proto, array_type=array_type, s=s, independent=True):
2872 b = array_type(s)
2873 b1a = pickle.PickleBuffer(b)
2874 b2a = pickle.PickleBuffer(b)
2875 p = self.dumps((b1a, b2a), proto)
2876 b1b, b2b = self.loads(p)
2877 if array_type is not bytes:
2878 self.assertIsNot(b1b, b2b)
2879 self.assert_is_copy(b1b, b)
2880 self.assert_is_copy(b2b, b)
2881
2882 def test_empty_picklebuffer_memoization(self):
2883 # gh-148914: Empty writable PickleBuffer memoized an empty bytearray
2884 # with the id of b'' (a singleton in CPython).
2885 if self.py_version < (3, 8):
2886 self.skipTest('not supported in Python < 3.8')
2887 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2888 for readonly in False, True:
2889 with self.subTest(proto=proto, readonly=readonly):
2890 b = b''
2891 ba = bytearray()
2892 buf = pickle.PickleBuffer(b if readonly else ba)
2893 p = self.dumps((buf, b, ba), proto)
2894 buf, b, ba = self.loads(p)
2895 array_type = bytes if readonly else bytearray
2896 self.assertIsInstance(buf, array_type)
2897 self.assertIsInstance(b, bytes)
2898 self.assertIsInstance(ba, bytearray)
2899 self.assertEqual(buf, b'')
2900 self.assertEqual(b, b'')
2901 self.assertEqual(ba, b'')
2902
2903 def test_ints(self):
2904 for proto in protocols:
2905 n = sys.maxsize
2906 while n:
2907 for expected in (-n, n):
2908 s = self.dumps(expected, proto)
2909 n2 = self.loads(s)
2910 self.assert_is_copy(expected, n2)
2911 n = n >> 1
2912
2913 def test_long(self):
2914 for proto in protocols:
2915 # 256 bytes is where LONG4 begins.
2916 for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257:
2917 nbase = 1 << nbits
2918 for npos in nbase-1, nbase, nbase+1:
2919 for n in npos, -npos:
2920 pickle = self.dumps(n, proto)
2921 got = self.loads(pickle)
2922 self.assert_is_copy(n, got)
2923 # Try a monster. This is quadratic-time in protos 0 & 1, so don't
2924 # bother with those.
2925 nbase = int("deadbeeffeedface", 16)
2926 nbase += nbase << 1000000
2927 for n in nbase, -nbase:
2928 p = self.dumps(n, 2)
2929 got = self.loads(p)
2930 # assert_is_copy is very expensive here as it precomputes
2931 # a failure message by computing the repr() of n and got,
2932 # we just do the check ourselves.
2933 self.assertIs(type(got), int)
2934 self.assertEqual(n, got)
2935
2936 def test_float(self):
2937 test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5,
2938 3.14, 263.44582062374053, 6.022e23, 1e30]
2939 test_values = test_values + [-x for x in test_values]
2940 for proto in protocols:
2941 for value in test_values:
2942 pickle = self.dumps(value, proto)
2943 got = self.loads(pickle)
2944 self.assert_is_copy(value, got)
2945
2946 @run_with_locales('LC_ALL', 'de_DE', 'fr_FR', '')
2947 def test_float_format(self):
2948 # make sure that floats are formatted locale independent with proto 0
2949 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
2950
2951 def test_reduce(self):
2952 for proto in protocols:
2953 with self.subTest(proto=proto):
2954 if self.py_version < (3, 4) and proto < 3:
2955 self.skipTest('str is not interoperable with Python < 3.4')
2956 inst = AAA()
2957 dumped = self.dumps(inst, proto)
2958 loaded = self.loads(dumped)
2959 self.assertEqual(loaded, REDUCE_A)
2960
2961 def test_getinitargs(self):
2962 if self.py_version < (3, 0):
2963 self.skipTest('"classic" classes are not interoperable with Python 2')
2964 for proto in protocols:
2965 inst = initarg(1, 2)
2966 dumped = self.dumps(inst, proto)
2967 loaded = self.loads(dumped)
2968 self.assert_is_copy(inst, loaded)
2969
2970 def test_metaclass(self):
2971 self.assertEqual(type(use_metaclass), metaclass)
2972 a = use_metaclass()
2973 for proto in protocols:
2974 s = self.dumps(a, proto)
2975 b = self.loads(s)
2976 self.assertEqual(a.__class__, b.__class__)
2977
2978 def test_dynamic_class(self):
2979 a = create_dynamic_class("my_dynamic_class", (object,))
2980 copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__)
2981 for proto in protocols:
2982 s = self.dumps(a, proto)
2983 b = self.loads(s)
2984 self.assertEqual(a, b)
2985 self.assertIs(type(a), type(b))
2986
2987 def test_structseq(self):
2988 import time
2989 import os
2990
2991 t = time.localtime()
2992 for proto in protocols:
2993 s = self.dumps(t, proto)
2994 u = self.loads(s)
2995 self.assert_is_copy(t, u)
2996 if self.py_version < (3, 4):
2997 # module 'os' has no attributes '_make_stat_result' and
2998 # '_make_statvfs_result'
2999 continue
3000 t = os.stat(os.curdir)
3001 s = self.dumps(t, proto)
3002 u = self.loads(s)
3003 self.assert_is_copy(t, u)
3004 if hasattr(os, "statvfs"):
3005 t = os.statvfs(os.curdir)
3006 s = self.dumps(t, proto)
3007 u = self.loads(s)
3008 self.assert_is_copy(t, u)
3009
3010 def test_ellipsis(self):
3011 if self.py_version < (3, 3):
3012 self.skipTest('not supported in Python < 3.3')
3013 for proto in protocols:
3014 with self.subTest(proto=proto):
3015 s = self.dumps(..., proto)
3016 u = self.loads(s)
3017 self.assertIs(..., u)
3018
3019 def test_notimplemented(self):
3020 if self.py_version < (3, 3):
3021 self.skipTest('not supported in Python < 3.3')
3022 for proto in protocols:
3023 with self.subTest(proto=proto):
3024 s = self.dumps(NotImplemented, proto)
3025 u = self.loads(s)
3026 self.assertIs(NotImplemented, u)
3027
3028 def test_singleton_types(self):
3029 # Issue #6477: Test that types of built-in singletons can be pickled.
3030 if self.py_version < (3, 3):
3031 self.skipTest('not supported in Python < 3.3')
3032 singletons = [None, ..., NotImplemented]
3033 for singleton in singletons:
3034 t = type(singleton)
3035 for proto in protocols:
3036 with self.subTest(name=t.__name__, proto=proto):
3037 s = self.dumps(t, proto)
3038 u = self.loads(s)
3039 self.assertIs(t, u)
3040
3041 def test_builtin_types(self):
3042 new_names = {
3043 'bytes': (3, 0),
3044 'BuiltinImporter': (3, 3),
3045 'str': (3, 4), # not interoperable with Python < 3.4
3046 }
3047 for t in builtins.__dict__.values():
3048 if isinstance(t, type) and not issubclass(t, BaseException):
3049 if t.__name__ in new_names and self.py_version < new_names[t.__name__]:
3050 continue
3051 for proto in protocols:
3052 with self.subTest(name=t.__name__, proto=proto):
3053 s = self.dumps(t, proto)
3054 self.assertIs(self.loads(s), t)
3055
3056 def test_builtin_exceptions(self):
3057 new_names = {
3058 'BlockingIOError': (3, 3),
3059 'BrokenPipeError': (3, 3),
3060 'ChildProcessError': (3, 3),
3061 'ConnectionError': (3, 3),
3062 'ConnectionAbortedError': (3, 3),
3063 'ConnectionRefusedError': (3, 3),
3064 'ConnectionResetError': (3, 3),
3065 'FileExistsError': (3, 3),
3066 'FileNotFoundError': (3, 3),
3067 'InterruptedError': (3, 3),
3068 'IsADirectoryError': (3, 3),
3069 'NotADirectoryError': (3, 3),
3070 'PermissionError': (3, 3),
3071 'ProcessLookupError': (3, 3),
3072 'TimeoutError': (3, 3),
3073 'RecursionError': (3, 5),
3074 'StopAsyncIteration': (3, 5),
3075 'ModuleNotFoundError': (3, 6),
3076 'EncodingWarning': (3, 10),
3077 'BaseExceptionGroup': (3, 11),
3078 'ExceptionGroup': (3, 11),
3079 '_IncompleteInputError': (3, 13),
3080 'PythonFinalizationError': (3, 13),
3081 }
3082 for t in builtins.__dict__.values():
3083 if isinstance(t, type) and issubclass(t, BaseException):
3084 if t.__name__ in new_names and self.py_version < new_names[t.__name__]:
3085 continue
3086 for proto in protocols:
3087 with self.subTest(name=t.__name__, proto=proto):
3088 if self.py_version < (3, 3) and proto < 3:
3089 self.skipTest('exception classes are not interoperable with Python < 3.3')
3090 s = self.dumps(t, proto)
3091 u = self.loads(s)
3092 if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError:
3093 self.assertIs(u, OSError)
3094 elif proto <= 2 and issubclass(t, ImportError):
3095 self.assertIs(u, ImportError)
3096 else:
3097 self.assertIs(u, t)
3098
3099 def test_builtin_functions(self):
3100 new_names = {
3101 '__build_class__': (3, 0),
3102 'ascii': (3, 0),
3103 'exec': (3, 0),
3104 'breakpoint': (3, 7),
3105 'aiter': (3, 10),
3106 'anext': (3, 10),
3107 }
3108 for t in builtins.__dict__.values():
3109 if isinstance(t, types.BuiltinFunctionType):
3110 if t.__name__ in new_names and self.py_version < new_names[t.__name__]:
3111 continue
3112 for proto in protocols:
3113 with self.subTest(name=t.__name__, proto=proto):
3114 s = self.dumps(t, proto)
3115 self.assertIs(self.loads(s), t)
3116
3117 # Tests for protocol 2
3118
3119 def test_proto(self):
3120 for proto in protocols:
3121 pickled = self.dumps(None, proto)
3122 if proto >= 2:
3123 proto_header = pickle.PROTO + bytes([proto])
3124 self.assertStartsWith(pickled, proto_header)
3125 else:
3126 self.assertEqual(count_opcode(pickle.PROTO, pickled), 0)
3127
3128 def test_bad_proto(self):
3129 if self.py_version < (3, 8):
3130 self.skipTest('no protocol validation in Python < 3.8')
3131 oob = protocols[-1] + 1 # a future protocol
3132 build_none = pickle.NONE + pickle.STOP
3133 badpickle = pickle.PROTO + bytes([oob]) + build_none
3134 try:
3135 self.loads(badpickle)
3136 except ValueError as err:
3137 self.assertIn("unsupported pickle protocol", str(err))
3138 else:
3139 self.fail("expected bad protocol number to raise ValueError")
3140
3141 def test_long1(self):
3142 x = 12345678910111213141516178920
3143 for proto in protocols:
3144 s = self.dumps(x, proto)
3145 y = self.loads(s)
3146 self.assert_is_copy(x, y)
3147 self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2)
3148
3149 def test_long4(self):
3150 x = 12345678910111213141516178920 << (256*8)
3151 for proto in protocols:
3152 s = self.dumps(x, proto)
3153 y = self.loads(s)
3154 self.assert_is_copy(x, y)
3155 self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2)
3156
3157 def test_short_tuples(self):
3158 # Map (proto, len(tuple)) to expected opcode.
3159 expected_opcode = {(0, 0): pickle.TUPLE,
3160 (0, 1): pickle.TUPLE,
3161 (0, 2): pickle.TUPLE,
3162 (0, 3): pickle.TUPLE,
3163 (0, 4): pickle.TUPLE,
3164
3165 (1, 0): pickle.EMPTY_TUPLE,
3166 (1, 1): pickle.TUPLE,
3167 (1, 2): pickle.TUPLE,
3168 (1, 3): pickle.TUPLE,
3169 (1, 4): pickle.TUPLE,
3170
3171 (2, 0): pickle.EMPTY_TUPLE,
3172 (2, 1): pickle.TUPLE1,
3173 (2, 2): pickle.TUPLE2,
3174 (2, 3): pickle.TUPLE3,
3175 (2, 4): pickle.TUPLE,
3176
3177 (3, 0): pickle.EMPTY_TUPLE,
3178 (3, 1): pickle.TUPLE1,
3179 (3, 2): pickle.TUPLE2,
3180 (3, 3): pickle.TUPLE3,
3181 (3, 4): pickle.TUPLE,
3182 }
3183 a = ()
3184 b = (1,)
3185 c = (1, 2)
3186 d = (1, 2, 3)
3187 e = (1, 2, 3, 4)
3188 for proto in protocols:
3189 for x in a, b, c, d, e:
3190 s = self.dumps(x, proto)
3191 y = self.loads(s)
3192 self.assert_is_copy(x, y)
3193 expected = expected_opcode[min(proto, 3), len(x)]
3194 self.assertTrue(opcode_in_pickle(expected, s))
3195
3196 def test_singletons(self):
3197 # Map (proto, singleton) to expected opcode.
3198 expected_opcode = {(0, None): pickle.NONE,
3199 (1, None): pickle.NONE,
3200 (2, None): pickle.NONE,
3201 (3, None): pickle.NONE,
3202
3203 (0, True): pickle.INT,
3204 (1, True): pickle.INT,
3205 (2, True): pickle.NEWTRUE,
3206 (3, True): pickle.NEWTRUE,
3207
3208 (0, False): pickle.INT,
3209 (1, False): pickle.INT,
3210 (2, False): pickle.NEWFALSE,
3211 (3, False): pickle.NEWFALSE,
3212 }
3213 for proto in protocols:
3214 for x in None, False, True:
3215 s = self.dumps(x, proto)
3216 y = self.loads(s)
3217 self.assertTrue(x is y, (proto, x, s, y))
3218 expected = expected_opcode[min(proto, 3), x]
3219 self.assertTrue(opcode_in_pickle(expected, s))
3220
3221 def test_newobj_tuple(self):
3222 x = MyTuple([1, 2, 3])
3223 x.foo = 42
3224 x.bar = "hello"
3225 for proto in protocols:
3226 s = self.dumps(x, proto)
3227 y = self.loads(s)
3228 self.assert_is_copy(x, y)
3229
3230 def test_newobj_list(self):
3231 x = MyList([1, 2, 3])
3232 x.foo = 42
3233 x.bar = "hello"
3234 for proto in protocols:
3235 s = self.dumps(x, proto)
3236 y = self.loads(s)
3237 self.assert_is_copy(x, y)
3238
3239 def test_newobj_generic(self):
3240 for proto in protocols:
3241 for C in myclasses:
3242 with self.subTest(proto=proto, C=C):
3243 if self.py_version < (3, 0) and proto < 2 and C in (MyInt, MyStr):
3244 self.skipTest('int and str subclasses are not interoperable with Python 2')
3245 if (3, 0) <= self.py_version < (3, 4) and proto < 2 and C in (MyStr, MyUnicode):
3246 self.skipTest('str subclasses are not interoperable with Python < 3.4')
3247 B = C.__base__
3248 x = C(C.sample)
3249 x.foo = 42
3250 s = self.dumps(x, proto)
3251 y = self.loads(s)
3252 detail = (proto, C, B, x, y, type(y))
3253 self.assert_is_copy(x, y) # XXX revisit
3254 self.assertEqual(B(x), B(y), detail)
3255 self.assertEqual(x.__dict__, y.__dict__, detail)
3256
3257 def test_newobj_proxies(self):
3258 # NEWOBJ should use the __class__ rather than the raw type
3259 classes = myclasses[:]
3260 # Cannot create weakproxies to these classes
3261 for c in (MyInt, MyLong, MyTuple):
3262 classes.remove(c)
3263 for proto in protocols:
3264 for C in classes:
3265 with self.subTest(proto=proto, C=C):
3266 if self.py_version < (3, 4) and proto < 3 and C in (MyStr, MyUnicode):
3267 self.skipTest('str subclasses are not interoperable with Python < 3.4')
3268 B = C.__base__
3269 x = C(C.sample)
3270 x.foo = 42
3271 p = weakref.proxy(x)
3272 s = self.dumps(p, proto)
3273 y = self.loads(s)
3274 self.assertEqual(type(y), type(x)) # rather than type(p)
3275 detail = (proto, C, B, x, y, type(y))
3276 self.assertEqual(B(x), B(y), detail)
3277 self.assertEqual(x.__dict__, y.__dict__, detail)
3278
3279 def test_newobj_overridden_new(self):
3280 # Test that Python class with C implemented __new__ is pickleable
3281 for proto in protocols:
3282 with self.subTest(proto=proto):
3283 if self.py_version < (3, 0) and proto < 2:
3284 self.skipTest('int subclasses are not interoperable with Python 2')
3285 x = MyIntWithNew2(1)
3286 x.foo = 42
3287 s = self.dumps(x, proto)
3288 y = self.loads(s)
3289 self.assertIs(type(y), MyIntWithNew2)
3290 self.assertEqual(int(y), 1)
3291 self.assertEqual(y.foo, 42)
3292
3293 def test_newobj_not_class(self):
3294 # Issue 24552
3295 if self.py_version < (3, 4):
3296 self.skipTest('not supported in Python < 3.4')
3297 o = SimpleNewObj.__new__(SimpleNewObj)
3298 b = self.dumps(o, 4)
3299 with support.swap_attr(picklecommon, 'SimpleNewObj', 42):
3300 self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b)
3301
3302 # Register a type with copyreg, with extension code extcode. Pickle
3303 # an object of that type. Check that the resulting pickle uses opcode
3304 # (EXT[124]) under proto 2, and not in proto 1.
3305
3306 def produce_global_ext(self, extcode, opcode):
3307 e = ExtensionSaver(extcode)
3308 try:
3309 copyreg.add_extension(MyList.__module__, "MyList", extcode)
3310 x = MyList([1, 2, 3])
3311 x.foo = 42
3312 x.bar = "hello"
3313
3314 # Dump using protocol 1 for comparison.
3315 s1 = self.dumps(x, 1)
3316 self.assertIn(MyList.__module__.encode(), s1)
3317 self.assertIn(b"MyList", s1)
3318 self.assertFalse(opcode_in_pickle(opcode, s1))
3319
3320 y = self.loads(s1)
3321 self.assert_is_copy(x, y)
3322
3323 # Dump using protocol 2 for test.
3324 s2 = self.dumps(x, 2)
3325 self.assertNotIn(MyList.__module__.encode(), s2)
3326 self.assertNotIn(b"MyList", s2)
3327 self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2))
3328
3329 y = self.loads(s2)
3330 self.assert_is_copy(x, y)
3331 finally:
3332 e.restore()
3333
3334 def test_global_ext1(self):
3335 self.produce_global_ext(0x00000001, pickle.EXT1) # smallest EXT1 code
3336 self.produce_global_ext(0x000000ff, pickle.EXT1) # largest EXT1 code
3337
3338 def test_global_ext2(self):
3339 self.produce_global_ext(0x00000100, pickle.EXT2) # smallest EXT2 code
3340 self.produce_global_ext(0x0000ffff, pickle.EXT2) # largest EXT2 code
3341 self.produce_global_ext(0x0000abcd, pickle.EXT2) # check endianness
3342
3343 def test_global_ext4(self):
3344 self.produce_global_ext(0x00010000, pickle.EXT4) # smallest EXT4 code
3345 self.produce_global_ext(0x7fffffff, pickle.EXT4) # largest EXT4 code
3346 self.produce_global_ext(0x12abcdef, pickle.EXT4) # check endianness
3347
3348 def test_list_chunking(self):
3349 n = 10 # too small to chunk
3350 x = list(range(n))
3351 for proto in protocols:
3352 s = self.dumps(x, proto)
3353 y = self.loads(s)
3354 self.assert_is_copy(x, y)
3355 num_appends = count_opcode(pickle.APPENDS, s)
3356 self.assertEqual(num_appends, proto > 0)
3357
3358 n = 2500 # expect at least two chunks when proto > 0
3359 x = list(range(n))
3360 for proto in protocols:
3361 s = self.dumps(x, proto)
3362 y = self.loads(s)
3363 self.assert_is_copy(x, y)
3364 num_appends = count_opcode(pickle.APPENDS, s)
3365 if proto == 0:
3366 self.assertEqual(num_appends, 0)
3367 else:
3368 self.assertTrue(num_appends >= 2)
3369
3370 def test_dict_chunking(self):
3371 n = 10 # too small to chunk
3372 x = dict.fromkeys(range(n))
3373 for proto in protocols:
3374 s = self.dumps(x, proto)
3375 self.assertIsInstance(s, bytes_types)
3376 y = self.loads(s)
3377 self.assert_is_copy(x, y)
3378 num_setitems = count_opcode(pickle.SETITEMS, s)
3379 self.assertEqual(num_setitems, proto > 0)
3380
3381 n = 2500 # expect at least two chunks when proto > 0
3382 x = dict.fromkeys(range(n))
3383 for proto in protocols:
3384 s = self.dumps(x, proto)
3385 y = self.loads(s)
3386 self.assert_is_copy(x, y)
3387 num_setitems = count_opcode(pickle.SETITEMS, s)
3388 if proto == 0:
3389 self.assertEqual(num_setitems, 0)
3390 else:
3391 self.assertTrue(num_setitems >= 2)
3392
3393 def test_set_chunking(self):
3394 n = 10 # too small to chunk
3395 x = set(range(n))
3396 for proto in protocols:
3397 s = self.dumps(x, proto)
3398 y = self.loads(s)
3399 self.assert_is_copy(x, y)
3400 num_additems = count_opcode(pickle.ADDITEMS, s)
3401 if proto < 4:
3402 self.assertEqual(num_additems, 0)
3403 else:
3404 self.assertEqual(num_additems, 1)
3405
3406 n = 2500 # expect at least two chunks when proto >= 4
3407 x = set(range(n))
3408 for proto in protocols:
3409 s = self.dumps(x, proto)
3410 y = self.loads(s)
3411 self.assert_is_copy(x, y)
3412 num_additems = count_opcode(pickle.ADDITEMS, s)
3413 if proto < 4:
3414 self.assertEqual(num_additems, 0)
3415 else:
3416 self.assertGreaterEqual(num_additems, 2)
3417
3418 def test_simple_newobj(self):
3419 x = SimpleNewObj.__new__(SimpleNewObj, 0xface) # avoid __init__
3420 x.abc = 666
3421 for proto in protocols:
3422 with self.subTest(proto=proto):
3423 if self.py_version < (3, 0) and proto < 2:
3424 self.skipTest('int subclasses are not interoperable with Python 2')
3425 s = self.dumps(x, proto)
3426 if proto < 1:
3427 if self.py_version >= (3, 7):
3428 self.assertIn(b'\nI64206', s) # INT
3429 else: # for test_xpickle
3430 self.assertIn(b'64206', s) # INT or LONG
3431 else:
3432 self.assertIn(b'M\xce\xfa', s) # BININT2
3433 if not (self.py_version < (3, 5) and proto == 4):
3434 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
3435 2 <= proto)
3436 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s))
3437 y = self.loads(s) # will raise TypeError if __init__ called
3438 self.assert_is_copy(x, y)
3439
3440 def test_complex_newobj(self):
3441 x = ComplexNewObj.__new__(ComplexNewObj, 0xface) # avoid __init__
3442 x.abc = 666
3443 for proto in protocols:
3444 with self.subTest(proto=proto):
3445 if self.py_version < (3, 0) and proto < 2:
3446 self.skipTest('int subclasses are not interoperable with Python 2')
3447 s = self.dumps(x, proto)
3448 if proto < 1:
3449 if self.py_version >= (3, 7):
3450 self.assertIn(b'\nI64206', s) # INT
3451 else: # for test_xpickle
3452 self.assertIn(b'64206', s) # INT or LONG
3453 elif proto < 2:
3454 self.assertIn(b'M\xce\xfa', s) # BININT2
3455 elif proto < 4:
3456 if self.py_version >= (3, 0):
3457 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE
3458 else: # for test_xpickle
3459 self.assertIn(b'U\x04FACE', s) # SHORT_BINSTRING
3460 else:
3461 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE
3462 if not (self.py_version < (3, 5) and proto == 4):
3463 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
3464 2 <= proto)
3465 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s))
3466 y = self.loads(s) # will raise TypeError if __init__ called
3467 self.assert_is_copy(x, y)
3468
3469 def test_complex_newobj_ex(self):
3470 if self.py_version < (3, 4):
3471 self.skipTest('not supported in Python < 3.4')
3472 x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface) # avoid __init__
3473 x.abc = 666
3474 for proto in protocols:
3475 with self.subTest(proto=proto):
3476 if self.py_version < (3, 6) and proto < 4:
3477 self.skipTest('requires protocol 4 in Python < 3.6')
3478 s = self.dumps(x, proto)
3479 if proto < 1:
3480 if self.py_version >= (3, 7):
3481 self.assertIn(b'\nI64206', s) # INT
3482 else: # for test_xpickle
3483 self.assertIn(b'64206', s) # INT or LONG
3484 elif proto < 2:
3485 self.assertIn(b'M\xce\xfa', s) # BININT2
3486 elif proto < 4:
3487 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE
3488 else:
3489 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE
3490 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s))
3491 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s),
3492 4 <= proto)
3493 y = self.loads(s) # will raise TypeError if __init__ called
3494 self.assert_is_copy(x, y)
3495
3496 def test_newobj_list_slots(self):
3497 x = SlotList([1, 2, 3])
3498 x.foo = 42
3499 x.bar = "hello"
3500 s = self.dumps(x, 2)
3501 y = self.loads(s)
3502 self.assert_is_copy(x, y)
3503
3504 def test_reduce_overrides_default_reduce_ex(self):
3505 for proto in protocols:
3506 x = REX_one()
3507 self.assertEqual(x._reduce_called, 0)
3508 s = self.dumps(x, proto)
3509 self.assertEqual(x._reduce_called, 1)
3510 y = self.loads(s)
3511 self.assertEqual(y._reduce_called, 0)
3512
3513 def test_reduce_ex_called(self):
3514 for proto in protocols:
3515 x = REX_two()
3516 self.assertEqual(x._proto, None)
3517 s = self.dumps(x, proto)
3518 self.assertEqual(x._proto, proto)
3519 y = self.loads(s)
3520 self.assertEqual(y._proto, None)
3521
3522 def test_reduce_ex_overrides_reduce(self):
3523 for proto in protocols:
3524 x = REX_three()
3525 self.assertEqual(x._proto, None)
3526 s = self.dumps(x, proto)
3527 self.assertEqual(x._proto, proto)
3528 y = self.loads(s)
3529 self.assertEqual(y._proto, None)
3530
3531 def test_reduce_ex_calls_base(self):
3532 for proto in protocols:
3533 x = REX_four()
3534 self.assertEqual(x._proto, None)
3535 s = self.dumps(x, proto)
3536 self.assertEqual(x._proto, proto)
3537 y = self.loads(s)
3538 self.assertEqual(y._proto, proto)
3539
3540 def test_reduce_calls_base(self):
3541 for proto in protocols:
3542 x = REX_five()
3543 self.assertEqual(x._reduce_called, 0)
3544 s = self.dumps(x, proto)
3545 self.assertEqual(x._reduce_called, 1)
3546 y = self.loads(s)
3547 self.assertEqual(y._reduce_called, 1)
3548
3549 def test_pickle_setstate_None(self):
3550 c = C_None_setstate()
3551 p = self.dumps(c)
3552 with self.assertRaises(TypeError):
3553 self.loads(p)
3554
3555 def test_many_puts_and_gets(self):
3556 # Test that internal data structures correctly deal with lots of
3557 # puts/gets.
3558 keys = ("aaa" + str(i) for i in range(100))
3559 large_dict = dict((k, [4, 5, 6]) for k in keys)
3560 obj = [dict(large_dict), dict(large_dict), dict(large_dict)]
3561
3562 for proto in protocols:
3563 with self.subTest(proto=proto):
3564 dumped = self.dumps(obj, proto)
3565 loaded = self.loads(dumped)
3566 self.assert_is_copy(obj, loaded)
3567
3568 def test_attribute_name_interning(self):
3569 # Test that attribute names of pickled objects are interned when
3570 # unpickling.
3571 if self.py_version < (3, 0):
3572 self.skipTest('"classic" classes are not interoperable with Python 2')
3573 for proto in protocols:
3574 x = C()
3575 x.foo = 42
3576 x.bar = "hello"
3577 s = self.dumps(x, proto)
3578 y = self.loads(s)
3579 x_keys = sorted(x.__dict__)
3580 y_keys = sorted(y.__dict__)
3581 for x_key, y_key in zip(x_keys, y_keys):
3582 self.assertIs(x_key, y_key)
3583
3584 def test_pickle_to_2x(self):
3585 # Pickle non-trivial data with protocol 2, expecting that it yields
3586 # the same result as Python 2.x did.
3587 # NOTE: this test is a bit too strong since we can produce different
3588 # bytecode that 2.x will still understand.
3589 dumped = self.dumps(range(5), 2)
3590 self.assertEqual(dumped, DATA_XRANGE)
3591 dumped = self.dumps(set([3]), 2)
3592 self.assertEqual(dumped, DATA_SET2)
3593
3594 def test_large_pickles(self):
3595 # Test the correctness of internal buffering routines when handling
3596 # large data.
3597 for proto in protocols:
3598 data = (1, min, b'xy' * (30 * 1024), len)
3599 dumped = self.dumps(data, proto)
3600 loaded = self.loads(dumped)
3601 self.assertEqual(len(loaded), len(data))
3602 if self.py_version < (3, 0):
3603 data = (1, min, 'xy' * (30 * 1024), len)
3604 self.assertEqual(loaded, data)
3605
3606 def test_int_pickling_efficiency(self):
3607 # Test compacity of int representation (see issue #12744)
3608 if self.py_version < (3, 3):
3609 self.skipTest('not supported in Python < 3.3')
3610 for proto in protocols:
3611 with self.subTest(proto=proto):
3612 pickles = [self.dumps(2**n, proto) for n in range(70)]
3613 sizes = list(map(len, pickles))
3614 # the size function is monotonic
3615 self.assertEqual(sorted(sizes), sizes)
3616 if proto >= 2:
3617 for p in pickles:
3618 self.assertFalse(opcode_in_pickle(pickle.LONG, p))
3619
3620 def _check_pickling_with_opcode(self, obj, opcode, proto):
3621 pickled = self.dumps(obj, proto)
3622 self.assertTrue(opcode_in_pickle(opcode, pickled))
3623 unpickled = self.loads(pickled)
3624 self.assertEqual(obj, unpickled)
3625
3626 def test_appends_on_non_lists(self):
3627 # Issue #17720
3628 obj = REX_six([1, 2, 3])
3629 for proto in protocols:
3630 with self.subTest(proto=proto):
3631 if proto == 0:
3632 self._check_pickling_with_opcode(obj, pickle.APPEND, proto)
3633 else:
3634 if self.py_version < (3, 0):
3635 self.skipTest('not supported in Python 2')
3636 self._check_pickling_with_opcode(obj, pickle.APPENDS, proto)
3637
3638 def test_setitems_on_non_dicts(self):
3639 obj = REX_seven({1: -1, 2: -2, 3: -3})
3640 for proto in protocols:
3641 if proto == 0:
3642 self._check_pickling_with_opcode(obj, pickle.SETITEM, proto)
3643 else:
3644 self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto)
3645
3646 # Exercise framing (proto >= 4) for significant workloads
3647
3648 FRAME_SIZE_MIN = 4
3649 FRAME_SIZE_TARGET = 64 * 1024
3650
3651 def check_frame_opcodes(self, pickled):
3652 """
3653 Check the arguments of FRAME opcodes in a protocol 4+ pickle.
3654
3655 Note that binary objects that are larger than FRAME_SIZE_TARGET are not
3656 framed by default and are therefore considered a frame by themselves in
3657 the following consistency check.
3658 """
3659 frame_end = frameless_start = None
3660 frameless_opcodes = {'BINBYTES', 'BINUNICODE', 'BINBYTES8',
3661 'BINUNICODE8', 'BYTEARRAY8'}
3662 for op, arg, pos in pickletools.genops(pickled):
3663 if frame_end is not None:
3664 self.assertLessEqual(pos, frame_end)
3665 if pos == frame_end:
3666 frame_end = None
3667
3668 if frame_end is not None: # framed
3669 self.assertNotEqual(op.name, 'FRAME')
3670 if op.name in frameless_opcodes:
3671 # Only short bytes and str objects should be written
3672 # in a frame
3673 self.assertLessEqual(len(arg), self.FRAME_SIZE_TARGET)
3674
3675 else: # not framed
3676 if (op.name == 'FRAME' or
3677 (op.name in frameless_opcodes and
3678 len(arg) > self.FRAME_SIZE_TARGET)):
3679 # Frame or large bytes or str object
3680 if frameless_start is not None:
3681 # Only short data should be written outside of a frame
3682 self.assertLess(pos - frameless_start,
3683 self.FRAME_SIZE_MIN)
3684 frameless_start = None
3685 elif frameless_start is None and op.name != 'PROTO':
3686 frameless_start = pos
3687
3688 if op.name == 'FRAME':
3689 self.assertGreaterEqual(arg, self.FRAME_SIZE_MIN)
3690 frame_end = pos + 9 + arg
3691
3692 pos = len(pickled)
3693 if frame_end is not None:
3694 self.assertEqual(frame_end, pos)
3695 elif frameless_start is not None:
3696 self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN)
3697
3698 @support.skip_if_pgo_task
3699 @support.requires_resource('cpu')
3700 def test_framing_many_objects(self):
3701 if self.py_version < (3, 4):
3702 self.skipTest('not supported in Python < 3.4')
3703 obj = list(range(10**5))
3704 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
3705 with self.subTest(proto=proto):
3706 pickled = self.dumps(obj, proto)
3707 unpickled = self.loads(pickled)
3708 self.assertEqual(obj, unpickled)
3709 bytes_per_frame = (len(pickled) /
3710 count_opcode(pickle.FRAME, pickled))
3711 self.assertGreater(bytes_per_frame,
3712 self.FRAME_SIZE_TARGET / 2)
3713 self.assertLessEqual(bytes_per_frame,
3714 self.FRAME_SIZE_TARGET * 1)
3715 self.check_frame_opcodes(pickled)
3716
3717 def test_framing_large_objects(self):
3718 if self.py_version < (3, 4):
3719 self.skipTest('not supported in Python < 3.4')
3720 N = 1024 * 1024
3721 small_items = [[i] for i in range(10)]
3722 obj = [b'x' * N, *small_items, b'y' * N, 'z' * N]
3723 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
3724 for fast in [False, True]:
3725 with self.subTest(proto=proto, fast=fast):
3726 if not fast:
3727 # fast=False by default.
3728 # This covers in-memory pickling with pickle.dumps().
3729 pickled = self.dumps(obj, proto)
3730 else:
3731 # Pickler is required when fast=True.
3732 if not hasattr(self, 'pickler'):
3733 continue
3734 buf = io.BytesIO()
3735 pickler = self.pickler(buf, protocol=proto)
3736 pickler.fast = fast
3737 pickler.dump(obj)
3738 pickled = buf.getvalue()
3739 unpickled = self.loads(pickled)
3740 # More informative error message in case of failure.
3741 self.assertEqual([len(x) for x in obj],
3742 [len(x) for x in unpickled])
3743 # Perform full equality check if the lengths match.
3744 self.assertEqual(obj, unpickled)
3745 if self.py_version >= (3, 7):
3746 n_frames = count_opcode(pickle.FRAME, pickled)
3747 # A single frame for small objects between
3748 # first two large objects.
3749 self.assertEqual(n_frames, 1)
3750 self.check_frame_opcodes(pickled)
3751
3752 def test_optional_frames(self):
3753 if self.py_version < (3, 4):
3754 self.skipTest('not supported in Python < 3.4')
3755
3756 def remove_frames(pickled, keep_frame=None):
3757 """Remove frame opcodes from the given pickle."""
3758 frame_starts = []
3759 # 1 byte for the opcode and 8 for the argument
3760 frame_opcode_size = 9
3761 for opcode, _, pos in pickletools.genops(pickled):
3762 if opcode.name == 'FRAME':
3763 frame_starts.append(pos)
3764
3765 newpickle = bytearray()
3766 last_frame_end = 0
3767 for i, pos in enumerate(frame_starts):
3768 if keep_frame and keep_frame(i):
3769 continue
3770 newpickle += pickled[last_frame_end:pos]
3771 last_frame_end = pos + frame_opcode_size
3772 newpickle += pickled[last_frame_end:]
3773 return newpickle
3774
3775 frame_size = self.FRAME_SIZE_TARGET
3776 num_frames = 20
3777 # Large byte objects (dict values) intermittent with small objects
3778 # (dict keys)
3779 for bytes_type in (bytes, bytearray):
3780 obj = {i: bytes_type([i]) * frame_size for i in range(num_frames)}
3781
3782 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
3783 pickled = self.dumps(obj, proto)
3784
3785 frameless_pickle = remove_frames(pickled)
3786 self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0)
3787 self.assertEqual(obj, self.loads(frameless_pickle))
3788
3789 some_frames_pickle = remove_frames(pickled, lambda i: i % 2)
3790 self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle),
3791 count_opcode(pickle.FRAME, pickled))
3792 self.assertEqual(obj, self.loads(some_frames_pickle))
3793
3794 @support.skip_if_pgo_task
3795 def test_framed_write_sizes_with_delayed_writer(self):
3796 if self.py_version < (3, 4):
3797 self.skipTest('not supported in Python < 3.4')
3798
3799 class ChunkAccumulator:
3800 """Accumulate pickler output in a list of raw chunks."""
3801 def __init__(self):
3802 self.chunks = []
3803 def write(self, chunk):
3804 self.chunks.append(chunk)
3805 def concatenate_chunks(self):
3806 return b"".join(self.chunks)
3807
3808 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
3809 objects = [(str(i).encode('ascii'), i % 42, {'i': str(i)})
3810 for i in range(int(1e4))]
3811 # Add a large unique ASCII string
3812 objects.append('0123456789abcdef' *
3813 (self.FRAME_SIZE_TARGET // 16 + 1))
3814
3815 # Protocol 4 packs groups of small objects into frames and issues
3816 # calls to write only once or twice per frame:
3817 # The C pickler issues one call to write per-frame (header and
3818 # contents) while Python pickler issues two calls to write: one for
3819 # the frame header and one for the frame binary contents.
3820 writer = ChunkAccumulator()
3821 self.pickler(writer, proto).dump(objects)
3822
3823 # Actually read the binary content of the chunks after the end
3824 # of the call to dump: any memoryview passed to write should not
3825 # be released otherwise this delayed access would not be possible.
3826 pickled = writer.concatenate_chunks()
3827 reconstructed = self.loads(pickled)
3828 self.assertEqual(reconstructed, objects)
3829 self.assertGreater(len(writer.chunks), 1)
3830
3831 # memoryviews should own the memory.
3832 del objects
3833 support.gc_collect()
3834 self.assertEqual(writer.concatenate_chunks(), pickled)
3835
3836 n_frames = (len(pickled) - 1) // self.FRAME_SIZE_TARGET + 1
3837 # There should be at least one call to write per frame
3838 self.assertGreaterEqual(len(writer.chunks), n_frames)
3839
3840 # but not too many either: there can be one for the proto,
3841 # one per-frame header, one per frame for the actual contents,
3842 # and two for the header.
3843 self.assertLessEqual(len(writer.chunks), 2 * n_frames + 3)
3844
3845 chunk_sizes = [len(c) for c in writer.chunks]
3846 large_sizes = [s for s in chunk_sizes
3847 if s >= self.FRAME_SIZE_TARGET]
3848 medium_sizes = [s for s in chunk_sizes
3849 if 9 < s < self.FRAME_SIZE_TARGET]
3850 small_sizes = [s for s in chunk_sizes if s <= 9]
3851
3852 # Large chunks should not be too large:
3853 for chunk_size in large_sizes:
3854 self.assertLess(chunk_size, 2 * self.FRAME_SIZE_TARGET,
3855 chunk_sizes)
3856 # There shouldn't bee too many small chunks: the protocol header,
3857 # the frame headers and the large string headers are written
3858 # in small chunks.
3859 self.assertLessEqual(len(small_sizes),
3860 len(large_sizes) + len(medium_sizes) + 3,
3861 chunk_sizes)
3862
3863 def test_nested_names(self):
3864 if self.py_version < (3, 4):
3865 self.skipTest('not supported in Python < 3.4')
3866 # required protocol 4 in Python 3.4
3867 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3868 if self.py_version < (3, 5) and proto < 4:
3869 continue
3870 for obj in [Nested.A, Nested.A.B, Nested.A.B.C]:
3871 with self.subTest(proto=proto, obj=obj):
3872 unpickled = self.loads(self.dumps(obj, proto))
3873 self.assertIs(obj, unpickled)
3874
3875 def test_recursive_nested_names(self):
3876 global Recursive
3877 class Recursive:
3878 pass
3879 Recursive.mod = sys.modules[Recursive.__module__]
3880 Recursive.__qualname__ = 'Recursive.mod.Recursive'
3881 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3882 with self.subTest(proto=proto):
3883 unpickled = self.loads(self.dumps(Recursive, proto))
3884 self.assertIs(unpickled, Recursive)
3885 del Recursive.mod # break reference loop
3886
3887 def test_recursive_nested_names2(self):
3888 global Recursive
3889 class Recursive:
3890 pass
3891 Recursive.ref = Recursive
3892 Recursive.__qualname__ = 'Recursive.ref'
3893 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3894 with self.subTest(proto=proto):
3895 unpickled = self.loads(self.dumps(Recursive, proto))
3896 self.assertIs(unpickled, Recursive)
3897 del Recursive.ref # break reference loop
3898
3899 def test_py_methods(self):
3900 if self.py_version < (3, 4):
3901 self.skipTest('not supported in Python < 3.4')
3902 py_methods = (
3903 PyMethodsTest.wine,
3904 PyMethodsTest().biscuits,
3905 )
3906 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3907 for method in py_methods:
3908 with self.subTest(proto=proto, method=method):
3909 unpickled = self.loads(self.dumps(method, proto))
3910 self.assertEqual(method(), unpickled())
3911
3912 # required protocol 4 in Python 3.4
3913 py_methods = (
3914 PyMethodsTest.cheese,
3915 PyMethodsTest.Nested.ketchup,
3916 PyMethodsTest.Nested.maple,
3917 PyMethodsTest.Nested().pie
3918 )
3919 py_unbound_methods = (
3920 (PyMethodsTest.biscuits, PyMethodsTest),
3921 (PyMethodsTest.Nested.pie, PyMethodsTest.Nested)
3922 )
3923 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3924 if self.py_version < (3, 5) and proto < 4:
3925 continue
3926 for method in py_methods:
3927 with self.subTest(proto=proto, method=method):
3928 unpickled = self.loads(self.dumps(method, proto))
3929 self.assertEqual(method(), unpickled())
3930 for method, cls in py_unbound_methods:
3931 obj = cls()
3932 with self.subTest(proto=proto, method=method):
3933 unpickled = self.loads(self.dumps(method, proto))
3934 self.assertEqual(method(obj), unpickled(obj))
3935
3936 descriptors = (
3937 PyMethodsTest.__dict__['cheese'], # static method descriptor
3938 PyMethodsTest.__dict__['wine'], # class method descriptor
3939 )
3940 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3941 for descr in descriptors:
3942 with self.subTest(proto=proto, descr=descr):
3943 self.assertRaises(TypeError, self.dumps, descr, proto)
3944
3945 def test_c_methods(self):
3946 if self.py_version < (3, 4):
3947 self.skipTest('not supported in Python < 3.4')
3948 c_methods = (
3949 # bound built-in method
3950 ("abcd".index, ("c",)),
3951 # unbound built-in method
3952 (str.index, ("abcd", "c")),
3953 # bound "slot" method
3954 ([1, 2, 3].__len__, ()),
3955 # unbound "slot" method
3956 (list.__len__, ([1, 2, 3],)),
3957 # bound "coexist" method
3958 ({1, 2}.__contains__, (2,)),
3959 # unbound "coexist" method
3960 (set.__contains__, ({1, 2}, 2)),
3961 # built-in class method
3962 (dict.fromkeys, (("a", 1), ("b", 2))),
3963 # built-in static method
3964 (bytearray.maketrans, (b"abc", b"xyz")),
3965 # subclass methods
3966 (Subclass([1,2,2]).count, (2,)),
3967 (Subclass.count, (Subclass([1,2,2]), 2)),
3968 (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")),
3969 )
3970 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3971 for method, args in c_methods:
3972 with self.subTest(proto=proto, method=method):
3973 unpickled = self.loads(self.dumps(method, proto))
3974 self.assertEqual(method(*args), unpickled(*args))
3975
3976 # required protocol 4 in Python 3.4
3977 c_methods = (
3978 (Subclass.Nested("sweet").count, ("e",)),
3979 )
3980 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3981 if self.py_version < (3, 5) and proto < 4:
3982 continue
3983 for method, args in c_methods:
3984 with self.subTest(proto=proto, method=method):
3985 unpickled = self.loads(self.dumps(method, proto))
3986 self.assertEqual(method(*args), unpickled(*args))
3987
3988 descriptors = (
3989 bytearray.__dict__['maketrans'], # built-in static method descriptor
3990 dict.__dict__['fromkeys'], # built-in class method descriptor
3991 )
3992 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3993 for descr in descriptors:
3994 with self.subTest(proto=proto, descr=descr):
3995 self.assertRaises(TypeError, self.dumps, descr, proto)
3996
3997 def test_object_with_attrs(self):
3998 obj = Object()
3999 obj.a = 1
4000 for proto in protocols:
4001 with self.subTest(proto=proto):
4002 unpickled = self.loads(self.dumps(obj, proto))
4003 self.assertEqual(unpickled.a, obj.a)
4004
4005 def test_object_with_slots(self):
4006 obj = WithSlots()
4007 obj.a = 1
4008 self.assertRaises(TypeError, self.dumps, obj, 0)
4009 self.assertRaises(TypeError, self.dumps, obj, 1)
4010 for proto in protocols[2:]:
4011 with self.subTest(proto=proto):
4012 unpickled = self.loads(self.dumps(obj, proto))
4013 self.assertEqual(unpickled.a, obj.a)
4014 self.assertNotHasAttr(unpickled, 'b')
4015
4016 obj = WithSlotsSubclass()
4017 obj.a = 1
4018 obj.c = 2
4019 self.assertRaises(TypeError, self.dumps, obj, 0)
4020 self.assertRaises(TypeError, self.dumps, obj, 1)
4021 for proto in protocols[2:]:
4022 with self.subTest(proto=proto):
4023 unpickled = self.loads(self.dumps(obj, proto))
4024 self.assertEqual(unpickled.a, obj.a)
4025 self.assertEqual(unpickled.c, obj.c)
4026 self.assertNotHasAttr(unpickled, 'b')
4027
4028 obj = WithSlotsAndDict()
4029 obj.a = 1
4030 obj.c = 2
4031 self.assertRaises(TypeError, self.dumps, obj, 0)
4032 self.assertRaises(TypeError, self.dumps, obj, 1)
4033 for proto in protocols[2:]:
4034 with self.subTest(proto=proto):
4035 unpickled = self.loads(self.dumps(obj, proto))
4036 self.assertEqual(unpickled.a, obj.a)
4037 self.assertEqual(unpickled.c, obj.c)
4038 self.assertEqual(unpickled.__dict__, obj.__dict__)
4039 self.assertNotHasAttr(unpickled, 'b')
4040
4041 def test_object_with_private_attrs(self):
4042 obj = WithPrivateAttrs(1)
4043 for proto in protocols:
4044 with self.subTest(proto=proto):
4045 unpickled = self.loads(self.dumps(obj, proto))
4046 self.assertEqual(unpickled.get(), obj.get())
4047
4048 obj = WithPrivateAttrsSubclass(1, 2)
4049 for proto in protocols:
4050 with self.subTest(proto=proto):
4051 unpickled = self.loads(self.dumps(obj, proto))
4052 self.assertEqual(unpickled.get(), obj.get())
4053 self.assertEqual(unpickled.get2(), obj.get2())
4054
4055 def test_object_with_private_slots(self):
4056 obj = WithPrivateSlots(1)
4057 self.assertRaises(TypeError, self.dumps, obj, 0)
4058 self.assertRaises(TypeError, self.dumps, obj, 1)
4059 for proto in protocols[2:]:
4060 with self.subTest(proto=proto):
4061 unpickled = self.loads(self.dumps(obj, proto))
4062 self.assertEqual(unpickled.get(), obj.get())
4063
4064 obj = WithPrivateSlotsSubclass(1, 2)
4065 self.assertRaises(TypeError, self.dumps, obj, 0)
4066 self.assertRaises(TypeError, self.dumps, obj, 1)
4067 for proto in protocols[2:]:
4068 with self.subTest(proto=proto):
4069 unpickled = self.loads(self.dumps(obj, proto))
4070 self.assertEqual(unpickled.get(), obj.get())
4071 self.assertEqual(unpickled.get2(), obj.get2())
4072
4073 def test_compat_pickle(self):
4074 if self.py_version < (3, 4):
4075 self.skipTest("doesn't work in Python < 3.4'")
4076 tests = [
4077 (range(1, 7), '__builtin__', 'xrange'),
4078 (map(int, '123'), 'itertools', 'imap'),
4079 (functools.reduce, '__builtin__', 'reduce'),
4080 (dbm.whichdb, 'whichdb', 'whichdb'),
4081 (Exception(), 'exceptions', 'Exception'),
4082 (collections.UserDict(), 'UserDict', 'IterableUserDict'),
4083 (collections.UserList(), 'UserList', 'UserList'),
4084 (collections.defaultdict(), 'collections', 'defaultdict'),
4085 ]
4086 for val, mod, name in tests:
4087 for proto in range(3):
4088 with self.subTest(type=type(val), proto=proto):
4089 pickled = self.dumps(val, proto)
4090 self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled)
4091 self.assertIs(type(self.loads(pickled)), type(val))
4092
4093 #
4094 # PEP 574 tests below
4095 #
4096
4097 def buffer_like_objects(self):
4098 # Yield buffer-like objects with the bytestring "abcdef" in them
4099 bytestring = b"abcdefgh"
4100 yield ZeroCopyBytes(bytestring)
4101 yield ZeroCopyBytearray(bytestring)
4102 if _testbuffer is not None:
4103 items = list(bytestring)
4104 value = int.from_bytes(bytestring, byteorder='little')
4105 for flags in (0, _testbuffer.ND_WRITABLE):
4106 # 1-D, contiguous
4107 yield PicklableNDArray(items, format='B', shape=(8,),
4108 flags=flags)
4109 # 2-D, C-contiguous
4110 yield PicklableNDArray(items, format='B', shape=(4, 2),
4111 strides=(2, 1), flags=flags)
4112 # 2-D, Fortran-contiguous
4113 yield PicklableNDArray(items, format='B',
4114 shape=(4, 2), strides=(1, 4),
4115 flags=flags)
4116
4117 def test_in_band_buffers(self):
4118 # Test in-band buffers (PEP 574)
4119 for obj in self.buffer_like_objects():
4120 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
4121 data = self.dumps(obj, proto)
4122 if obj.c_contiguous and proto >= 5:
4123 # The raw memory bytes are serialized in physical order
4124 self.assertIn(b"abcdefgh", data)
4125 self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 0)
4126 if proto >= 5:
4127 self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data),
4128 1 if obj.readonly else 0)
4129 self.assertEqual(count_opcode(pickle.BYTEARRAY8, data),
4130 0 if obj.readonly else 1)
4131 # Return a true value from buffer_callback should have
4132 # the same effect
4133 def buffer_callback(obj):
4134 return True
4135 data2 = self.dumps(obj, proto,
4136 buffer_callback=buffer_callback)
4137 self.assertEqual(data2, data)
4138
4139 new = self.loads(data)
4140 # It's a copy
4141 self.assertIsNot(new, obj)
4142 self.assertIs(type(new), type(obj))
4143 self.assertEqual(new, obj)
4144
4145 # XXX Unfortunately cannot test non-contiguous array
4146 # (see comment in PicklableNDArray.__reduce_ex__)
4147
4148 def test_oob_buffers(self):
4149 # Test out-of-band buffers (PEP 574)
4150 for obj in self.buffer_like_objects():
4151 for proto in range(0, 5):
4152 # Need protocol >= 5 for buffer_callback
4153 with self.assertRaises(ValueError):
4154 self.dumps(obj, proto,
4155 buffer_callback=[].append)
4156 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
4157 buffers = []
4158 buffer_callback = lambda pb: buffers.append(pb.raw())
4159 data = self.dumps(obj, proto,
4160 buffer_callback=buffer_callback)
4161 self.assertNotIn(b"abcdefgh", data)
4162 self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data), 0)
4163 self.assertEqual(count_opcode(pickle.BYTEARRAY8, data), 0)
4164 self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 1)
4165 self.assertEqual(count_opcode(pickle.READONLY_BUFFER, data),
4166 1 if obj.readonly else 0)
4167
4168 if obj.c_contiguous:
4169 self.assertEqual(bytes(buffers[0]), b"abcdefgh")
4170 # Need buffers argument to unpickle properly
4171 with self.assertRaises(pickle.UnpicklingError):
4172 self.loads(data)
4173
4174 new = self.loads(data, buffers=buffers)
4175 if obj.zero_copy_reconstruct:
4176 # Zero-copy achieved
4177 self.assertIs(new, obj)
4178 else:
4179 self.assertIs(type(new), type(obj))
4180 self.assertEqual(new, obj)
4181 # Non-sequence buffers accepted too
4182 new = self.loads(data, buffers=iter(buffers))
4183 if obj.zero_copy_reconstruct:
4184 # Zero-copy achieved
4185 self.assertIs(new, obj)
4186 else:
4187 self.assertIs(type(new), type(obj))
4188 self.assertEqual(new, obj)
4189
4190 def test_oob_buffers_writable_to_readonly(self):
4191 # Test reconstructing readonly object from writable buffer
4192 obj = ZeroCopyBytes(b"foobar")
4193 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
4194 buffers = []
4195 buffer_callback = buffers.append
4196 data = self.dumps(obj, proto, buffer_callback=buffer_callback)
4197
4198 buffers = map(bytearray, buffers)
4199 new = self.loads(data, buffers=buffers)
4200 self.assertIs(type(new), type(obj))
4201 self.assertEqual(new, obj)
4202
4203 def test_buffers_error(self):
4204 pb = pickle.PickleBuffer(b"foobar")
4205 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
4206 data = self.dumps(pb, proto, buffer_callback=[].append)
4207 # Non iterable buffers
4208 with self.assertRaises(TypeError):
4209 self.loads(data, buffers=object())
4210 # Buffer iterable exhausts too early
4211 with self.assertRaises(pickle.UnpicklingError):
4212 self.loads(data, buffers=[])
4213
4214 def test_inband_accept_default_buffers_argument(self):
4215 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
4216 data_pickled = self.dumps(1, proto, buffer_callback=None)
4217 data = self.loads(data_pickled, buffers=None)
4218
4219 @unittest.skipIf(np is None, "Test needs Numpy")
4220 def test_buffers_numpy(self):
4221 def check_no_copy(x, y):
4222 np.testing.assert_equal(x, y)
4223 self.assertEqual(x.ctypes.data, y.ctypes.data)
4224
4225 def check_copy(x, y):
4226 np.testing.assert_equal(x, y)
4227 self.assertNotEqual(x.ctypes.data, y.ctypes.data)
4228
4229 def check_array(arr):
4230 # In-band
4231 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
4232 data = self.dumps(arr, proto)
4233 new = self.loads(data)
4234 check_copy(arr, new)
4235 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
4236 buffer_callback = lambda _: True
4237 data = self.dumps(arr, proto, buffer_callback=buffer_callback)
4238 new = self.loads(data)
4239 check_copy(arr, new)
4240 # Out-of-band
4241 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
4242 buffers = []
4243 buffer_callback = buffers.append
4244 data = self.dumps(arr, proto, buffer_callback=buffer_callback)
4245 new = self.loads(data, buffers=buffers)
4246 if arr.flags.c_contiguous or arr.flags.f_contiguous:
4247 check_no_copy(arr, new)
4248 else:
4249 check_copy(arr, new)
4250
4251 # 1-D
4252 arr = np.arange(6)
4253 check_array(arr)
4254 # 1-D, non-contiguous
4255 check_array(arr[::2])
4256 # 2-D, C-contiguous
4257 arr = np.arange(12).reshape((3, 4))
4258 check_array(arr)
4259 # 2-D, F-contiguous
4260 check_array(arr.T)
4261 # 2-D, non-contiguous
4262 check_array(arr[::2])
4263
4264 def test_concurrent_mutation_in_buffer_with_bytearray(self):
4265 def factory():
4266 s = b"a" * 16
4267 return bytearray(s), s
4268 self.do_test_concurrent_mutation_in_buffer_callback(factory)
4269
4270 def test_concurrent_mutation_in_buffer_with_memoryview(self):
4271 def factory():
4272 obj = memoryview(b"a" * 32)[10:26]
4273 sub = b"a" * len(obj)
4274 return obj, sub
4275 self.do_test_concurrent_mutation_in_buffer_callback(factory)
4276
4277 def do_test_concurrent_mutation_in_buffer_callback(self, factory):
4278 # See: https://github.com/python/cpython/issues/143308.
4279 class R:
4280 def __bool__(self):
4281 buf.release()
4282 return True
4283
4284 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
4285 obj, sub = factory()
4286 buf = pickle.PickleBuffer(obj)
4287 buffer_callback = lambda _: R()
4288
4289 with self.subTest(proto=proto, obj=obj, sub=sub):
4290 res = self.dumps(buf, proto, buffer_callback=buffer_callback)
4291 self.assertIn(sub, res)
4292
4293 def test_evil_class_mutating_dict(self):
4294 # https://github.com/python/cpython/issues/92930
4295 from random import getrandbits
4296
4297 global Bad
4298 class Bad:
4299 def __eq__(self, other):
4300 return ENABLED
4301 def __hash__(self):
4302 return 42
4303 def __reduce__(self):
4304 if getrandbits(6) == 0:
4305 collection.clear()
4306 return (Bad, ())
4307
4308 for proto in protocols:
4309 for _ in range(20):
4310 ENABLED = False
4311 collection = {Bad(): Bad() for _ in range(20)}
4312 for bad in collection:
4313 bad.bad = bad
4314 bad.collection = collection
4315 ENABLED = True
4316 try:
4317 data = self.dumps(collection, proto)
4318 self.loads(data)
4319 except RuntimeError as e:
4320 expected = "changed size during iteration"
4321 self.assertIn(expected, str(e))
4322
4323 def fast_save_enter(self, create_data, minprotocol=0):
4324 # gh-146059: Check that fast_save_leave() is called when
4325 # fast_save_enter() is called.
4326 if not hasattr(self, "pickler"):
4327 self.skipTest("need Pickler class")
4328
4329 data = [create_data(i) for i in range(FAST_NESTING_LIMIT * 2)]
4330 protocols = range(minprotocol, pickle.HIGHEST_PROTOCOL + 1)
4331 for proto in protocols:
4332 with self.subTest(proto=proto):
4333 buf = io.BytesIO()
4334 pickler = self.pickler(buf, protocol=proto)
4335 # Enable fast mode (disables memo, enables cycle detection)
4336 pickler.fast = 1
4337 pickler.dump(data)
4338
4339 buf.seek(0)
4340 data2 = self.unpickler(buf).load()
4341 self.assertEqual(data2, data)
4342
4343 def test_fast_save_enter_tuple(self):
4344 self.fast_save_enter(lambda i: (i,))
4345
4346 def test_fast_save_enter_list(self):
4347 self.fast_save_enter(lambda i: [i])
4348
4349 def test_fast_save_enter_frozenset(self):
4350 self.fast_save_enter(lambda i: frozenset([i]))
4351
4352 def test_fast_save_enter_set(self):
4353 self.fast_save_enter(lambda i: set([i]))
4354
4355 def test_fast_save_enter_frozendict(self):
4356 if self.py_version < (3, 15):
4357 self.skipTest('need frozendict')
4358 self.fast_save_enter(lambda i: frozendict(key=i), minprotocol=2)
4359
4360 def test_fast_save_enter_dict(self):
4361 self.fast_save_enter(lambda i: {"key": i})
4362
4363 def deep_nested_struct(self, create_nested,
4364 minprotocol=0, compare_equal=True,
4365 depth=FAST_NESTING_LIMIT * 2):
4366 # gh-146059: Check that fast_save_leave() is called when
4367 # fast_save_enter() is called.
4368 if not hasattr(self, "pickler"):
4369 self.skipTest("need Pickler class")
4370
4371 data = None
4372 for i in range(depth):
4373 data = create_nested(data)
4374 protocols = range(minprotocol, pickle.HIGHEST_PROTOCOL + 1)
4375 for proto in protocols:
4376 with self.subTest(proto=proto):
4377 buf = io.BytesIO()
4378 pickler = self.pickler(buf, protocol=proto)
4379 # Enable fast mode (disables memo, enables cycle detection)
4380 pickler.fast = 1
4381 pickler.dump(data)
4382
4383 buf.seek(0)
4384 data2 = self.unpickler(buf).load()
4385 if compare_equal:
4386 self.assertEqual(data2, data)
4387
4388 def test_deep_nested_struct_tuple(self):
4389 self.deep_nested_struct(lambda data: (data,))
4390
4391 def test_deep_nested_struct_list(self):
4392 self.deep_nested_struct(lambda data: [data])
4393
4394 def test_deep_nested_struct_frozenset(self):
4395 self.deep_nested_struct(lambda data: frozenset((1, data)))
4396
4397 def test_deep_nested_struct_set(self):
4398 self.deep_nested_struct(lambda data: {K(data)},
4399 depth=FAST_NESTING_LIMIT+1,
4400 compare_equal=False)
4401
4402 def test_deep_nested_struct_frozendict(self):
4403 if self.py_version < (3, 15):
4404 self.skipTest('need frozendict')
4405 self.deep_nested_struct(lambda data: frozendict(x=data),
4406 minprotocol=2)
4407
4408 def test_deep_nested_struct_dict(self):
4409 self.deep_nested_struct(lambda data: {'x': data})
4410
4411
4412 class BigmemPickleTests:
4413
4414 # Binary protocols can serialize longs of up to 2 GiB-1
4415
4416 @bigmemtest(size=_2G, memuse=3.6, dry_run=False)
4417 def test_huge_long_32b(self, size):
4418 data = 1 << (8 * size)
4419 try:
4420 for proto in protocols:
4421 if proto < 2:
4422 continue
4423 with self.subTest(proto=proto):
4424 with self.assertRaises((ValueError, OverflowError)):
4425 self.dumps(data, protocol=proto)
4426 finally:
4427 data = None
4428
4429 # Protocol 3 can serialize up to 4 GiB-1 as a bytes object
4430 # (older protocols don't have a dedicated opcode for bytes and are
4431 # too inefficient)
4432
4433 @bigmemtest(size=_2G, memuse=2.5, dry_run=False)
4434 def test_huge_bytes_32b(self, size):
4435 data = b"abcd" * (size // 4)
4436 try:
4437 for proto in protocols:
4438 if proto < 3:
4439 continue
4440 with self.subTest(proto=proto):
4441 try:
4442 pickled = self.dumps(data, protocol=proto)
4443 header = (pickle.BINBYTES +
4444 struct.pack("<I", len(data)))
4445 data_start = pickled.index(data)
4446 self.assertEqual(
4447 header,
4448 pickled[data_start-len(header):data_start])
4449 finally:
4450 pickled = None
4451 finally:
4452 data = None
4453
4454 @bigmemtest(size=_4G, memuse=2.5, dry_run=False)
4455 def test_huge_bytes_64b(self, size):
4456 data = b"acbd" * (size // 4)
4457 try:
4458 for proto in protocols:
4459 if proto < 3:
4460 continue
4461 with self.subTest(proto=proto):
4462 if proto == 3:
4463 # Protocol 3 does not support large bytes objects.
4464 # Verify that we do not crash when processing one.
4465 with self.assertRaises((ValueError, OverflowError)):
4466 self.dumps(data, protocol=proto)
4467 continue
4468 try:
4469 pickled = self.dumps(data, protocol=proto)
4470 header = (pickle.BINBYTES8 +
4471 struct.pack("<Q", len(data)))
4472 data_start = pickled.index(data)
4473 self.assertEqual(
4474 header,
4475 pickled[data_start-len(header):data_start])
4476 finally:
4477 pickled = None
4478 finally:
4479 data = None
4480
4481 # All protocols use 1-byte per printable ASCII character; we add another
4482 # byte because the encoded form has to be copied into the internal buffer.
4483
4484 @bigmemtest(size=_2G, memuse=8, dry_run=False)
4485 def test_huge_str_32b(self, size):
4486 data = "abcd" * (size // 4)
4487 try:
4488 for proto in protocols:
4489 if proto == 0:
4490 continue
4491 with self.subTest(proto=proto):
4492 try:
4493 pickled = self.dumps(data, protocol=proto)
4494 header = (pickle.BINUNICODE +
4495 struct.pack("<I", len(data)))
4496 data_start = pickled.index(b'abcd')
4497 self.assertEqual(
4498 header,
4499 pickled[data_start-len(header):data_start])
4500 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
4501 pickled.index(b"abcd")), len(data))
4502 finally:
4503 pickled = None
4504 finally:
4505 data = None
4506
4507 # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes
4508 # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge
4509 # unicode strings however.
4510
4511 @bigmemtest(size=_4G, memuse=8, dry_run=False)
4512 def test_huge_str_64b(self, size):
4513 data = "abcd" * (size // 4)
4514 try:
4515 for proto in protocols:
4516 if proto == 0:
4517 continue
4518 with self.subTest(proto=proto):
4519 if proto < 4:
4520 with self.assertRaises((ValueError, OverflowError)):
4521 self.dumps(data, protocol=proto)
4522 continue
4523 try:
4524 pickled = self.dumps(data, protocol=proto)
4525 header = (pickle.BINUNICODE8 +
4526 struct.pack("<Q", len(data)))
4527 data_start = pickled.index(b'abcd')
4528 self.assertEqual(
4529 header,
4530 pickled[data_start-len(header):data_start])
4531 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
4532 pickled.index(b"abcd")), len(data))
4533 finally:
4534 pickled = None
4535 finally:
4536 data = None
4537
4538
4539 class CustomError(Exception):
4540 pass
4541
4542 class Unpickleable:
4543 def __reduce__(self):
4544 raise CustomError
4545
4546 UNPICKLEABLE = Unpickleable()
4547
4548 # For test_unpickleable_reconstructor and test_unpickleable_state_setter
4549 class UnpickleableCallable(Unpickleable):
4550 def __call__(self, *args, **kwargs):
4551 pass
4552
4553 # For test_bad_getattr
4554 class BadGetattr:
4555 def __getattr__(self, key):
4556 self.foo
4557
4558 # For test_bad_newobj_class and test_bad_newobj_ex__class
4559 class NoNew:
4560 def __getattribute__(self, name):
4561 if name == '__new__':
4562 raise AttributeError
4563 return super().__getattribute__(name)
4564
4565
4566 class AbstractPickleModuleTests:
4567
4568 def test_dump_closed_file(self):
4569 f = open(TESTFN, "wb")
4570 try:
4571 f.close()
4572 self.assertRaises(ValueError, self.dump, 123, f)
4573 finally:
4574 os_helper.unlink(TESTFN)
4575
4576 def test_load_closed_file(self):
4577 f = open(TESTFN, "wb")
4578 try:
4579 f.close()
4580 self.assertRaises(ValueError, self.dump, 123, f)
4581 finally:
4582 os_helper.unlink(TESTFN)
4583
4584 def test_load_from_and_dump_to_file(self):
4585 stream = io.BytesIO()
4586 data = [123, {}, 124]
4587 self.dump(data, stream)
4588 stream.seek(0)
4589 unpickled = self.load(stream)
4590 self.assertEqual(unpickled, data)
4591
4592 def test_highest_protocol(self):
4593 # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
4594 self.assertEqual(pickle.HIGHEST_PROTOCOL, 5)
4595
4596 def test_callapi(self):
4597 f = io.BytesIO()
4598 # With and without keyword arguments
4599 self.dump(123, f, -1)
4600 self.dump(123, file=f, protocol=-1)
4601 self.dumps(123, -1)
4602 self.dumps(123, protocol=-1)
4603 self.Pickler(f, -1)
4604 self.Pickler(f, protocol=-1)
4605
4606 def test_dump_text_file(self):
4607 f = open(TESTFN, "w")
4608 try:
4609 for proto in protocols:
4610 self.assertRaises(TypeError, self.dump, 123, f, proto)
4611 finally:
4612 f.close()
4613 os_helper.unlink(TESTFN)
4614
4615 def test_incomplete_input(self):
4616 s = io.BytesIO(b"X''.")
4617 self.assertRaises((EOFError, struct.error, pickle.UnpicklingError), self.load, s)
4618
4619 def test_bad_init(self):
4620 # Test issue3664 (pickle can segfault from a badly initialized Pickler).
4621 # Override initialization without calling __init__() of the superclass.
4622 class BadPickler(self.Pickler):
4623 def __init__(self): pass
4624
4625 class BadUnpickler(self.Unpickler):
4626 def __init__(self): pass
4627
4628 self.assertRaises(pickle.PicklingError, BadPickler().dump, 0)
4629 self.assertRaises(pickle.UnpicklingError, BadUnpickler().load)
4630
4631 def test_unpickler_bad_file(self):
4632 # bpo-38384: Crash in _pickle if the read attribute raises an error.
4633 def raises_oserror(self, *args, **kwargs):
4634 raise OSError
4635 @property
4636 def bad_property(self):
4637 raise CustomError
4638
4639 # File without read and readline
4640 class F:
4641 pass
4642 self.assertRaises((AttributeError, TypeError), self.Unpickler, F())
4643
4644 # File without read
4645 class F:
4646 readline = raises_oserror
4647 self.assertRaises((AttributeError, TypeError), self.Unpickler, F())
4648
4649 # File without readline
4650 class F:
4651 read = raises_oserror
4652 self.assertRaises((AttributeError, TypeError), self.Unpickler, F())
4653
4654 # File with bad read
4655 class F:
4656 read = bad_property
4657 readline = raises_oserror
4658 self.assertRaises(CustomError, self.Unpickler, F())
4659
4660 # File with bad readline
4661 class F:
4662 readline = bad_property
4663 read = raises_oserror
4664 self.assertRaises(CustomError, self.Unpickler, F())
4665
4666 # File with bad readline, no read
4667 class F:
4668 readline = bad_property
4669 self.assertRaises(CustomError, self.Unpickler, F())
4670
4671 # File with bad read, no readline
4672 class F:
4673 read = bad_property
4674 self.assertRaises((AttributeError, CustomError), self.Unpickler, F())
4675
4676 # File with bad peek
4677 class F:
4678 peek = bad_property
4679 read = raises_oserror
4680 readline = raises_oserror
4681 try:
4682 self.Unpickler(F())
4683 except CustomError:
4684 pass
4685
4686 # File with bad readinto
4687 class F:
4688 readinto = bad_property
4689 read = raises_oserror
4690 readline = raises_oserror
4691 try:
4692 self.Unpickler(F())
4693 except CustomError:
4694 pass
4695
4696 def test_pickler_bad_file(self):
4697 # File without write
4698 class F:
4699 pass
4700 self.assertRaises(TypeError, self.Pickler, F())
4701
4702 # File with bad write
4703 class F:
4704 @property
4705 def write(self):
4706 raise CustomError
4707 self.assertRaises(CustomError, self.Pickler, F())
4708
4709 def check_dumps_loads_oob_buffers(self, dumps, loads):
4710 # No need to do the full gamut of tests here, just enough to
4711 # check that dumps() and loads() redirect their arguments
4712 # to the underlying Pickler and Unpickler, respectively.
4713 obj = ZeroCopyBytes(b"foo")
4714
4715 for proto in range(0, 5):
4716 # Need protocol >= 5 for buffer_callback
4717 with self.assertRaises(ValueError):
4718 dumps(obj, protocol=proto,
4719 buffer_callback=[].append)
4720 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
4721 buffers = []
4722 buffer_callback = buffers.append
4723 data = dumps(obj, protocol=proto,
4724 buffer_callback=buffer_callback)
4725 self.assertNotIn(b"foo", data)
4726 self.assertEqual(bytes(buffers[0]), b"foo")
4727 # Need buffers argument to unpickle properly
4728 with self.assertRaises(pickle.UnpicklingError):
4729 loads(data)
4730 new = loads(data, buffers=buffers)
4731 self.assertIs(new, obj)
4732
4733 def test_dumps_loads_oob_buffers(self):
4734 # Test out-of-band buffers (PEP 574) with top-level dumps() and loads()
4735 self.check_dumps_loads_oob_buffers(self.dumps, self.loads)
4736
4737 def test_dump_load_oob_buffers(self):
4738 # Test out-of-band buffers (PEP 574) with top-level dump() and load()
4739 def dumps(obj, **kwargs):
4740 f = io.BytesIO()
4741 self.dump(obj, f, **kwargs)
4742 return f.getvalue()
4743
4744 def loads(data, **kwargs):
4745 f = io.BytesIO(data)
4746 return self.load(f, **kwargs)
4747
4748 self.check_dumps_loads_oob_buffers(dumps, loads)
4749
4750
4751 class AbstractPersistentPicklerTests:
4752
4753 # This class defines persistent_id() and persistent_load()
4754 # functions that should be used by the pickler. All even integers
4755 # are pickled using persistent ids.
4756
4757 def persistent_id(self, object):
4758 if isinstance(object, int) and object % 2 == 0:
4759 self.id_count += 1
4760 return str(object)
4761 elif object == "test_false_value":
4762 self.false_count += 1
4763 return ""
4764 else:
4765 return None
4766
4767 def persistent_load(self, oid):
4768 if not oid:
4769 self.load_false_count += 1
4770 return "test_false_value"
4771 else:
4772 self.load_count += 1
4773 object = int(oid)
4774 assert object % 2 == 0
4775 return object
4776
4777 def test_persistence(self):
4778 L = list(range(10)) + ["test_false_value"]
4779 for proto in protocols:
4780 self.id_count = 0
4781 self.false_count = 0
4782 self.load_false_count = 0
4783 self.load_count = 0
4784 self.assertEqual(self.loads(self.dumps(L, proto)), L)
4785 self.assertEqual(self.id_count, 5)
4786 self.assertEqual(self.false_count, 1)
4787 self.assertEqual(self.load_count, 5)
4788 self.assertEqual(self.load_false_count, 1)
4789
4790
4791 class AbstractIdentityPersistentPicklerTests:
4792
4793 def persistent_id(self, obj):
4794 return obj
4795
4796 def persistent_load(self, pid):
4797 return pid
4798
4799 def _check_return_correct_type(self, obj, proto):
4800 unpickled = self.loads(self.dumps(obj, proto))
4801 self.assertIsInstance(unpickled, type(obj))
4802 self.assertEqual(unpickled, obj)
4803
4804 def test_return_correct_type(self):
4805 for proto in protocols:
4806 # Protocol 0 supports only ASCII strings.
4807 if proto == 0:
4808 self._check_return_correct_type("abc", 0)
4809 else:
4810 for obj in [b"abc\n", "abc\n", -1, -1.1 * 0.1, str]:
4811 self._check_return_correct_type(obj, proto)
4812
4813 def test_protocol0_is_ascii_only(self):
4814 non_ascii_str = "\N{EMPTY SET}"
4815 with self.assertRaises(pickle.PicklingError) as cm:
4816 self.dumps(non_ascii_str, 0)
4817 self.assertEqual(str(cm.exception),
4818 'persistent IDs in protocol 0 must be ASCII strings')
4819 pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.'
4820 with self.assertRaises(pickle.UnpicklingError) as cm:
4821 self.loads(pickled)
4822 self.assertEqual(str(cm.exception),
4823 'persistent IDs in protocol 0 must be ASCII strings')
4824
4825
4826 class AbstractPicklerUnpicklerObjectTests:
4827
4828 pickler_class = None
4829 unpickler_class = None
4830
4831 def setUp(self):
4832 assert self.pickler_class
4833 assert self.unpickler_class
4834
4835 def test_clear_pickler_memo(self):
4836 # To test whether clear_memo() has any effect, we pickle an object,
4837 # then pickle it again without clearing the memo; the two serialized
4838 # forms should be different. If we clear_memo() and then pickle the
4839 # object again, the third serialized form should be identical to the
4840 # first one we obtained.
4841 data = ["abcdefg", "abcdefg", 44]
4842 for proto in protocols:
4843 f = io.BytesIO()
4844 pickler = self.pickler_class(f, proto)
4845
4846 pickler.dump(data)
4847 first_pickled = f.getvalue()
4848
4849 # Reset BytesIO object.
4850 f.seek(0)
4851 f.truncate()
4852
4853 pickler.dump(data)
4854 second_pickled = f.getvalue()
4855
4856 # Reset the Pickler and BytesIO objects.
4857 pickler.clear_memo()
4858 f.seek(0)
4859 f.truncate()
4860
4861 pickler.dump(data)
4862 third_pickled = f.getvalue()
4863
4864 self.assertNotEqual(first_pickled, second_pickled)
4865 self.assertEqual(first_pickled, third_pickled)
4866
4867 def test_priming_pickler_memo(self):
4868 # Verify that we can set the Pickler's memo attribute.
4869 data = ["abcdefg", "abcdefg", 44]
4870 f = io.BytesIO()
4871 pickler = self.pickler_class(f)
4872
4873 pickler.dump(data)
4874 first_pickled = f.getvalue()
4875
4876 f = io.BytesIO()
4877 primed = self.pickler_class(f)
4878 primed.memo = pickler.memo
4879
4880 primed.dump(data)
4881 primed_pickled = f.getvalue()
4882
4883 self.assertNotEqual(first_pickled, primed_pickled)
4884
4885 def test_priming_unpickler_memo(self):
4886 # Verify that we can set the Unpickler's memo attribute.
4887 data = ["abcdefg", "abcdefg", 44]
4888 f = io.BytesIO()
4889 pickler = self.pickler_class(f)
4890
4891 pickler.dump(data)
4892 first_pickled = f.getvalue()
4893
4894 f = io.BytesIO()
4895 primed = self.pickler_class(f)
4896 primed.memo = pickler.memo
4897
4898 primed.dump(data)
4899 primed_pickled = f.getvalue()
4900
4901 unpickler = self.unpickler_class(io.BytesIO(first_pickled))
4902 unpickled_data1 = unpickler.load()
4903
4904 self.assertEqual(unpickled_data1, data)
4905
4906 primed = self.unpickler_class(io.BytesIO(primed_pickled))
4907 primed.memo = unpickler.memo
4908 unpickled_data2 = primed.load()
4909
4910 primed.memo.clear()
4911
4912 self.assertEqual(unpickled_data2, data)
4913 self.assertTrue(unpickled_data2 is unpickled_data1)
4914
4915 def test_reusing_unpickler_objects(self):
4916 data1 = ["abcdefg", "abcdefg", 44]
4917 f = io.BytesIO()
4918 pickler = self.pickler_class(f)
4919 pickler.dump(data1)
4920 pickled1 = f.getvalue()
4921
4922 data2 = ["abcdefg", 44, 44]
4923 f = io.BytesIO()
4924 pickler = self.pickler_class(f)
4925 pickler.dump(data2)
4926 pickled2 = f.getvalue()
4927
4928 f = io.BytesIO()
4929 f.write(pickled1)
4930 f.seek(0)
4931 unpickler = self.unpickler_class(f)
4932 self.assertEqual(unpickler.load(), data1)
4933
4934 f.seek(0)
4935 f.truncate()
4936 f.write(pickled2)
4937 f.seek(0)
4938 self.assertEqual(unpickler.load(), data2)
4939
4940 def _check_multiple_unpicklings(self, ioclass, *, seekable=True):
4941 for proto in protocols:
4942 with self.subTest(proto=proto):
4943 data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len]
4944 f = ioclass()
4945 pickler = self.pickler_class(f, protocol=proto)
4946 pickler.dump(data1)
4947 pickled = f.getvalue()
4948
4949 N = 5
4950 f = ioclass(pickled * N)
4951 unpickler = self.unpickler_class(f)
4952 for i in range(N):
4953 if seekable:
4954 pos = f.tell()
4955 self.assertEqual(unpickler.load(), data1)
4956 if seekable:
4957 self.assertEqual(f.tell(), pos + len(pickled))
4958 self.assertRaises(EOFError, unpickler.load)
4959
4960 def test_multiple_unpicklings_seekable(self):
4961 self._check_multiple_unpicklings(io.BytesIO)
4962
4963 def test_multiple_unpicklings_unseekable(self):
4964 self._check_multiple_unpicklings(UnseekableIO, seekable=False)
4965
4966 def test_multiple_unpicklings_minimal(self):
4967 # File-like object that doesn't support peek() and readinto()
4968 # (bpo-39681)
4969 self._check_multiple_unpicklings(MinimalIO, seekable=False)
4970
4971 def test_unpickling_buffering_readline(self):
4972 # Issue #12687: the unpickler's buffering logic could fail with
4973 # text mode opcodes.
4974 data = list(range(10))
4975 for proto in protocols:
4976 for buf_size in range(1, 11):
4977 f = io.BufferedRandom(io.BytesIO(), buffer_size=buf_size)
4978 pickler = self.pickler_class(f, protocol=proto)
4979 pickler.dump(data)
4980 f.seek(0)
4981 unpickler = self.unpickler_class(f)
4982 self.assertEqual(unpickler.load(), data)
4983
4984 def test_pickle_invalid_reducer_override(self):
4985 # gh-103035
4986 obj = object()
4987
4988 f = io.BytesIO()
4989 class MyPickler(self.pickler_class):
4990 pass
4991 pickler = MyPickler(f)
4992 pickler.dump(obj)
4993
4994 pickler.clear_memo()
4995 pickler.reducer_override = None
4996 with self.assertRaises(TypeError):
4997 pickler.dump(obj)
4998
4999 pickler.clear_memo()
5000 pickler.reducer_override = 10
5001 with self.assertRaises(TypeError):
5002 pickler.dump(obj)
5003
5004 # Tests for dispatch_table attribute
5005
5006 REDUCE_A = 'reduce_A'
5007
5008 class AAA(object):
5009 def __reduce__(self):
5010 return str, (REDUCE_A,)
5011
5012 class BBB(object):
5013 def __init__(self):
5014 # Add an instance attribute to enable state-saving routines at pickling
5015 # time.
5016 self.a = "some attribute"
5017
5018 def __setstate__(self, state):
5019 self.a = "BBB.__setstate__"
5020
5021
5022 def setstate_bbb(obj, state):
5023 """Custom state setter for BBB objects
5024
5025 Such callable may be created by other persons than the ones who created the
5026 BBB class. If passed as the state_setter item of a custom reducer, this
5027 allows for custom state setting behavior of BBB objects. One can think of
5028 it as the analogous of list_setitems or dict_setitems but for foreign
5029 classes/functions.
5030 """
5031 obj.a = "custom state_setter"
5032
5033
5034
5035 class AbstractCustomPicklerClass:
5036 """Pickler implementing a reducing hook using reducer_override."""
5037 def reducer_override(self, obj):
5038 obj_name = getattr(obj, "__name__", None)
5039
5040 if obj_name == 'f':
5041 # asking the pickler to save f as 5
5042 return int, (5, )
5043
5044 if obj_name == 'MyClass':
5045 return str, ('some str',)
5046
5047 elif obj_name == 'g':
5048 # in this case, the callback returns an invalid result (not a 2-5
5049 # tuple or a string), the pickler should raise a proper error.
5050 return False
5051
5052 elif obj_name == 'h':
5053 # Simulate a case when the reducer fails. The error should
5054 # be propagated to the original ``dump`` call.
5055 raise ValueError('The reducer just failed')
5056
5057 return NotImplemented
5058
5059 class AbstractHookTests:
5060 def test_pickler_hook(self):
5061 # test the ability of a custom, user-defined CPickler subclass to
5062 # override the default reducing routines of any type using the method
5063 # reducer_override
5064
5065 def f():
5066 pass
5067
5068 def g():
5069 pass
5070
5071 def h():
5072 pass
5073
5074 class MyClass:
5075 pass
5076
5077 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
5078 with self.subTest(proto=proto):
5079 bio = io.BytesIO()
5080 p = self.pickler_class(bio, proto)
5081
5082 p.dump([f, MyClass, math.log])
5083 new_f, some_str, math_log = pickle.loads(bio.getvalue())
5084
5085 self.assertEqual(new_f, 5)
5086 self.assertEqual(some_str, 'some str')
5087 # math.log does not have its usual reducer overridden, so the
5088 # custom reduction callback should silently direct the pickler
5089 # to the default pickling by attribute, by returning
5090 # NotImplemented
5091 self.assertIs(math_log, math.log)
5092
5093 with self.assertRaises(pickle.PicklingError) as cm:
5094 p.dump(g)
5095 self.assertRegex(str(cm.exception),
5096 r'(__reduce__|<bound method .*reducer_override.*>)'
5097 r' must return (a )?string or tuple')
5098
5099 with self.assertRaisesRegex(
5100 ValueError, 'The reducer just failed'):
5101 p.dump(h)
5102
5103 @support.cpython_only
5104 def test_reducer_override_no_reference_cycle(self):
5105 # bpo-39492: reducer_override used to induce a spurious reference cycle
5106 # inside the Pickler object, that could prevent all serialized objects
5107 # from being garbage-collected without explicitly invoking gc.collect.
5108
5109 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
5110 with self.subTest(proto=proto):
5111 def f():
5112 pass
5113
5114 wr = weakref.ref(f)
5115
5116 bio = io.BytesIO()
5117 p = self.pickler_class(bio, proto)
5118 p.dump(f)
5119 new_f = pickle.loads(bio.getvalue())
5120 assert new_f == 5
5121
5122 del p
5123 del f
5124
5125 self.assertIsNone(wr())
5126
5127
5128 class AbstractDispatchTableTests:
5129
5130 def test_default_dispatch_table(self):
5131 # No dispatch_table attribute by default
5132 f = io.BytesIO()
5133 p = self.pickler_class(f, 0)
5134 with self.assertRaises(AttributeError):
5135 p.dispatch_table
5136 self.assertNotHasAttr(p, 'dispatch_table')
5137
5138 def test_class_dispatch_table(self):
5139 # A dispatch_table attribute can be specified class-wide
5140 dt = self.get_dispatch_table()
5141
5142 class MyPickler(self.pickler_class):
5143 dispatch_table = dt
5144
5145 def dumps(obj, protocol=None):
5146 f = io.BytesIO()
5147 p = MyPickler(f, protocol)
5148 self.assertEqual(p.dispatch_table, dt)
5149 p.dump(obj)
5150 return f.getvalue()
5151
5152 self._test_dispatch_table(dumps, dt)
5153
5154 def test_instance_dispatch_table(self):
5155 # A dispatch_table attribute can also be specified instance-wide
5156 dt = self.get_dispatch_table()
5157
5158 def dumps(obj, protocol=None):
5159 f = io.BytesIO()
5160 p = self.pickler_class(f, protocol)
5161 p.dispatch_table = dt
5162 self.assertEqual(p.dispatch_table, dt)
5163 p.dump(obj)
5164 return f.getvalue()
5165
5166 self._test_dispatch_table(dumps, dt)
5167
5168 def test_dispatch_table_None_item(self):
5169 # gh-93627
5170 obj = object()
5171 f = io.BytesIO()
5172 pickler = self.pickler_class(f)
5173 pickler.dispatch_table = {type(obj): None}
5174 with self.assertRaises(TypeError):
5175 pickler.dump(obj)
5176
5177 def _test_dispatch_table(self, dumps, dispatch_table):
5178 def custom_load_dump(obj):
5179 return pickle.loads(dumps(obj, 0))
5180
5181 def default_load_dump(obj):
5182 return pickle.loads(pickle.dumps(obj, 0))
5183
5184 # pickling complex numbers using protocol 0 relies on copyreg
5185 # so check pickling a complex number still works
5186 z = 1 + 2j
5187 self.assertEqual(custom_load_dump(z), z)
5188 self.assertEqual(default_load_dump(z), z)
5189
5190 # modify pickling of complex
5191 REDUCE_1 = 'reduce_1'
5192 def reduce_1(obj):
5193 return str, (REDUCE_1,)
5194 dispatch_table[complex] = reduce_1
5195 self.assertEqual(custom_load_dump(z), REDUCE_1)
5196 self.assertEqual(default_load_dump(z), z)
5197
5198 # check picklability of AAA and BBB
5199 a = AAA()
5200 b = BBB()
5201 self.assertEqual(custom_load_dump(a), REDUCE_A)
5202 self.assertIsInstance(custom_load_dump(b), BBB)
5203 self.assertEqual(default_load_dump(a), REDUCE_A)
5204 self.assertIsInstance(default_load_dump(b), BBB)
5205
5206 # modify pickling of BBB
5207 dispatch_table[BBB] = reduce_1
5208 self.assertEqual(custom_load_dump(a), REDUCE_A)
5209 self.assertEqual(custom_load_dump(b), REDUCE_1)
5210 self.assertEqual(default_load_dump(a), REDUCE_A)
5211 self.assertIsInstance(default_load_dump(b), BBB)
5212
5213 # revert pickling of BBB and modify pickling of AAA
5214 REDUCE_2 = 'reduce_2'
5215 def reduce_2(obj):
5216 return str, (REDUCE_2,)
5217 dispatch_table[AAA] = reduce_2
5218 del dispatch_table[BBB]
5219 self.assertEqual(custom_load_dump(a), REDUCE_2)
5220 self.assertIsInstance(custom_load_dump(b), BBB)
5221 self.assertEqual(default_load_dump(a), REDUCE_A)
5222 self.assertIsInstance(default_load_dump(b), BBB)
5223
5224 # End-to-end testing of save_reduce with the state_setter keyword
5225 # argument. This is a dispatch_table test as the primary goal of
5226 # state_setter is to tweak objects reduction behavior.
5227 # In particular, state_setter is useful when the default __setstate__
5228 # behavior is not flexible enough.
5229
5230 # No custom reducer for b has been registered for now, so
5231 # BBB.__setstate__ should be used at unpickling time
5232 self.assertEqual(default_load_dump(b).a, "BBB.__setstate__")
5233
5234 def reduce_bbb(obj):
5235 return BBB, (), obj.__dict__, None, None, setstate_bbb
5236
5237 dispatch_table[BBB] = reduce_bbb
5238
5239 # The custom reducer reduce_bbb includes a state setter, that should
5240 # have priority over BBB.__setstate__
5241 self.assertEqual(custom_load_dump(b).a, "custom state_setter")
5242
5243
5244 if __name__ == "__main__":
5245 # Print some stuff that can be used to rewrite DATA{0,1,2}
5246 from pickletools import dis
5247 x = create_data()
5248 for i in range(pickle.HIGHEST_PROTOCOL+1):
5249 p = pickle.dumps(x, i)
5250 print("DATA{0} = (".format(i))
5251 for j in range(0, len(p), 20):
5252 b = bytes(p[j:j+20])
5253 print(" {0!r}".format(b))
5254 print(")")
5255 print()
5256 print("# Disassembly of DATA{0}".format(i))
5257 print("DATA{0}_DIS = \"\"\"\\".format(i))
5258 dis(p)
5259 print("\"\"\"")
5260 print()