| 1 |
from _compat_pickle import (IMPORT_MAPPING, REVERSE_IMPORT_MAPPING, |
| 2 |
NAME_MAPPING, REVERSE_NAME_MAPPING) |
| 3 |
import builtins |
| 4 |
import collections |
| 5 |
import contextlib |
| 6 |
import io |
| 7 |
import pickle |
| 8 |
import struct |
| 9 |
import sys |
| 10 |
import tempfile |
| 11 |
import warnings |
| 12 |
import weakref |
| 13 |
from textwrap import dedent |
| 14 |
|
| 15 |
import doctest |
| 16 |
import unittest |
| 17 |
from test import support |
| 18 |
from test.support import cpython_only, import_helper, os_helper |
| 19 |
from test.support.import_helper import ensure_lazy_imports |
| 20 |
|
| 21 |
from test.pickletester import AbstractHookTests |
| 22 |
from test.pickletester import AbstractUnpickleTests |
| 23 |
from test.pickletester import AbstractPicklingErrorTests |
| 24 |
from test.pickletester import AbstractPickleTests |
| 25 |
from test.pickletester import AbstractPickleModuleTests |
| 26 |
from test.pickletester import AbstractPersistentPicklerTests |
| 27 |
from test.pickletester import AbstractIdentityPersistentPicklerTests |
| 28 |
from test.pickletester import AbstractPicklerUnpicklerObjectTests |
| 29 |
from test.pickletester import AbstractDispatchTableTests |
| 30 |
from test.pickletester import AbstractCustomPicklerClass |
| 31 |
from test.pickletester import BigmemPickleTests |
| 32 |
|
| 33 |
try: |
| 34 |
import _pickle |
| 35 |
has_c_implementation = True |
| 36 |
except ImportError: |
| 37 |
has_c_implementation = False |
| 38 |
|
| 39 |
|
| 40 |
class LazyImportTest(unittest.TestCase): |
| 41 |
@cpython_only |
| 42 |
def test_lazy_import(self): |
| 43 |
ensure_lazy_imports("pickle", {"re"}) |
| 44 |
|
| 45 |
|
| 46 |
class PyPickleTests(AbstractPickleModuleTests, unittest.TestCase): |
| 47 |
dump = staticmethod(pickle._dump) |
| 48 |
dumps = staticmethod(pickle._dumps) |
| 49 |
load = staticmethod(pickle._load) |
| 50 |
loads = staticmethod(pickle._loads) |
| 51 |
Pickler = pickle._Pickler |
| 52 |
Unpickler = pickle._Unpickler |
| 53 |
|
| 54 |
|
| 55 |
class PyUnpicklerTests(AbstractUnpickleTests, unittest.TestCase): |
| 56 |
|
| 57 |
unpickler = pickle._Unpickler |
| 58 |
bad_stack_errors = (IndexError,) |
| 59 |
truncated_errors = (pickle.UnpicklingError, EOFError, |
| 60 |
AttributeError, ValueError, |
| 61 |
struct.error, IndexError, ImportError) |
| 62 |
|
| 63 |
def loads(self, buf, **kwds): |
| 64 |
f = io.BytesIO(buf) |
| 65 |
u = self.unpickler(f, **kwds) |
| 66 |
return u.load() |
| 67 |
|
| 68 |
|
| 69 |
class PyPicklingErrorTests(AbstractPicklingErrorTests, unittest.TestCase): |
| 70 |
|
| 71 |
pickler = pickle._Pickler |
| 72 |
|
| 73 |
def dumps(self, arg, proto=None, **kwargs): |
| 74 |
f = io.BytesIO() |
| 75 |
p = self.pickler(f, proto, **kwargs) |
| 76 |
p.dump(arg) |
| 77 |
f.seek(0) |
| 78 |
return bytes(f.read()) |
| 79 |
|
| 80 |
|
| 81 |
class PyPicklerTests(AbstractPickleTests, unittest.TestCase): |
| 82 |
|
| 83 |
pickler = pickle._Pickler |
| 84 |
unpickler = pickle._Unpickler |
| 85 |
|
| 86 |
def dumps(self, arg, proto=None, **kwargs): |
| 87 |
f = io.BytesIO() |
| 88 |
p = self.pickler(f, proto, **kwargs) |
| 89 |
p.dump(arg) |
| 90 |
f.seek(0) |
| 91 |
return bytes(f.read()) |
| 92 |
|
| 93 |
def loads(self, buf, **kwds): |
| 94 |
f = io.BytesIO(buf) |
| 95 |
u = self.unpickler(f, **kwds) |
| 96 |
return u.load() |
| 97 |
|
| 98 |
|
| 99 |
class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, |
| 100 |
BigmemPickleTests, unittest.TestCase): |
| 101 |
|
| 102 |
bad_stack_errors = (pickle.UnpicklingError, IndexError) |
| 103 |
truncated_errors = (pickle.UnpicklingError, EOFError, |
| 104 |
AttributeError, ValueError, |
| 105 |
struct.error, IndexError, ImportError) |
| 106 |
|
| 107 |
def dumps(self, arg, protocol=None, **kwargs): |
| 108 |
return pickle.dumps(arg, protocol, **kwargs) |
| 109 |
|
| 110 |
def loads(self, buf, **kwds): |
| 111 |
return pickle.loads(buf, **kwds) |
| 112 |
|
| 113 |
test_framed_write_sizes_with_delayed_writer = None |
| 114 |
test_find_class = None |
| 115 |
test_custom_find_class = None |
| 116 |
|
| 117 |
|
| 118 |
class PersistentPicklerUnpicklerMixin(object): |
| 119 |
|
| 120 |
def dumps(self, arg, proto=None): |
| 121 |
class PersPickler(self.pickler): |
| 122 |
def persistent_id(subself, obj): |
| 123 |
return self.persistent_id(obj) |
| 124 |
f = io.BytesIO() |
| 125 |
p = PersPickler(f, proto) |
| 126 |
p.dump(arg) |
| 127 |
return f.getvalue() |
| 128 |
|
| 129 |
def loads(self, buf, **kwds): |
| 130 |
class PersUnpickler(self.unpickler): |
| 131 |
def persistent_load(subself, obj): |
| 132 |
return self.persistent_load(obj) |
| 133 |
f = io.BytesIO(buf) |
| 134 |
u = PersUnpickler(f, **kwds) |
| 135 |
return u.load() |
| 136 |
|
| 137 |
|
| 138 |
class PyPersPicklerTests(AbstractPersistentPicklerTests, |
| 139 |
PersistentPicklerUnpicklerMixin, unittest.TestCase): |
| 140 |
|
| 141 |
pickler = pickle._Pickler |
| 142 |
unpickler = pickle._Unpickler |
| 143 |
|
| 144 |
|
| 145 |
class PyIdPersPicklerTests(AbstractIdentityPersistentPicklerTests, |
| 146 |
PersistentPicklerUnpicklerMixin, unittest.TestCase): |
| 147 |
|
| 148 |
pickler = pickle._Pickler |
| 149 |
unpickler = pickle._Unpickler |
| 150 |
persistent_load_error = pickle.UnpicklingError |
| 151 |
|
| 152 |
@support.cpython_only |
| 153 |
def test_pickler_reference_cycle(self): |
| 154 |
def check(Pickler): |
| 155 |
for proto in range(pickle.HIGHEST_PROTOCOL + 1): |
| 156 |
f = io.BytesIO() |
| 157 |
pickler = Pickler(f, proto) |
| 158 |
pickler.dump('abc') |
| 159 |
self.assertEqual(self.loads(f.getvalue()), 'abc') |
| 160 |
pickler = Pickler(io.BytesIO()) |
| 161 |
self.assertEqual(pickler.persistent_id('def'), 'def') |
| 162 |
r = weakref.ref(pickler) |
| 163 |
del pickler |
| 164 |
self.assertIsNone(r()) |
| 165 |
|
| 166 |
class PersPickler(self.pickler): |
| 167 |
def persistent_id(subself, obj): |
| 168 |
return obj |
| 169 |
check(PersPickler) |
| 170 |
|
| 171 |
class PersPickler(self.pickler): |
| 172 |
@classmethod |
| 173 |
def persistent_id(cls, obj): |
| 174 |
return obj |
| 175 |
check(PersPickler) |
| 176 |
|
| 177 |
class PersPickler(self.pickler): |
| 178 |
@staticmethod |
| 179 |
def persistent_id(obj): |
| 180 |
return obj |
| 181 |
check(PersPickler) |
| 182 |
|
| 183 |
@support.cpython_only |
| 184 |
def test_custom_pickler_dispatch_table_memleak(self): |
| 185 |
# See https://github.com/python/cpython/issues/89988 |
| 186 |
|
| 187 |
class Pickler(self.pickler): |
| 188 |
def __init__(self, *args, **kwargs): |
| 189 |
self.dispatch_table = table |
| 190 |
super().__init__(*args, **kwargs) |
| 191 |
|
| 192 |
class DispatchTable: |
| 193 |
pass |
| 194 |
|
| 195 |
table = DispatchTable() |
| 196 |
pickler = Pickler(io.BytesIO()) |
| 197 |
self.assertIs(pickler.dispatch_table, table) |
| 198 |
table_ref = weakref.ref(table) |
| 199 |
self.assertIsNotNone(table_ref()) |
| 200 |
del pickler |
| 201 |
del table |
| 202 |
support.gc_collect() |
| 203 |
self.assertIsNone(table_ref()) |
| 204 |
|
| 205 |
@support.cpython_only |
| 206 |
def test_unpickler_reference_cycle(self): |
| 207 |
def check(Unpickler): |
| 208 |
for proto in range(pickle.HIGHEST_PROTOCOL + 1): |
| 209 |
unpickler = Unpickler(io.BytesIO(self.dumps('abc', proto))) |
| 210 |
self.assertEqual(unpickler.load(), 'abc') |
| 211 |
unpickler = Unpickler(io.BytesIO()) |
| 212 |
self.assertEqual(unpickler.persistent_load('def'), 'def') |
| 213 |
r = weakref.ref(unpickler) |
| 214 |
del unpickler |
| 215 |
self.assertIsNone(r()) |
| 216 |
|
| 217 |
class PersUnpickler(self.unpickler): |
| 218 |
def persistent_load(subself, pid): |
| 219 |
return pid |
| 220 |
check(PersUnpickler) |
| 221 |
|
| 222 |
class PersUnpickler(self.unpickler): |
| 223 |
@classmethod |
| 224 |
def persistent_load(cls, pid): |
| 225 |
return pid |
| 226 |
check(PersUnpickler) |
| 227 |
|
| 228 |
class PersUnpickler(self.unpickler): |
| 229 |
@staticmethod |
| 230 |
def persistent_load(pid): |
| 231 |
return pid |
| 232 |
check(PersUnpickler) |
| 233 |
|
| 234 |
def test_pickler_super(self): |
| 235 |
class PersPickler(self.pickler): |
| 236 |
def persistent_id(subself, obj): |
| 237 |
called.append(obj) |
| 238 |
self.assertIsNone(super().persistent_id(obj)) |
| 239 |
return obj |
| 240 |
|
| 241 |
for proto in range(pickle.HIGHEST_PROTOCOL + 1): |
| 242 |
f = io.BytesIO() |
| 243 |
pickler = PersPickler(f, proto) |
| 244 |
called = [] |
| 245 |
pickler.dump('abc') |
| 246 |
self.assertEqual(called, ['abc']) |
| 247 |
self.assertEqual(self.loads(f.getvalue()), 'abc') |
| 248 |
|
| 249 |
def test_unpickler_super(self): |
| 250 |
class PersUnpickler(self.unpickler): |
| 251 |
def persistent_load(subself, pid): |
| 252 |
called.append(pid) |
| 253 |
with self.assertRaises(self.persistent_load_error): |
| 254 |
super().persistent_load(pid) |
| 255 |
return pid |
| 256 |
|
| 257 |
for proto in range(pickle.HIGHEST_PROTOCOL + 1): |
| 258 |
unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) |
| 259 |
called = [] |
| 260 |
self.assertEqual(unpickler.load(), 'abc') |
| 261 |
self.assertEqual(called, ['abc']) |
| 262 |
|
| 263 |
def test_pickler_instance_attribute(self): |
| 264 |
def persistent_id(obj): |
| 265 |
called.append(obj) |
| 266 |
return obj |
| 267 |
|
| 268 |
for proto in range(pickle.HIGHEST_PROTOCOL + 1): |
| 269 |
f = io.BytesIO() |
| 270 |
pickler = self.pickler(f, proto) |
| 271 |
called = [] |
| 272 |
old_persistent_id = pickler.persistent_id |
| 273 |
pickler.persistent_id = persistent_id |
| 274 |
self.assertEqual(pickler.persistent_id, persistent_id) |
| 275 |
pickler.dump('abc') |
| 276 |
self.assertEqual(called, ['abc']) |
| 277 |
self.assertEqual(self.loads(f.getvalue()), 'abc') |
| 278 |
del pickler.persistent_id |
| 279 |
self.assertEqual(pickler.persistent_id, old_persistent_id) |
| 280 |
|
| 281 |
def test_unpickler_instance_attribute(self): |
| 282 |
def persistent_load(pid): |
| 283 |
called.append(pid) |
| 284 |
return pid |
| 285 |
|
| 286 |
for proto in range(pickle.HIGHEST_PROTOCOL + 1): |
| 287 |
unpickler = self.unpickler(io.BytesIO(self.dumps('abc', proto))) |
| 288 |
called = [] |
| 289 |
old_persistent_load = unpickler.persistent_load |
| 290 |
unpickler.persistent_load = persistent_load |
| 291 |
self.assertEqual(unpickler.persistent_load, persistent_load) |
| 292 |
self.assertEqual(unpickler.load(), 'abc') |
| 293 |
self.assertEqual(called, ['abc']) |
| 294 |
del unpickler.persistent_load |
| 295 |
self.assertEqual(unpickler.persistent_load, old_persistent_load) |
| 296 |
|
| 297 |
def test_pickler_super_instance_attribute(self): |
| 298 |
class PersPickler(self.pickler): |
| 299 |
def persistent_id(subself, obj): |
| 300 |
raise AssertionError('should never be called') |
| 301 |
def _persistent_id(subself, obj): |
| 302 |
called.append(obj) |
| 303 |
self.assertIsNone(super().persistent_id(obj)) |
| 304 |
return obj |
| 305 |
|
| 306 |
for proto in range(pickle.HIGHEST_PROTOCOL + 1): |
| 307 |
f = io.BytesIO() |
| 308 |
pickler = PersPickler(f, proto) |
| 309 |
called = [] |
| 310 |
old_persistent_id = pickler.persistent_id |
| 311 |
pickler.persistent_id = pickler._persistent_id |
| 312 |
self.assertEqual(pickler.persistent_id, pickler._persistent_id) |
| 313 |
pickler.dump('abc') |
| 314 |
self.assertEqual(called, ['abc']) |
| 315 |
self.assertEqual(self.loads(f.getvalue()), 'abc') |
| 316 |
del pickler.persistent_id |
| 317 |
self.assertEqual(pickler.persistent_id, old_persistent_id) |
| 318 |
|
| 319 |
def test_unpickler_super_instance_attribute(self): |
| 320 |
class PersUnpickler(self.unpickler): |
| 321 |
def persistent_load(subself, pid): |
| 322 |
raise AssertionError('should never be called') |
| 323 |
def _persistent_load(subself, pid): |
| 324 |
called.append(pid) |
| 325 |
with self.assertRaises(self.persistent_load_error): |
| 326 |
super().persistent_load(pid) |
| 327 |
return pid |
| 328 |
|
| 329 |
for proto in range(pickle.HIGHEST_PROTOCOL + 1): |
| 330 |
unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) |
| 331 |
called = [] |
| 332 |
old_persistent_load = unpickler.persistent_load |
| 333 |
unpickler.persistent_load = unpickler._persistent_load |
| 334 |
self.assertEqual(unpickler.persistent_load, unpickler._persistent_load) |
| 335 |
self.assertEqual(unpickler.load(), 'abc') |
| 336 |
self.assertEqual(called, ['abc']) |
| 337 |
del unpickler.persistent_load |
| 338 |
self.assertEqual(unpickler.persistent_load, old_persistent_load) |
| 339 |
|
| 340 |
|
| 341 |
class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests, unittest.TestCase): |
| 342 |
|
| 343 |
pickler_class = pickle._Pickler |
| 344 |
unpickler_class = pickle._Unpickler |
| 345 |
|
| 346 |
|
| 347 |
class PyDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): |
| 348 |
|
| 349 |
pickler_class = pickle._Pickler |
| 350 |
|
| 351 |
def get_dispatch_table(self): |
| 352 |
return pickle.dispatch_table.copy() |
| 353 |
|
| 354 |
|
| 355 |
class PyChainDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): |
| 356 |
|
| 357 |
pickler_class = pickle._Pickler |
| 358 |
|
| 359 |
def get_dispatch_table(self): |
| 360 |
return collections.ChainMap({}, pickle.dispatch_table) |
| 361 |
|
| 362 |
|
| 363 |
class PyPicklerHookTests(AbstractHookTests, unittest.TestCase): |
| 364 |
class CustomPyPicklerClass(pickle._Pickler, |
| 365 |
AbstractCustomPicklerClass): |
| 366 |
pass |
| 367 |
pickler_class = CustomPyPicklerClass |
| 368 |
|
| 369 |
|
| 370 |
if has_c_implementation: |
| 371 |
class CPickleTests(AbstractPickleModuleTests, unittest.TestCase): |
| 372 |
from _pickle import dump, dumps, load, loads, Pickler, Unpickler |
| 373 |
|
| 374 |
class CUnpicklerTests(PyUnpicklerTests): |
| 375 |
unpickler = _pickle.Unpickler |
| 376 |
bad_stack_errors = (pickle.UnpicklingError,) |
| 377 |
truncated_errors = (pickle.UnpicklingError,) |
| 378 |
|
| 379 |
class CPicklingErrorTests(PyPicklingErrorTests): |
| 380 |
pickler = _pickle.Pickler |
| 381 |
|
| 382 |
class CPicklerTests(PyPicklerTests): |
| 383 |
pickler = _pickle.Pickler |
| 384 |
unpickler = _pickle.Unpickler |
| 385 |
|
| 386 |
class CPersPicklerTests(PyPersPicklerTests): |
| 387 |
pickler = _pickle.Pickler |
| 388 |
unpickler = _pickle.Unpickler |
| 389 |
|
| 390 |
class CIdPersPicklerTests(PyIdPersPicklerTests): |
| 391 |
pickler = _pickle.Pickler |
| 392 |
unpickler = _pickle.Unpickler |
| 393 |
persistent_load_error = _pickle.UnpicklingError |
| 394 |
|
| 395 |
class CDumpPickle_LoadPickle(PyPicklerTests): |
| 396 |
pickler = _pickle.Pickler |
| 397 |
unpickler = pickle._Unpickler |
| 398 |
|
| 399 |
class DumpPickle_CLoadPickle(PyPicklerTests): |
| 400 |
pickler = pickle._Pickler |
| 401 |
unpickler = _pickle.Unpickler |
| 402 |
|
| 403 |
class CPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests, unittest.TestCase): |
| 404 |
pickler_class = _pickle.Pickler |
| 405 |
unpickler_class = _pickle.Unpickler |
| 406 |
|
| 407 |
def test_issue18339(self): |
| 408 |
unpickler = self.unpickler_class(io.BytesIO()) |
| 409 |
with self.assertRaises(TypeError): |
| 410 |
unpickler.memo = object |
| 411 |
# used to cause a segfault |
| 412 |
with self.assertRaises(ValueError): |
| 413 |
unpickler.memo = {-1: None} |
| 414 |
unpickler.memo = {1: None} |
| 415 |
|
| 416 |
def test_concurrent_pickler_dump(self): |
| 417 |
f = io.BytesIO() |
| 418 |
pickler = self.pickler_class(f) |
| 419 |
class X: |
| 420 |
def __reduce__(slf): |
| 421 |
self.assertRaises(RuntimeError, pickler.dump, 42) |
| 422 |
return list, () |
| 423 |
pickler.dump(X()) # should not crash |
| 424 |
self.assertEqual(pickle.loads(f.getvalue()), []) |
| 425 |
|
| 426 |
def test_concurrent_pickler_dump_and_init(self): |
| 427 |
f = io.BytesIO() |
| 428 |
pickler = self.pickler_class(f) |
| 429 |
class X: |
| 430 |
def __reduce__(slf): |
| 431 |
self.assertRaises(RuntimeError, pickler.__init__, f) |
| 432 |
return list, () |
| 433 |
pickler.dump([X()]) # should not fail |
| 434 |
self.assertEqual(pickle.loads(f.getvalue()), [[]]) |
| 435 |
|
| 436 |
def test_concurrent_unpickler_load(self): |
| 437 |
global reducer |
| 438 |
def reducer(): |
| 439 |
self.assertRaises(RuntimeError, unpickler.load) |
| 440 |
return 42 |
| 441 |
f = io.BytesIO(b'(c%b\nreducer\n(tRl.' % (__name__.encode(),)) |
| 442 |
unpickler = self.unpickler_class(f) |
| 443 |
unpickled = unpickler.load() # should not fail |
| 444 |
self.assertEqual(unpickled, [42]) |
| 445 |
|
| 446 |
def test_concurrent_unpickler_load_and_init(self): |
| 447 |
global reducer |
| 448 |
def reducer(): |
| 449 |
self.assertRaises(RuntimeError, unpickler.__init__, f) |
| 450 |
return 42 |
| 451 |
f = io.BytesIO(b'(c%b\nreducer\n(tRl.' % (__name__.encode(),)) |
| 452 |
unpickler = self.unpickler_class(f) |
| 453 |
unpickled = unpickler.load() # should not crash |
| 454 |
self.assertEqual(unpickled, [42]) |
| 455 |
|
| 456 |
class CDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): |
| 457 |
pickler_class = pickle.Pickler |
| 458 |
def get_dispatch_table(self): |
| 459 |
return pickle.dispatch_table.copy() |
| 460 |
|
| 461 |
class CChainDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): |
| 462 |
pickler_class = pickle.Pickler |
| 463 |
def get_dispatch_table(self): |
| 464 |
return collections.ChainMap({}, pickle.dispatch_table) |
| 465 |
|
| 466 |
class CPicklerHookTests(AbstractHookTests, unittest.TestCase): |
| 467 |
class CustomCPicklerClass(_pickle.Pickler, AbstractCustomPicklerClass): |
| 468 |
pass |
| 469 |
pickler_class = CustomCPicklerClass |
| 470 |
|
| 471 |
@support.cpython_only |
| 472 |
class HeapTypesTests(unittest.TestCase): |
| 473 |
def setUp(self): |
| 474 |
pickler = _pickle.Pickler(io.BytesIO()) |
| 475 |
unpickler = _pickle.Unpickler(io.BytesIO()) |
| 476 |
|
| 477 |
self._types = ( |
| 478 |
_pickle.Pickler, |
| 479 |
_pickle.Unpickler, |
| 480 |
type(pickler.memo), |
| 481 |
type(unpickler.memo), |
| 482 |
|
| 483 |
# We cannot test the _pickle.Pdata; |
| 484 |
# there's no way to get to it. |
| 485 |
) |
| 486 |
|
| 487 |
def test_have_gc(self): |
| 488 |
import gc |
| 489 |
for tp in self._types: |
| 490 |
with self.subTest(tp=tp): |
| 491 |
self.assertTrue(gc.is_tracked(tp)) |
| 492 |
|
| 493 |
def test_immutable(self): |
| 494 |
for tp in self._types: |
| 495 |
with self.subTest(tp=tp): |
| 496 |
with self.assertRaisesRegex(TypeError, "immutable"): |
| 497 |
tp.foo = "bar" |
| 498 |
|
| 499 |
@support.cpython_only |
| 500 |
class SizeofTests(unittest.TestCase): |
| 501 |
check_sizeof = support.check_sizeof |
| 502 |
|
| 503 |
def test_pickler(self): |
| 504 |
basesize = support.calcobjsize('7P2n3i2n4i2P') |
| 505 |
p = _pickle.Pickler(io.BytesIO()) |
| 506 |
self.assertEqual(object.__sizeof__(p), basesize) |
| 507 |
MT_size = struct.calcsize('3nP0n') |
| 508 |
ME_size = struct.calcsize('Pn0P') |
| 509 |
check = self.check_sizeof |
| 510 |
check(p, basesize + |
| 511 |
MT_size + 8 * ME_size + # Minimal memo table size. |
| 512 |
sys.getsizeof(b'x'*4096)) # Minimal write buffer size. |
| 513 |
for i in range(6): |
| 514 |
p.dump(chr(i)) |
| 515 |
check(p, basesize + |
| 516 |
MT_size + 32 * ME_size + # Size of memo table required to |
| 517 |
# save references to 6 objects. |
| 518 |
0) # Write buffer is cleared after every dump(). |
| 519 |
|
| 520 |
def test_unpickler(self): |
| 521 |
basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n3i') |
| 522 |
unpickler = _pickle.Unpickler |
| 523 |
P = struct.calcsize('P') # Size of memo table entry. |
| 524 |
n = struct.calcsize('n') # Size of mark table entry. |
| 525 |
check = self.check_sizeof |
| 526 |
for encoding in 'ASCII', 'UTF-16', 'latin-1': |
| 527 |
for errors in 'strict', 'replace': |
| 528 |
u = unpickler(io.BytesIO(), |
| 529 |
encoding=encoding, errors=errors) |
| 530 |
self.assertEqual(object.__sizeof__(u), basesize) |
| 531 |
check(u, basesize + |
| 532 |
32 * P + # Minimal memo table size. |
| 533 |
len(encoding) + 1 + len(errors) + 1) |
| 534 |
|
| 535 |
stdsize = basesize + len('ASCII') + 1 + len('strict') + 1 |
| 536 |
def check_unpickler(data, memo_size, marks_size): |
| 537 |
dump = pickle.dumps(data) |
| 538 |
u = unpickler(io.BytesIO(dump), |
| 539 |
encoding='ASCII', errors='strict') |
| 540 |
u.load() |
| 541 |
check(u, stdsize + memo_size * P + marks_size * n) |
| 542 |
|
| 543 |
check_unpickler(0, 32, 0) |
| 544 |
# 20 is minimal non-empty mark stack size. |
| 545 |
check_unpickler([0] * 100, 32, 20) |
| 546 |
# 128 is memo table size required to save references to 100 objects. |
| 547 |
check_unpickler([chr(i) for i in range(100)], 128, 20) |
| 548 |
def recurse(deep): |
| 549 |
data = 0 |
| 550 |
for i in range(deep): |
| 551 |
data = [data, data] |
| 552 |
return data |
| 553 |
check_unpickler(recurse(0), 32, 0) |
| 554 |
check_unpickler(recurse(1), 32, 20) |
| 555 |
check_unpickler(recurse(20), 32, 20) |
| 556 |
check_unpickler(recurse(50), 64, 60) |
| 557 |
if not (support.is_wasi and support.Py_DEBUG): |
| 558 |
# stack depth too shallow in pydebug WASI. |
| 559 |
check_unpickler(recurse(100), 128, 140) |
| 560 |
|
| 561 |
u = unpickler(io.BytesIO(pickle.dumps('a', 0)), |
| 562 |
encoding='ASCII', errors='strict') |
| 563 |
u.load() |
| 564 |
check(u, stdsize + 32 * P + 2 + 1) |
| 565 |
|
| 566 |
|
| 567 |
ALT_IMPORT_MAPPING = { |
| 568 |
('_elementtree', 'xml.etree.ElementTree'), |
| 569 |
('cPickle', 'pickle'), |
| 570 |
('StringIO', 'io'), |
| 571 |
('cStringIO', 'io'), |
| 572 |
} |
| 573 |
|
| 574 |
ALT_NAME_MAPPING = { |
| 575 |
('__builtin__', 'basestring', 'builtins', 'str'), |
| 576 |
('exceptions', 'StandardError', 'builtins', 'Exception'), |
| 577 |
('UserDict', 'UserDict', 'collections', 'UserDict'), |
| 578 |
('socket', '_socketobject', 'socket', 'SocketType'), |
| 579 |
} |
| 580 |
|
| 581 |
def mapping(module, name): |
| 582 |
if (module, name) in NAME_MAPPING: |
| 583 |
module, name = NAME_MAPPING[(module, name)] |
| 584 |
elif module in IMPORT_MAPPING: |
| 585 |
module = IMPORT_MAPPING[module] |
| 586 |
return module, name |
| 587 |
|
| 588 |
def reverse_mapping(module, name): |
| 589 |
if (module, name) in REVERSE_NAME_MAPPING: |
| 590 |
module, name = REVERSE_NAME_MAPPING[(module, name)] |
| 591 |
elif module in REVERSE_IMPORT_MAPPING: |
| 592 |
module = REVERSE_IMPORT_MAPPING[module] |
| 593 |
return module, name |
| 594 |
|
| 595 |
def getmodule(module): |
| 596 |
try: |
| 597 |
return sys.modules[module] |
| 598 |
except KeyError: |
| 599 |
try: |
| 600 |
with warnings.catch_warnings(): |
| 601 |
action = 'always' if support.verbose else 'ignore' |
| 602 |
warnings.simplefilter(action, DeprecationWarning) |
| 603 |
__import__(module) |
| 604 |
except AttributeError as exc: |
| 605 |
if support.verbose: |
| 606 |
print("Can't import module %r: %s" % (module, exc)) |
| 607 |
raise ImportError |
| 608 |
except ImportError as exc: |
| 609 |
if support.verbose: |
| 610 |
print(exc) |
| 611 |
raise |
| 612 |
return sys.modules[module] |
| 613 |
|
| 614 |
def getattribute(module, name): |
| 615 |
obj = getmodule(module) |
| 616 |
for n in name.split('.'): |
| 617 |
obj = getattr(obj, n) |
| 618 |
return obj |
| 619 |
|
| 620 |
def get_exceptions(mod): |
| 621 |
for name in dir(mod): |
| 622 |
attr = getattr(mod, name) |
| 623 |
if isinstance(attr, type) and issubclass(attr, BaseException): |
| 624 |
yield name, attr |
| 625 |
|
| 626 |
class CompatPickleTests(unittest.TestCase): |
| 627 |
def test_import(self): |
| 628 |
modules = set(IMPORT_MAPPING.values()) |
| 629 |
modules |= set(REVERSE_IMPORT_MAPPING) |
| 630 |
modules |= {module for module, name in REVERSE_NAME_MAPPING} |
| 631 |
modules |= {module for module, name in NAME_MAPPING.values()} |
| 632 |
for module in modules: |
| 633 |
try: |
| 634 |
getmodule(module) |
| 635 |
except ImportError: |
| 636 |
pass |
| 637 |
|
| 638 |
def test_import_mapping(self): |
| 639 |
for module3, module2 in REVERSE_IMPORT_MAPPING.items(): |
| 640 |
with self.subTest((module3, module2)): |
| 641 |
try: |
| 642 |
getmodule(module3) |
| 643 |
except ImportError: |
| 644 |
pass |
| 645 |
if module3[:1] != '_': |
| 646 |
self.assertIn(module2, IMPORT_MAPPING) |
| 647 |
self.assertEqual(IMPORT_MAPPING[module2], module3) |
| 648 |
|
| 649 |
def test_name_mapping(self): |
| 650 |
for (module3, name3), (module2, name2) in REVERSE_NAME_MAPPING.items(): |
| 651 |
with self.subTest(((module3, name3), (module2, name2))): |
| 652 |
if (module2, name2) == ('exceptions', 'OSError'): |
| 653 |
attr = getattribute(module3, name3) |
| 654 |
self.assertIsSubclass(attr, OSError) |
| 655 |
elif (module2, name2) == ('exceptions', 'ImportError'): |
| 656 |
attr = getattribute(module3, name3) |
| 657 |
self.assertIsSubclass(attr, ImportError) |
| 658 |
else: |
| 659 |
module, name = mapping(module2, name2) |
| 660 |
if module3[:1] != '_': |
| 661 |
self.assertEqual((module, name), (module3, name3)) |
| 662 |
try: |
| 663 |
attr = getattribute(module3, name3) |
| 664 |
except ImportError: |
| 665 |
pass |
| 666 |
else: |
| 667 |
self.assertEqual(getattribute(module, name), attr) |
| 668 |
|
| 669 |
def test_reverse_import_mapping(self): |
| 670 |
for module2, module3 in IMPORT_MAPPING.items(): |
| 671 |
with self.subTest((module2, module3)): |
| 672 |
try: |
| 673 |
getmodule(module3) |
| 674 |
except ImportError as exc: |
| 675 |
if support.verbose: |
| 676 |
print(exc) |
| 677 |
if ((module2, module3) not in ALT_IMPORT_MAPPING and |
| 678 |
REVERSE_IMPORT_MAPPING.get(module3, None) != module2): |
| 679 |
for (m3, n3), (m2, n2) in REVERSE_NAME_MAPPING.items(): |
| 680 |
if (module3, module2) == (m3, m2): |
| 681 |
break |
| 682 |
else: |
| 683 |
self.fail('No reverse mapping from %r to %r' % |
| 684 |
(module3, module2)) |
| 685 |
module = REVERSE_IMPORT_MAPPING.get(module3, module3) |
| 686 |
module = IMPORT_MAPPING.get(module, module) |
| 687 |
self.assertEqual(module, module3) |
| 688 |
|
| 689 |
def test_reverse_name_mapping(self): |
| 690 |
for (module2, name2), (module3, name3) in NAME_MAPPING.items(): |
| 691 |
with self.subTest(((module2, name2), (module3, name3))): |
| 692 |
try: |
| 693 |
attr = getattribute(module3, name3) |
| 694 |
except ImportError: |
| 695 |
pass |
| 696 |
module, name = reverse_mapping(module3, name3) |
| 697 |
if (module2, name2, module3, name3) not in ALT_NAME_MAPPING: |
| 698 |
self.assertEqual((module, name), (module2, name2)) |
| 699 |
module, name = mapping(module, name) |
| 700 |
self.assertEqual((module, name), (module3, name3)) |
| 701 |
|
| 702 |
def test_exceptions(self): |
| 703 |
self.assertEqual(mapping('exceptions', 'StandardError'), |
| 704 |
('builtins', 'Exception')) |
| 705 |
self.assertEqual(mapping('exceptions', 'Exception'), |
| 706 |
('builtins', 'Exception')) |
| 707 |
self.assertEqual(reverse_mapping('builtins', 'Exception'), |
| 708 |
('exceptions', 'Exception')) |
| 709 |
self.assertEqual(mapping('exceptions', 'OSError'), |
| 710 |
('builtins', 'OSError')) |
| 711 |
self.assertEqual(reverse_mapping('builtins', 'OSError'), |
| 712 |
('exceptions', 'OSError')) |
| 713 |
|
| 714 |
for name, exc in get_exceptions(builtins): |
| 715 |
with self.subTest(name): |
| 716 |
if exc in (BlockingIOError, |
| 717 |
ResourceWarning, |
| 718 |
StopAsyncIteration, |
| 719 |
PythonFinalizationError, |
| 720 |
RecursionError, |
| 721 |
EncodingWarning, |
| 722 |
BaseExceptionGroup, |
| 723 |
ExceptionGroup, |
| 724 |
_IncompleteInputError): |
| 725 |
continue |
| 726 |
if exc is not OSError and issubclass(exc, OSError): |
| 727 |
self.assertEqual(reverse_mapping('builtins', name), |
| 728 |
('exceptions', 'OSError')) |
| 729 |
elif exc is not ImportError and issubclass(exc, ImportError): |
| 730 |
self.assertEqual(reverse_mapping('builtins', name), |
| 731 |
('exceptions', 'ImportError')) |
| 732 |
self.assertEqual(mapping('exceptions', name), |
| 733 |
('exceptions', name)) |
| 734 |
else: |
| 735 |
self.assertEqual(reverse_mapping('builtins', name), |
| 736 |
('exceptions', name)) |
| 737 |
self.assertEqual(mapping('exceptions', name), |
| 738 |
('builtins', name)) |
| 739 |
|
| 740 |
def test_multiprocessing_exceptions(self): |
| 741 |
module = import_helper.import_module('multiprocessing.context') |
| 742 |
for name, exc in get_exceptions(module): |
| 743 |
if issubclass(exc, Warning): |
| 744 |
continue |
| 745 |
with self.subTest(name): |
| 746 |
self.assertEqual(reverse_mapping('multiprocessing.context', name), |
| 747 |
('multiprocessing', name)) |
| 748 |
self.assertEqual(mapping('multiprocessing', name), |
| 749 |
('multiprocessing.context', name)) |
| 750 |
|
| 751 |
|
| 752 |
class CommandLineTest(unittest.TestCase): |
| 753 |
def setUp(self): |
| 754 |
self.filename = tempfile.mktemp() |
| 755 |
self.addCleanup(os_helper.unlink, self.filename) |
| 756 |
|
| 757 |
@staticmethod |
| 758 |
def text_normalize(string): |
| 759 |
"""Dedent *string* and strip it from its surrounding whitespaces. |
| 760 |
|
| 761 |
This method is used by the other utility functions so that any |
| 762 |
string to write or to match against can be freely indented. |
| 763 |
""" |
| 764 |
return dedent(string).strip() |
| 765 |
|
| 766 |
def set_pickle_data(self, data): |
| 767 |
with open(self.filename, 'wb') as f: |
| 768 |
pickle.dump(data, f) |
| 769 |
|
| 770 |
def invoke_pickle(self, *flags): |
| 771 |
output = io.StringIO() |
| 772 |
with contextlib.redirect_stdout(output): |
| 773 |
pickle._main(args=[*flags, self.filename]) |
| 774 |
return self.text_normalize(output.getvalue()) |
| 775 |
|
| 776 |
def test_invocation(self): |
| 777 |
# test 'python -m pickle pickle_file' |
| 778 |
data = { |
| 779 |
'a': [1, 2.0, 3+4j], |
| 780 |
'b': ('character string', b'byte string'), |
| 781 |
'c': 'string' |
| 782 |
} |
| 783 |
expect = ''' |
| 784 |
{'a': [1, 2.0, (3+4j)], |
| 785 |
'b': ('character string', b'byte string'), |
| 786 |
'c': 'string'} |
| 787 |
''' |
| 788 |
self.set_pickle_data(data) |
| 789 |
|
| 790 |
with self.subTest(data=data): |
| 791 |
res = self.invoke_pickle() |
| 792 |
expect = self.text_normalize(expect) |
| 793 |
self.assertListEqual(res.splitlines(), expect.splitlines()) |
| 794 |
|
| 795 |
@support.force_not_colorized |
| 796 |
def test_unknown_flag(self): |
| 797 |
stderr = io.StringIO() |
| 798 |
with self.assertRaises(SystemExit): |
| 799 |
# check that the parser help is shown |
| 800 |
with contextlib.redirect_stderr(stderr): |
| 801 |
_ = self.invoke_pickle('--unknown') |
| 802 |
self.assertStartsWith(stderr.getvalue(), 'usage: ') |
| 803 |
|
| 804 |
|
| 805 |
def load_tests(loader, tests, pattern): |
| 806 |
tests.addTest(doctest.DocTestSuite(pickle)) |
| 807 |
return tests |
| 808 |
|
| 809 |
|
| 810 |
if __name__ == "__main__": |
| 811 |
unittest.main() |