| 1 |
# This test covers backwards compatibility with previous versions of Python |
| 2 |
# by bouncing pickled objects through Python versions by running xpickle_worker.py. |
| 3 |
import io |
| 4 |
import os |
| 5 |
import pickle |
| 6 |
import struct |
| 7 |
import subprocess |
| 8 |
import sys |
| 9 |
import unittest |
| 10 |
|
| 11 |
|
| 12 |
from test import support |
| 13 |
from test import pickletester |
| 14 |
|
| 15 |
try: |
| 16 |
import _pickle |
| 17 |
has_c_implementation = True |
| 18 |
except ModuleNotFoundError: |
| 19 |
has_c_implementation = False |
| 20 |
|
| 21 |
support.requires('xpickle') |
| 22 |
|
| 23 |
is_windows = sys.platform.startswith('win') |
| 24 |
|
| 25 |
# Map python version to a tuple containing the name of a corresponding valid |
| 26 |
# Python binary to execute and its arguments. |
| 27 |
py_executable_map = {} |
| 28 |
|
| 29 |
protocols_map = { |
| 30 |
3: (3, 0), |
| 31 |
4: (3, 4), |
| 32 |
5: (3, 8), |
| 33 |
} |
| 34 |
|
| 35 |
def highest_proto_for_py_version(py_version): |
| 36 |
"""Finds the highest supported pickle protocol for a given Python version. |
| 37 |
Args: |
| 38 |
py_version: a 2-tuple of the major, minor version. Eg. Python 3.7 would |
| 39 |
be (3, 7) |
| 40 |
Returns: |
| 41 |
int for the highest supported pickle protocol |
| 42 |
""" |
| 43 |
proto = 2 |
| 44 |
for p, v in protocols_map.items(): |
| 45 |
if py_version < v: |
| 46 |
break |
| 47 |
proto = p |
| 48 |
return proto |
| 49 |
|
| 50 |
def have_python_version(py_version): |
| 51 |
"""Check whether a Python binary exists for the given py_version and has |
| 52 |
support. This respects your PATH. |
| 53 |
For Windows, it will first try to use the py launcher specified in PEP 397. |
| 54 |
Otherwise (and for all other platforms), it will attempt to check for |
| 55 |
python<py_version[0]>.<py_version[1]>. |
| 56 |
|
| 57 |
Eg. given a *py_version* of (3, 7), the function will attempt to try |
| 58 |
'py -3.7' (for Windows) first, then 'python3.7', and return |
| 59 |
['py', '-3.7'] (on Windows) or ['python3.7'] on other platforms. |
| 60 |
|
| 61 |
Args: |
| 62 |
py_version: a 2-tuple of the major, minor version. Eg. python 3.7 would |
| 63 |
be (3, 7) |
| 64 |
Returns: |
| 65 |
List/Tuple containing the Python binary name and its required arguments, |
| 66 |
or None if no valid binary names found. |
| 67 |
""" |
| 68 |
python_str = ".".join(map(str, py_version)) |
| 69 |
targets = [('py', f'-{python_str}'), (f'python{python_str}',)] |
| 70 |
if py_version not in py_executable_map: |
| 71 |
for target in targets[0 if is_windows else 1:]: |
| 72 |
try: |
| 73 |
worker = subprocess.Popen([*target, '-c', 'pass'], |
| 74 |
stdout=subprocess.DEVNULL, |
| 75 |
stderr=subprocess.DEVNULL, |
| 76 |
shell=is_windows) |
| 77 |
worker.communicate() |
| 78 |
if worker.returncode == 0: |
| 79 |
py_executable_map[py_version] = target |
| 80 |
break |
| 81 |
except FileNotFoundError: |
| 82 |
pass |
| 83 |
|
| 84 |
return py_executable_map.get(py_version, None) |
| 85 |
|
| 86 |
|
| 87 |
def read_exact(f, n): |
| 88 |
buf = b'' |
| 89 |
while len(buf) < n: |
| 90 |
chunk = f.read(n - len(buf)) |
| 91 |
if not chunk: |
| 92 |
raise EOFError |
| 93 |
buf += chunk |
| 94 |
return buf |
| 95 |
|
| 96 |
|
| 97 |
class AbstractCompatTests(pickletester.AbstractPickleTests): |
| 98 |
py_version = None |
| 99 |
worker = None |
| 100 |
|
| 101 |
@classmethod |
| 102 |
def setUpClass(cls): |
| 103 |
assert cls.py_version is not None, 'Needs a python version tuple' |
| 104 |
if not have_python_version(cls.py_version): |
| 105 |
py_version_str = ".".join(map(str, cls.py_version)) |
| 106 |
raise unittest.SkipTest(f'Python {py_version_str} not available') |
| 107 |
cls.addClassCleanup(cls.finish_worker) |
| 108 |
# Override the default pickle protocol to match what xpickle worker |
| 109 |
# will be running. |
| 110 |
highest_protocol = highest_proto_for_py_version(cls.py_version) |
| 111 |
cls.enterClassContext(support.swap_attr(pickletester, 'protocols', |
| 112 |
range(highest_protocol + 1))) |
| 113 |
cls.enterClassContext(support.swap_attr(pickle, 'HIGHEST_PROTOCOL', |
| 114 |
highest_protocol)) |
| 115 |
|
| 116 |
@classmethod |
| 117 |
def start_worker(cls, python): |
| 118 |
target = os.path.join(os.path.dirname(__file__), 'xpickle_worker.py') |
| 119 |
worker = subprocess.Popen([*python, target], |
| 120 |
stdin=subprocess.PIPE, |
| 121 |
stdout=subprocess.PIPE, |
| 122 |
stderr=subprocess.PIPE, |
| 123 |
# For windows bpo-17023. |
| 124 |
shell=is_windows) |
| 125 |
cls.worker = worker |
| 126 |
return worker |
| 127 |
|
| 128 |
@classmethod |
| 129 |
def finish_worker(cls): |
| 130 |
worker = cls.worker |
| 131 |
if worker is None: |
| 132 |
return |
| 133 |
cls.worker = None |
| 134 |
worker.stdin.close() |
| 135 |
worker.stdout.close() |
| 136 |
worker.stderr.close() |
| 137 |
worker.terminate() |
| 138 |
worker.wait() |
| 139 |
|
| 140 |
@classmethod |
| 141 |
def send_to_worker(cls, python, data): |
| 142 |
"""Bounce a pickled object through another version of Python. |
| 143 |
This will send data to a child process where it will |
| 144 |
be unpickled, then repickled and sent back to the parent process. |
| 145 |
Args: |
| 146 |
python: list containing the python binary to start and its arguments |
| 147 |
data: bytes object to send to the child process |
| 148 |
Returns: |
| 149 |
The pickled data received from the child process. |
| 150 |
""" |
| 151 |
worker = cls.worker |
| 152 |
if worker is None: |
| 153 |
worker = cls.start_worker(python) |
| 154 |
|
| 155 |
try: |
| 156 |
worker.stdin.write(struct.pack('!i', len(data)) + data) |
| 157 |
worker.stdin.flush() |
| 158 |
|
| 159 |
size, = struct.unpack('!i', read_exact(worker.stdout, 4)) |
| 160 |
if size > 0: |
| 161 |
return read_exact(worker.stdout, size) |
| 162 |
# if the worker fails, it will write the exception to stdout |
| 163 |
if size < 0: |
| 164 |
stdout = read_exact(worker.stdout, -size) |
| 165 |
try: |
| 166 |
exception = pickle.loads(stdout) |
| 167 |
except (pickle.UnpicklingError, EOFError): |
| 168 |
pass |
| 169 |
else: |
| 170 |
if isinstance(exception, Exception): |
| 171 |
# To allow for tests which test for errors. |
| 172 |
raise exception |
| 173 |
_, stderr = worker.communicate() |
| 174 |
raise RuntimeError(stderr) |
| 175 |
except: |
| 176 |
cls.finish_worker() |
| 177 |
raise |
| 178 |
|
| 179 |
def dumps(self, arg, proto=0, **kwargs): |
| 180 |
# Skip tests that require buffer_callback arguments since |
| 181 |
# there isn't a reliable way to marshal/pickle the callback and ensure |
| 182 |
# it works in a different Python version. |
| 183 |
if 'buffer_callback' in kwargs: |
| 184 |
self.skipTest('Test does not support "buffer_callback" argument.') |
| 185 |
f = io.BytesIO() |
| 186 |
p = self.pickler(f, proto, **kwargs) |
| 187 |
p.dump(arg) |
| 188 |
data = struct.pack('!i', proto) + f.getvalue() |
| 189 |
python = py_executable_map[self.py_version] |
| 190 |
return self.send_to_worker(python, data) |
| 191 |
|
| 192 |
def loads(self, buf, **kwds): |
| 193 |
f = io.BytesIO(buf) |
| 194 |
u = self.unpickler(f, **kwds) |
| 195 |
return u.load() |
| 196 |
|
| 197 |
# A scaled-down version of test_bytes from pickletester, to reduce |
| 198 |
# the number of calls to self.dumps() and hence reduce the number of |
| 199 |
# child python processes forked. This allows the test to complete |
| 200 |
# much faster (the one from pickletester takes 3-4 minutes when running |
| 201 |
# under text_xpickle). |
| 202 |
def test_bytes(self): |
| 203 |
if self.py_version < (3, 0): |
| 204 |
self.skipTest('not supported in Python < 3.0') |
| 205 |
for proto in pickletester.protocols: |
| 206 |
for s in b'', b'xyz', b'xyz'*100: |
| 207 |
p = self.dumps(s, proto) |
| 208 |
self.assert_is_copy(s, self.loads(p)) |
| 209 |
s = bytes(range(256)) |
| 210 |
p = self.dumps(s, proto) |
| 211 |
self.assert_is_copy(s, self.loads(p)) |
| 212 |
s = bytes([i for i in range(256) for _ in range(2)]) |
| 213 |
p = self.dumps(s, proto) |
| 214 |
self.assert_is_copy(s, self.loads(p)) |
| 215 |
|
| 216 |
# These tests are disabled because they require some special setup |
| 217 |
# on the worker that's hard to keep in sync. |
| 218 |
test_global_ext1 = None |
| 219 |
test_global_ext2 = None |
| 220 |
test_global_ext4 = None |
| 221 |
|
| 222 |
# These tests fail because they require classes from pickletester |
| 223 |
# which cannot be properly imported by the xpickle worker. |
| 224 |
test_recursive_nested_names = None |
| 225 |
test_recursive_nested_names2 = None |
| 226 |
|
| 227 |
# Attribute lookup problems are expected, disable the test |
| 228 |
test_dynamic_class = None |
| 229 |
test_evil_class_mutating_dict = None |
| 230 |
|
| 231 |
# Expected exception is raised during unpickling in a subprocess. |
| 232 |
test_pickle_setstate_None = None |
| 233 |
|
| 234 |
# Other Python version may not have NumPy. |
| 235 |
test_buffers_numpy = None |
| 236 |
|
| 237 |
# Skip tests that require buffer_callback arguments since |
| 238 |
# there isn't a reliable way to marshal/pickle the callback and ensure |
| 239 |
# it works in a different Python version. |
| 240 |
test_in_band_buffers = None |
| 241 |
test_buffers_error = None |
| 242 |
test_oob_buffers = None |
| 243 |
test_oob_buffers_writable_to_readonly = None |
| 244 |
|
| 245 |
class PyPicklePythonCompat(AbstractCompatTests): |
| 246 |
pickler = pickle._Pickler |
| 247 |
unpickler = pickle._Unpickler |
| 248 |
|
| 249 |
if has_c_implementation: |
| 250 |
class CPicklePythonCompat(AbstractCompatTests): |
| 251 |
pickler = _pickle.Pickler |
| 252 |
unpickler = _pickle.Unpickler |
| 253 |
|
| 254 |
|
| 255 |
def make_test(py_version, base): |
| 256 |
class_dict = {'py_version': py_version} |
| 257 |
name = base.__name__.replace('Python', 'Python%d%d' % py_version) |
| 258 |
return type(name, (base, unittest.TestCase), class_dict) |
| 259 |
|
| 260 |
def load_tests(loader, tests, pattern): |
| 261 |
def add_tests(py_version): |
| 262 |
test_class = make_test(py_version, PyPicklePythonCompat) |
| 263 |
tests.addTest(loader.loadTestsFromTestCase(test_class)) |
| 264 |
if has_c_implementation: |
| 265 |
test_class = make_test(py_version, CPicklePythonCompat) |
| 266 |
tests.addTest(loader.loadTestsFromTestCase(test_class)) |
| 267 |
|
| 268 |
value = support.get_resource_value('xpickle') |
| 269 |
if value is None: |
| 270 |
major = sys.version_info.major |
| 271 |
assert major == 3 |
| 272 |
add_tests((2, 7)) |
| 273 |
for minor in range(2, sys.version_info.minor): |
| 274 |
add_tests((major, minor)) |
| 275 |
else: |
| 276 |
add_tests(tuple(map(int, value.split('.')))) |
| 277 |
return tests |
| 278 |
|
| 279 |
|
| 280 |
if __name__ == '__main__': |
| 281 |
unittest.main() |