Debian Codegraph @

Lib/test/test_xpickle.py

raw
1 # This test covers backwards compatibility with previous versions of Python
2 # by bouncing pickled objects through Python versions by running xpickle_worker.py.
3 import io
4 import os
5 import pickle
6 import struct
7 import subprocess
8 import sys
9 import unittest
10
11
12 from test import support
13 from test import pickletester
14
15 try:
16 import _pickle
17 has_c_implementation = True
18 except ModuleNotFoundError:
19 has_c_implementation = False
20
21 support.requires('xpickle')
22
23 is_windows = sys.platform.startswith('win')
24
25 # Map python version to a tuple containing the name of a corresponding valid
26 # Python binary to execute and its arguments.
27 py_executable_map = {}
28
29 protocols_map = {
30 3: (3, 0),
31 4: (3, 4),
32 5: (3, 8),
33 }
34
35 def highest_proto_for_py_version(py_version):
36 """Finds the highest supported pickle protocol for a given Python version.
37 Args:
38 py_version: a 2-tuple of the major, minor version. Eg. Python 3.7 would
39 be (3, 7)
40 Returns:
41 int for the highest supported pickle protocol
42 """
43 proto = 2
44 for p, v in protocols_map.items():
45 if py_version < v:
46 break
47 proto = p
48 return proto
49
50 def have_python_version(py_version):
51 """Check whether a Python binary exists for the given py_version and has
52 support. This respects your PATH.
53 For Windows, it will first try to use the py launcher specified in PEP 397.
54 Otherwise (and for all other platforms), it will attempt to check for
55 python<py_version[0]>.<py_version[1]>.
56
57 Eg. given a *py_version* of (3, 7), the function will attempt to try
58 'py -3.7' (for Windows) first, then 'python3.7', and return
59 ['py', '-3.7'] (on Windows) or ['python3.7'] on other platforms.
60
61 Args:
62 py_version: a 2-tuple of the major, minor version. Eg. python 3.7 would
63 be (3, 7)
64 Returns:
65 List/Tuple containing the Python binary name and its required arguments,
66 or None if no valid binary names found.
67 """
68 python_str = ".".join(map(str, py_version))
69 targets = [('py', f'-{python_str}'), (f'python{python_str}',)]
70 if py_version not in py_executable_map:
71 for target in targets[0 if is_windows else 1:]:
72 try:
73 worker = subprocess.Popen([*target, '-c', 'pass'],
74 stdout=subprocess.DEVNULL,
75 stderr=subprocess.DEVNULL,
76 shell=is_windows)
77 worker.communicate()
78 if worker.returncode == 0:
79 py_executable_map[py_version] = target
80 break
81 except FileNotFoundError:
82 pass
83
84 return py_executable_map.get(py_version, None)
85
86
87 def read_exact(f, n):
88 buf = b''
89 while len(buf) < n:
90 chunk = f.read(n - len(buf))
91 if not chunk:
92 raise EOFError
93 buf += chunk
94 return buf
95
96
97 class AbstractCompatTests(pickletester.AbstractPickleTests):
98 py_version = None
99 worker = None
100
101 @classmethod
102 def setUpClass(cls):
103 assert cls.py_version is not None, 'Needs a python version tuple'
104 if not have_python_version(cls.py_version):
105 py_version_str = ".".join(map(str, cls.py_version))
106 raise unittest.SkipTest(f'Python {py_version_str} not available')
107 cls.addClassCleanup(cls.finish_worker)
108 # Override the default pickle protocol to match what xpickle worker
109 # will be running.
110 highest_protocol = highest_proto_for_py_version(cls.py_version)
111 cls.enterClassContext(support.swap_attr(pickletester, 'protocols',
112 range(highest_protocol + 1)))
113 cls.enterClassContext(support.swap_attr(pickle, 'HIGHEST_PROTOCOL',
114 highest_protocol))
115
116 @classmethod
117 def start_worker(cls, python):
118 target = os.path.join(os.path.dirname(__file__), 'xpickle_worker.py')
119 worker = subprocess.Popen([*python, target],
120 stdin=subprocess.PIPE,
121 stdout=subprocess.PIPE,
122 stderr=subprocess.PIPE,
123 # For windows bpo-17023.
124 shell=is_windows)
125 cls.worker = worker
126 return worker
127
128 @classmethod
129 def finish_worker(cls):
130 worker = cls.worker
131 if worker is None:
132 return
133 cls.worker = None
134 worker.stdin.close()
135 worker.stdout.close()
136 worker.stderr.close()
137 worker.terminate()
138 worker.wait()
139
140 @classmethod
141 def send_to_worker(cls, python, data):
142 """Bounce a pickled object through another version of Python.
143 This will send data to a child process where it will
144 be unpickled, then repickled and sent back to the parent process.
145 Args:
146 python: list containing the python binary to start and its arguments
147 data: bytes object to send to the child process
148 Returns:
149 The pickled data received from the child process.
150 """
151 worker = cls.worker
152 if worker is None:
153 worker = cls.start_worker(python)
154
155 try:
156 worker.stdin.write(struct.pack('!i', len(data)) + data)
157 worker.stdin.flush()
158
159 size, = struct.unpack('!i', read_exact(worker.stdout, 4))
160 if size > 0:
161 return read_exact(worker.stdout, size)
162 # if the worker fails, it will write the exception to stdout
163 if size < 0:
164 stdout = read_exact(worker.stdout, -size)
165 try:
166 exception = pickle.loads(stdout)
167 except (pickle.UnpicklingError, EOFError):
168 pass
169 else:
170 if isinstance(exception, Exception):
171 # To allow for tests which test for errors.
172 raise exception
173 _, stderr = worker.communicate()
174 raise RuntimeError(stderr)
175 except:
176 cls.finish_worker()
177 raise
178
179 def dumps(self, arg, proto=0, **kwargs):
180 # Skip tests that require buffer_callback arguments since
181 # there isn't a reliable way to marshal/pickle the callback and ensure
182 # it works in a different Python version.
183 if 'buffer_callback' in kwargs:
184 self.skipTest('Test does not support "buffer_callback" argument.')
185 f = io.BytesIO()
186 p = self.pickler(f, proto, **kwargs)
187 p.dump(arg)
188 data = struct.pack('!i', proto) + f.getvalue()
189 python = py_executable_map[self.py_version]
190 return self.send_to_worker(python, data)
191
192 def loads(self, buf, **kwds):
193 f = io.BytesIO(buf)
194 u = self.unpickler(f, **kwds)
195 return u.load()
196
197 # A scaled-down version of test_bytes from pickletester, to reduce
198 # the number of calls to self.dumps() and hence reduce the number of
199 # child python processes forked. This allows the test to complete
200 # much faster (the one from pickletester takes 3-4 minutes when running
201 # under text_xpickle).
202 def test_bytes(self):
203 if self.py_version < (3, 0):
204 self.skipTest('not supported in Python < 3.0')
205 for proto in pickletester.protocols:
206 for s in b'', b'xyz', b'xyz'*100:
207 p = self.dumps(s, proto)
208 self.assert_is_copy(s, self.loads(p))
209 s = bytes(range(256))
210 p = self.dumps(s, proto)
211 self.assert_is_copy(s, self.loads(p))
212 s = bytes([i for i in range(256) for _ in range(2)])
213 p = self.dumps(s, proto)
214 self.assert_is_copy(s, self.loads(p))
215
216 # These tests are disabled because they require some special setup
217 # on the worker that's hard to keep in sync.
218 test_global_ext1 = None
219 test_global_ext2 = None
220 test_global_ext4 = None
221
222 # These tests fail because they require classes from pickletester
223 # which cannot be properly imported by the xpickle worker.
224 test_recursive_nested_names = None
225 test_recursive_nested_names2 = None
226
227 # Attribute lookup problems are expected, disable the test
228 test_dynamic_class = None
229 test_evil_class_mutating_dict = None
230
231 # Expected exception is raised during unpickling in a subprocess.
232 test_pickle_setstate_None = None
233
234 # Other Python version may not have NumPy.
235 test_buffers_numpy = None
236
237 # Skip tests that require buffer_callback arguments since
238 # there isn't a reliable way to marshal/pickle the callback and ensure
239 # it works in a different Python version.
240 test_in_band_buffers = None
241 test_buffers_error = None
242 test_oob_buffers = None
243 test_oob_buffers_writable_to_readonly = None
244
245 class PyPicklePythonCompat(AbstractCompatTests):
246 pickler = pickle._Pickler
247 unpickler = pickle._Unpickler
248
249 if has_c_implementation:
250 class CPicklePythonCompat(AbstractCompatTests):
251 pickler = _pickle.Pickler
252 unpickler = _pickle.Unpickler
253
254
255 def make_test(py_version, base):
256 class_dict = {'py_version': py_version}
257 name = base.__name__.replace('Python', 'Python%d%d' % py_version)
258 return type(name, (base, unittest.TestCase), class_dict)
259
260 def load_tests(loader, tests, pattern):
261 def add_tests(py_version):
262 test_class = make_test(py_version, PyPicklePythonCompat)
263 tests.addTest(loader.loadTestsFromTestCase(test_class))
264 if has_c_implementation:
265 test_class = make_test(py_version, CPicklePythonCompat)
266 tests.addTest(loader.loadTestsFromTestCase(test_class))
267
268 value = support.get_resource_value('xpickle')
269 if value is None:
270 major = sys.version_info.major
271 assert major == 3
272 add_tests((2, 7))
273 for minor in range(2, sys.version_info.minor):
274 add_tests((major, minor))
275 else:
276 add_tests(tuple(map(int, value.split('.'))))
277 return tests
278
279
280 if __name__ == '__main__':
281 unittest.main()