Debian Codegraph @

Lib/test/test_interpreters/utils.py

raw
1 from collections import namedtuple
2 import contextlib
3 import errno
4 import json
5 import logging
6 import os
7 import os.path
8 #import select
9 import subprocess
10 import sys
11 import tempfile
12 from textwrap import dedent
13 import threading
14 import types
15 import unittest
16 import warnings
17
18 from test import support
19
20 # We would use test.support.import_helper.import_module(),
21 # but the indirect import of test.support.os_helper causes refleaks.
22 try:
23 import _interpreters
24 except ImportError as exc:
25 raise unittest.SkipTest(str(exc))
26 from concurrent import interpreters
27
28
29 try:
30 import _testinternalcapi
31 import _testcapi
32 except ImportError:
33 _testinternalcapi = None
34 _testcapi = None
35
36 def requires_test_modules(func):
37 return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func)
38
39
40 def _dump_script(text):
41 lines = text.splitlines()
42 print()
43 print('-' * 20)
44 for i, line in enumerate(lines, 1):
45 print(f' {i:>{len(str(len(lines)))}} {line}')
46 print('-' * 20)
47
48
49 def _close_file(file):
50 try:
51 if hasattr(file, 'close'):
52 file.close()
53 else:
54 os.close(file)
55 except OSError as exc:
56 if exc.errno != errno.EBADF:
57 raise # re-raise
58 # It was closed already.
59
60
61 def pack_exception(exc=None):
62 captured = _interpreters.capture_exception(exc)
63 data = dict(captured.__dict__)
64 data['type'] = dict(captured.type.__dict__)
65 return json.dumps(data)
66
67
68 def unpack_exception(packed):
69 try:
70 data = json.loads(packed)
71 except json.decoder.JSONDecodeError as e:
72 logging.getLogger(__name__).warning('incomplete exception data', exc_info=e)
73 print(packed if isinstance(packed, str) else packed.decode('utf-8'))
74 return None
75 exc = types.SimpleNamespace(**data)
76 exc.type = types.SimpleNamespace(**exc.type)
77 return exc;
78
79
80 class CapturingResults:
81
82 STDIO = dedent("""\
83 with open({w_pipe}, 'wb', buffering=0) as _spipe_{stream}:
84 _captured_std{stream} = io.StringIO()
85 with contextlib.redirect_std{stream}(_captured_std{stream}):
86 #########################
87 # begin wrapped script
88
89 {indented}
90
91 # end wrapped script
92 #########################
93 text = _captured_std{stream}.getvalue()
94 _spipe_{stream}.write(text.encode('utf-8'))
95 """)[:-1]
96 EXC = dedent("""\
97 with open({w_pipe}, 'wb', buffering=0) as _spipe_exc:
98 try:
99 #########################
100 # begin wrapped script
101
102 {indented}
103
104 # end wrapped script
105 #########################
106 except Exception as exc:
107 text = _interp_utils.pack_exception(exc)
108 _spipe_exc.write(text.encode('utf-8'))
109 """)[:-1]
110
111 @classmethod
112 def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False):
113 script = dedent(script).strip(os.linesep)
114 imports = [
115 f'import {__name__} as _interp_utils',
116 ]
117 wrapped = script
118
119 # Handle exc.
120 if exc:
121 exc = os.pipe()
122 r_exc, w_exc = exc
123 indented = wrapped.replace('\n', '\n ')
124 wrapped = cls.EXC.format(
125 w_pipe=w_exc,
126 indented=indented,
127 )
128 else:
129 exc = None
130
131 # Handle stdout.
132 if stdout:
133 imports.extend([
134 'import contextlib, io',
135 ])
136 stdout = os.pipe()
137 r_out, w_out = stdout
138 indented = wrapped.replace('\n', '\n ')
139 wrapped = cls.STDIO.format(
140 w_pipe=w_out,
141 indented=indented,
142 stream='out',
143 )
144 else:
145 stdout = None
146
147 # Handle stderr.
148 if stderr == 'stdout':
149 stderr = None
150 elif stderr:
151 if not stdout:
152 imports.extend([
153 'import contextlib, io',
154 ])
155 stderr = os.pipe()
156 r_err, w_err = stderr
157 indented = wrapped.replace('\n', '\n ')
158 wrapped = cls.STDIO.format(
159 w_pipe=w_err,
160 indented=indented,
161 stream='err',
162 )
163 else:
164 stderr = None
165
166 if wrapped == script:
167 raise NotImplementedError
168 else:
169 for line in imports:
170 wrapped = f'{line}{os.linesep}{wrapped}'
171
172 results = cls(stdout, stderr, exc)
173 return wrapped, results
174
175 def __init__(self, out, err, exc):
176 self._rf_out = None
177 self._rf_err = None
178 self._rf_exc = None
179 self._w_out = None
180 self._w_err = None
181 self._w_exc = None
182
183 if out is not None:
184 r_out, w_out = out
185 self._rf_out = open(r_out, 'rb', buffering=0)
186 self._w_out = w_out
187
188 if err is not None:
189 r_err, w_err = err
190 self._rf_err = open(r_err, 'rb', buffering=0)
191 self._w_err = w_err
192
193 if exc is not None:
194 r_exc, w_exc = exc
195 self._rf_exc = open(r_exc, 'rb', buffering=0)
196 self._w_exc = w_exc
197
198 self._buf_out = b''
199 self._buf_err = b''
200 self._buf_exc = b''
201 self._exc = None
202
203 self._closed = False
204
205 def __enter__(self):
206 return self
207
208 def __exit__(self, *args):
209 self.close()
210
211 @property
212 def closed(self):
213 return self._closed
214
215 def close(self):
216 if self._closed:
217 return
218 self._closed = True
219
220 if self._w_out is not None:
221 _close_file(self._w_out)
222 self._w_out = None
223 if self._w_err is not None:
224 _close_file(self._w_err)
225 self._w_err = None
226 if self._w_exc is not None:
227 _close_file(self._w_exc)
228 self._w_exc = None
229
230 self._capture()
231
232 if self._rf_out is not None:
233 _close_file(self._rf_out)
234 self._rf_out = None
235 if self._rf_err is not None:
236 _close_file(self._rf_err)
237 self._rf_err = None
238 if self._rf_exc is not None:
239 _close_file(self._rf_exc)
240 self._rf_exc = None
241
242 def _capture(self):
243 # Ideally this is called only after the script finishes
244 # (and thus has closed the write end of the pipe.
245 if self._rf_out is not None:
246 chunk = self._rf_out.read(100)
247 while chunk:
248 self._buf_out += chunk
249 chunk = self._rf_out.read(100)
250 if self._rf_err is not None:
251 chunk = self._rf_err.read(100)
252 while chunk:
253 self._buf_err += chunk
254 chunk = self._rf_err.read(100)
255 if self._rf_exc is not None:
256 chunk = self._rf_exc.read(100)
257 while chunk:
258 self._buf_exc += chunk
259 chunk = self._rf_exc.read(100)
260
261 def _unpack_stdout(self):
262 return self._buf_out.decode('utf-8')
263
264 def _unpack_stderr(self):
265 return self._buf_err.decode('utf-8')
266
267 def _unpack_exc(self):
268 if self._exc is not None:
269 return self._exc
270 if not self._buf_exc:
271 return None
272 self._exc = unpack_exception(self._buf_exc)
273 return self._exc
274
275 def stdout(self):
276 if self.closed:
277 return self.final().stdout
278 self._capture()
279 return self._unpack_stdout()
280
281 def stderr(self):
282 if self.closed:
283 return self.final().stderr
284 self._capture()
285 return self._unpack_stderr()
286
287 def exc(self):
288 if self.closed:
289 return self.final().exc
290 self._capture()
291 return self._unpack_exc()
292
293 def final(self, *, force=False):
294 try:
295 return self._final
296 except AttributeError:
297 if not self._closed:
298 if not force:
299 raise Exception('no final results available yet')
300 else:
301 return CapturedResults.Proxy(self)
302 self._final = CapturedResults(
303 self._unpack_stdout(),
304 self._unpack_stderr(),
305 self._unpack_exc(),
306 )
307 return self._final
308
309
310 class CapturedResults(namedtuple('CapturedResults', 'stdout stderr exc')):
311
312 class Proxy:
313 def __init__(self, capturing):
314 self._capturing = capturing
315 def _finish(self):
316 if self._capturing is None:
317 return
318 self._final = self._capturing.final()
319 self._capturing = None
320 def __iter__(self):
321 self._finish()
322 yield from self._final
323 def __len__(self):
324 self._finish()
325 return len(self._final)
326 def __getattr__(self, name):
327 self._finish()
328 if name.startswith('_'):
329 raise AttributeError(name)
330 return getattr(self._final, name)
331
332 def raise_if_failed(self):
333 if self.exc is not None:
334 raise interpreters.ExecutionFailed(self.exc)
335
336
337 def _captured_script(script, *, stdout=True, stderr=False, exc=False):
338 return CapturingResults.wrap_script(
339 script,
340 stdout=stdout,
341 stderr=stderr,
342 exc=exc,
343 )
344
345
346 def clean_up_interpreters():
347 for interp in interpreters.list_all():
348 if interp.id == 0: # main
349 continue
350 try:
351 interp.close()
352 except _interpreters.InterpreterError:
353 pass # already destroyed
354
355
356 def _run_output(interp, request, init=None):
357 script, results = _captured_script(request)
358 with results:
359 if init:
360 interp.prepare_main(init)
361 interp.exec(script)
362 return results.stdout()
363
364
365 @contextlib.contextmanager
366 def _running(interp):
367 r, w = os.pipe()
368 def run():
369 interp.exec(dedent(f"""
370 # wait for "signal"
371 with open({r}) as rpipe:
372 rpipe.read()
373 """))
374
375 t = threading.Thread(target=run)
376 t.start()
377
378 yield
379
380 with open(w, 'w') as spipe:
381 spipe.write('done')
382 t.join()
383
384
385 class TestBase(unittest.TestCase):
386
387 def tearDown(self):
388 clean_up_interpreters()
389
390 def pipe(self):
391 def ensure_closed(fd):
392 try:
393 os.close(fd)
394 except OSError:
395 pass
396 r, w = os.pipe()
397 self.addCleanup(lambda: ensure_closed(r))
398 self.addCleanup(lambda: ensure_closed(w))
399 return r, w
400
401 def temp_dir(self):
402 tempdir = tempfile.mkdtemp()
403 tempdir = os.path.realpath(tempdir)
404 from test.support import os_helper
405 self.addCleanup(lambda: os_helper.rmtree(tempdir))
406 return tempdir
407
408 @contextlib.contextmanager
409 def captured_thread_exception(self):
410 ctx = types.SimpleNamespace(caught=None)
411 def excepthook(args):
412 ctx.caught = args
413 orig_excepthook = threading.excepthook
414 threading.excepthook = excepthook
415 try:
416 yield ctx
417 finally:
418 threading.excepthook = orig_excepthook
419
420 def make_script(self, filename, dirname=None, text=None):
421 if text:
422 text = dedent(text)
423 if dirname is None:
424 dirname = self.temp_dir()
425 filename = os.path.join(dirname, filename)
426
427 os.makedirs(os.path.dirname(filename), exist_ok=True)
428 with open(filename, 'w', encoding='utf-8') as outfile:
429 outfile.write(text or '')
430 return filename
431
432 def make_module(self, name, pathentry=None, text=None):
433 if text:
434 text = dedent(text)
435 if pathentry is None:
436 pathentry = self.temp_dir()
437 else:
438 os.makedirs(pathentry, exist_ok=True)
439 *subnames, basename = name.split('.')
440
441 dirname = pathentry
442 for subname in subnames:
443 dirname = os.path.join(dirname, subname)
444 if os.path.isdir(dirname):
445 pass
446 elif os.path.exists(dirname):
447 raise Exception(dirname)
448 else:
449 os.mkdir(dirname)
450 initfile = os.path.join(dirname, '__init__.py')
451 if not os.path.exists(initfile):
452 with open(initfile, 'w'):
453 pass
454 filename = os.path.join(dirname, basename + '.py')
455
456 with open(filename, 'w', encoding='utf-8') as outfile:
457 outfile.write(text or '')
458 return filename
459
460 @support.requires_subprocess()
461 def run_python(self, *argv):
462 proc = subprocess.run(
463 [sys.executable, *argv],
464 capture_output=True,
465 text=True,
466 )
467 return proc.returncode, proc.stdout, proc.stderr
468
469 def assert_python_ok(self, *argv):
470 exitcode, stdout, stderr = self.run_python(*argv)
471 self.assertNotEqual(exitcode, 1)
472 return stdout, stderr
473
474 def assert_python_failure(self, *argv):
475 exitcode, stdout, stderr = self.run_python(*argv)
476 self.assertNotEqual(exitcode, 0)
477 return stdout, stderr
478
479 def assert_ns_equal(self, ns1, ns2, msg=None):
480 # This is mostly copied from TestCase.assertDictEqual.
481 self.assertEqual(type(ns1), type(ns2))
482 if ns1 == ns2:
483 return
484
485 import difflib
486 import pprint
487 from unittest.util import _common_shorten_repr
488 standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2)
489 diff = ('\n' + '\n'.join(difflib.ndiff(
490 pprint.pformat(vars(ns1)).splitlines(),
491 pprint.pformat(vars(ns2)).splitlines())))
492 diff = f'namespace({diff})'
493 standardMsg = self._truncateMessage(standardMsg, diff)
494 self.fail(self._formatMessage(msg, standardMsg))
495
496 def _run_string(self, interp, script):
497 wrapped, results = _captured_script(script, exc=False)
498 #_dump_script(wrapped)
499 with results:
500 if isinstance(interp, interpreters.Interpreter):
501 interp.exec(script)
502 else:
503 err = _interpreters.run_string(interp, wrapped)
504 if err is not None:
505 return None, err
506 return results.stdout(), None
507
508 def run_and_capture(self, interp, script):
509 text, err = self._run_string(interp, script)
510 if err is not None:
511 raise interpreters.ExecutionFailed(err)
512 else:
513 return text
514
515 def interp_exists(self, interpid):
516 try:
517 _interpreters.whence(interpid)
518 except _interpreters.InterpreterNotFoundError:
519 return False
520 else:
521 return True
522
523 @requires_test_modules
524 @contextlib.contextmanager
525 def interpreter_from_capi(self, config=None, whence=None):
526 if config is False:
527 if whence is None:
528 whence = _interpreters.WHENCE_LEGACY_CAPI
529 else:
530 assert whence in (_interpreters.WHENCE_LEGACY_CAPI,
531 _interpreters.WHENCE_UNKNOWN), repr(whence)
532 config = None
533 elif config is True:
534 config = _interpreters.new_config('default')
535 elif config is None:
536 if whence not in (
537 _interpreters.WHENCE_LEGACY_CAPI,
538 _interpreters.WHENCE_UNKNOWN,
539 ):
540 config = _interpreters.new_config('legacy')
541 elif isinstance(config, str):
542 config = _interpreters.new_config(config)
543
544 if whence is None:
545 whence = _interpreters.WHENCE_XI
546
547 interpid = _testinternalcapi.create_interpreter(config, whence=whence)
548 try:
549 yield interpid
550 finally:
551 try:
552 _testinternalcapi.destroy_interpreter(interpid)
553 except _interpreters.InterpreterNotFoundError:
554 pass
555
556 @contextlib.contextmanager
557 def interpreter_obj_from_capi(self, config='legacy'):
558 with self.interpreter_from_capi(config) as interpid:
559 interp = interpreters.Interpreter(
560 interpid,
561 _whence=_interpreters.WHENCE_CAPI,
562 _ownsref=False,
563 )
564 yield interp, interpid
565
566 @contextlib.contextmanager
567 def capturing(self, script):
568 wrapped, capturing = _captured_script(script, stdout=True, exc=True)
569 #_dump_script(wrapped)
570 with capturing:
571 yield wrapped, capturing.final(force=True)
572
573 @requires_test_modules
574 def run_from_capi(self, interpid, script, *, main=False):
575 with self.capturing(script) as (wrapped, results):
576 rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main)
577 assert rc == 0, rc
578 results.raise_if_failed()
579 return results.stdout
580
581 @contextlib.contextmanager
582 def _running(self, run_interp, exec_interp):
583 token = b'\0'
584 r_in, w_in = self.pipe()
585 r_out, w_out = self.pipe()
586
587 def close():
588 _close_file(r_in)
589 _close_file(w_in)
590 _close_file(r_out)
591 _close_file(w_out)
592
593 # Start running (and wait).
594 script = dedent(f"""
595 import os
596 try:
597 # handshake
598 token = os.read({r_in}, 1)
599 os.write({w_out}, token)
600 # Wait for the "done" message.
601 os.read({r_in}, 1)
602 except BrokenPipeError:
603 pass
604 except OSError as exc:
605 if exc.errno != 9:
606 raise # re-raise
607 # It was closed already.
608 """)
609 failed = None
610 def run():
611 nonlocal failed
612 try:
613 run_interp(script)
614 except Exception as exc:
615 failed = exc
616 close()
617 t = threading.Thread(target=run)
618 t.start()
619
620 # handshake
621 try:
622 os.write(w_in, token)
623 token2 = os.read(r_out, 1)
624 assert token2 == token, (token2, token)
625 except OSError:
626 t.join()
627 if failed is not None:
628 raise failed
629
630 # CM __exit__()
631 try:
632 try:
633 yield
634 finally:
635 # Send "done".
636 os.write(w_in, b'\0')
637 finally:
638 close()
639 t.join()
640 if failed is not None:
641 raise failed
642
643 @contextlib.contextmanager
644 def running(self, interp):
645 if isinstance(interp, int):
646 interpid = interp
647 def exec_interp(script):
648 exc = _interpreters.exec(interpid, script)
649 assert exc is None, exc
650 run_interp = exec_interp
651 else:
652 def run_interp(script):
653 text = self.run_and_capture(interp, script)
654 assert text == '', repr(text)
655 def exec_interp(script):
656 interp.exec(script)
657 with self._running(run_interp, exec_interp):
658 yield
659
660 @requires_test_modules
661 @contextlib.contextmanager
662 def running_from_capi(self, interpid, *, main=False):
663 def run_interp(script):
664 text = self.run_from_capi(interpid, script, main=main)
665 assert text == '', repr(text)
666 def exec_interp(script):
667 rc = _testinternalcapi.exec_interpreter(interpid, script)
668 assert rc == 0, rc
669 with self._running(run_interp, exec_interp):
670 yield
671
672 @requires_test_modules
673 def run_temp_from_capi(self, script, config='legacy'):
674 if config is False:
675 # Force using Py_NewInterpreter().
676 run_in_interp = (lambda s, c: _testcapi.run_in_subinterp(s))
677 config = None
678 else:
679 run_in_interp = _testinternalcapi.run_in_subinterp_with_config
680 if config is True:
681 config = 'default'
682 if isinstance(config, str):
683 config = _interpreters.new_config(config)
684 with self.capturing(script) as (wrapped, results):
685 rc = run_in_interp(wrapped, config)
686 assert rc == 0, rc
687 results.raise_if_failed()
688 return results.stdout