Debian Codegraph @

Lib/test/test_asyncio/test_subprocess.py

raw
1 import os
2 import signal
3 import sys
4 import textwrap
5 import unittest
6 import warnings
7 from unittest import mock
8
9 import asyncio
10 from asyncio import base_subprocess
11 from asyncio import subprocess
12 from test.test_asyncio import utils as test_utils
13 from test import support
14 from test.support import os_helper, warnings_helper, gc_collect
15
16 if not support.has_subprocess_support:
17 raise unittest.SkipTest("test module requires subprocess")
18
19 if support.MS_WINDOWS:
20 import msvcrt
21 else:
22 from asyncio import unix_events
23
24
25 if support.check_sanitizer(address=True):
26 raise unittest.SkipTest("Exposes ASAN flakiness in GitHub CI")
27
28 # Program blocking
29 PROGRAM_BLOCKED = [sys.executable, '-c', 'import time; time.sleep(3600)']
30
31 # Program copying input to output
32 PROGRAM_CAT = [
33 sys.executable, '-c',
34 ';'.join(('import sys',
35 'data = sys.stdin.buffer.read()',
36 'sys.stdout.buffer.write(data)'))]
37
38
39 def tearDownModule():
40 asyncio.events._set_event_loop_policy(None)
41
42
43 class TestSubprocessTransport(base_subprocess.BaseSubprocessTransport):
44 def _start(self, *args, **kwargs):
45 self._proc = mock.Mock()
46 self._proc.stdin = None
47 self._proc.stdout = None
48 self._proc.stderr = None
49 self._proc.pid = -1
50
51
52 class SubprocessTransportTests(test_utils.TestCase):
53 def setUp(self):
54 super().setUp()
55 self.loop = self.new_test_loop()
56 self.set_event_loop(self.loop)
57
58 def create_transport(self, waiter=None):
59 protocol = mock.Mock()
60 transport = TestSubprocessTransport(
61 self.loop, protocol, ['test'], False,
62 None, None, None, 0, waiter=waiter)
63 return (transport, protocol)
64
65 def test_proc_exited(self):
66 waiter = self.loop.create_future()
67 transport, protocol = self.create_transport(waiter)
68 transport._process_exited(6)
69 self.loop.run_until_complete(waiter)
70
71 self.assertEqual(transport.get_returncode(), 6)
72
73 self.assertTrue(protocol.connection_made.called)
74 self.assertTrue(protocol.process_exited.called)
75 self.assertTrue(protocol.connection_lost.called)
76 self.assertEqual(protocol.connection_lost.call_args[0], (None,))
77
78 self.assertFalse(transport.is_closing())
79 self.assertIsNone(transport._loop)
80 self.assertIsNone(transport._proc)
81 self.assertIsNone(transport._protocol)
82
83 # methods must raise ProcessLookupError if the process exited
84 self.assertRaises(ProcessLookupError,
85 transport.send_signal, signal.SIGTERM)
86 self.assertRaises(ProcessLookupError, transport.terminate)
87 self.assertRaises(ProcessLookupError, transport.kill)
88
89 transport.close()
90
91 def test_subprocess_repr(self):
92 waiter = self.loop.create_future()
93 transport, protocol = self.create_transport(waiter)
94 transport._process_exited(6)
95 self.loop.run_until_complete(waiter)
96
97 self.assertEqual(
98 repr(transport),
99 "<TestSubprocessTransport pid=-1 returncode=6>"
100 )
101 transport._returncode = None
102 self.assertEqual(
103 repr(transport),
104 "<TestSubprocessTransport pid=-1 running>"
105 )
106 transport._pid = None
107 transport._returncode = None
108 self.assertEqual(
109 repr(transport),
110 "<TestSubprocessTransport not started>"
111 )
112 transport.close()
113
114 def test_proc_exited_no_invalid_state_error_on_exit_waiters(self):
115 # gh-145541: when _connect_pipes hasn't completed (so
116 # _pipes_connected is False) and the process exits, _try_finish()
117 # sets the result on exit waiters. Then _call_connection_lost() must
118 # not call set_result() again on the same waiters.
119 self.loop.set_exception_handler(
120 lambda loop, context: self.fail(
121 f"unexpected exception: {context}")
122 )
123 waiter = self.loop.create_future()
124 transport, protocol = self.create_transport(waiter)
125
126 # Simulate a waiter registered via _wait() before the process exits.
127 exit_waiter = self.loop.create_future()
128 transport._exit_waiters.append(exit_waiter)
129
130 # _connect_pipes hasn't completed, so _pipes_connected is False.
131 self.assertFalse(transport._pipes_connected)
132
133 # Simulate process exit. _try_finish() will set the result on
134 # exit_waiter because _pipes_connected is False, and then schedule
135 # _call_connection_lost() because _pipes is empty (vacuously all
136 # disconnected). _call_connection_lost() must skip exit_waiter
137 # because it's already done.
138 transport._process_exited(6)
139 self.loop.run_until_complete(waiter)
140
141 self.assertEqual(exit_waiter.result(), 6)
142
143 transport.close()
144
145
146 class SubprocessMixin:
147
148 def test_stdin_stdout(self):
149 args = PROGRAM_CAT
150
151 async def run(data):
152 proc = await asyncio.create_subprocess_exec(
153 *args,
154 stdin=subprocess.PIPE,
155 stdout=subprocess.PIPE,
156 )
157
158 # feed data
159 proc.stdin.write(data)
160 await proc.stdin.drain()
161 proc.stdin.close()
162
163 # get output and exitcode
164 data = await proc.stdout.read()
165 exitcode = await proc.wait()
166 return (exitcode, data)
167
168 task = run(b'some data')
169 task = asyncio.wait_for(task, 60.0)
170 exitcode, stdout = self.loop.run_until_complete(task)
171 self.assertEqual(exitcode, 0)
172 self.assertEqual(stdout, b'some data')
173
174 def test_communicate(self):
175 args = PROGRAM_CAT
176
177 async def run(data):
178 proc = await asyncio.create_subprocess_exec(
179 *args,
180 stdin=subprocess.PIPE,
181 stdout=subprocess.PIPE,
182 )
183 stdout, stderr = await proc.communicate(data)
184 return proc.returncode, stdout
185
186 task = run(b'some data')
187 task = asyncio.wait_for(task, support.LONG_TIMEOUT)
188 exitcode, stdout = self.loop.run_until_complete(task)
189 self.assertEqual(exitcode, 0)
190 self.assertEqual(stdout, b'some data')
191
192 def test_communicate_none_input(self):
193 args = PROGRAM_CAT
194
195 async def run():
196 proc = await asyncio.create_subprocess_exec(
197 *args,
198 stdin=subprocess.PIPE,
199 stdout=subprocess.PIPE,
200 )
201 stdout, stderr = await proc.communicate()
202 return proc.returncode, stdout
203
204 task = run()
205 task = asyncio.wait_for(task, support.LONG_TIMEOUT)
206 exitcode, stdout = self.loop.run_until_complete(task)
207 self.assertEqual(exitcode, 0)
208 self.assertEqual(stdout, b'')
209
210 def test_shell(self):
211 proc = self.loop.run_until_complete(
212 asyncio.create_subprocess_shell('exit 7')
213 )
214 exitcode = self.loop.run_until_complete(proc.wait())
215 self.assertEqual(exitcode, 7)
216
217 def test_start_new_session(self):
218 # start the new process in a new session
219 proc = self.loop.run_until_complete(
220 asyncio.create_subprocess_shell(
221 'exit 8',
222 start_new_session=True,
223 )
224 )
225 exitcode = self.loop.run_until_complete(proc.wait())
226 self.assertEqual(exitcode, 8)
227
228 def test_kill(self):
229 args = PROGRAM_BLOCKED
230 proc = self.loop.run_until_complete(
231 asyncio.create_subprocess_exec(*args)
232 )
233 proc.kill()
234 returncode = self.loop.run_until_complete(proc.wait())
235 if sys.platform == 'win32':
236 self.assertIsInstance(returncode, int)
237 # expect 1 but sometimes get 0
238 else:
239 self.assertEqual(-signal.SIGKILL, returncode)
240
241 def test_kill_issue43884(self):
242 if sys.platform == 'win32':
243 blocking_shell_command = f'"{sys.executable}" -c "import time; time.sleep(2)"'
244 else:
245 blocking_shell_command = 'sleep 1; sleep 1'
246 creationflags = 0
247 if sys.platform == 'win32':
248 from subprocess import CREATE_NEW_PROCESS_GROUP
249 # On windows create a new process group so that killing process
250 # kills the process and all its children.
251 creationflags = CREATE_NEW_PROCESS_GROUP
252 proc = self.loop.run_until_complete(
253 asyncio.create_subprocess_shell(blocking_shell_command, stdout=asyncio.subprocess.PIPE,
254 creationflags=creationflags)
255 )
256 self.loop.run_until_complete(asyncio.sleep(1))
257 if sys.platform == 'win32':
258 proc.send_signal(signal.CTRL_BREAK_EVENT)
259 # On windows it is an alias of terminate which sets the return code
260 proc.kill()
261 returncode = self.loop.run_until_complete(proc.wait())
262 if sys.platform == 'win32':
263 self.assertIsInstance(returncode, int)
264 # expect 1 but sometimes get 0
265 else:
266 self.assertEqual(-signal.SIGKILL, returncode)
267
268 def test_terminate(self):
269 args = PROGRAM_BLOCKED
270 proc = self.loop.run_until_complete(
271 asyncio.create_subprocess_exec(*args)
272 )
273 proc.terminate()
274 returncode = self.loop.run_until_complete(proc.wait())
275 if sys.platform == 'win32':
276 self.assertIsInstance(returncode, int)
277 # expect 1 but sometimes get 0
278 else:
279 self.assertEqual(-signal.SIGTERM, returncode)
280
281 @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP")
282 def test_send_signal(self):
283 # bpo-31034: Make sure that we get the default signal handler (killing
284 # the process). The parent process may have decided to ignore SIGHUP,
285 # and signal handlers are inherited.
286 old_handler = signal.signal(signal.SIGHUP, signal.SIG_DFL)
287 try:
288 code = 'import time; print("sleeping", flush=True); time.sleep(3600)'
289 args = [sys.executable, '-c', code]
290 proc = self.loop.run_until_complete(
291 asyncio.create_subprocess_exec(
292 *args,
293 stdout=subprocess.PIPE,
294 )
295 )
296
297 async def send_signal(proc):
298 # basic synchronization to wait until the program is sleeping
299 line = await proc.stdout.readline()
300 self.assertEqual(line, b'sleeping\n')
301
302 proc.send_signal(signal.SIGHUP)
303 returncode = await proc.wait()
304 return returncode
305
306 returncode = self.loop.run_until_complete(send_signal(proc))
307 self.assertEqual(-signal.SIGHUP, returncode)
308 finally:
309 signal.signal(signal.SIGHUP, old_handler)
310
311 def test_stdin_broken_pipe(self):
312 # buffer large enough to feed the whole pipe buffer
313 large_data = b'x' * support.PIPE_MAX_SIZE
314
315 rfd, wfd = os.pipe()
316 self.addCleanup(os.close, rfd)
317 self.addCleanup(os.close, wfd)
318 if support.MS_WINDOWS:
319 handle = msvcrt.get_osfhandle(rfd)
320 os.set_handle_inheritable(handle, True)
321 code = textwrap.dedent(f'''
322 import os, msvcrt
323 handle = {handle}
324 fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
325 os.read(fd, 1)
326 ''')
327 from subprocess import STARTUPINFO
328 startupinfo = STARTUPINFO()
329 startupinfo.lpAttributeList = {"handle_list": [handle]}
330 kwargs = dict(startupinfo=startupinfo)
331 else:
332 code = f'import os; fd = {rfd}; os.read(fd, 1)'
333 kwargs = dict(pass_fds=(rfd,))
334
335 # the program ends before the stdin can be fed
336 proc = self.loop.run_until_complete(
337 asyncio.create_subprocess_exec(
338 sys.executable, '-c', code,
339 stdin=subprocess.PIPE,
340 **kwargs
341 )
342 )
343
344 async def write_stdin(proc, data):
345 proc.stdin.write(data)
346 # Only exit the child process once the write buffer is filled
347 os.write(wfd, b'go')
348 await proc.stdin.drain()
349
350 coro = write_stdin(proc, large_data)
351 # drain() must raise BrokenPipeError or ConnectionResetError
352 with test_utils.disable_logger():
353 self.assertRaises((BrokenPipeError, ConnectionResetError),
354 self.loop.run_until_complete, coro)
355 self.loop.run_until_complete(proc.wait())
356
357 def test_communicate_ignore_broken_pipe(self):
358 # buffer large enough to feed the whole pipe buffer
359 large_data = b'x' * support.PIPE_MAX_SIZE
360
361 # the program ends before the stdin can be fed
362 proc = self.loop.run_until_complete(
363 asyncio.create_subprocess_exec(
364 sys.executable, '-c', 'pass',
365 stdin=subprocess.PIPE,
366 )
367 )
368
369 # communicate() must ignore BrokenPipeError when feeding stdin
370 self.loop.set_exception_handler(lambda loop, msg: None)
371 self.loop.run_until_complete(proc.communicate(large_data))
372 self.loop.run_until_complete(proc.wait())
373
374 def test_pause_reading(self):
375 limit = 10
376 size = (limit * 2 + 1)
377
378 async def test_pause_reading():
379 code = '\n'.join((
380 'import sys',
381 'sys.stdout.write("x" * %s)' % size,
382 'sys.stdout.flush()',
383 ))
384
385 connect_read_pipe = self.loop.connect_read_pipe
386
387 async def connect_read_pipe_mock(*args, **kw):
388 transport, protocol = await connect_read_pipe(*args, **kw)
389 transport.pause_reading = mock.Mock()
390 transport.resume_reading = mock.Mock()
391 return (transport, protocol)
392
393 self.loop.connect_read_pipe = connect_read_pipe_mock
394
395 proc = await asyncio.create_subprocess_exec(
396 sys.executable, '-c', code,
397 stdin=asyncio.subprocess.PIPE,
398 stdout=asyncio.subprocess.PIPE,
399 limit=limit,
400 )
401 stdout_transport = proc._transport.get_pipe_transport(1)
402
403 stdout, stderr = await proc.communicate()
404
405 # The child process produced more than limit bytes of output,
406 # the stream reader transport should pause the protocol to not
407 # allocate too much memory.
408 return (stdout, stdout_transport)
409
410 # Issue #22685: Ensure that the stream reader pauses the protocol
411 # when the child process produces too much data
412 stdout, transport = self.loop.run_until_complete(test_pause_reading())
413
414 self.assertEqual(stdout, b'x' * size)
415 self.assertTrue(transport.pause_reading.called)
416 self.assertTrue(transport.resume_reading.called)
417
418 def test_stdin_not_inheritable(self):
419 # asyncio issue #209: stdin must not be inheritable, otherwise
420 # the Process.communicate() hangs
421 async def len_message(message):
422 code = 'import sys; data = sys.stdin.read(); print(len(data))'
423 proc = await asyncio.create_subprocess_exec(
424 sys.executable, '-c', code,
425 stdin=asyncio.subprocess.PIPE,
426 stdout=asyncio.subprocess.PIPE,
427 stderr=asyncio.subprocess.PIPE,
428 close_fds=False,
429 )
430 stdout, stderr = await proc.communicate(message)
431 exitcode = await proc.wait()
432 return (stdout, exitcode)
433
434 output, exitcode = self.loop.run_until_complete(len_message(b'abc'))
435 self.assertEqual(output.rstrip(), b'3')
436 self.assertEqual(exitcode, 0)
437
438 def test_empty_input(self):
439
440 async def empty_input():
441 code = 'import sys; data = sys.stdin.read(); print(len(data))'
442 proc = await asyncio.create_subprocess_exec(
443 sys.executable, '-c', code,
444 stdin=asyncio.subprocess.PIPE,
445 stdout=asyncio.subprocess.PIPE,
446 stderr=asyncio.subprocess.PIPE,
447 close_fds=False,
448 )
449 stdout, stderr = await proc.communicate(b'')
450 exitcode = await proc.wait()
451 return (stdout, exitcode)
452
453 output, exitcode = self.loop.run_until_complete(empty_input())
454 self.assertEqual(output.rstrip(), b'0')
455 self.assertEqual(exitcode, 0)
456
457 def test_devnull_input(self):
458
459 async def empty_input():
460 code = 'import sys; data = sys.stdin.read(); print(len(data))'
461 proc = await asyncio.create_subprocess_exec(
462 sys.executable, '-c', code,
463 stdin=asyncio.subprocess.DEVNULL,
464 stdout=asyncio.subprocess.PIPE,
465 stderr=asyncio.subprocess.PIPE,
466 close_fds=False,
467 )
468 stdout, stderr = await proc.communicate()
469 exitcode = await proc.wait()
470 return (stdout, exitcode)
471
472 output, exitcode = self.loop.run_until_complete(empty_input())
473 self.assertEqual(output.rstrip(), b'0')
474 self.assertEqual(exitcode, 0)
475
476 def test_devnull_output(self):
477
478 async def empty_output():
479 code = 'import sys; data = sys.stdin.read(); print(len(data))'
480 proc = await asyncio.create_subprocess_exec(
481 sys.executable, '-c', code,
482 stdin=asyncio.subprocess.PIPE,
483 stdout=asyncio.subprocess.DEVNULL,
484 stderr=asyncio.subprocess.PIPE,
485 close_fds=False,
486 )
487 stdout, stderr = await proc.communicate(b"abc")
488 exitcode = await proc.wait()
489 return (stdout, exitcode)
490
491 output, exitcode = self.loop.run_until_complete(empty_output())
492 self.assertEqual(output, None)
493 self.assertEqual(exitcode, 0)
494
495 def test_devnull_error(self):
496
497 async def empty_error():
498 code = 'import sys; data = sys.stdin.read(); print(len(data))'
499 proc = await asyncio.create_subprocess_exec(
500 sys.executable, '-c', code,
501 stdin=asyncio.subprocess.PIPE,
502 stdout=asyncio.subprocess.PIPE,
503 stderr=asyncio.subprocess.DEVNULL,
504 close_fds=False,
505 )
506 stdout, stderr = await proc.communicate(b"abc")
507 exitcode = await proc.wait()
508 return (stderr, exitcode)
509
510 output, exitcode = self.loop.run_until_complete(empty_error())
511 self.assertEqual(output, None)
512 self.assertEqual(exitcode, 0)
513
514 @unittest.skipIf(sys.platform not in ('linux', 'android'),
515 "Don't have /dev/stdin")
516 def test_devstdin_input(self):
517
518 async def devstdin_input(message):
519 code = 'file = open("/dev/stdin"); data = file.read(); print(len(data))'
520 proc = await asyncio.create_subprocess_exec(
521 sys.executable, '-c', code,
522 stdin=asyncio.subprocess.PIPE,
523 stdout=asyncio.subprocess.PIPE,
524 stderr=asyncio.subprocess.PIPE,
525 close_fds=False,
526 )
527 stdout, stderr = await proc.communicate(message)
528 exitcode = await proc.wait()
529 return (stdout, exitcode)
530
531 output, exitcode = self.loop.run_until_complete(devstdin_input(b'abc'))
532 self.assertEqual(output.rstrip(), b'3')
533 self.assertEqual(exitcode, 0)
534
535 def test_cancel_process_wait(self):
536 # Issue #23140: cancel Process.wait()
537
538 async def cancel_wait():
539 proc = await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED)
540
541 # Create an internal future waiting on the process exit
542 task = self.loop.create_task(proc.wait())
543 self.loop.call_soon(task.cancel)
544 try:
545 await task
546 except asyncio.CancelledError:
547 pass
548
549 # Cancel the future
550 task.cancel()
551
552 # Kill the process and wait until it is done
553 proc.kill()
554 await proc.wait()
555
556 self.loop.run_until_complete(cancel_wait())
557
558 def test_cancel_make_subprocess_transport_exec(self):
559
560 async def cancel_make_transport():
561 coro = asyncio.create_subprocess_exec(*PROGRAM_BLOCKED)
562 task = self.loop.create_task(coro)
563
564 self.loop.call_soon(task.cancel)
565 try:
566 await task
567 except asyncio.CancelledError:
568 pass
569
570 # ignore the log:
571 # "Exception during subprocess creation, kill the subprocess"
572 with test_utils.disable_logger():
573 self.loop.run_until_complete(cancel_make_transport())
574
575 def test_cancel_post_init(self):
576
577 async def cancel_make_transport():
578 coro = self.loop.subprocess_exec(asyncio.SubprocessProtocol,
579 *PROGRAM_BLOCKED)
580 task = self.loop.create_task(coro)
581
582 self.loop.call_soon(task.cancel)
583 try:
584 await task
585 except asyncio.CancelledError:
586 pass
587
588 # ignore the log:
589 # "Exception during subprocess creation, kill the subprocess"
590 with test_utils.disable_logger():
591 self.loop.run_until_complete(cancel_make_transport())
592 test_utils.run_briefly(self.loop)
593
594 def test_close_kill_running(self):
595
596 async def kill_running():
597 create = self.loop.subprocess_exec(asyncio.SubprocessProtocol,
598 *PROGRAM_BLOCKED)
599 transport, protocol = await create
600
601 kill_called = False
602 def kill():
603 nonlocal kill_called
604 kill_called = True
605 orig_kill()
606
607 proc = transport.get_extra_info('subprocess')
608 orig_kill = proc.kill
609 proc.kill = kill
610 returncode = transport.get_returncode()
611 transport.close()
612 await asyncio.wait_for(transport._wait(), 5)
613 return (returncode, kill_called)
614
615 # Ignore "Close running child process: kill ..." log
616 with test_utils.disable_logger():
617 try:
618 returncode, killed = self.loop.run_until_complete(
619 kill_running()
620 )
621 except asyncio.TimeoutError:
622 self.skipTest(
623 "Timeout failure on waiting for subprocess stopping"
624 )
625 self.assertIsNone(returncode)
626
627 # transport.close() must kill the process if it is still running
628 self.assertTrue(killed)
629 test_utils.run_briefly(self.loop)
630
631 def test_close_dont_kill_finished(self):
632
633 async def kill_running():
634 create = self.loop.subprocess_exec(asyncio.SubprocessProtocol,
635 *PROGRAM_BLOCKED)
636 transport, protocol = await create
637 proc = transport.get_extra_info('subprocess')
638
639 # kill the process (but asyncio is not notified immediately)
640 proc.kill()
641 proc.wait()
642
643 proc.kill = mock.Mock()
644 proc_returncode = proc.poll()
645 transport_returncode = transport.get_returncode()
646 transport.close()
647 return (proc_returncode, transport_returncode, proc.kill.called)
648
649 # Ignore "Unknown child process pid ..." log of SafeChildWatcher,
650 # emitted because the test already consumes the exit status:
651 # proc.wait()
652 with test_utils.disable_logger():
653 result = self.loop.run_until_complete(kill_running())
654 test_utils.run_briefly(self.loop)
655
656 proc_returncode, transport_return_code, killed = result
657
658 self.assertIsNotNone(proc_returncode)
659 self.assertIsNone(transport_return_code)
660
661 # transport.close() must not kill the process if it finished, even if
662 # the transport was not notified yet
663 self.assertFalse(killed)
664
665 async def _test_popen_error(self, stdin):
666 if sys.platform == 'win32':
667 target = 'asyncio.windows_utils.Popen'
668 else:
669 target = 'subprocess.Popen'
670 with mock.patch(target) as popen:
671 exc = ZeroDivisionError
672 popen.side_effect = exc
673
674 with warnings.catch_warnings(record=True) as warns:
675 with self.assertRaises(exc):
676 await asyncio.create_subprocess_exec(
677 sys.executable,
678 '-c',
679 'pass',
680 stdin=stdin
681 )
682 self.assertEqual(warns, [])
683
684 def test_popen_error(self):
685 # Issue #24763: check that the subprocess transport is closed
686 # when BaseSubprocessTransport fails
687 self.loop.run_until_complete(self._test_popen_error(stdin=None))
688
689 def test_popen_error_with_stdin_pipe(self):
690 # Issue #35721: check that newly created socket pair is closed when
691 # Popen fails
692 self.loop.run_until_complete(
693 self._test_popen_error(stdin=subprocess.PIPE))
694
695 def test_read_stdout_after_process_exit(self):
696
697 async def execute():
698 code = '\n'.join(['import sys',
699 'for _ in range(64):',
700 ' sys.stdout.write("x" * 4096)',
701 'sys.stdout.flush()',
702 'sys.exit(1)'])
703
704 process = await asyncio.create_subprocess_exec(
705 sys.executable, '-c', code,
706 stdout=asyncio.subprocess.PIPE,
707 )
708
709 while True:
710 data = await process.stdout.read(65536)
711 if data:
712 await asyncio.sleep(0.3)
713 else:
714 break
715
716 self.loop.run_until_complete(execute())
717
718 def test_create_subprocess_exec_text_mode_fails(self):
719 async def execute():
720 with self.assertRaises(ValueError):
721 await subprocess.create_subprocess_exec(sys.executable,
722 text=True)
723
724 with self.assertRaises(ValueError):
725 await subprocess.create_subprocess_exec(sys.executable,
726 encoding="utf-8")
727
728 with self.assertRaises(ValueError):
729 await subprocess.create_subprocess_exec(sys.executable,
730 errors="strict")
731
732 self.loop.run_until_complete(execute())
733
734 def test_create_subprocess_shell_text_mode_fails(self):
735
736 async def execute():
737 with self.assertRaises(ValueError):
738 await subprocess.create_subprocess_shell(sys.executable,
739 text=True)
740
741 with self.assertRaises(ValueError):
742 await subprocess.create_subprocess_shell(sys.executable,
743 encoding="utf-8")
744
745 with self.assertRaises(ValueError):
746 await subprocess.create_subprocess_shell(sys.executable,
747 errors="strict")
748
749 self.loop.run_until_complete(execute())
750
751 def test_create_subprocess_exec_with_path(self):
752 async def execute():
753 p = await subprocess.create_subprocess_exec(
754 os_helper.FakePath(sys.executable), '-c', 'pass')
755 await p.wait()
756 p = await subprocess.create_subprocess_exec(
757 sys.executable, '-c', 'pass', os_helper.FakePath('.'))
758 await p.wait()
759
760 self.assertIsNone(self.loop.run_until_complete(execute()))
761
762 async def check_stdout_output(self, coro, output):
763 proc = await coro
764 stdout, _ = await proc.communicate()
765 self.assertEqual(stdout, output)
766 self.assertEqual(proc.returncode, 0)
767 task = asyncio.create_task(proc.wait())
768 await asyncio.sleep(0)
769 self.assertEqual(task.result(), proc.returncode)
770
771 def test_create_subprocess_env_shell(self) -> None:
772 async def main() -> None:
773 executable = sys.executable
774 if sys.platform == "win32":
775 executable = f'"{executable}"'
776 cmd = f'''{executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"'''
777 env = os.environ.copy()
778 env["FOO"] = "bar"
779 proc = await asyncio.create_subprocess_shell(
780 cmd, env=env, stdout=subprocess.PIPE
781 )
782 return proc
783
784 self.loop.run_until_complete(self.check_stdout_output(main(), b'bar'))
785
786 def test_create_subprocess_env_exec(self) -> None:
787 async def main() -> None:
788 cmd = [sys.executable, "-c",
789 "import os, sys; sys.stdout.write(os.getenv('FOO'))"]
790 env = os.environ.copy()
791 env["FOO"] = "baz"
792 proc = await asyncio.create_subprocess_exec(
793 *cmd, env=env, stdout=subprocess.PIPE
794 )
795 return proc
796
797 self.loop.run_until_complete(self.check_stdout_output(main(), b'baz'))
798
799
800 def test_subprocess_concurrent_wait(self) -> None:
801 async def main() -> None:
802 proc = await asyncio.create_subprocess_exec(
803 *PROGRAM_CAT,
804 stdin=subprocess.PIPE,
805 stdout=subprocess.PIPE,
806 )
807 stdout, _ = await proc.communicate(b'some data')
808 self.assertEqual(stdout, b"some data")
809 self.assertEqual(proc.returncode, 0)
810 self.assertEqual(await asyncio.gather(*[proc.wait() for _ in range(10)]),
811 [proc.returncode] * 10)
812
813 self.loop.run_until_complete(main())
814
815 def test_subprocess_protocol_events(self):
816 # gh-108973: Test that all subprocess protocol methods are called.
817 # The protocol methods are not called in a deterministic order.
818 # The order depends on the event loop and the operating system.
819 events = []
820 fds = [1, 2]
821 expected = [
822 ('pipe_data_received', 1, b'stdout'),
823 ('pipe_data_received', 2, b'stderr'),
824 ('pipe_connection_lost', 1),
825 ('pipe_connection_lost', 2),
826 'process_exited',
827 ]
828 per_fd_expected = [
829 'pipe_data_received',
830 'pipe_connection_lost',
831 ]
832
833 class MyProtocol(asyncio.SubprocessProtocol):
834 def __init__(self, exit_future: asyncio.Future) -> None:
835 self.exit_future = exit_future
836
837 def pipe_data_received(self, fd, data) -> None:
838 events.append(('pipe_data_received', fd, data))
839 self.exit_maybe()
840
841 def pipe_connection_lost(self, fd, exc) -> None:
842 events.append(('pipe_connection_lost', fd))
843 self.exit_maybe()
844
845 def process_exited(self) -> None:
846 events.append('process_exited')
847 self.exit_maybe()
848
849 def exit_maybe(self):
850 # Only exit when we got all expected events
851 if len(events) >= len(expected):
852 self.exit_future.set_result(True)
853
854 async def main() -> None:
855 loop = asyncio.get_running_loop()
856 exit_future = asyncio.Future()
857 code = 'import sys; sys.stdout.write("stdout"); sys.stderr.write("stderr")'
858 transport, _ = await loop.subprocess_exec(lambda: MyProtocol(exit_future),
859 sys.executable, '-c', code, stdin=None)
860 await exit_future
861 transport.close()
862
863 return events
864
865 events = self.loop.run_until_complete(main())
866
867 # First, make sure that we received all events
868 self.assertSetEqual(set(events), set(expected))
869
870 # Second, check order of pipe events per file descriptor
871 per_fd_events = {fd: [] for fd in fds}
872 for event in events:
873 if event == 'process_exited':
874 continue
875 name, fd = event[:2]
876 per_fd_events[fd].append(name)
877
878 for fd in fds:
879 self.assertEqual(per_fd_events[fd], per_fd_expected, (fd, events))
880
881 def test_subprocess_communicate_stdout(self):
882 # See https://github.com/python/cpython/issues/100133
883 async def get_command_stdout(cmd, *args):
884 proc = await asyncio.create_subprocess_exec(
885 cmd, *args, stdout=asyncio.subprocess.PIPE,
886 )
887 stdout, _ = await proc.communicate()
888 return stdout.decode().strip()
889
890 async def main():
891 outputs = [f'foo{i}' for i in range(10)]
892 res = await asyncio.gather(*[get_command_stdout(sys.executable, '-c',
893 f'print({out!r})') for out in outputs])
894 self.assertEqual(res, outputs)
895
896 self.loop.run_until_complete(main())
897
898 @unittest.skipIf(sys.platform != 'linux', "Linux only")
899 def test_subprocess_send_signal_race(self):
900 # See https://github.com/python/cpython/issues/87744
901 async def main():
902 for _ in range(10):
903 proc = await asyncio.create_subprocess_exec('sleep', '0.1')
904 await asyncio.sleep(0.1)
905 try:
906 proc.send_signal(signal.SIGUSR1)
907 except ProcessLookupError:
908 pass
909 self.assertNotEqual(await proc.wait(), 255)
910
911 self.loop.run_until_complete(main())
912
913 @warnings_helper.ignore_warnings(category=ResourceWarning)
914 def test_subprocess_read_pipe_cancelled(self):
915 async def main():
916 loop = asyncio.get_running_loop()
917 loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError)
918 with self.assertRaises(asyncio.CancelledError):
919 await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stderr=asyncio.subprocess.PIPE)
920
921 asyncio.run(main())
922 gc_collect()
923
924 @warnings_helper.ignore_warnings(category=ResourceWarning)
925 def test_subprocess_write_pipe_cancelled(self):
926 async def main():
927 loop = asyncio.get_running_loop()
928 loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError)
929 with self.assertRaises(asyncio.CancelledError):
930 await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stdin=asyncio.subprocess.PIPE)
931
932 asyncio.run(main())
933 gc_collect()
934
935 @warnings_helper.ignore_warnings(category=ResourceWarning)
936 def test_subprocess_read_write_pipe_cancelled(self):
937 async def main():
938 loop = asyncio.get_running_loop()
939 loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError)
940 loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError)
941 with self.assertRaises(asyncio.CancelledError):
942 await asyncio.create_subprocess_exec(
943 *PROGRAM_BLOCKED,
944 stdin=asyncio.subprocess.PIPE,
945 stdout=asyncio.subprocess.PIPE,
946 stderr=asyncio.subprocess.PIPE,
947 )
948
949 asyncio.run(main())
950 gc_collect()
951
952 if sys.platform != 'win32':
953 # Unix
954 class SubprocessWatcherMixin(SubprocessMixin):
955
956 def setUp(self):
957 super().setUp()
958 self.loop = asyncio.new_event_loop()
959 self.set_event_loop(self.loop)
960
961 def test_watcher_implementation(self):
962 loop = self.loop
963 watcher = loop._watcher
964 if unix_events.can_use_pidfd():
965 self.assertIsInstance(watcher, unix_events._PidfdChildWatcher)
966 else:
967 self.assertIsInstance(watcher, unix_events._ThreadedChildWatcher)
968
969
970 class SubprocessThreadedWatcherTests(SubprocessWatcherMixin,
971 test_utils.TestCase):
972 def setUp(self):
973 self._original_can_use_pidfd = unix_events.can_use_pidfd
974 # Force the use of the threaded child watcher
975 unix_events.can_use_pidfd = mock.Mock(return_value=False)
976 super().setUp()
977
978 def tearDown(self):
979 unix_events.can_use_pidfd = self._original_can_use_pidfd
980 return super().tearDown()
981
982 @unittest.skipUnless(
983 unix_events.can_use_pidfd(),
984 "operating system does not support pidfds",
985 )
986 class SubprocessPidfdWatcherTests(SubprocessWatcherMixin,
987 test_utils.TestCase):
988
989 pass
990
991 else:
992 # Windows
993 class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase):
994
995 def setUp(self):
996 super().setUp()
997 self.loop = asyncio.ProactorEventLoop()
998 self.set_event_loop(self.loop)
999
1000
1001 if __name__ == '__main__':
1002 unittest.main()