Debian Codegraph @

Lib/test/test_external_inspection.py

raw
1 import unittest
2 import os
3 import textwrap
4 import importlib
5 import sys
6 import socket
7 import threading
8 import time
9 from asyncio import staggered, taskgroups, base_events, tasks
10 from unittest.mock import ANY
11 from test.support import (
12 os_helper,
13 SHORT_TIMEOUT,
14 busy_retry,
15 requires_gil_enabled,
16 )
17 from test.support.import_helper import import_module
18 from test.support.script_helper import make_script
19 from test.support.socket_helper import find_unused_port
20
21 import subprocess
22
23 PROCESS_VM_READV_SUPPORTED = False
24
25 try:
26 from _remote_debugging import PROCESS_VM_READV_SUPPORTED
27 from _remote_debugging import RemoteUnwinder
28 from _remote_debugging import FrameInfo, CoroInfo, TaskInfo
29 except ImportError:
30 raise unittest.SkipTest(
31 "Test only runs when _remote_debugging is available"
32 )
33
34
35 def _make_test_script(script_dir, script_basename, source):
36 to_return = make_script(script_dir, script_basename, source)
37 importlib.invalidate_caches()
38 return to_return
39
40
41 skip_if_not_supported = unittest.skipIf(
42 (
43 sys.platform != "darwin"
44 and sys.platform != "linux"
45 and sys.platform != "win32"
46 ),
47 "Test only runs on Linux, Windows and MacOS",
48 )
49
50
51 def get_stack_trace(pid):
52 unwinder = RemoteUnwinder(pid, all_threads=True, debug=True)
53 return unwinder.get_stack_trace()
54
55
56 def get_async_stack_trace(pid):
57 unwinder = RemoteUnwinder(pid, debug=True)
58 return unwinder.get_async_stack_trace()
59
60
61 def get_all_awaited_by(pid):
62 unwinder = RemoteUnwinder(pid, debug=True)
63 return unwinder.get_all_awaited_by()
64
65
66 class TestGetStackTrace(unittest.TestCase):
67 maxDiff = None
68
69 @skip_if_not_supported
70 @unittest.skipIf(
71 sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
72 "Test only runs on Linux with process_vm_readv support",
73 )
74 def test_remote_stack_trace(self):
75 # Spawn a process with some realistic Python code
76 port = find_unused_port()
77 script = textwrap.dedent(
78 f"""\
79 import time, sys, socket, threading
80 # Connect to the test process
81 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
82 sock.connect(('localhost', {port}))
83
84 def bar():
85 for x in range(100):
86 if x == 50:
87 baz()
88
89 def baz():
90 foo()
91
92 def foo():
93 sock.sendall(b"ready:thread\\n"); time.sleep(10_000) # same line number
94
95 t = threading.Thread(target=bar)
96 t.start()
97 sock.sendall(b"ready:main\\n"); t.join() # same line number
98 """
99 )
100 stack_trace = None
101 with os_helper.temp_dir() as work_dir:
102 script_dir = os.path.join(work_dir, "script_pkg")
103 os.mkdir(script_dir)
104
105 # Create a socket server to communicate with the target process
106 server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
107 server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
108 server_socket.bind(("localhost", port))
109 server_socket.settimeout(SHORT_TIMEOUT)
110 server_socket.listen(1)
111
112 script_name = _make_test_script(script_dir, "script", script)
113 client_socket = None
114 try:
115 p = subprocess.Popen([sys.executable, script_name])
116 client_socket, _ = server_socket.accept()
117 server_socket.close()
118 response = b""
119 while (
120 b"ready:main" not in response
121 or b"ready:thread" not in response
122 ):
123 response += client_socket.recv(1024)
124 stack_trace = get_stack_trace(p.pid)
125 except PermissionError:
126 self.skipTest(
127 "Insufficient permissions to read the stack trace"
128 )
129 finally:
130 if client_socket is not None:
131 client_socket.close()
132 p.kill()
133 p.terminate()
134 p.wait(timeout=SHORT_TIMEOUT)
135
136 thread_expected_stack_trace = [
137 FrameInfo([script_name, 15, "foo"]),
138 FrameInfo([script_name, 12, "baz"]),
139 FrameInfo([script_name, 9, "bar"]),
140 FrameInfo([threading.__file__, ANY, "Thread.run"]),
141 ]
142 # Is possible that there are more threads, so we check that the
143 # expected stack traces are in the result (looking at you Windows!)
144 self.assertIn((ANY, thread_expected_stack_trace), stack_trace)
145
146 # Check that the main thread stack trace is in the result
147 frame = FrameInfo([script_name, 19, "<module>"])
148 for _, stack in stack_trace:
149 if frame in stack:
150 break
151 else:
152 self.fail("Main thread stack trace not found in result")
153
154 @skip_if_not_supported
155 @unittest.skipIf(
156 sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
157 "Test only runs on Linux with process_vm_readv support",
158 )
159 def test_self_trace_after_ctypes_import(self):
160 """Test that RemoteUnwinder works on the same process after _ctypes import.
161
162 When _ctypes is imported, it may call dlopen on the libpython shared
163 library, creating a duplicate mapping in the process address space.
164 The remote debugging code must skip these uninitialized duplicate
165 mappings and find the real PyRuntime. See gh-144563.
166 """
167
168 # Skip the test if the _ctypes module is missing.
169 import_module("_ctypes")
170
171 # Run the test in a subprocess to avoid side effects
172 script = textwrap.dedent("""\
173 import os
174 import _remote_debugging
175
176 # Should work before _ctypes import
177 unwinder = _remote_debugging.RemoteUnwinder(os.getpid())
178
179 import _ctypes
180
181 # Should still work after _ctypes import (gh-144563)
182 unwinder = _remote_debugging.RemoteUnwinder(os.getpid())
183 """)
184
185 result = subprocess.run(
186 [sys.executable, "-c", script],
187 capture_output=True,
188 text=True,
189 timeout=SHORT_TIMEOUT,
190 )
191 self.assertEqual(
192 result.returncode, 0,
193 f"stdout: {result.stdout}\nstderr: {result.stderr}"
194 )
195
196 @skip_if_not_supported
197 @unittest.skipIf(
198 sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
199 "Test only runs on Linux with process_vm_readv support",
200 )
201 def test_async_remote_stack_trace(self):
202 # Spawn a process with some realistic Python code
203 port = find_unused_port()
204 script = textwrap.dedent(
205 f"""\
206 import asyncio
207 import time
208 import sys
209 import socket
210 # Connect to the test process
211 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
212 sock.connect(('localhost', {port}))
213
214 def c5():
215 sock.sendall(b"ready"); time.sleep(10_000) # same line number
216
217 async def c4():
218 await asyncio.sleep(0)
219 c5()
220
221 async def c3():
222 await c4()
223
224 async def c2():
225 await c3()
226
227 async def c1(task):
228 await task
229
230 async def main():
231 async with asyncio.TaskGroup() as tg:
232 task = tg.create_task(c2(), name="c2_root")
233 tg.create_task(c1(task), name="sub_main_1")
234 tg.create_task(c1(task), name="sub_main_2")
235
236 def new_eager_loop():
237 loop = asyncio.new_event_loop()
238 eager_task_factory = asyncio.create_eager_task_factory(
239 asyncio.Task)
240 loop.set_task_factory(eager_task_factory)
241 return loop
242
243 asyncio.run(main(), loop_factory={{TASK_FACTORY}})
244 """
245 )
246 stack_trace = None
247 for task_factory_variant in "asyncio.new_event_loop", "new_eager_loop":
248 with (
249 self.subTest(task_factory_variant=task_factory_variant),
250 os_helper.temp_dir() as work_dir,
251 ):
252 script_dir = os.path.join(work_dir, "script_pkg")
253 os.mkdir(script_dir)
254 server_socket = socket.socket(
255 socket.AF_INET, socket.SOCK_STREAM
256 )
257 server_socket.setsockopt(
258 socket.SOL_SOCKET, socket.SO_REUSEADDR, 1
259 )
260 server_socket.bind(("localhost", port))
261 server_socket.settimeout(SHORT_TIMEOUT)
262 server_socket.listen(1)
263 script_name = _make_test_script(
264 script_dir,
265 "script",
266 script.format(TASK_FACTORY=task_factory_variant),
267 )
268 client_socket = None
269 try:
270 p = subprocess.Popen([sys.executable, script_name])
271 client_socket, _ = server_socket.accept()
272 server_socket.close()
273 response = client_socket.recv(1024)
274 self.assertEqual(response, b"ready")
275 stack_trace = get_async_stack_trace(p.pid)
276 except PermissionError:
277 self.skipTest(
278 "Insufficient permissions to read the stack trace"
279 )
280 finally:
281 if client_socket is not None:
282 client_socket.close()
283 p.kill()
284 p.terminate()
285 p.wait(timeout=SHORT_TIMEOUT)
286
287 # First check all the tasks are present
288 tasks_names = [
289 task.task_name for task in stack_trace[0].awaited_by
290 ]
291 for task_name in ["c2_root", "sub_main_1", "sub_main_2"]:
292 self.assertIn(task_name, tasks_names)
293
294 # Now ensure that the awaited_by_relationships are correct
295 id_to_task = {
296 task.task_id: task for task in stack_trace[0].awaited_by
297 }
298 task_name_to_awaited_by = {
299 task.task_name: set(
300 id_to_task[awaited.task_name].task_name
301 for awaited in task.awaited_by
302 )
303 for task in stack_trace[0].awaited_by
304 }
305 self.assertEqual(
306 task_name_to_awaited_by,
307 {
308 "c2_root": {"Task-1", "sub_main_1", "sub_main_2"},
309 "Task-1": set(),
310 "sub_main_1": {"Task-1"},
311 "sub_main_2": {"Task-1"},
312 },
313 )
314
315 # Now ensure that the coroutine stacks are correct
316 coroutine_stacks = {
317 task.task_name: sorted(
318 tuple(tuple(frame) for frame in coro.call_stack)
319 for coro in task.coroutine_stack
320 )
321 for task in stack_trace[0].awaited_by
322 }
323 self.assertEqual(
324 coroutine_stacks,
325 {
326 "Task-1": [
327 (
328 tuple(
329 [
330 taskgroups.__file__,
331 ANY,
332 "TaskGroup._aexit",
333 ]
334 ),
335 tuple(
336 [
337 taskgroups.__file__,
338 ANY,
339 "TaskGroup.__aexit__",
340 ]
341 ),
342 tuple([script_name, 26, "main"]),
343 )
344 ],
345 "c2_root": [
346 (
347 tuple([script_name, 10, "c5"]),
348 tuple([script_name, 14, "c4"]),
349 tuple([script_name, 17, "c3"]),
350 tuple([script_name, 20, "c2"]),
351 )
352 ],
353 "sub_main_1": [(tuple([script_name, 23, "c1"]),)],
354 "sub_main_2": [(tuple([script_name, 23, "c1"]),)],
355 },
356 )
357
358 # Now ensure the coroutine stacks for the awaited_by relationships are correct.
359 awaited_by_coroutine_stacks = {
360 task.task_name: sorted(
361 (
362 id_to_task[coro.task_name].task_name,
363 tuple(tuple(frame) for frame in coro.call_stack),
364 )
365 for coro in task.awaited_by
366 )
367 for task in stack_trace[0].awaited_by
368 }
369 self.assertEqual(
370 awaited_by_coroutine_stacks,
371 {
372 "Task-1": [],
373 "c2_root": [
374 (
375 "Task-1",
376 (
377 tuple(
378 [
379 taskgroups.__file__,
380 ANY,
381 "TaskGroup._aexit",
382 ]
383 ),
384 tuple(
385 [
386 taskgroups.__file__,
387 ANY,
388 "TaskGroup.__aexit__",
389 ]
390 ),
391 tuple([script_name, 26, "main"]),
392 ),
393 ),
394 ("sub_main_1", (tuple([script_name, 23, "c1"]),)),
395 ("sub_main_2", (tuple([script_name, 23, "c1"]),)),
396 ],
397 "sub_main_1": [
398 (
399 "Task-1",
400 (
401 tuple(
402 [
403 taskgroups.__file__,
404 ANY,
405 "TaskGroup._aexit",
406 ]
407 ),
408 tuple(
409 [
410 taskgroups.__file__,
411 ANY,
412 "TaskGroup.__aexit__",
413 ]
414 ),
415 tuple([script_name, 26, "main"]),
416 ),
417 )
418 ],
419 "sub_main_2": [
420 (
421 "Task-1",
422 (
423 tuple(
424 [
425 taskgroups.__file__,
426 ANY,
427 "TaskGroup._aexit",
428 ]
429 ),
430 tuple(
431 [
432 taskgroups.__file__,
433 ANY,
434 "TaskGroup.__aexit__",
435 ]
436 ),
437 tuple([script_name, 26, "main"]),
438 ),
439 )
440 ],
441 },
442 )
443
444 @skip_if_not_supported
445 @unittest.skipIf(
446 sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
447 "Test only runs on Linux with process_vm_readv support",
448 )
449 def test_asyncgen_remote_stack_trace(self):
450 # Spawn a process with some realistic Python code
451 port = find_unused_port()
452 script = textwrap.dedent(
453 f"""\
454 import asyncio
455 import time
456 import sys
457 import socket
458 # Connect to the test process
459 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
460 sock.connect(('localhost', {port}))
461
462 async def gen_nested_call():
463 sock.sendall(b"ready"); time.sleep(10_000) # same line number
464
465 async def gen():
466 for num in range(2):
467 yield num
468 if num == 1:
469 await gen_nested_call()
470
471 async def main():
472 async for el in gen():
473 pass
474
475 asyncio.run(main())
476 """
477 )
478 stack_trace = None
479 with os_helper.temp_dir() as work_dir:
480 script_dir = os.path.join(work_dir, "script_pkg")
481 os.mkdir(script_dir)
482 # Create a socket server to communicate with the target process
483 server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
484 server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
485 server_socket.bind(("localhost", port))
486 server_socket.settimeout(SHORT_TIMEOUT)
487 server_socket.listen(1)
488 script_name = _make_test_script(script_dir, "script", script)
489 client_socket = None
490 try:
491 p = subprocess.Popen([sys.executable, script_name])
492 client_socket, _ = server_socket.accept()
493 server_socket.close()
494 response = client_socket.recv(1024)
495 self.assertEqual(response, b"ready")
496 stack_trace = get_async_stack_trace(p.pid)
497 except PermissionError:
498 self.skipTest(
499 "Insufficient permissions to read the stack trace"
500 )
501 finally:
502 if client_socket is not None:
503 client_socket.close()
504 p.kill()
505 p.terminate()
506 p.wait(timeout=SHORT_TIMEOUT)
507
508 # For this simple asyncgen test, we only expect one task with the full coroutine stack
509 self.assertEqual(len(stack_trace[0].awaited_by), 1)
510 task = stack_trace[0].awaited_by[0]
511 self.assertEqual(task.task_name, "Task-1")
512
513 # Check the coroutine stack - based on actual output, only shows main
514 coroutine_stack = sorted(
515 tuple(tuple(frame) for frame in coro.call_stack)
516 for coro in task.coroutine_stack
517 )
518 self.assertEqual(
519 coroutine_stack,
520 [
521 (
522 tuple([script_name, 10, "gen_nested_call"]),
523 tuple([script_name, 16, "gen"]),
524 tuple([script_name, 19, "main"]),
525 )
526 ],
527 )
528
529 # No awaited_by relationships expected for this simple case
530 self.assertEqual(task.awaited_by, [])
531
532 @skip_if_not_supported
533 @unittest.skipIf(
534 sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
535 "Test only runs on Linux with process_vm_readv support",
536 )
537 def test_async_gather_remote_stack_trace(self):
538 # Spawn a process with some realistic Python code
539 port = find_unused_port()
540 script = textwrap.dedent(
541 f"""\
542 import asyncio
543 import time
544 import sys
545 import socket
546 # Connect to the test process
547 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
548 sock.connect(('localhost', {port}))
549
550 async def deep():
551 await asyncio.sleep(0)
552 sock.sendall(b"ready"); time.sleep(10_000) # same line number
553
554 async def c1():
555 await asyncio.sleep(0)
556 await deep()
557
558 async def c2():
559 await asyncio.sleep(0)
560
561 async def main():
562 await asyncio.gather(c1(), c2())
563
564 asyncio.run(main())
565 """
566 )
567 stack_trace = None
568 with os_helper.temp_dir() as work_dir:
569 script_dir = os.path.join(work_dir, "script_pkg")
570 os.mkdir(script_dir)
571 # Create a socket server to communicate with the target process
572 server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
573 server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
574 server_socket.bind(("localhost", port))
575 server_socket.settimeout(SHORT_TIMEOUT)
576 server_socket.listen(1)
577 script_name = _make_test_script(script_dir, "script", script)
578 client_socket = None
579 try:
580 p = subprocess.Popen([sys.executable, script_name])
581 client_socket, _ = server_socket.accept()
582 server_socket.close()
583 response = client_socket.recv(1024)
584 self.assertEqual(response, b"ready")
585 stack_trace = get_async_stack_trace(p.pid)
586 except PermissionError:
587 self.skipTest(
588 "Insufficient permissions to read the stack trace"
589 )
590 finally:
591 if client_socket is not None:
592 client_socket.close()
593 p.kill()
594 p.terminate()
595 p.wait(timeout=SHORT_TIMEOUT)
596
597 # First check all the tasks are present
598 tasks_names = [
599 task.task_name for task in stack_trace[0].awaited_by
600 ]
601 for task_name in ["Task-1", "Task-2"]:
602 self.assertIn(task_name, tasks_names)
603
604 # Now ensure that the awaited_by_relationships are correct
605 id_to_task = {
606 task.task_id: task for task in stack_trace[0].awaited_by
607 }
608 task_name_to_awaited_by = {
609 task.task_name: set(
610 id_to_task[awaited.task_name].task_name
611 for awaited in task.awaited_by
612 )
613 for task in stack_trace[0].awaited_by
614 }
615 self.assertEqual(
616 task_name_to_awaited_by,
617 {
618 "Task-1": set(),
619 "Task-2": {"Task-1"},
620 },
621 )
622
623 # Now ensure that the coroutine stacks are correct
624 coroutine_stacks = {
625 task.task_name: sorted(
626 tuple(tuple(frame) for frame in coro.call_stack)
627 for coro in task.coroutine_stack
628 )
629 for task in stack_trace[0].awaited_by
630 }
631 self.assertEqual(
632 coroutine_stacks,
633 {
634 "Task-1": [(tuple([script_name, 21, "main"]),)],
635 "Task-2": [
636 (
637 tuple([script_name, 11, "deep"]),
638 tuple([script_name, 15, "c1"]),
639 )
640 ],
641 },
642 )
643
644 # Now ensure the coroutine stacks for the awaited_by relationships are correct.
645 awaited_by_coroutine_stacks = {
646 task.task_name: sorted(
647 (
648 id_to_task[coro.task_name].task_name,
649 tuple(tuple(frame) for frame in coro.call_stack),
650 )
651 for coro in task.awaited_by
652 )
653 for task in stack_trace[0].awaited_by
654 }
655 self.assertEqual(
656 awaited_by_coroutine_stacks,
657 {
658 "Task-1": [],
659 "Task-2": [
660 ("Task-1", (tuple([script_name, 21, "main"]),))
661 ],
662 },
663 )
664
665 @skip_if_not_supported
666 @unittest.skipIf(
667 sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
668 "Test only runs on Linux with process_vm_readv support",
669 )
670 def test_async_staggered_race_remote_stack_trace(self):
671 # Spawn a process with some realistic Python code
672 port = find_unused_port()
673 script = textwrap.dedent(
674 f"""\
675 import asyncio.staggered
676 import time
677 import sys
678 import socket
679 # Connect to the test process
680 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
681 sock.connect(('localhost', {port}))
682
683 async def deep():
684 await asyncio.sleep(0)
685 sock.sendall(b"ready"); time.sleep(10_000) # same line number
686
687 async def c1():
688 await asyncio.sleep(0)
689 await deep()
690
691 async def c2():
692 await asyncio.sleep(10_000)
693
694 async def main():
695 await asyncio.staggered.staggered_race(
696 [c1, c2],
697 delay=None,
698 )
699
700 asyncio.run(main())
701 """
702 )
703 stack_trace = None
704 with os_helper.temp_dir() as work_dir:
705 script_dir = os.path.join(work_dir, "script_pkg")
706 os.mkdir(script_dir)
707 # Create a socket server to communicate with the target process
708 server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
709 server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
710 server_socket.bind(("localhost", port))
711 server_socket.settimeout(SHORT_TIMEOUT)
712 server_socket.listen(1)
713 script_name = _make_test_script(script_dir, "script", script)
714 client_socket = None
715 try:
716 p = subprocess.Popen([sys.executable, script_name])
717 client_socket, _ = server_socket.accept()
718 server_socket.close()
719 response = client_socket.recv(1024)
720 self.assertEqual(response, b"ready")
721 stack_trace = get_async_stack_trace(p.pid)
722 except PermissionError:
723 self.skipTest(
724 "Insufficient permissions to read the stack trace"
725 )
726 finally:
727 if client_socket is not None:
728 client_socket.close()
729 p.kill()
730 p.terminate()
731 p.wait(timeout=SHORT_TIMEOUT)
732
733 # First check all the tasks are present
734 tasks_names = [
735 task.task_name for task in stack_trace[0].awaited_by
736 ]
737 for task_name in ["Task-1", "Task-2"]:
738 self.assertIn(task_name, tasks_names)
739
740 # Now ensure that the awaited_by_relationships are correct
741 id_to_task = {
742 task.task_id: task for task in stack_trace[0].awaited_by
743 }
744 task_name_to_awaited_by = {
745 task.task_name: set(
746 id_to_task[awaited.task_name].task_name
747 for awaited in task.awaited_by
748 )
749 for task in stack_trace[0].awaited_by
750 }
751 self.assertEqual(
752 task_name_to_awaited_by,
753 {
754 "Task-1": set(),
755 "Task-2": {"Task-1"},
756 },
757 )
758
759 # Now ensure that the coroutine stacks are correct
760 coroutine_stacks = {
761 task.task_name: sorted(
762 tuple(tuple(frame) for frame in coro.call_stack)
763 for coro in task.coroutine_stack
764 )
765 for task in stack_trace[0].awaited_by
766 }
767 self.assertEqual(
768 coroutine_stacks,
769 {
770 "Task-1": [
771 (
772 tuple([staggered.__file__, ANY, "staggered_race"]),
773 tuple([script_name, 21, "main"]),
774 )
775 ],
776 "Task-2": [
777 (
778 tuple([script_name, 11, "deep"]),
779 tuple([script_name, 15, "c1"]),
780 tuple(
781 [
782 staggered.__file__,
783 ANY,
784 "staggered_race.<locals>.run_one_coro",
785 ]
786 ),
787 )
788 ],
789 },
790 )
791
792 # Now ensure the coroutine stacks for the awaited_by relationships are correct.
793 awaited_by_coroutine_stacks = {
794 task.task_name: sorted(
795 (
796 id_to_task[coro.task_name].task_name,
797 tuple(tuple(frame) for frame in coro.call_stack),
798 )
799 for coro in task.awaited_by
800 )
801 for task in stack_trace[0].awaited_by
802 }
803 self.assertEqual(
804 awaited_by_coroutine_stacks,
805 {
806 "Task-1": [],
807 "Task-2": [
808 (
809 "Task-1",
810 (
811 tuple(
812 [staggered.__file__, ANY, "staggered_race"]
813 ),
814 tuple([script_name, 21, "main"]),
815 ),
816 )
817 ],
818 },
819 )
820
821 @skip_if_not_supported
822 @unittest.skipIf(
823 sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
824 "Test only runs on Linux with process_vm_readv support",
825 )
826 def test_async_global_awaited_by(self):
827 port = find_unused_port()
828 script = textwrap.dedent(
829 f"""\
830 import asyncio
831 import os
832 import random
833 import sys
834 import socket
835 from string import ascii_lowercase, digits
836 from test.support import socket_helper, SHORT_TIMEOUT
837
838 HOST = '127.0.0.1'
839 PORT = socket_helper.find_unused_port()
840 connections = 0
841
842 # Connect to the test process
843 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
844 sock.connect(('localhost', {port}))
845
846 class EchoServerProtocol(asyncio.Protocol):
847 def connection_made(self, transport):
848 global connections
849 connections += 1
850 self.transport = transport
851
852 def data_received(self, data):
853 self.transport.write(data)
854 self.transport.close()
855
856 async def echo_client(message):
857 reader, writer = await asyncio.open_connection(HOST, PORT)
858 writer.write(message.encode())
859 await writer.drain()
860
861 data = await reader.read(100)
862 assert message == data.decode()
863 writer.close()
864 await writer.wait_closed()
865 # Signal we are ready to sleep
866 sock.sendall(b"ready")
867 await asyncio.sleep(SHORT_TIMEOUT)
868
869 async def echo_client_spam(server):
870 async with asyncio.TaskGroup() as tg:
871 while connections < 1000:
872 msg = list(ascii_lowercase + digits)
873 random.shuffle(msg)
874 tg.create_task(echo_client("".join(msg)))
875 await asyncio.sleep(0)
876 # at least a 1000 tasks created. Each task will signal
877 # when is ready to avoid the race caused by the fact that
878 # tasks are waited on tg.__exit__ and we cannot signal when
879 # that happens otherwise
880 # at this point all client tasks completed without assertion errors
881 # let's wrap up the test
882 server.close()
883 await server.wait_closed()
884
885 async def main():
886 loop = asyncio.get_running_loop()
887 server = await loop.create_server(EchoServerProtocol, HOST, PORT)
888 async with server:
889 async with asyncio.TaskGroup() as tg:
890 tg.create_task(server.serve_forever(), name="server task")
891 tg.create_task(echo_client_spam(server), name="echo client spam")
892
893 asyncio.run(main())
894 """
895 )
896 stack_trace = None
897 with os_helper.temp_dir() as work_dir:
898 script_dir = os.path.join(work_dir, "script_pkg")
899 os.mkdir(script_dir)
900 # Create a socket server to communicate with the target process
901 server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
902 server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
903 server_socket.bind(("localhost", port))
904 server_socket.settimeout(SHORT_TIMEOUT)
905 server_socket.listen(1)
906 script_name = _make_test_script(script_dir, "script", script)
907 client_socket = None
908 try:
909 p = subprocess.Popen([sys.executable, script_name])
910 client_socket, _ = server_socket.accept()
911 server_socket.close()
912 for _ in range(1000):
913 expected_response = b"ready"
914 response = client_socket.recv(len(expected_response))
915 self.assertEqual(response, expected_response)
916 for _ in busy_retry(SHORT_TIMEOUT):
917 try:
918 all_awaited_by = get_all_awaited_by(p.pid)
919 except RuntimeError as re:
920 # This call reads a linked list in another process with
921 # no synchronization. That occasionally leads to invalid
922 # reads. Here we avoid making the test flaky.
923 msg = str(re)
924 if msg.startswith("Task list appears corrupted"):
925 continue
926 elif msg.startswith(
927 "Invalid linked list structure reading remote memory"
928 ):
929 continue
930 elif msg.startswith("Unknown error reading memory"):
931 continue
932 elif msg.startswith("Unhandled frame owner"):
933 continue
934 raise # Unrecognized exception, safest not to ignore it
935 else:
936 break
937 # expected: a list of two elements: 1 thread, 1 interp
938 self.assertEqual(len(all_awaited_by), 2)
939 # expected: a tuple with the thread ID and the awaited_by list
940 self.assertEqual(len(all_awaited_by[0]), 2)
941 # expected: no tasks in the fallback per-interp task list
942 self.assertEqual(all_awaited_by[1], (0, []))
943 entries = all_awaited_by[0][1]
944 # expected: at least 1000 pending tasks
945 self.assertGreaterEqual(len(entries), 1000)
946 # the first three tasks stem from the code structure
947 main_stack = [
948 FrameInfo([taskgroups.__file__, ANY, "TaskGroup._aexit"]),
949 FrameInfo(
950 [taskgroups.__file__, ANY, "TaskGroup.__aexit__"]
951 ),
952 FrameInfo([script_name, 60, "main"]),
953 ]
954 self.assertIn(
955 TaskInfo(
956 [ANY, "Task-1", [CoroInfo([main_stack, ANY])], []]
957 ),
958 entries,
959 )
960 self.assertIn(
961 TaskInfo(
962 [
963 ANY,
964 "server task",
965 [
966 CoroInfo(
967 [
968 [
969 FrameInfo(
970 [
971 base_events.__file__,
972 ANY,
973 "Server.serve_forever",
974 ]
975 )
976 ],
977 ANY,
978 ]
979 )
980 ],
981 [
982 CoroInfo(
983 [
984 [
985 FrameInfo(
986 [
987 taskgroups.__file__,
988 ANY,
989 "TaskGroup._aexit",
990 ]
991 ),
992 FrameInfo(
993 [
994 taskgroups.__file__,
995 ANY,
996 "TaskGroup.__aexit__",
997 ]
998 ),
999 FrameInfo(
1000 [script_name, ANY, "main"]
1001 ),
1002 ],
1003 ANY,
1004 ]
1005 )
1006 ],
1007 ]
1008 ),
1009 entries,
1010 )
1011 self.assertIn(
1012 TaskInfo(
1013 [
1014 ANY,
1015 "Task-4",
1016 [
1017 CoroInfo(
1018 [
1019 [
1020 FrameInfo(
1021 [tasks.__file__, ANY, "sleep"]
1022 ),
1023 FrameInfo(
1024 [
1025 script_name,
1026 38,
1027 "echo_client",
1028 ]
1029 ),
1030 ],
1031 ANY,
1032 ]
1033 )
1034 ],
1035 [
1036 CoroInfo(
1037 [
1038 [
1039 FrameInfo(
1040 [
1041 taskgroups.__file__,
1042 ANY,
1043 "TaskGroup._aexit",
1044 ]
1045 ),
1046 FrameInfo(
1047 [
1048 taskgroups.__file__,
1049 ANY,
1050 "TaskGroup.__aexit__",
1051 ]
1052 ),
1053 FrameInfo(
1054 [
1055 script_name,
1056 41,
1057 "echo_client_spam",
1058 ]
1059 ),
1060 ],
1061 ANY,
1062 ]
1063 )
1064 ],
1065 ]
1066 ),
1067 entries,
1068 )
1069
1070 expected_awaited_by = [
1071 CoroInfo(
1072 [
1073 [
1074 FrameInfo(
1075 [
1076 taskgroups.__file__,
1077 ANY,
1078 "TaskGroup._aexit",
1079 ]
1080 ),
1081 FrameInfo(
1082 [
1083 taskgroups.__file__,
1084 ANY,
1085 "TaskGroup.__aexit__",
1086 ]
1087 ),
1088 FrameInfo(
1089 [script_name, 41, "echo_client_spam"]
1090 ),
1091 ],
1092 ANY,
1093 ]
1094 )
1095 ]
1096 tasks_with_awaited = [
1097 task
1098 for task in entries
1099 if task.awaited_by == expected_awaited_by
1100 ]
1101 self.assertGreaterEqual(len(tasks_with_awaited), 1000)
1102
1103 # the final task will have some random number, but it should for
1104 # sure be one of the echo client spam horde (In windows this is not true
1105 # for some reason)
1106 if sys.platform != "win32":
1107 self.assertEqual(
1108 tasks_with_awaited[-1].awaited_by,
1109 entries[-1].awaited_by,
1110 )
1111 except PermissionError:
1112 self.skipTest(
1113 "Insufficient permissions to read the stack trace"
1114 )
1115 finally:
1116 if client_socket is not None:
1117 client_socket.close()
1118 p.kill()
1119 p.terminate()
1120 p.wait(timeout=SHORT_TIMEOUT)
1121
1122 @skip_if_not_supported
1123 @unittest.skipIf(
1124 sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
1125 "Test only runs on Linux with process_vm_readv support",
1126 )
1127 def test_self_trace(self):
1128 stack_trace = get_stack_trace(os.getpid())
1129 # Is possible that there are more threads, so we check that the
1130 # expected stack traces are in the result (looking at you Windows!)
1131 this_tread_stack = None
1132 for thread_id, stack in stack_trace:
1133 if thread_id == threading.get_native_id():
1134 this_tread_stack = stack
1135 break
1136 self.assertIsNotNone(this_tread_stack)
1137 self.assertEqual(
1138 stack[:2],
1139 [
1140 FrameInfo(
1141 [
1142 __file__,
1143 get_stack_trace.__code__.co_firstlineno + 2,
1144 "get_stack_trace",
1145 ]
1146 ),
1147 FrameInfo(
1148 [
1149 __file__,
1150 self.test_self_trace.__code__.co_firstlineno + 6,
1151 "TestGetStackTrace.test_self_trace",
1152 ]
1153 ),
1154 ],
1155 )
1156
1157 @skip_if_not_supported
1158 @unittest.skipIf(
1159 sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
1160 "Test only runs on Linux with process_vm_readv support",
1161 )
1162 @requires_gil_enabled("Free threaded builds don't have an 'active thread'")
1163 def test_only_active_thread(self):
1164 # Test that only_active_thread parameter works correctly
1165 port = find_unused_port()
1166 script = textwrap.dedent(
1167 f"""\
1168 import time, sys, socket, threading
1169
1170 # Connect to the test process
1171 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1172 sock.connect(('localhost', {port}))
1173
1174 def worker_thread(name, barrier, ready_event):
1175 barrier.wait() # Synchronize thread start
1176 ready_event.wait() # Wait for main thread signal
1177 # Sleep to keep thread alive
1178 time.sleep(10_000)
1179
1180 def main_work():
1181 # Do busy work to hold the GIL
1182 sock.sendall(b"working\\n")
1183 count = 0
1184 while count < 100000000:
1185 count += 1
1186 if count % 10000000 == 0:
1187 pass # Keep main thread busy
1188 sock.sendall(b"done\\n")
1189
1190 # Create synchronization primitives
1191 num_threads = 3
1192 barrier = threading.Barrier(num_threads + 1) # +1 for main thread
1193 ready_event = threading.Event()
1194
1195 # Start worker threads
1196 threads = []
1197 for i in range(num_threads):
1198 t = threading.Thread(target=worker_thread, args=(f"Worker-{{i}}", barrier, ready_event))
1199 t.start()
1200 threads.append(t)
1201
1202 # Wait for all threads to be ready
1203 barrier.wait()
1204
1205 # Signal ready to parent process
1206 sock.sendall(b"ready\\n")
1207
1208 # Signal threads to start waiting
1209 ready_event.set()
1210
1211 # Now do busy work to hold the GIL
1212 main_work()
1213 """
1214 )
1215
1216 with os_helper.temp_dir() as work_dir:
1217 script_dir = os.path.join(work_dir, "script_pkg")
1218 os.mkdir(script_dir)
1219
1220 # Create a socket server to communicate with the target process
1221 server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1222 server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
1223 server_socket.bind(("localhost", port))
1224 server_socket.settimeout(SHORT_TIMEOUT)
1225 server_socket.listen(1)
1226
1227 script_name = _make_test_script(script_dir, "script", script)
1228 client_socket = None
1229 try:
1230 p = subprocess.Popen([sys.executable, script_name])
1231 client_socket, _ = server_socket.accept()
1232 server_socket.close()
1233
1234 # Wait for ready signal
1235 response = b""
1236 while b"ready" not in response:
1237 response += client_socket.recv(1024)
1238
1239 # Wait for the main thread to start its busy work
1240 while b"working" not in response:
1241 response += client_socket.recv(1024)
1242
1243 # Get stack trace with all threads
1244 unwinder_all = RemoteUnwinder(p.pid, all_threads=True)
1245 for _ in range(10):
1246 # Wait for the main thread to start its busy work
1247 all_traces = unwinder_all.get_stack_trace()
1248 found = False
1249 for thread_id, stack in all_traces:
1250 if not stack:
1251 continue
1252 current_frame = stack[0]
1253 if (
1254 current_frame.funcname == "main_work"
1255 and current_frame.lineno > 15
1256 ):
1257 found = True
1258
1259 if found:
1260 break
1261 # Give a bit of time to take the next sample
1262 time.sleep(0.1)
1263 else:
1264 self.fail(
1265 "Main thread did not start its busy work on time"
1266 )
1267
1268 # Get stack trace with only GIL holder
1269 unwinder_gil = RemoteUnwinder(p.pid, only_active_thread=True)
1270 gil_traces = unwinder_gil.get_stack_trace()
1271
1272 except PermissionError:
1273 self.skipTest(
1274 "Insufficient permissions to read the stack trace"
1275 )
1276 finally:
1277 if client_socket is not None:
1278 client_socket.close()
1279 p.kill()
1280 p.terminate()
1281 p.wait(timeout=SHORT_TIMEOUT)
1282
1283 # Verify we got multiple threads in all_traces
1284 self.assertGreater(
1285 len(all_traces), 1, "Should have multiple threads"
1286 )
1287
1288 # Verify we got exactly one thread in gil_traces
1289 self.assertEqual(
1290 len(gil_traces), 1, "Should have exactly one GIL holder"
1291 )
1292
1293 # The GIL holder should be in the all_traces list
1294 gil_thread_id = gil_traces[0][0]
1295 all_thread_ids = [trace[0] for trace in all_traces]
1296 self.assertIn(
1297 gil_thread_id,
1298 all_thread_ids,
1299 "GIL holder should be among all threads",
1300 )
1301
1302
1303 if __name__ == "__main__":
1304 unittest.main()