Debian Codegraph @

Lib/test/test_importlib/test_threaded_import.py

raw
1 # This is a variant of the very old (early 90's) file
2 # Demo/threads/bug.py. It simply provokes a number of threads into
3 # trying to import the same module "at the same time".
4 # There are no pleasant failure modes -- most likely is that Python
5 # complains several times about module random having no attribute
6 # randrange, and then Python hangs.
7
8 import _imp as imp
9 import os
10 import importlib
11 import sys
12 import time
13 import shutil
14 import threading
15 import unittest
16 from test import support
17 from test.support import verbose
18 from test.support.import_helper import forget, mock_register_at_fork
19 from test.support.os_helper import (TESTFN, unlink, rmtree)
20 from test.support import script_helper, threading_helper
21
22 threading_helper.requires_working_threading(module=True)
23
24 def task(N, done, done_tasks, errors):
25 try:
26 # We don't use modulefinder but still import it in order to stress
27 # importing of different modules from several threads.
28 if len(done_tasks) % 2:
29 import modulefinder
30 import random
31 else:
32 import random
33 import modulefinder
34 # This will fail if random is not completely initialized
35 x = random.randrange(1, 3)
36 except Exception as e:
37 errors.append(e.with_traceback(None))
38 finally:
39 done_tasks.append(threading.get_ident())
40 finished = len(done_tasks) == N
41 if finished:
42 done.set()
43
44 # Create a circular import structure: A -> C -> B -> D -> A
45 # NOTE: `time` is already loaded and therefore doesn't threaten to deadlock.
46
47 circular_imports_modules = {
48 'A': """if 1:
49 import time
50 time.sleep(%(delay)s)
51 x = 'a'
52 import C
53 """,
54 'B': """if 1:
55 import time
56 time.sleep(%(delay)s)
57 x = 'b'
58 import D
59 """,
60 'C': """import B""",
61 'D': """import A""",
62 }
63
64 class Finder:
65 """A dummy finder to detect concurrent access to its find_spec()
66 method."""
67
68 def __init__(self):
69 self.numcalls = 0
70 self.x = 0
71 self.lock = threading.Lock()
72
73 def find_spec(self, name, path=None, target=None):
74 # Simulate some thread-unsafe behaviour. If calls to find_spec()
75 # are properly serialized, `x` will end up the same as `numcalls`.
76 # Otherwise not.
77 assert imp.lock_held()
78 with self.lock:
79 self.numcalls += 1
80 x = self.x
81 time.sleep(0.01)
82 self.x = x + 1
83
84 class FlushingFinder:
85 """A dummy finder which flushes sys.path_importer_cache when it gets
86 called."""
87
88 def find_spec(self, name, path=None, target=None):
89 sys.path_importer_cache.clear()
90
91
92 class ThreadedImportTests(unittest.TestCase):
93
94 def setUp(self):
95 self.old_random = sys.modules.pop('random', None)
96
97 def tearDown(self):
98 # If the `random` module was already initialized, we restore the
99 # old module at the end so that pickling tests don't fail.
100 # See http://bugs.python.org/issue3657#msg110461
101 if self.old_random is not None:
102 sys.modules['random'] = self.old_random
103
104 @mock_register_at_fork
105 def check_parallel_module_init(self, mock_os):
106 if imp.lock_held():
107 # This triggers on, e.g., from test import autotest.
108 raise unittest.SkipTest("can't run when import lock is held")
109
110 done = threading.Event()
111 for N in (20, 50) * 3:
112 if verbose:
113 print("Trying", N, "threads ...", end=' ')
114 # Make sure that random and modulefinder get reimported freshly
115 for modname in ['random', 'modulefinder']:
116 try:
117 del sys.modules[modname]
118 except KeyError:
119 pass
120 errors = []
121 done_tasks = []
122 done.clear()
123 t0 = time.monotonic()
124 with threading_helper.start_threads(
125 threading.Thread(target=task, args=(N, done, done_tasks, errors,))
126 for i in range(N)):
127 pass
128 completed = done.wait(10 * 60)
129 dt = time.monotonic() - t0
130 if verbose:
131 print("%.1f ms" % (dt*1e3), flush=True, end=" ")
132 dbg_info = 'done: %s/%s' % (len(done_tasks), N)
133 self.assertFalse(errors, dbg_info)
134 self.assertTrue(completed, dbg_info)
135 if verbose:
136 print("OK.")
137
138 @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False)
139 def test_parallel_module_init(self, size):
140 self.check_parallel_module_init()
141
142 @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False)
143 def test_parallel_meta_path(self, size):
144 finder = Finder()
145 sys.meta_path.insert(0, finder)
146 try:
147 self.check_parallel_module_init()
148 self.assertGreater(finder.numcalls, 0)
149 self.assertEqual(finder.x, finder.numcalls)
150 finally:
151 sys.meta_path.remove(finder)
152
153 @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False)
154 def test_parallel_path_hooks(self, size):
155 # Here the Finder instance is only used to check concurrent calls
156 # to path_hook().
157 finder = Finder()
158 # In order for our path hook to be called at each import, we need
159 # to flush the path_importer_cache, which we do by registering a
160 # dedicated meta_path entry.
161 flushing_finder = FlushingFinder()
162 def path_hook(path):
163 finder.find_spec('')
164 raise ImportError
165 sys.path_hooks.insert(0, path_hook)
166 sys.meta_path.append(flushing_finder)
167 try:
168 # Flush the cache a first time
169 flushing_finder.find_spec('')
170 numtests = self.check_parallel_module_init()
171 self.assertGreater(finder.numcalls, 0)
172 self.assertEqual(finder.x, finder.numcalls)
173 finally:
174 sys.meta_path.remove(flushing_finder)
175 sys.path_hooks.remove(path_hook)
176
177 def test_import_hangers(self):
178 # In case this test is run again, make sure the helper module
179 # gets loaded from scratch again.
180 try:
181 del sys.modules['test.test_importlib.threaded_import_hangers']
182 except KeyError:
183 pass
184 import test.test_importlib.threaded_import_hangers
185 self.assertFalse(test.test_importlib.threaded_import_hangers.errors)
186
187 def test_circular_imports(self):
188 # The goal of this test is to exercise implementations of the import
189 # lock which use a per-module lock, rather than a global lock.
190 # In these implementations, there is a possible deadlock with
191 # circular imports, for example:
192 # - thread 1 imports A (grabbing the lock for A) which imports B
193 # - thread 2 imports B (grabbing the lock for B) which imports A
194 # Such implementations should be able to detect such situations and
195 # resolve them one way or the other, without freezing.
196 # NOTE: our test constructs a slightly less trivial import cycle,
197 # in order to better stress the deadlock avoidance mechanism.
198 delay = 0.5
199 os.mkdir(TESTFN)
200 self.addCleanup(shutil.rmtree, TESTFN)
201 sys.path.insert(0, TESTFN)
202 self.addCleanup(sys.path.remove, TESTFN)
203 for name, contents in circular_imports_modules.items():
204 contents = contents % {'delay': delay}
205 with open(os.path.join(TESTFN, name + ".py"), "wb") as f:
206 f.write(contents.encode('utf-8'))
207 self.addCleanup(forget, name)
208
209 importlib.invalidate_caches()
210 results = []
211 def import_ab():
212 import A
213 results.append(getattr(A, 'x', None))
214 def import_ba():
215 import B
216 results.append(getattr(B, 'x', None))
217 t1 = threading.Thread(target=import_ab)
218 t2 = threading.Thread(target=import_ba)
219 t1.start()
220 t2.start()
221 t1.join()
222 t2.join()
223 self.assertEqual(set(results), {'a', 'b'})
224
225 @mock_register_at_fork
226 def test_side_effect_import(self, mock_os):
227 code = """if 1:
228 import threading
229 def target():
230 import random
231 t = threading.Thread(target=target)
232 t.start()
233 t.join()
234 t = None"""
235 sys.path.insert(0, os.curdir)
236 self.addCleanup(sys.path.remove, os.curdir)
237 filename = TESTFN + ".py"
238 with open(filename, "wb") as f:
239 f.write(code.encode('utf-8'))
240 self.addCleanup(unlink, filename)
241 self.addCleanup(forget, TESTFN)
242 self.addCleanup(rmtree, '__pycache__')
243 importlib.invalidate_caches()
244 with threading_helper.wait_threads_exit():
245 __import__(TESTFN)
246 del sys.modules[TESTFN]
247
248 @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False)
249 def test_concurrent_futures_circular_import(self, size):
250 # Regression test for bpo-43515
251 fn = os.path.join(os.path.dirname(__file__),
252 'partial', 'cfimport.py')
253 script_helper.assert_python_ok(fn)
254
255 @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False)
256 def test_multiprocessing_pool_circular_import(self, size):
257 # Regression test for bpo-41567
258 fn = os.path.join(os.path.dirname(__file__),
259 'partial', 'pool_in_threads.py')
260 script_helper.assert_python_ok(fn)
261
262 def test_import_failure_race_condition(self):
263 # Regression test for race condition where a thread could receive
264 # a partially-initialized module when another thread's import fails.
265 # The race occurs when:
266 # 1. Thread 1 starts importing, adds module to sys.modules
267 # 2. Thread 2 sees the module in sys.modules
268 # 3. Thread 1's import fails, removes module from sys.modules
269 # 4. Thread 2 should NOT return the stale module reference
270 os.mkdir(TESTFN)
271 self.addCleanup(shutil.rmtree, TESTFN)
272 sys.path.insert(0, TESTFN)
273 self.addCleanup(sys.path.remove, TESTFN)
274
275 # Create a module that partially initializes then fails
276 modname = 'failing_import_module'
277 with open(os.path.join(TESTFN, modname + '.py'), 'w') as f:
278 f.write('''
279 import time
280 PARTIAL_ATTR = 'initialized'
281 time.sleep(0.05) # Widen race window
282 raise RuntimeError("Intentional import failure")
283 ''')
284 self.addCleanup(forget, modname)
285 importlib.invalidate_caches()
286
287 errors = []
288 results = []
289
290 def do_import(delay=0):
291 time.sleep(delay)
292 try:
293 mod = __import__(modname)
294 # If we got a module, verify it's in sys.modules
295 if modname not in sys.modules:
296 errors.append(
297 f"Got module {mod!r} but {modname!r} not in sys.modules"
298 )
299 elif sys.modules[modname] is not mod:
300 errors.append(
301 f"Got different module than sys.modules[{modname!r}]"
302 )
303 else:
304 results.append(('success', mod))
305 except RuntimeError:
306 results.append(('RuntimeError',))
307 except Exception as e:
308 errors.append(f"Unexpected exception: {e}")
309
310 # Run multiple iterations to increase chance of hitting the race
311 for _ in range(10):
312 errors.clear()
313 results.clear()
314 if modname in sys.modules:
315 del sys.modules[modname]
316
317 t1 = threading.Thread(target=do_import, args=(0,))
318 t2 = threading.Thread(target=do_import, args=(0.01,))
319 t1.start()
320 t2.start()
321 t1.join()
322 t2.join()
323
324 # Neither thread should have errors about stale modules
325 self.assertEqual(errors, [], f"Race condition detected: {errors}")
326
327
328 def setUpModule():
329 thread_info = threading_helper.threading_setup()
330 unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info)
331 try:
332 old_switchinterval = sys.getswitchinterval()
333 unittest.addModuleCleanup(sys.setswitchinterval, old_switchinterval)
334 support.setswitchinterval(1e-5)
335 except AttributeError:
336 pass
337
338
339 if __name__ == "__main__":
340 unittest.main()