Debian Codegraph @

Lib/concurrent/futures/thread.py

raw
1 # Copyright 2009 Brian Quinlan. All Rights Reserved.
2 # Licensed to PSF under a Contributor Agreement.
3
4 """Implements ThreadPoolExecutor."""
5
6 __author__ = 'Brian Quinlan ([email protected])'
7
8 from concurrent.futures import _base
9 import itertools
10 import queue
11 import threading
12 import types
13 import weakref
14 import os
15
16
17 _threads_queues = weakref.WeakKeyDictionary()
18 _shutdown = False
19 # Lock that ensures that new workers are not created while the interpreter is
20 # shutting down. Must be held while mutating _threads_queues and _shutdown.
21 _global_shutdown_lock = threading.Lock()
22
23 def _python_exit():
24 global _shutdown
25 with _global_shutdown_lock:
26 _shutdown = True
27 items = list(_threads_queues.items())
28 for t, q in items:
29 q.put(None)
30 for t, q in items:
31 t.join()
32
33 # Register for `_python_exit()` to be called just before joining all
34 # non-daemon threads. This is used instead of `atexit.register()` for
35 # compatibility with subinterpreters, which no longer support daemon threads.
36 # See bpo-39812 for context.
37 threading._register_atexit(_python_exit)
38
39 # At fork, reinitialize the `_global_shutdown_lock` lock in the child process
40 if hasattr(os, 'register_at_fork'):
41 os.register_at_fork(before=_global_shutdown_lock.acquire,
42 after_in_child=_global_shutdown_lock._at_fork_reinit,
43 after_in_parent=_global_shutdown_lock.release)
44 os.register_at_fork(after_in_child=_threads_queues.clear)
45
46
47 class WorkerContext:
48
49 @classmethod
50 def prepare(cls, initializer, initargs):
51 if initializer is not None:
52 if not callable(initializer):
53 raise TypeError("initializer must be a callable")
54 def create_context():
55 return cls(initializer, initargs)
56 def resolve_task(fn, args, kwargs):
57 return (fn, args, kwargs)
58 return create_context, resolve_task
59
60 def __init__(self, initializer, initargs):
61 self.initializer = initializer
62 self.initargs = initargs
63
64 def initialize(self):
65 if self.initializer is not None:
66 self.initializer(*self.initargs)
67
68 def finalize(self):
69 pass
70
71 def run(self, task):
72 fn, args, kwargs = task
73 return fn(*args, **kwargs)
74
75
76 class _WorkItem:
77 def __init__(self, future, task):
78 self.future = future
79 self.task = task
80
81 def run(self, ctx):
82 if not self.future.set_running_or_notify_cancel():
83 return
84
85 try:
86 result = ctx.run(self.task)
87 except BaseException as exc:
88 self.future.set_exception(exc)
89 # Break a reference cycle with the exception 'exc'
90 self = None
91 else:
92 self.future.set_result(result)
93
94 __class_getitem__ = classmethod(types.GenericAlias)
95
96
97 def _worker(executor_reference, ctx, work_queue):
98 try:
99 ctx.initialize()
100 except BaseException:
101 _base.LOGGER.critical('Exception in initializer:', exc_info=True)
102 executor = executor_reference()
103 if executor is not None:
104 executor._initializer_failed()
105 return
106 try:
107 while True:
108 try:
109 work_item = work_queue.get_nowait()
110 except queue.Empty:
111 # attempt to increment idle count if queue is empty
112 executor = executor_reference()
113 if executor is not None:
114 executor._idle_semaphore.release()
115 del executor
116 work_item = work_queue.get(block=True)
117
118 if work_item is not None:
119 work_item.run(ctx)
120 # Delete references to object. See GH-60488
121 del work_item
122 continue
123
124 executor = executor_reference()
125 # Exit if:
126 # - The interpreter is shutting down OR
127 # - The executor that owns the worker has been collected OR
128 # - The executor that owns the worker has been shutdown.
129 if _shutdown or executor is None or executor._shutdown:
130 # Flag the executor as shutting down as early as possible if it
131 # is not gc-ed yet.
132 if executor is not None:
133 executor._shutdown = True
134 # Notice other workers
135 work_queue.put(None)
136 return
137 del executor
138 except BaseException:
139 _base.LOGGER.critical('Exception in worker', exc_info=True)
140 finally:
141 ctx.finalize()
142
143
144 class BrokenThreadPool(_base.BrokenExecutor):
145 """
146 Raised when a worker thread in a ThreadPoolExecutor failed initializing.
147 """
148
149
150 class ThreadPoolExecutor(_base.Executor):
151
152 BROKEN = BrokenThreadPool
153
154 # Used to assign unique thread names when thread_name_prefix is not supplied.
155 _counter = itertools.count().__next__
156
157 @classmethod
158 def prepare_context(cls, initializer, initargs):
159 return WorkerContext.prepare(initializer, initargs)
160
161 def __init__(self, max_workers=None, thread_name_prefix='',
162 initializer=None, initargs=(), **ctxkwargs):
163 """Initializes a new ThreadPoolExecutor instance.
164
165 Args:
166 max_workers: The maximum number of threads that can be used to
167 execute the given calls.
168 thread_name_prefix: An optional name prefix to give our threads.
169 initializer: A callable used to initialize worker threads.
170 initargs: A tuple of arguments to pass to the initializer.
171 ctxkwargs: Additional arguments to cls.prepare_context().
172 """
173 if max_workers is None:
174 # ThreadPoolExecutor is often used to:
175 # * CPU bound task which releases GIL
176 # * I/O bound task (which releases GIL, of course)
177 #
178 # We use process_cpu_count + 4 for both types of tasks.
179 # But we limit it to 32 to avoid consuming surprisingly large resource
180 # on many core machine.
181 max_workers = min(32, (os.process_cpu_count() or 1) + 4)
182 if max_workers <= 0:
183 raise ValueError("max_workers must be greater than 0")
184
185 (self._create_worker_context,
186 self._resolve_work_item_task,
187 ) = type(self).prepare_context(initializer, initargs, **ctxkwargs)
188
189 self._max_workers = max_workers
190 self._work_queue = queue.SimpleQueue()
191 self._idle_semaphore = threading.Semaphore(0)
192 self._threads = set()
193 self._broken = False
194 self._shutdown = False
195 self._shutdown_lock = threading.Lock()
196 self._thread_name_prefix = (thread_name_prefix or
197 ("ThreadPoolExecutor-%d" % self._counter()))
198
199 def submit(self, fn, /, *args, **kwargs):
200 with self._shutdown_lock, _global_shutdown_lock:
201 if self._broken:
202 raise self.BROKEN(self._broken)
203
204 if self._shutdown:
205 raise RuntimeError('cannot schedule new futures after shutdown')
206 if _shutdown:
207 raise RuntimeError('cannot schedule new futures after '
208 'interpreter shutdown')
209
210 f = _base.Future()
211 task = self._resolve_work_item_task(fn, args, kwargs)
212 w = _WorkItem(f, task)
213
214 self._work_queue.put(w)
215 self._adjust_thread_count()
216 return f
217 submit.__doc__ = _base.Executor.submit.__doc__
218
219 def _adjust_thread_count(self):
220 # if idle threads are available, don't spin new threads
221 if self._idle_semaphore.acquire(timeout=0):
222 return
223
224 # When the executor gets lost, the weakref callback will wake up
225 # the worker threads.
226 def weakref_cb(_, q=self._work_queue):
227 q.put(None)
228
229 num_threads = len(self._threads)
230 if num_threads < self._max_workers:
231 thread_name = '%s_%d' % (self._thread_name_prefix or self,
232 num_threads)
233 t = threading.Thread(name=thread_name, target=_worker,
234 args=(weakref.ref(self, weakref_cb),
235 self._create_worker_context(),
236 self._work_queue))
237 t.start()
238 self._threads.add(t)
239 _threads_queues[t] = self._work_queue
240
241 def _initializer_failed(self):
242 with self._shutdown_lock:
243 self._broken = ('A thread initializer failed, the thread pool '
244 'is not usable anymore')
245 # Drain work queue and mark pending futures failed
246 while True:
247 try:
248 work_item = self._work_queue.get_nowait()
249 except queue.Empty:
250 break
251 if work_item is not None:
252 work_item.future.set_exception(self.BROKEN(self._broken))
253
254 def shutdown(self, wait=True, *, cancel_futures=False):
255 with self._shutdown_lock:
256 self._shutdown = True
257 if cancel_futures:
258 # Drain all work items from the queue, and then cancel their
259 # associated futures.
260 while True:
261 try:
262 work_item = self._work_queue.get_nowait()
263 except queue.Empty:
264 break
265 if work_item is not None:
266 work_item.future.cancel()
267
268 # Send a wake-up to prevent threads calling
269 # _work_queue.get(block=True) from permanently blocking.
270 self._work_queue.put(None)
271 if wait:
272 for t in self._threads:
273 t.join()
274 shutdown.__doc__ = _base.Executor.shutdown.__doc__