Debian Codegraph @

src/pygame_sdl2/threads/__init__.py

raw
1 """
2 * Experimental *
3
4 Like the map function, but can use a pool of threads.
5
6 Really easy to use threads. eg. tmap(f, alist)
7
8 If you know how to use the map function, you can use threads.
9 """
10
11 __author__ = "Rene Dudfield"
12 __version__ = "0.3.0"
13 __license__ = 'Python license'
14
15 import traceback, sys
16
17 from pygame.compat import geterror
18
19 if sys.version_info[0] == 3:
20 from queue import Queue
21 from queue import Empty
22 elif (sys.version_info[0] == 2 and sys.version_info[1] < 5):
23 from Py25Queue import Queue
24 from Py25Queue import Empty
25 else:
26 # use up to date version
27 from Queue import Queue
28 from Queue import Empty
29
30 import threading
31 Thread = threading.Thread
32
33 STOP = object()
34 FINISH = object()
35
36 # DONE_ONE = object()
37 # DONE_TWO = object()
38
39 # a default worker queue.
40 _wq = None
41
42 # if we are using threads or not. This is the number of workers.
43 _use_workers = 0
44
45 # Set this to the maximum for the amount of Cores/CPUs
46 # Note, that the tests early out.
47 # So it should only test the best number of workers +2
48 MAX_WORKERS_TO_TEST = 64
49
50
51
52 def init(number_of_workers = 0):
53 """ Does a little test to see if threading is worth it.
54 Sets up a global worker queue if it's worth it.
55
56 Calling init() is not required, but is generally better to do.
57 """
58 global _wq, _use_workers
59
60 if number_of_workers:
61 _use_workers = number_of_workers
62 else:
63 _use_workers = benchmark_workers()
64
65 # if it is best to use zero workers, then use that.
66 _wq = WorkerQueue(_use_workers)
67
68
69
70
71 def quit():
72 """ cleans up everything.
73 """
74 global _wq, _use_workers
75 _wq.stop()
76 _wq = None
77 _use_workers = False
78
79
80 def benchmark_workers(a_bench_func = None, the_data = None):
81 """ does a little test to see if workers are at all faster.
82 Returns the number of workers which works best.
83 Takes a little bit of time to run, so you should only really call
84 it once.
85 You can pass in benchmark data, and functions if you want.
86 a_bench_func - f(data)
87 the_data - data to work on.
88 """
89 global _use_workers
90
91 #TODO: try and make this scale better with slower/faster cpus.
92 # first find some variables so that using 0 workers takes about 1.0 seconds.
93 # then go from there.
94
95
96 # note, this will only work with pygame 1.8rc3+
97 # replace the doit() and the_data with something that releases the GIL
98
99
100 import pygame
101 import pygame.transform
102 import time
103
104 if not a_bench_func:
105 def doit(x):
106 return pygame.transform.scale(x, (544, 576))
107 else:
108 doit = a_bench_func
109
110 if not the_data:
111 thedata = []
112 for x in range(10):
113 thedata.append(pygame.Surface((155,155), 0, 32))
114 else:
115 thedata = the_data
116
117 best = time.time() + 100000000
118 best_number = 0
119 last_best = -1
120
121 for num_workers in range(0, MAX_WORKERS_TO_TEST):
122
123 wq = WorkerQueue(num_workers)
124 t1 = time.time()
125 for xx in range(20):
126 print ("active count:%s" % threading.activeCount())
127 results = tmap(doit, thedata, worker_queue = wq)
128 t2 = time.time()
129
130 wq.stop()
131
132
133 total_time = t2 - t1
134 print ("total time num_workers:%s: time:%s:" % (num_workers, total_time))
135
136 if total_time < best:
137 last_best = best_number
138 best_number =num_workers
139 best = total_time
140
141 if num_workers - best_number > 1:
142 # We tried to add more, but it didn't like it.
143 # so we stop with testing at this number.
144 break
145
146
147 return best_number
148
149
150
151
152 class WorkerQueue(object):
153
154 def __init__(self, num_workers = 20):
155 self.queue = Queue()
156 self.pool = []
157 self._setup_workers(num_workers)
158
159 def _setup_workers(self, num_workers):
160 """ Sets up the worker threads
161 NOTE: undefined behaviour if you call this again.
162 """
163 self.pool = []
164
165 for _ in range(num_workers):
166 self.pool.append(Thread(target=self.threadloop))
167
168 for a_thread in self.pool:
169 a_thread.setDaemon(True)
170 a_thread.start()
171
172
173 def do(self, f, *args, **kwArgs):
174 """ puts a function on a queue for running later.
175 """
176 self.queue.put((f, args, kwArgs))
177
178
179 def stop(self):
180 """ Stops the WorkerQueue, waits for all of the threads to finish up.
181 """
182 self.queue.put(STOP)
183 for thread in self.pool:
184 thread.join()
185
186
187 def threadloop(self): #, finish = False):
188 """ Loops until all of the tasks are finished.
189 """
190 while True:
191 args = self.queue.get()
192 if args is STOP:
193 self.queue.put(STOP)
194 self.queue.task_done()
195 break
196 else:
197 try:
198 args[0](*args[1], **args[2])
199 finally:
200 # clean up the queue, raise the exception.
201 self.queue.task_done()
202 #raise
203
204
205 def wait(self):
206 """ waits until all tasks are complete.
207 """
208 self.queue.join()
209
210 class FuncResult:
211 """ Used for wrapping up a function call so that the results are stored
212 inside the instances result attribute.
213 """
214 def __init__(self, f, callback = None, errback = None):
215 """ f - is the function we that we call
216 callback(result) - this is called when the function(f) returns
217 errback(exception) - this is called when the function(f) raises
218 an exception.
219 """
220 self.f = f
221 self.exception = None
222 self.callback = callback
223 self.errback = errback
224
225 def __call__(self, *args, **kwargs):
226
227 #we try to call the function here. If it fails we store the exception.
228 try:
229 self.result = self.f(*args, **kwargs)
230 if self.callback:
231 self.callback(self.result)
232 except Exception:
233 self.exception = geterror()
234 if self.errback:
235 self.errback(self.exception)
236
237
238 def tmap(f, seq_args, num_workers = 20, worker_queue = None, wait = True, stop_on_error = True):
239 """ like map, but uses a thread pool to execute.
240 num_workers - the number of worker threads that will be used. If pool
241 is passed in, then the num_workers arg is ignored.
242 worker_queue - you can optionally pass in an existing WorkerQueue.
243 wait - True means that the results are returned when everything is finished.
244 False means that we return the [worker_queue, results] right away instead.
245 results, is returned as a list of FuncResult instances.
246 stop_on_error -
247 """
248
249 if worker_queue:
250 wq = worker_queue
251 else:
252 # see if we have a global queue to work with.
253 if _wq:
254 wq = _wq
255 else:
256 if num_workers == 0:
257 return map(f, seq_args)
258
259 wq = WorkerQueue(num_workers)
260
261 # we short cut it here if the number of workers is 0.
262 # normal map should be faster in this case.
263 if len(wq.pool) == 0:
264 return map(f, seq_args)
265
266 #print ("queue size:%s" % wq.queue.qsize())
267
268
269 #TODO: divide the data (seq_args) into even chunks and
270 # then pass each thread a map(f, equal_part(seq_args))
271 # That way there should be less locking, and overhead.
272
273
274
275 results = []
276 for sa in seq_args:
277 results.append(FuncResult(f))
278 wq.do(results[-1], sa)
279
280
281 #wq.stop()
282
283 if wait:
284 #print ("wait")
285 wq.wait()
286 #print ("after wait")
287 #print ("queue size:%s" % wq.queue.qsize())
288 if wq.queue.qsize():
289 raise Exception("buggy threadmap")
290 # if we created a worker queue, we need to stop it.
291 if not worker_queue and not _wq:
292 #print ("stoping")
293 wq.stop()
294 if wq.queue.qsize():
295 um = wq.queue.get()
296 if not um is STOP:
297 raise Exception("buggy threadmap")
298
299
300 # see if there were any errors. If so raise the first one. This matches map behaviour.
301 # TODO: the traceback doesn't show up nicely.
302 # NOTE: TODO: we might want to return the results anyway? This should be an option.
303 if stop_on_error:
304 error_ones = list(filter(lambda x:x.exception, results))
305 if error_ones:
306 raise error_ones[0].exception
307
308 return map(lambda x:x.result, results)
309 else:
310 return [wq, results]