Debian Codegraph @

jenkins_jobs/parallel.py

raw
1 #!/usr/bin/env python
2 # Copyright (C) 2015 OpenStack, LLC.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
14 # under the License.
15
16 # Concurrent execution helper functions and classes
17
18 from functools import wraps
19 import logging
20 from multiprocessing import cpu_count
21 import threading
22 import traceback
23
24 try:
25 import Queue as queue
26 except ImportError:
27 import queue
28
29 logger = logging.getLogger(__name__)
30
31
32 class TaskFunc(dict):
33 """
34 Simple class to wrap around the information needed to run a function.
35 """
36
37 def __init__(self, n_ord, func, args=None, kwargs=None):
38 self["func"] = func
39 self["args"] = args or []
40 self["kwargs"] = kwargs or {}
41 self["ord"] = n_ord
42
43
44 class Worker(threading.Thread):
45 """
46 Class that actually does the work, gets a TaskFunc through the queue,
47 runs its function with the passed parameters and returns the result
48 If the string 'done' is passed instead of a TaskFunc instance, the thread
49 will end.
50 """
51
52 def __init__(self, in_queue, out_queue):
53 threading.Thread.__init__(self)
54 self.in_queue = in_queue
55 self.out_queue = out_queue
56
57 def run(self):
58 while True:
59 task = self.in_queue.get()
60 if task == "done":
61 return
62 try:
63 res = task["func"](*task["args"], **task["kwargs"])
64 except Exception as exc:
65 res = exc
66 traceback.print_exc()
67 self.out_queue.put((task["ord"], res))
68
69
70 def concurrent(func):
71 @wraps(func)
72 def concurrentized(*args, **kwargs):
73 """
74 This function will spawn workers and run the decorated function
75 concurrently on the workers. It will not ensure the thread safety of
76 the decorated function (the decorated function should be thread safe by
77 itself). It accepts two special parameters:
78
79 :arg list concurrentize: list of the arguments to pass to each of the
80 runs, the results of each run will be returned in the same order.
81 :arg int n_workers: number of workers to use, by default and if '0'
82 passed will autodetect the number of cores and use that, if '1'
83 passed, it will not use any workers and just run as if were not
84 concurrentized everything.
85
86 Example:
87
88 > @concurrent
89 > def sample(param1, param2, param3):
90 > return param1 + param2 + param3
91 >
92 > sample('param1', param2='val2',
93 > concurrent=[
94 > {'param3': 'val3'},
95 > {'param3': 'val4'},
96 > {'param3': 'val5'},
97 > ])
98 >
99 ['param1val2val3', 'param1val2val4', 'param1val2val5']
100
101 This will run the function `concurrentized_function` 3 times, in
102 concurrent (depending on the number of detected cores) and return an
103 array with the results of the executions in the same order the
104 parameters were passed.
105 """
106 n_workers = kwargs.pop("n_workers", 0)
107 p_kwargs = kwargs.pop("concurrent", [])
108 # if only one parameter is passed inside the concurrent dict, run the
109 # original function as is, no need for pools
110 if len(p_kwargs) == 1:
111 kwargs.update(p_kwargs[0])
112 if len(p_kwargs) in (1, 0):
113 return func(*args, **kwargs)
114
115 # prepare the workers
116 # If no number of workers passed or passed 0
117 if not n_workers:
118 n_workers = cpu_count()
119 logging.debug("Running concurrent %d workers", n_workers)
120 worker_pool = []
121 in_queue = queue.Queue()
122 out_queue = queue.Queue()
123 for n_worker in range(n_workers):
124 new_worker = Worker(in_queue, out_queue)
125 new_worker.setDaemon(True)
126 logging.debug("Spawning worker %d", n_worker)
127 new_worker.start()
128 worker_pool.append(new_worker)
129
130 # Feed the workers
131 n_ord = 0
132 for f_kwargs in p_kwargs:
133 f_kwargs.update(kwargs)
134 in_queue.put(TaskFunc(n_ord, func, args, f_kwargs))
135 n_ord += 1
136 for _ in range(n_workers):
137 in_queue.put("done")
138
139 # Wait for the results
140 logging.debug("Waiting for workers to finish processing")
141 results = []
142 for _ in p_kwargs:
143 new_res = out_queue.get()
144 results.append(new_res)
145 # cleanup
146 for worker in worker_pool:
147 worker.join()
148 # Reorder the results
149 results = [r[1] for r in sorted(results)]
150 logging.debug("Concurrent task finished")
151 return results
152
153 return concurrentized