| 1 |
#!/usr/bin/env python |
| 2 |
# Copyright (C) 2015 OpenStack, LLC. |
| 3 |
# |
| 4 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
# you may not use this file except in compliance with the License. |
| 6 |
# You may obtain a copy of the License at |
| 7 |
# |
| 8 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
# |
| 10 |
# Unless required by applicable law or agreed to in writing, software |
| 11 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 12 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 13 |
# License for the specific language governing permissions and limitations |
| 14 |
# under the License. |
| 15 |
|
| 16 |
# Concurrent execution helper functions and classes |
| 17 |
|
| 18 |
from functools import wraps |
| 19 |
import logging |
| 20 |
from multiprocessing import cpu_count |
| 21 |
import threading |
| 22 |
import traceback |
| 23 |
|
| 24 |
try: |
| 25 |
import Queue as queue |
| 26 |
except ImportError: |
| 27 |
import queue |
| 28 |
|
| 29 |
logger = logging.getLogger(__name__) |
| 30 |
|
| 31 |
|
| 32 |
class TaskFunc(dict): |
| 33 |
""" |
| 34 |
Simple class to wrap around the information needed to run a function. |
| 35 |
""" |
| 36 |
|
| 37 |
def __init__(self, n_ord, func, args=None, kwargs=None): |
| 38 |
self["func"] = func |
| 39 |
self["args"] = args or [] |
| 40 |
self["kwargs"] = kwargs or {} |
| 41 |
self["ord"] = n_ord |
| 42 |
|
| 43 |
|
| 44 |
class Worker(threading.Thread): |
| 45 |
""" |
| 46 |
Class that actually does the work, gets a TaskFunc through the queue, |
| 47 |
runs its function with the passed parameters and returns the result |
| 48 |
If the string 'done' is passed instead of a TaskFunc instance, the thread |
| 49 |
will end. |
| 50 |
""" |
| 51 |
|
| 52 |
def __init__(self, in_queue, out_queue): |
| 53 |
threading.Thread.__init__(self) |
| 54 |
self.in_queue = in_queue |
| 55 |
self.out_queue = out_queue |
| 56 |
|
| 57 |
def run(self): |
| 58 |
while True: |
| 59 |
task = self.in_queue.get() |
| 60 |
if task == "done": |
| 61 |
return |
| 62 |
try: |
| 63 |
res = task["func"](*task["args"], **task["kwargs"]) |
| 64 |
except Exception as exc: |
| 65 |
res = exc |
| 66 |
traceback.print_exc() |
| 67 |
self.out_queue.put((task["ord"], res)) |
| 68 |
|
| 69 |
|
| 70 |
def concurrent(func): |
| 71 |
@wraps(func) |
| 72 |
def concurrentized(*args, **kwargs): |
| 73 |
""" |
| 74 |
This function will spawn workers and run the decorated function |
| 75 |
concurrently on the workers. It will not ensure the thread safety of |
| 76 |
the decorated function (the decorated function should be thread safe by |
| 77 |
itself). It accepts two special parameters: |
| 78 |
|
| 79 |
:arg list concurrentize: list of the arguments to pass to each of the |
| 80 |
runs, the results of each run will be returned in the same order. |
| 81 |
:arg int n_workers: number of workers to use, by default and if '0' |
| 82 |
passed will autodetect the number of cores and use that, if '1' |
| 83 |
passed, it will not use any workers and just run as if were not |
| 84 |
concurrentized everything. |
| 85 |
|
| 86 |
Example: |
| 87 |
|
| 88 |
> @concurrent |
| 89 |
> def sample(param1, param2, param3): |
| 90 |
> return param1 + param2 + param3 |
| 91 |
> |
| 92 |
> sample('param1', param2='val2', |
| 93 |
> concurrent=[ |
| 94 |
> {'param3': 'val3'}, |
| 95 |
> {'param3': 'val4'}, |
| 96 |
> {'param3': 'val5'}, |
| 97 |
> ]) |
| 98 |
> |
| 99 |
['param1val2val3', 'param1val2val4', 'param1val2val5'] |
| 100 |
|
| 101 |
This will run the function `concurrentized_function` 3 times, in |
| 102 |
concurrent (depending on the number of detected cores) and return an |
| 103 |
array with the results of the executions in the same order the |
| 104 |
parameters were passed. |
| 105 |
""" |
| 106 |
n_workers = kwargs.pop("n_workers", 0) |
| 107 |
p_kwargs = kwargs.pop("concurrent", []) |
| 108 |
# if only one parameter is passed inside the concurrent dict, run the |
| 109 |
# original function as is, no need for pools |
| 110 |
if len(p_kwargs) == 1: |
| 111 |
kwargs.update(p_kwargs[0]) |
| 112 |
if len(p_kwargs) in (1, 0): |
| 113 |
return func(*args, **kwargs) |
| 114 |
|
| 115 |
# prepare the workers |
| 116 |
# If no number of workers passed or passed 0 |
| 117 |
if not n_workers: |
| 118 |
n_workers = cpu_count() |
| 119 |
logging.debug("Running concurrent %d workers", n_workers) |
| 120 |
worker_pool = [] |
| 121 |
in_queue = queue.Queue() |
| 122 |
out_queue = queue.Queue() |
| 123 |
for n_worker in range(n_workers): |
| 124 |
new_worker = Worker(in_queue, out_queue) |
| 125 |
new_worker.setDaemon(True) |
| 126 |
logging.debug("Spawning worker %d", n_worker) |
| 127 |
new_worker.start() |
| 128 |
worker_pool.append(new_worker) |
| 129 |
|
| 130 |
# Feed the workers |
| 131 |
n_ord = 0 |
| 132 |
for f_kwargs in p_kwargs: |
| 133 |
f_kwargs.update(kwargs) |
| 134 |
in_queue.put(TaskFunc(n_ord, func, args, f_kwargs)) |
| 135 |
n_ord += 1 |
| 136 |
for _ in range(n_workers): |
| 137 |
in_queue.put("done") |
| 138 |
|
| 139 |
# Wait for the results |
| 140 |
logging.debug("Waiting for workers to finish processing") |
| 141 |
results = [] |
| 142 |
for _ in p_kwargs: |
| 143 |
new_res = out_queue.get() |
| 144 |
results.append(new_res) |
| 145 |
# cleanup |
| 146 |
for worker in worker_pool: |
| 147 |
worker.join() |
| 148 |
# Reorder the results |
| 149 |
results = [r[1] for r in sorted(results)] |
| 150 |
logging.debug("Concurrent task finished") |
| 151 |
return results |
| 152 |
|
| 153 |
return concurrentized |