Debian Codegraph @

taskflow/engines/action_engine/executor.py

raw
1 # Copyright (C) 2013 Yahoo! Inc. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14
15 import abc
16
17 import futurist
18
19 from taskflow import task as ta
20 from taskflow.types import failure
21 from taskflow.types import notifier
22
23 # Execution and reversion outcomes.
24 EXECUTED = 'executed'
25 REVERTED = 'reverted'
26
27
28 def _execute_retry(retry, arguments):
29 try:
30 result = retry.execute(**arguments)
31 except Exception:
32 result = failure.Failure()
33 return (EXECUTED, result)
34
35
36 def _revert_retry(retry, arguments):
37 try:
38 result = retry.revert(**arguments)
39 except Exception:
40 result = failure.Failure()
41 return (REVERTED, result)
42
43
44 def _execute_task(task, arguments, progress_callback=None):
45 with notifier.register_deregister(task.notifier,
46 ta.EVENT_UPDATE_PROGRESS,
47 callback=progress_callback):
48 try:
49 task.pre_execute()
50 result = task.execute(**arguments)
51 except Exception:
52 # NOTE(imelnikov): wrap current exception with Failure
53 # object and return it.
54 result = failure.Failure()
55 finally:
56 task.post_execute()
57 return (EXECUTED, result)
58
59
60 def _revert_task(task, arguments, result, failures, progress_callback=None):
61 arguments = arguments.copy()
62 arguments[ta.REVERT_RESULT] = result
63 arguments[ta.REVERT_FLOW_FAILURES] = failures
64 with notifier.register_deregister(task.notifier,
65 ta.EVENT_UPDATE_PROGRESS,
66 callback=progress_callback):
67 try:
68 task.pre_revert()
69 result = task.revert(**arguments)
70 except Exception:
71 # NOTE(imelnikov): wrap current exception with Failure
72 # object and return it.
73 result = failure.Failure()
74 finally:
75 task.post_revert()
76 return (REVERTED, result)
77
78
79 class SerialRetryExecutor:
80 """Executes and reverts retries."""
81
82 def __init__(self):
83 self._executor = futurist.SynchronousExecutor()
84
85 def start(self):
86 """Prepare to execute retries."""
87 self._executor.restart()
88
89 def stop(self):
90 """Finalize retry executor."""
91 self._executor.shutdown()
92
93 def execute_retry(self, retry, arguments):
94 """Schedules retry execution."""
95 fut = self._executor.submit(_execute_retry, retry, arguments)
96 fut.atom = retry
97 return fut
98
99 def revert_retry(self, retry, arguments):
100 """Schedules retry reversion."""
101 fut = self._executor.submit(_revert_retry, retry, arguments)
102 fut.atom = retry
103 return fut
104
105
106 class TaskExecutor(metaclass=abc.ABCMeta):
107 """Executes and reverts tasks.
108
109 This class takes task and its arguments and executes or reverts it.
110 It encapsulates knowledge on how task should be executed or reverted:
111 right now, on separate thread, on another machine, etc.
112 """
113
114 @abc.abstractmethod
115 def execute_task(self, task, task_uuid, arguments,
116 progress_callback=None):
117 """Schedules task execution."""
118
119 @abc.abstractmethod
120 def revert_task(self, task, task_uuid, arguments, result, failures,
121 progress_callback=None):
122 """Schedules task reversion."""
123
124 def start(self):
125 """Prepare to execute tasks."""
126
127 def stop(self):
128 """Finalize task executor."""
129
130
131 class SerialTaskExecutor(TaskExecutor):
132 """Executes tasks one after another."""
133
134 def __init__(self):
135 self._executor = futurist.SynchronousExecutor()
136
137 def start(self):
138 self._executor.restart()
139
140 def stop(self):
141 self._executor.shutdown()
142
143 def execute_task(self, task, task_uuid, arguments, progress_callback=None):
144 fut = self._executor.submit(_execute_task,
145 task, arguments,
146 progress_callback=progress_callback)
147 fut.atom = task
148 return fut
149
150 def revert_task(self, task, task_uuid, arguments, result, failures,
151 progress_callback=None):
152 fut = self._executor.submit(_revert_task,
153 task, arguments, result, failures,
154 progress_callback=progress_callback)
155 fut.atom = task
156 return fut
157
158
159 class ParallelTaskExecutor(TaskExecutor):
160 """Executes tasks in parallel.
161
162 Submits tasks to an executor which should provide an interface similar
163 to concurrent.Futures.Executor.
164 """
165
166 constructor_options = [
167 ('max_workers', lambda v: v if v is None else int(v)),
168 ]
169 """
170 Optional constructor keyword arguments this executor supports. These will
171 typically be passed via engine options (by a engine user) and converted
172 into the correct type before being sent into this
173 classes ``__init__`` method.
174 """
175
176 def __init__(self, executor=None, max_workers=None):
177 self._executor = executor
178 self._max_workers = max_workers
179 self._own_executor = executor is None
180
181 @abc.abstractmethod
182 def _create_executor(self, max_workers=None):
183 """Called when an executor has not been provided to make one."""
184
185 def _submit_task(self, func, task, *args, **kwargs):
186 fut = self._executor.submit(func, task, *args, **kwargs)
187 fut.atom = task
188 return fut
189
190 def execute_task(self, task, task_uuid, arguments, progress_callback=None):
191 return self._submit_task(_execute_task, task, arguments,
192 progress_callback=progress_callback)
193
194 def revert_task(self, task, task_uuid, arguments, result, failures,
195 progress_callback=None):
196 return self._submit_task(_revert_task, task, arguments, result,
197 failures, progress_callback=progress_callback)
198
199 def start(self):
200 if self._own_executor:
201 self._executor = self._create_executor(
202 max_workers=self._max_workers)
203
204 def stop(self):
205 if self._own_executor:
206 self._executor.shutdown(wait=True)
207 self._executor = None
208
209
210 class ParallelThreadTaskExecutor(ParallelTaskExecutor):
211 """Executes tasks in parallel using a thread pool executor."""
212
213 def _create_executor(self, max_workers=None):
214 return futurist.ThreadPoolExecutor(max_workers=max_workers)
215
216
217 class ParallelGreenThreadTaskExecutor(ParallelThreadTaskExecutor):
218 """Executes tasks in parallel using a greenthread pool executor."""
219
220 DEFAULT_WORKERS = 1000
221 """
222 Default number of workers when ``None`` is passed; being that
223 greenthreads don't map to native threads or processors very well this
224 is more of a guess/somewhat arbitrary, but it does match what the eventlet
225 greenpool default size is (so at least it's consistent with what eventlet
226 does).
227 """
228
229 def _create_executor(self, max_workers=None):
230 if max_workers is None:
231 max_workers = self.DEFAULT_WORKERS
232 return futurist.GreenThreadPoolExecutor(max_workers=max_workers)