Debian Codegraph @

processview/core/manager/manager.py

raw
1 from __future__ import annotations
2
3 from processview.utils.singleton import singleton
4 from processview.core.dataset import Dataset
5 from processview.core.dataset import DatasetIdentifier
6 from collections import OrderedDict
7 from silx.utils.enum import Enum as _Enum
8 import threading
9 from typing import Optional, Type, Union
10 from datetime import datetime
11 import weakref
12 import logging
13
14 _logger = logging.getLogger(__name__)
15
16
17 class DatasetState(_Enum):
18 """possible dataset status relatif to a Process"""
19
20 ON_GOING = "on going"
21 SUCCEED = "succeed"
22 FAILED = "failed"
23 PENDING = "pending"
24 SKIPPED = "skipped"
25 WAIT_USER_VALIDATION = "waiting user validation"
26 CANCELLED = "cancelled"
27
28
29 @singleton
30 class ProcessManager:
31 """
32 Manager to register and observe `SuperviseProcess`.
33 """
34
35 def __init__(self):
36 self._processes = {}
37 self._dataset_process_states: dict[int, dict[str, tuple]] = {}
38 """key is processID, Value is a tuple of
39 (dataset state, details)"""
40 self._processID = 0
41 self._updateCallback = set()
42 """list of callback to trigger when an update is generated"""
43 self._newProcessCallback = set()
44 """list of callback to trigger when a process is added"""
45 self._processRemovedCallback = set()
46 """list of callback to trigger when a process is removed"""
47 self._process_renamed_callback = set()
48 """list of callback to trigger when a process is renamed"""
49 self.lock = threading.Lock()
50
51 def met(self, process, dataset) -> bool:
52 """check if a dataset has already met a dataset"""
53 if isinstance(dataset, DatasetIdentifier):
54 dataset_id = dataset
55 elif dataset is None:
56 return False
57 else:
58 dataset_id = dataset.get_identifier()
59 if process.process_id in self._dataset_process_states:
60 return dataset_id in self._dataset_process_states[process.process_id]
61
62 def process_renamed(self, process):
63 """
64 the process name only matter for the GUI side
65 """
66 for callback in self._process_renamed_callback:
67 callback(process)
68
69 def register(self, process) -> int:
70 """
71 Register a process to the manager
72
73 :param SuperviseProcess process:
74 """
75 from processview.core.superviseprocess import (
76 SuperviseProcess,
77 ) # avoid cyclic import
78
79 if not isinstance(process, SuperviseProcess):
80 raise TypeError(
81 f"{process} is expected to be an isntance of {SuperviseProcess}"
82 )
83 process_id = self._processID
84 self._processID += 1
85 self._processes[process_id] = weakref.ref(process)
86 process._set_process_id(process_id)
87
88 with self.lock:
89 for callback in self._newProcessCallback:
90 callback()
91 return process_id
92
93 def _find_dataset_id(self, data) -> Dataset:
94 if isinstance(data, Dataset):
95 return data.get_identifier()
96 elif isinstance(data, DatasetIdentifier):
97 return data
98 else:
99 raise TypeError(
100 "dataset should be an instance of Dataset or "
101 "DatasetIdentifier. Get {} instead".format(type(data))
102 )
103
104 def _find_process(self, process):
105 from processview.core.superviseprocess import (
106 SuperviseProcess,
107 ) # avoid cyclic import
108
109 if isinstance(process, SuperviseProcess):
110 return process
111 elif isinstance(process, str):
112 for p in self.get_processes():
113 if p.name == process:
114 return p
115 else:
116 raise TypeError(
117 "process should be an instance of SuperviseProcess or"
118 " str. Get {} instead".format(type(process))
119 )
120
121 def unregister(self, process):
122 """
123 Unregister a process to the manager
124
125 :param BaseProcess process:
126 """
127 with self.lock:
128 if process.process_id in self._processes and process.is_master_process:
129 del self._processes[process.process_id]
130
131 for callback in self._processRemovedCallback:
132 callback()
133
134 def get_processes(self) -> tuple:
135 """
136
137 :return: tuple of processes currently registered
138 """
139 processes = []
140 for _, p in self._processes.items():
141 if p() is not None:
142 processes.append(p())
143 return tuple(processes)
144
145 def get_process(self, process_id: int):
146 if process_id in self._processes:
147 return self._processes[process_id]()
148
149 def get_datasets(self) -> tuple:
150 """
151
152 :return: tuple of datasets
153 """
154 res = set()
155 for _, dataset_states in self._dataset_process_states.items():
156 [res.add(dataset) for dataset in dataset_states]
157 return tuple(res)
158
159 def remove_dataset(self, dataset: Dataset):
160 """
161 Remove a dataset from the list of registered dataset
162 """
163 dataset_id = self._find_dataset_id(dataset)
164 for process_id in self._dataset_process_states:
165 self._dataset_process_states[process_id].pop(dataset_id, None)
166
167 def notify_dataset_state(
168 self,
169 dataset: Dataset | DatasetIdentifier,
170 process,
171 state: DatasetState,
172 details=None,
173 ) -> None:
174 """
175 Update dataset state
176
177 :param Dataset dataset: dataset which state is updated
178 :param BaseProcess process: Process concern by the new state
179 :param DatasetState state: current State
180 :param str info: details about the error or success
181 :return:
182 """
183 from processview.core.superviseprocess import (
184 SuperviseProcess,
185 ) # avoid cyclic import
186
187 if not isinstance(process, SuperviseProcess):
188 _logger.warning(
189 f"Unable to update dataset {dataset} state. Process does not inherit from SuperviseProcess"
190 )
191 if process.process_id not in self._processes:
192 self.register(process)
193 if process.process_id not in self._dataset_process_states:
194 self._dataset_process_states[process.process_id] = OrderedDict()
195 if details is None:
196 details = ""
197 if isinstance(dataset, Dataset):
198 dataset_id = dataset.get_identifier()
199 else:
200 dataset_id = dataset
201 if not isinstance(dataset_id, DatasetIdentifier):
202 raise TypeError(
203 f"dataset should be an instance of {Dataset} or {DatasetIdentifier}. Got {type(dataset)}"
204 )
205 # as we store the identifier and not a string anymore we might need to remove another instanciation
206 process_states = self._dataset_process_states[process.process_id]
207 to_remove = set()
208 for id in process_states:
209 if id == dataset_id:
210 to_remove.add(id)
211 for id in to_remove:
212 del process_states[id]
213 self._dataset_process_states[process.process_id] = process_states
214 self._dataset_process_states[process.process_id][dataset_id] = (
215 state,
216 datetime.now(),
217 details,
218 )
219 self.updated()
220
221 def get_dataset_state(self, dataset_id, process) -> Union[None, DatasetState]:
222 """
223
224 :param Dataset dataset_id:
225 :param BaseProcess process:
226 :return: DatasetState relative to provided process if know
227 :rtype: Union[None, DatasetState]
228 """
229 dataset_id = self._find_dataset_id(dataset_id)
230 assert isinstance(dataset_id, DatasetIdentifier)
231 process = self._find_process(process)
232 if process is None:
233 _logger.warning("process {} is no more supervised".format(process))
234 return
235 if dataset_id is None:
236 _logger.warning("dataset {} is no more supervised".format(dataset_id))
237 elif process.process_id in self._dataset_process_states:
238 if dataset_id in self._dataset_process_states[process.process_id]:
239 return self._dataset_process_states[process.process_id][dataset_id][0]
240 return None
241
242 def get_dataset_details(self, dataset_id, process) -> Union[None, str]:
243 """
244
245 :param Dataset dataset_id:
246 :param BaseProcess process:
247 :return: DatasetState relative to provided process if know
248 """
249 dataset_id = self._find_dataset_id(dataset_id)
250 process = self._find_process(process)
251 if process is None:
252 _logger.warning("process {} is no more supervised".format(process))
253 return
254 if dataset_id is None:
255 _logger.warning("dataset {} is no more supervised".format(dataset_id))
256 if process.process_id in self._dataset_process_states:
257 if dataset_id in self._dataset_process_states[process.process_id]:
258 return self._dataset_process_states[process.process_id][dataset_id][2]
259 return None
260
261 def get_dataset_time_stamp(self, dataset_id, process) -> Union[None, str]:
262 """
263
264 :param Dataset dataset_id:
265 :param BaseProcess process:
266 :return: DatasetState relative to provided process if know
267 """
268 dataset_id = self._find_dataset_id(dataset_id)
269 process = self._find_process(process)
270 if process is None:
271 _logger.warning("process {} is no more supervised".format(process))
272 return
273 if dataset_id is None:
274 _logger.warning("dataset {} is no more supervised".format(dataset_id))
275 if process.process_id in self._dataset_process_states:
276 if dataset_id in self._dataset_process_states[process.process_id]:
277 return self._dataset_process_states[process.process_id][dataset_id][1]
278 return None
279
280 def get_dataset_stream(
281 self, dataset, time_stamp=False
282 ) -> tuple[int, DatasetState, list]:
283 """
284
285 :param Dataset dataset: dataset the stream is focus on
286 :param bool time_stamp: if True then return timestamp in the list of
287 elements
288 :return: stream of (process ID, DatasetState) for a given dataset
289 """
290 stream = []
291 for process_id, dataset_states in self._dataset_process_states.items():
292 dst_id = dataset.get_identifier()
293 if dst_id in dataset_states:
294 state, _timestamp, _ = dataset_states[dst_id]
295 stream.append((process_id, state, _timestamp))
296 # order the stream
297 stream = sorted(stream, key=lambda elmt: elmt[2])
298 if not time_stamp:
299 stream = [s[:-1] for s in stream]
300 return tuple(stream)
301
302 def get_process_history(self, process, time_stamp=False) -> tuple:
303 """
304 Return the know history of the process.
305
306 :param BaseProcess process:
307 :param bool time_stamp: if True then return timestamp in the list of
308 elements
309 :return: tuple of (DatasetIdentifier, state, [timestamp])
310 """
311 history = []
312 if process.process_id in self._dataset_process_states:
313 dataset_states = self._dataset_process_states[process.process_id]
314 for dataset_id, info in dataset_states.items():
315 state, _timestamp, details = info
316 history.append((dataset_id, state, _timestamp))
317
318 history = sorted(history, key=lambda elmt: elmt[2])
319 if not time_stamp:
320 history = [s[:-1] for s in history]
321 return tuple(history)
322
323 def updated(self):
324 """Function 'open' for monckey patch"""
325 with self.lock:
326 for callback in self._updateCallback:
327 callback()
328
329 def add_update_callback(self, callback) -> None:
330 """
331
332 :param callback: add a callback to be trigger when dataset state change
333 """
334 with self.lock:
335 self._updateCallback.add(callback)
336
337 def remove_update_callback(self, callback) -> None:
338 """
339
340 :param callback: remove a callback from the stack of callback to be
341 call when dataset state change
342 """
343 with self.lock:
344 if callback in self._updateCallback:
345 self._updateCallback.remove(callback)
346
347 def add_new_process_callback(self, callback):
348 self._newProcessCallback.add(callback)
349
350 def remove_new_process_callback(self, callback):
351 with self.lock:
352 if callback in self._newProcessCallback:
353 self._newProcessCallback.remove(callback)
354
355 def add_process_removed_callback(self, callback):
356 self._processRemovedCallback.add(callback)
357
358 def remove_process_removed_callback(self, callback):
359 with self.lock:
360 if callback in self._processRemovedCallback:
361 self._processRemovedCallback.remove(callback)
362
363 def add_process_name_changed_callback(self, callback):
364 self._process_renamed_callback.add(callback)
365
366 def clear(self):
367 """
368 clear registered processes and dataset states
369
370 :return:
371 """
372 self._processes = {}
373 self._dataset_process_states = {}