Debian Codegraph @

pymeasure/experiment/results.py

raw
1 #
2 # This file is part of the PyMeasure package.
3 #
4 # Copyright (c) 2013-2024 PyMeasure Developers
5 #
6 # Permission is hereby granted, free of charge, to any person obtaining a copy
7 # of this software and associated documentation files (the "Software"), to deal
8 # in the Software without restriction, including without limitation the rights
9 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 # copies of the Software, and to permit persons to whom the Software is
11 # furnished to do so, subject to the following conditions:
12 #
13 # The above copyright notice and this permission notice shall be included in
14 # all copies or substantial portions of the Software.
15 #
16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 # THE SOFTWARE.
23 #
24
25 from decimal import Decimal
26 import logging
27 import os
28 import re
29 import sys
30 from importlib import import_module
31 from importlib.machinery import SourceFileLoader
32 from datetime import datetime
33 from string import Formatter
34
35 import pandas as pd
36 import pint
37
38 from .procedure import Procedure, UnknownProcedure
39 from pymeasure.units import ureg
40
41 log = logging.getLogger(__name__)
42 log.addHandler(logging.NullHandler())
43
44
45 def replace_placeholders(string, procedure, date_format="%Y-%m-%d", time_format="%H:%M:%S"):
46 """Replace placeholders in string with values from procedure parameters.
47
48 Replaces the placeholders in the provided string with the values of the
49 associated parameters, as provided by the procedure. This uses the standard
50 python string.format syntax. Apart from the parameter in the procedure (which
51 should be called by their full names) "date" and "time" are also added as optional
52 placeholders.
53
54 :param string:
55 The string in which the placeholders are to be replaced. Python string.format
56 syntax is used, e.g. "{Parameter Name}" to insert a FloatParameter called
57 "Parameter Name", or "{Parameter Name:.2f}" to also specifically format the
58 parameter.
59
60 :param procedure:
61 The procedure from which to get the parameter values.
62
63 :param date_format:
64 A string to represent how the additional placeholder "date" will be formatted.
65
66 :param time_format:
67 A string to represent how the additional placeholder "time" will be formatted.
68
69 """
70 now = datetime.now()
71
72 parameters = procedure.placeholder_objects()
73 placeholders = {param.name: param.value for param in parameters.values()}
74
75 placeholders["date"] = now.strftime(date_format)
76 placeholders["time"] = now.strftime(time_format)
77
78 # Check keys against available parameters
79 invalid_keys = [i[1] for i in Formatter().parse(string)
80 if i[1] is not None and i[1] not in placeholders]
81 if invalid_keys:
82 raise KeyError("The following placeholder-keys are not valid: '%s'; "
83 "valid keys are: '%s'." % (
84 "', '".join(invalid_keys),
85 "', '".join(placeholders.keys())
86 ))
87
88 return string.format(**placeholders)
89
90
91 def unique_filename(directory, prefix='DATA', suffix='', ext='csv',
92 dated_folder=False, index=True, datetimeformat="%Y-%m-%d",
93 procedure=None):
94 """ Returns a unique filename based on the directory and prefix
95 """
96 now = datetime.now()
97 directory = os.path.abspath(directory)
98
99 if procedure is not None:
100 prefix = replace_placeholders(prefix, procedure)
101 suffix = replace_placeholders(suffix, procedure)
102
103 if dated_folder:
104 directory = os.path.join(directory, now.strftime('%Y-%m-%d'))
105 if not os.path.exists(directory):
106 os.makedirs(directory)
107 if index:
108 i = 1
109 basename = f"{prefix}{now.strftime(datetimeformat)}"
110 basepath = os.path.join(directory, basename)
111 filename = "%s_%d%s.%s" % (basepath, i, suffix, ext)
112 while os.path.exists(filename):
113 i += 1
114 filename = "%s_%d%s.%s" % (basepath, i, suffix, ext)
115 else:
116 basename = f"{prefix}{now.strftime(datetimeformat)}{suffix}.{ext}"
117 filename = os.path.join(directory, basename)
118 return filename
119
120
121 class CSVFormatter(logging.Formatter):
122 """ Formatter of data results """
123
124 def __init__(self, columns, delimiter=','):
125 """Creates a csv formatter for a given list of columns (=header).
126
127 :param columns: list of column names.
128 :type columns: list
129 :param delimiter: delimiter between columns.
130 :type delimiter: str
131 """
132 super().__init__()
133 self.columns = columns
134 self.units = Procedure.parse_columns(columns)
135 self.delimiter = delimiter
136
137 def format(self, record):
138 """Formats a record as csv.
139
140 :param record: record to format.
141 :type record: dict
142 :return: a string
143 """
144 line = []
145 for x in self.columns:
146 value = record.get(x, float("nan"))
147 if isinstance(value, (float, int, Decimal)) and type(value) is not bool:
148 line.append(f"{value}")
149 else:
150 units = self.units.get(x, None)
151 if units is not None:
152 if isinstance(value, str):
153 try:
154 value = ureg.Quantity(value)
155 except pint.UndefinedUnitError:
156 log.warning(
157 f"Value {value} for column {x} cannot be parsed to"
158 f" unit {units}.")
159 if isinstance(value, pint.Quantity):
160 try:
161 line.append(f"{value.m_as(units)}")
162 except pint.DimensionalityError:
163 line.append("nan")
164 log.warning(
165 f"Value {value} for column {x} does not have the "
166 f"right unit {units}.")
167 elif isinstance(value, bool):
168 line.append("nan")
169 log.warning(
170 f"Boolean for column {x} does not have unit {units}.")
171 else:
172 line.append("nan")
173 log.warning(
174 f"Value {value} for column {x} does not have the right"
175 f" type for unit {units}.")
176 else:
177 if isinstance(value, pint.Quantity):
178 if value.units == ureg.dimensionless:
179 line.append(f"{value.magnitude}")
180 else:
181 self.units[x] = value.to_base_units().units
182 line.append(f"{value.m_as(self.units[x])}")
183 log.info(f"Column {x} units was set to {self.units[x]}")
184 else:
185 line.append(f"{value}")
186 return self.delimiter.join(line)
187
188 def format_header(self):
189 return self.delimiter.join(self.columns)
190
191
192 class Results:
193 """ The Results class provides a convenient interface to reading and
194 writing data in connection with a :class:`.Procedure` object.
195
196 :cvar COMMENT: The character used to identify a comment (default: #)
197 :cvar DELIMITER: The character used to delimit the data (default: ,)
198 :cvar LINE_BREAK: The character used for line breaks (default \\n)
199 :cvar CHUNK_SIZE: The length of the data chuck that is read
200
201 :param procedure: Procedure object
202 :param data_filename: The data filename where the data is or should be
203 stored
204 """
205
206 COMMENT = '#'
207 DELIMITER = ','
208 LINE_BREAK = "\n"
209 CHUNK_SIZE = 1000
210
211 def __init__(self, procedure, data_filename):
212 if not isinstance(procedure, Procedure):
213 raise ValueError("Results require a Procedure object")
214 self.procedure = procedure
215 self.procedure_class = procedure.__class__
216 self.parameters = procedure.parameter_objects()
217 self._header_count = -1
218 self._metadata_count = -1
219
220 self.formatter = CSVFormatter(columns=self.procedure.DATA_COLUMNS)
221
222 if isinstance(data_filename, (list, tuple)):
223 data_filenames, data_filename = data_filename, data_filename[0]
224 else:
225 data_filenames = [data_filename]
226
227 self.data_filename = data_filename
228 self.data_filenames = data_filenames
229
230 if os.path.exists(data_filename): # Assume header is already written
231 self.reload()
232 self.procedure.status = Procedure.FINISHED
233 # TODO: Correctly store and retrieve status
234 else:
235 for filename in self.data_filenames:
236 with open(filename, 'w') as f:
237 f.write(self.header())
238 f.write(self.labels())
239 self._data = None
240
241 def __getstate__(self):
242 # Get all information needed to reconstruct procedure
243 self._parameters = self.procedure.parameter_values()
244 self._class = self.procedure.__class__.__name__
245 module = sys.modules[self.procedure.__module__]
246 self._package = module.__package__
247 self._module = module.__name__
248 self._file = module.__file__
249
250 state = self.__dict__.copy()
251 del state['procedure']
252 del state['procedure_class']
253 return state
254
255 def __setstate__(self, state):
256 self.__dict__.update(state)
257
258 # Restore the procedure
259 module = SourceFileLoader(self._module, self._file).load_module()
260 cls = getattr(module, self._class)
261
262 self.procedure = cls()
263 self.procedure.set_parameters(self._parameters)
264 self.procedure.refresh_parameters()
265
266 self.procedure_class = cls
267
268 del self._parameters
269 del self._class
270 del self._package
271 del self._module
272 del self._file
273
274 def header(self):
275 """ Returns a text header to accompany a datafile so that the procedure
276 can be reconstructed
277 """
278 h = []
279 procedure = re.search("'(?P<name>[^']+)'",
280 repr(self.procedure_class)).group("name")
281 h.append("Procedure: <%s>" % procedure)
282 h.append("Parameters:")
283 for name, parameter in self.parameters.items():
284 h.append("\t{}: {}".format(parameter.name, str(
285 parameter).encode("unicode_escape").decode("utf-8")))
286 h.append("Data:")
287 self._header_count = len(h)
288 h = [Results.COMMENT + line for line in h] # Comment each line
289 return Results.LINE_BREAK.join(h) + Results.LINE_BREAK
290
291 def labels(self):
292 """ Returns the columns labels as a string to be written
293 to the file
294 """
295 return self.formatter.format_header() + Results.LINE_BREAK
296
297 def format(self, data):
298 """ Returns a formatted string containing the data to be written
299 to a file
300 """
301 return self.formatter.format(data)
302
303 def parse(self, line):
304 """ Returns a dictionary containing the data from the line """
305 data = {}
306 items = line.split(Results.DELIMITER)
307 for i, key in enumerate(self.procedure.DATA_COLUMNS):
308 data[key] = items[i]
309 return data
310
311 def metadata(self):
312 """ Returns a text header for the metadata to write into the datafile """
313 if not self.procedure.metadata_objects():
314 return
315
316 m = ["Metadata:"]
317 for _, metadata in self.procedure.metadata_objects().items():
318 value = str(metadata).encode("unicode_escape").decode("utf-8")
319 m.append(f"\t{metadata.name}: {value}")
320
321 self._metadata_count = len(m)
322 m = [Results.COMMENT + line for line in m] # Comment each line
323 return Results.LINE_BREAK.join(m) + Results.LINE_BREAK
324
325 def store_metadata(self):
326 """ Inserts the metadata header (if any) into the datafile """
327 c_header = self.metadata()
328 if c_header is None:
329 return
330
331 for filename in self.data_filenames:
332 with open(filename, 'r+') as f:
333 contents = f.readlines()
334 contents.insert(self._header_count - 1, c_header)
335
336 f.seek(0)
337 f.writelines(contents)
338
339 self._header_count += self._metadata_count
340
341 @staticmethod
342 def parse_header(header, procedure_class=None):
343 """ Returns a Procedure object with the parameters as defined in the
344 header text.
345 """
346 if procedure_class is not None:
347 procedure = procedure_class()
348 else:
349 procedure = None
350
351 header = header.split(Results.LINE_BREAK)
352 procedure_module = None
353 parameters = {}
354 for line in header:
355 if line.startswith(Results.COMMENT):
356 line = line[1:] # Uncomment
357 else:
358 raise ValueError("Parsing a header which contains "
359 "uncommented sections")
360 if line.startswith("Procedure"):
361 regex = r"<(?:(?P<module>[^>]+)\.)?(?P<class>[^.>]+)>"
362 search = re.search(regex, line)
363 procedure_module = search.group("module")
364 procedure_class = search.group("class")
365 elif line.startswith("\t"):
366 separator = ": "
367 partitioned_line = line[1:].partition(separator)
368 if partitioned_line[1] != separator:
369 raise Exception("Error partitioning header line %s." % line)
370 else:
371 parameters[partitioned_line[0]] = partitioned_line[2]
372
373 if procedure is None:
374 if procedure_class is None:
375 raise ValueError("Header does not contain the Procedure class")
376 try:
377 procedure_module = import_module(procedure_module)
378 procedure_class = getattr(procedure_module, procedure_class)
379 procedure = procedure_class()
380 except ImportError:
381 procedure = UnknownProcedure(parameters)
382 log.warning("Unknown Procedure being used")
383
384 # Fill the procedure with the parameters found
385 for name, parameter in procedure.parameter_objects().items():
386 if parameter.name in parameters:
387 value = parameters[parameter.name]
388 setattr(procedure, name, value)
389 else:
390 log.warning(
391 f"Parameter \"{parameter.name}\" not found when loading " +
392 f"'{procedure_class}', setting default value")
393 setattr(procedure, name, parameter.default)
394
395 procedure.refresh_parameters() # Enforce update of meta data
396
397 # Fill the procedure with the metadata found
398 for name, metadata in procedure.metadata_objects().items():
399 if metadata.name in parameters:
400 value = parameters[metadata.name]
401 setattr(procedure, name, value)
402
403 # Set the value in the metadata
404 metadata._value = value
405 metadata.evaluated = True
406
407 return procedure
408
409 @staticmethod
410 def load(data_filename, procedure_class=None):
411 """ Returns a Results object with the associated Procedure object and
412 data
413 """
414 header = ""
415 header_read = False
416 header_count = 0
417 with open(data_filename) as f:
418 while not header_read:
419 line = f.readline()
420 if line.startswith(Results.COMMENT):
421 header += line.strip('\t\v\n\r\f') + Results.LINE_BREAK
422 header_count += 1
423 else:
424 header_read = True
425 procedure = Results.parse_header(header[:-1], procedure_class)
426 results = Results(procedure, data_filename)
427 results._header_count = header_count
428 return results
429
430 @property
431 def data(self):
432 # Need to update header count for correct referencing
433 if self._header_count == -1:
434 self._header_count = len(
435 self.header()[-1].split(Results.LINE_BREAK))
436 if self._data is None or len(self._data) == 0:
437 # Data has not been read
438 try:
439 self.reload()
440 except Exception:
441 # Empty dataframe
442 self._data = pd.DataFrame(columns=self.procedure.DATA_COLUMNS)
443 else: # Concatenate additional data, if any, to already loaded data
444 skiprows = len(self._data) + self._header_count
445 chunks = pd.read_csv(
446 self.data_filename,
447 comment=Results.COMMENT,
448 header=0,
449 names=self._data.columns,
450 chunksize=Results.CHUNK_SIZE, skiprows=skiprows, iterator=True
451 )
452 try:
453 tmp_frame = pd.concat(chunks, ignore_index=True)
454 # only append new data if there is any
455 # if no new data, tmp_frame dtype is object, which override's
456 # self._data's original dtype - this can cause problems plotting
457 # (e.g. if trying to plot int data on a log axis)
458 if len(tmp_frame) > 0:
459 self._data = pd.concat([self._data, tmp_frame],
460 ignore_index=True)
461 except Exception:
462 pass # All data is up to date
463 return self._data
464
465 def reload(self):
466 """ Preforms a full reloading of the file data, neglecting
467 any changes in the comments
468 """
469 chunks = pd.read_csv(
470 self.data_filename,
471 comment=Results.COMMENT,
472 chunksize=Results.CHUNK_SIZE,
473 iterator=True
474 )
475 try:
476 self._data = pd.concat(chunks, ignore_index=True)
477 except Exception:
478 self._data = chunks.read()
479
480 def __repr__(self):
481 return "<{}(filename='{}',procedure={},shape={})>".format(
482 self.__class__.__name__, self.data_filename,
483 self.procedure.__class__.__name__,
484 self.data.shape
485 )