| 1 |
# |
| 2 |
# This file is part of the PyMeasure package. |
| 3 |
# |
| 4 |
# Copyright (c) 2013-2024 PyMeasure Developers |
| 5 |
# |
| 6 |
# Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 |
# of this software and associated documentation files (the "Software"), to deal |
| 8 |
# in the Software without restriction, including without limitation the rights |
| 9 |
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 10 |
# copies of the Software, and to permit persons to whom the Software is |
| 11 |
# furnished to do so, subject to the following conditions: |
| 12 |
# |
| 13 |
# The above copyright notice and this permission notice shall be included in |
| 14 |
# all copies or substantial portions of the Software. |
| 15 |
# |
| 16 |
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 |
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 |
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 |
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 |
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 |
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| 22 |
# THE SOFTWARE. |
| 23 |
# |
| 24 |
|
| 25 |
from decimal import Decimal |
| 26 |
import logging |
| 27 |
import os |
| 28 |
import re |
| 29 |
import sys |
| 30 |
from importlib import import_module |
| 31 |
from importlib.machinery import SourceFileLoader |
| 32 |
from datetime import datetime |
| 33 |
from string import Formatter |
| 34 |
|
| 35 |
import pandas as pd |
| 36 |
import pint |
| 37 |
|
| 38 |
from .procedure import Procedure, UnknownProcedure |
| 39 |
from pymeasure.units import ureg |
| 40 |
|
| 41 |
log = logging.getLogger(__name__) |
| 42 |
log.addHandler(logging.NullHandler()) |
| 43 |
|
| 44 |
|
| 45 |
def replace_placeholders(string, procedure, date_format="%Y-%m-%d", time_format="%H:%M:%S"): |
| 46 |
"""Replace placeholders in string with values from procedure parameters. |
| 47 |
|
| 48 |
Replaces the placeholders in the provided string with the values of the |
| 49 |
associated parameters, as provided by the procedure. This uses the standard |
| 50 |
python string.format syntax. Apart from the parameter in the procedure (which |
| 51 |
should be called by their full names) "date" and "time" are also added as optional |
| 52 |
placeholders. |
| 53 |
|
| 54 |
:param string: |
| 55 |
The string in which the placeholders are to be replaced. Python string.format |
| 56 |
syntax is used, e.g. "{Parameter Name}" to insert a FloatParameter called |
| 57 |
"Parameter Name", or "{Parameter Name:.2f}" to also specifically format the |
| 58 |
parameter. |
| 59 |
|
| 60 |
:param procedure: |
| 61 |
The procedure from which to get the parameter values. |
| 62 |
|
| 63 |
:param date_format: |
| 64 |
A string to represent how the additional placeholder "date" will be formatted. |
| 65 |
|
| 66 |
:param time_format: |
| 67 |
A string to represent how the additional placeholder "time" will be formatted. |
| 68 |
|
| 69 |
""" |
| 70 |
now = datetime.now() |
| 71 |
|
| 72 |
parameters = procedure.placeholder_objects() |
| 73 |
placeholders = {param.name: param.value for param in parameters.values()} |
| 74 |
|
| 75 |
placeholders["date"] = now.strftime(date_format) |
| 76 |
placeholders["time"] = now.strftime(time_format) |
| 77 |
|
| 78 |
# Check keys against available parameters |
| 79 |
invalid_keys = [i[1] for i in Formatter().parse(string) |
| 80 |
if i[1] is not None and i[1] not in placeholders] |
| 81 |
if invalid_keys: |
| 82 |
raise KeyError("The following placeholder-keys are not valid: '%s'; " |
| 83 |
"valid keys are: '%s'." % ( |
| 84 |
"', '".join(invalid_keys), |
| 85 |
"', '".join(placeholders.keys()) |
| 86 |
)) |
| 87 |
|
| 88 |
return string.format(**placeholders) |
| 89 |
|
| 90 |
|
| 91 |
def unique_filename(directory, prefix='DATA', suffix='', ext='csv', |
| 92 |
dated_folder=False, index=True, datetimeformat="%Y-%m-%d", |
| 93 |
procedure=None): |
| 94 |
""" Returns a unique filename based on the directory and prefix |
| 95 |
""" |
| 96 |
now = datetime.now() |
| 97 |
directory = os.path.abspath(directory) |
| 98 |
|
| 99 |
if procedure is not None: |
| 100 |
prefix = replace_placeholders(prefix, procedure) |
| 101 |
suffix = replace_placeholders(suffix, procedure) |
| 102 |
|
| 103 |
if dated_folder: |
| 104 |
directory = os.path.join(directory, now.strftime('%Y-%m-%d')) |
| 105 |
if not os.path.exists(directory): |
| 106 |
os.makedirs(directory) |
| 107 |
if index: |
| 108 |
i = 1 |
| 109 |
basename = f"{prefix}{now.strftime(datetimeformat)}" |
| 110 |
basepath = os.path.join(directory, basename) |
| 111 |
filename = "%s_%d%s.%s" % (basepath, i, suffix, ext) |
| 112 |
while os.path.exists(filename): |
| 113 |
i += 1 |
| 114 |
filename = "%s_%d%s.%s" % (basepath, i, suffix, ext) |
| 115 |
else: |
| 116 |
basename = f"{prefix}{now.strftime(datetimeformat)}{suffix}.{ext}" |
| 117 |
filename = os.path.join(directory, basename) |
| 118 |
return filename |
| 119 |
|
| 120 |
|
| 121 |
class CSVFormatter(logging.Formatter): |
| 122 |
""" Formatter of data results """ |
| 123 |
|
| 124 |
def __init__(self, columns, delimiter=','): |
| 125 |
"""Creates a csv formatter for a given list of columns (=header). |
| 126 |
|
| 127 |
:param columns: list of column names. |
| 128 |
:type columns: list |
| 129 |
:param delimiter: delimiter between columns. |
| 130 |
:type delimiter: str |
| 131 |
""" |
| 132 |
super().__init__() |
| 133 |
self.columns = columns |
| 134 |
self.units = Procedure.parse_columns(columns) |
| 135 |
self.delimiter = delimiter |
| 136 |
|
| 137 |
def format(self, record): |
| 138 |
"""Formats a record as csv. |
| 139 |
|
| 140 |
:param record: record to format. |
| 141 |
:type record: dict |
| 142 |
:return: a string |
| 143 |
""" |
| 144 |
line = [] |
| 145 |
for x in self.columns: |
| 146 |
value = record.get(x, float("nan")) |
| 147 |
if isinstance(value, (float, int, Decimal)) and type(value) is not bool: |
| 148 |
line.append(f"{value}") |
| 149 |
else: |
| 150 |
units = self.units.get(x, None) |
| 151 |
if units is not None: |
| 152 |
if isinstance(value, str): |
| 153 |
try: |
| 154 |
value = ureg.Quantity(value) |
| 155 |
except pint.UndefinedUnitError: |
| 156 |
log.warning( |
| 157 |
f"Value {value} for column {x} cannot be parsed to" |
| 158 |
f" unit {units}.") |
| 159 |
if isinstance(value, pint.Quantity): |
| 160 |
try: |
| 161 |
line.append(f"{value.m_as(units)}") |
| 162 |
except pint.DimensionalityError: |
| 163 |
line.append("nan") |
| 164 |
log.warning( |
| 165 |
f"Value {value} for column {x} does not have the " |
| 166 |
f"right unit {units}.") |
| 167 |
elif isinstance(value, bool): |
| 168 |
line.append("nan") |
| 169 |
log.warning( |
| 170 |
f"Boolean for column {x} does not have unit {units}.") |
| 171 |
else: |
| 172 |
line.append("nan") |
| 173 |
log.warning( |
| 174 |
f"Value {value} for column {x} does not have the right" |
| 175 |
f" type for unit {units}.") |
| 176 |
else: |
| 177 |
if isinstance(value, pint.Quantity): |
| 178 |
if value.units == ureg.dimensionless: |
| 179 |
line.append(f"{value.magnitude}") |
| 180 |
else: |
| 181 |
self.units[x] = value.to_base_units().units |
| 182 |
line.append(f"{value.m_as(self.units[x])}") |
| 183 |
log.info(f"Column {x} units was set to {self.units[x]}") |
| 184 |
else: |
| 185 |
line.append(f"{value}") |
| 186 |
return self.delimiter.join(line) |
| 187 |
|
| 188 |
def format_header(self): |
| 189 |
return self.delimiter.join(self.columns) |
| 190 |
|
| 191 |
|
| 192 |
class Results: |
| 193 |
""" The Results class provides a convenient interface to reading and |
| 194 |
writing data in connection with a :class:`.Procedure` object. |
| 195 |
|
| 196 |
:cvar COMMENT: The character used to identify a comment (default: #) |
| 197 |
:cvar DELIMITER: The character used to delimit the data (default: ,) |
| 198 |
:cvar LINE_BREAK: The character used for line breaks (default \\n) |
| 199 |
:cvar CHUNK_SIZE: The length of the data chuck that is read |
| 200 |
|
| 201 |
:param procedure: Procedure object |
| 202 |
:param data_filename: The data filename where the data is or should be |
| 203 |
stored |
| 204 |
""" |
| 205 |
|
| 206 |
COMMENT = '#' |
| 207 |
DELIMITER = ',' |
| 208 |
LINE_BREAK = "\n" |
| 209 |
CHUNK_SIZE = 1000 |
| 210 |
|
| 211 |
def __init__(self, procedure, data_filename): |
| 212 |
if not isinstance(procedure, Procedure): |
| 213 |
raise ValueError("Results require a Procedure object") |
| 214 |
self.procedure = procedure |
| 215 |
self.procedure_class = procedure.__class__ |
| 216 |
self.parameters = procedure.parameter_objects() |
| 217 |
self._header_count = -1 |
| 218 |
self._metadata_count = -1 |
| 219 |
|
| 220 |
self.formatter = CSVFormatter(columns=self.procedure.DATA_COLUMNS) |
| 221 |
|
| 222 |
if isinstance(data_filename, (list, tuple)): |
| 223 |
data_filenames, data_filename = data_filename, data_filename[0] |
| 224 |
else: |
| 225 |
data_filenames = [data_filename] |
| 226 |
|
| 227 |
self.data_filename = data_filename |
| 228 |
self.data_filenames = data_filenames |
| 229 |
|
| 230 |
if os.path.exists(data_filename): # Assume header is already written |
| 231 |
self.reload() |
| 232 |
self.procedure.status = Procedure.FINISHED |
| 233 |
# TODO: Correctly store and retrieve status |
| 234 |
else: |
| 235 |
for filename in self.data_filenames: |
| 236 |
with open(filename, 'w') as f: |
| 237 |
f.write(self.header()) |
| 238 |
f.write(self.labels()) |
| 239 |
self._data = None |
| 240 |
|
| 241 |
def __getstate__(self): |
| 242 |
# Get all information needed to reconstruct procedure |
| 243 |
self._parameters = self.procedure.parameter_values() |
| 244 |
self._class = self.procedure.__class__.__name__ |
| 245 |
module = sys.modules[self.procedure.__module__] |
| 246 |
self._package = module.__package__ |
| 247 |
self._module = module.__name__ |
| 248 |
self._file = module.__file__ |
| 249 |
|
| 250 |
state = self.__dict__.copy() |
| 251 |
del state['procedure'] |
| 252 |
del state['procedure_class'] |
| 253 |
return state |
| 254 |
|
| 255 |
def __setstate__(self, state): |
| 256 |
self.__dict__.update(state) |
| 257 |
|
| 258 |
# Restore the procedure |
| 259 |
module = SourceFileLoader(self._module, self._file).load_module() |
| 260 |
cls = getattr(module, self._class) |
| 261 |
|
| 262 |
self.procedure = cls() |
| 263 |
self.procedure.set_parameters(self._parameters) |
| 264 |
self.procedure.refresh_parameters() |
| 265 |
|
| 266 |
self.procedure_class = cls |
| 267 |
|
| 268 |
del self._parameters |
| 269 |
del self._class |
| 270 |
del self._package |
| 271 |
del self._module |
| 272 |
del self._file |
| 273 |
|
| 274 |
def header(self): |
| 275 |
""" Returns a text header to accompany a datafile so that the procedure |
| 276 |
can be reconstructed |
| 277 |
""" |
| 278 |
h = [] |
| 279 |
procedure = re.search("'(?P<name>[^']+)'", |
| 280 |
repr(self.procedure_class)).group("name") |
| 281 |
h.append("Procedure: <%s>" % procedure) |
| 282 |
h.append("Parameters:") |
| 283 |
for name, parameter in self.parameters.items(): |
| 284 |
h.append("\t{}: {}".format(parameter.name, str( |
| 285 |
parameter).encode("unicode_escape").decode("utf-8"))) |
| 286 |
h.append("Data:") |
| 287 |
self._header_count = len(h) |
| 288 |
h = [Results.COMMENT + line for line in h] # Comment each line |
| 289 |
return Results.LINE_BREAK.join(h) + Results.LINE_BREAK |
| 290 |
|
| 291 |
def labels(self): |
| 292 |
""" Returns the columns labels as a string to be written |
| 293 |
to the file |
| 294 |
""" |
| 295 |
return self.formatter.format_header() + Results.LINE_BREAK |
| 296 |
|
| 297 |
def format(self, data): |
| 298 |
""" Returns a formatted string containing the data to be written |
| 299 |
to a file |
| 300 |
""" |
| 301 |
return self.formatter.format(data) |
| 302 |
|
| 303 |
def parse(self, line): |
| 304 |
""" Returns a dictionary containing the data from the line """ |
| 305 |
data = {} |
| 306 |
items = line.split(Results.DELIMITER) |
| 307 |
for i, key in enumerate(self.procedure.DATA_COLUMNS): |
| 308 |
data[key] = items[i] |
| 309 |
return data |
| 310 |
|
| 311 |
def metadata(self): |
| 312 |
""" Returns a text header for the metadata to write into the datafile """ |
| 313 |
if not self.procedure.metadata_objects(): |
| 314 |
return |
| 315 |
|
| 316 |
m = ["Metadata:"] |
| 317 |
for _, metadata in self.procedure.metadata_objects().items(): |
| 318 |
value = str(metadata).encode("unicode_escape").decode("utf-8") |
| 319 |
m.append(f"\t{metadata.name}: {value}") |
| 320 |
|
| 321 |
self._metadata_count = len(m) |
| 322 |
m = [Results.COMMENT + line for line in m] # Comment each line |
| 323 |
return Results.LINE_BREAK.join(m) + Results.LINE_BREAK |
| 324 |
|
| 325 |
def store_metadata(self): |
| 326 |
""" Inserts the metadata header (if any) into the datafile """ |
| 327 |
c_header = self.metadata() |
| 328 |
if c_header is None: |
| 329 |
return |
| 330 |
|
| 331 |
for filename in self.data_filenames: |
| 332 |
with open(filename, 'r+') as f: |
| 333 |
contents = f.readlines() |
| 334 |
contents.insert(self._header_count - 1, c_header) |
| 335 |
|
| 336 |
f.seek(0) |
| 337 |
f.writelines(contents) |
| 338 |
|
| 339 |
self._header_count += self._metadata_count |
| 340 |
|
| 341 |
@staticmethod |
| 342 |
def parse_header(header, procedure_class=None): |
| 343 |
""" Returns a Procedure object with the parameters as defined in the |
| 344 |
header text. |
| 345 |
""" |
| 346 |
if procedure_class is not None: |
| 347 |
procedure = procedure_class() |
| 348 |
else: |
| 349 |
procedure = None |
| 350 |
|
| 351 |
header = header.split(Results.LINE_BREAK) |
| 352 |
procedure_module = None |
| 353 |
parameters = {} |
| 354 |
for line in header: |
| 355 |
if line.startswith(Results.COMMENT): |
| 356 |
line = line[1:] # Uncomment |
| 357 |
else: |
| 358 |
raise ValueError("Parsing a header which contains " |
| 359 |
"uncommented sections") |
| 360 |
if line.startswith("Procedure"): |
| 361 |
regex = r"<(?:(?P<module>[^>]+)\.)?(?P<class>[^.>]+)>" |
| 362 |
search = re.search(regex, line) |
| 363 |
procedure_module = search.group("module") |
| 364 |
procedure_class = search.group("class") |
| 365 |
elif line.startswith("\t"): |
| 366 |
separator = ": " |
| 367 |
partitioned_line = line[1:].partition(separator) |
| 368 |
if partitioned_line[1] != separator: |
| 369 |
raise Exception("Error partitioning header line %s." % line) |
| 370 |
else: |
| 371 |
parameters[partitioned_line[0]] = partitioned_line[2] |
| 372 |
|
| 373 |
if procedure is None: |
| 374 |
if procedure_class is None: |
| 375 |
raise ValueError("Header does not contain the Procedure class") |
| 376 |
try: |
| 377 |
procedure_module = import_module(procedure_module) |
| 378 |
procedure_class = getattr(procedure_module, procedure_class) |
| 379 |
procedure = procedure_class() |
| 380 |
except ImportError: |
| 381 |
procedure = UnknownProcedure(parameters) |
| 382 |
log.warning("Unknown Procedure being used") |
| 383 |
|
| 384 |
# Fill the procedure with the parameters found |
| 385 |
for name, parameter in procedure.parameter_objects().items(): |
| 386 |
if parameter.name in parameters: |
| 387 |
value = parameters[parameter.name] |
| 388 |
setattr(procedure, name, value) |
| 389 |
else: |
| 390 |
log.warning( |
| 391 |
f"Parameter \"{parameter.name}\" not found when loading " + |
| 392 |
f"'{procedure_class}', setting default value") |
| 393 |
setattr(procedure, name, parameter.default) |
| 394 |
|
| 395 |
procedure.refresh_parameters() # Enforce update of meta data |
| 396 |
|
| 397 |
# Fill the procedure with the metadata found |
| 398 |
for name, metadata in procedure.metadata_objects().items(): |
| 399 |
if metadata.name in parameters: |
| 400 |
value = parameters[metadata.name] |
| 401 |
setattr(procedure, name, value) |
| 402 |
|
| 403 |
# Set the value in the metadata |
| 404 |
metadata._value = value |
| 405 |
metadata.evaluated = True |
| 406 |
|
| 407 |
return procedure |
| 408 |
|
| 409 |
@staticmethod |
| 410 |
def load(data_filename, procedure_class=None): |
| 411 |
""" Returns a Results object with the associated Procedure object and |
| 412 |
data |
| 413 |
""" |
| 414 |
header = "" |
| 415 |
header_read = False |
| 416 |
header_count = 0 |
| 417 |
with open(data_filename) as f: |
| 418 |
while not header_read: |
| 419 |
line = f.readline() |
| 420 |
if line.startswith(Results.COMMENT): |
| 421 |
header += line.strip('\t\v\n\r\f') + Results.LINE_BREAK |
| 422 |
header_count += 1 |
| 423 |
else: |
| 424 |
header_read = True |
| 425 |
procedure = Results.parse_header(header[:-1], procedure_class) |
| 426 |
results = Results(procedure, data_filename) |
| 427 |
results._header_count = header_count |
| 428 |
return results |
| 429 |
|
| 430 |
@property |
| 431 |
def data(self): |
| 432 |
# Need to update header count for correct referencing |
| 433 |
if self._header_count == -1: |
| 434 |
self._header_count = len( |
| 435 |
self.header()[-1].split(Results.LINE_BREAK)) |
| 436 |
if self._data is None or len(self._data) == 0: |
| 437 |
# Data has not been read |
| 438 |
try: |
| 439 |
self.reload() |
| 440 |
except Exception: |
| 441 |
# Empty dataframe |
| 442 |
self._data = pd.DataFrame(columns=self.procedure.DATA_COLUMNS) |
| 443 |
else: # Concatenate additional data, if any, to already loaded data |
| 444 |
skiprows = len(self._data) + self._header_count |
| 445 |
chunks = pd.read_csv( |
| 446 |
self.data_filename, |
| 447 |
comment=Results.COMMENT, |
| 448 |
header=0, |
| 449 |
names=self._data.columns, |
| 450 |
chunksize=Results.CHUNK_SIZE, skiprows=skiprows, iterator=True |
| 451 |
) |
| 452 |
try: |
| 453 |
tmp_frame = pd.concat(chunks, ignore_index=True) |
| 454 |
# only append new data if there is any |
| 455 |
# if no new data, tmp_frame dtype is object, which override's |
| 456 |
# self._data's original dtype - this can cause problems plotting |
| 457 |
# (e.g. if trying to plot int data on a log axis) |
| 458 |
if len(tmp_frame) > 0: |
| 459 |
self._data = pd.concat([self._data, tmp_frame], |
| 460 |
ignore_index=True) |
| 461 |
except Exception: |
| 462 |
pass # All data is up to date |
| 463 |
return self._data |
| 464 |
|
| 465 |
def reload(self): |
| 466 |
""" Preforms a full reloading of the file data, neglecting |
| 467 |
any changes in the comments |
| 468 |
""" |
| 469 |
chunks = pd.read_csv( |
| 470 |
self.data_filename, |
| 471 |
comment=Results.COMMENT, |
| 472 |
chunksize=Results.CHUNK_SIZE, |
| 473 |
iterator=True |
| 474 |
) |
| 475 |
try: |
| 476 |
self._data = pd.concat(chunks, ignore_index=True) |
| 477 |
except Exception: |
| 478 |
self._data = chunks.read() |
| 479 |
|
| 480 |
def __repr__(self): |
| 481 |
return "<{}(filename='{}',procedure={},shape={})>".format( |
| 482 |
self.__class__.__name__, self.data_filename, |
| 483 |
self.procedure.__class__.__name__, |
| 484 |
self.data.shape |
| 485 |
) |