Debian Codegraph @

pyvista/examples/_dataset_loader.py

raw
1 """Abstraction layer for downloading, reading, and loading dataset files.
2
3 The classes and methods in this module define an API for working with either
4 a single file or multiple files which may be downloaded and/or loaded as an
5 example dataset.
6
7 Many datasets have a straightforward input to output mapping:
8 file -> read -> dataset
9
10 However, some file formats require multiple input files for reading (e.g.
11 separate data and header files):
12 (file1, file1) -> read -> dataset
13
14 Or, a dataset may be combination of two separate datasets:
15 file1 -> read -> dataset1 ┬─> combined_dataset
16 file2 -> read -> dataset2 ┘
17
18 In some cases, the input may be a folder instead of a file (e.g. DICOM):
19 folder -> read -> dataset
20
21 In addition, there may be a need to customize the reading function to read
22 files with specific options enabled (e.g. set a time value), or perform
23 post-read processing to modify the dataset (e.g. set active scalars).
24
25 This module aims to serve these use cases and provide a flexible way of
26 downloading, reading, and processing files with a generic mapping:
27 file or files or folder -> fully processed dataset(s) in any form
28
29 """
30
31 # mypy: disable-error-code="redundant-expr"
32 from __future__ import annotations
33
34 from abc import abstractmethod
35 from collections.abc import Sequence
36 import functools
37 import os
38 from pathlib import Path
39 from typing import TYPE_CHECKING
40 from typing import Any
41 from typing import Generic
42 from typing import Protocol
43 from typing import TypeVar
44 from typing import cast
45 from typing import final
46 from typing import runtime_checkable
47
48 import pyvista as pv
49 from pyvista.core._typing_core import NumpyArray
50 from pyvista.core.utilities.fileio import get_ext
51
52 if TYPE_CHECKING:
53 from collections.abc import Callable
54
55 # Define TypeVars for two main class definitions used by this module:
56 # 1. classes for single file inputs: T -> T
57 # 2. classes for multi-file inputs: (T, ...) -> (T, ...)
58 # Any properties with these typevars should have a one-to-one mapping for all files
59 _FilePropStrType_co = TypeVar(
60 '_FilePropStrType_co',
61 str,
62 tuple[str, ...],
63 covariant=True,
64 )
65 _FilePropIntType_co = TypeVar(
66 '_FilePropIntType_co',
67 int,
68 tuple[int, ...],
69 covariant=True,
70 )
71
72 DatasetObject = pv.DataSet | pv.Texture | NumpyArray[Any] | pv.MultiBlock
73 DatasetType = type[pv.DataSet] | type[pv.Texture] | type[NumpyArray[Any]] | type[pv.MultiBlock]
74
75
76 class _BaseFilePropsProtocol(Generic[_FilePropStrType_co, _FilePropIntType_co]):
77 @property
78 @abstractmethod
79 def path(self) -> _FilePropStrType_co:
80 """Return the path(s) of all files."""
81
82 @property
83 def num_files(self) -> int:
84 """Return the number of files from path or paths.
85
86 If a path is a folder, the number of files contained in the folder is returned.
87 """
88 path = self.path
89 paths = [path] if isinstance(path, str) else path
90 return sum(1 if os.path.isfile(p) else len(_get_all_nested_filepaths(p)) for p in paths)
91
92 @property
93 def unique_extension(self) -> str | tuple[str, ...]:
94 """Return the unique file extension(s) from all files."""
95 return _get_unique_extension(self.path)
96
97 @property
98 @abstractmethod
99 def _filesize_bytes(self) -> _FilePropIntType_co:
100 """Return the file size(s) of all files in bytes."""
101
102 @property
103 @abstractmethod
104 def _filesize_format(self) -> _FilePropStrType_co:
105 """Return the formatted size of all file(s)."""
106
107 @property
108 @abstractmethod
109 def _total_size_bytes(self) -> int:
110 """Return the total size of all files in bytes."""
111
112 @property
113 @abstractmethod
114 def total_size(self) -> str:
115 """Return the total size of all files formatted as a string."""
116
117 @property
118 @abstractmethod
119 def _reader(
120 self,
121 ) -> pv.BaseReader[Any] | tuple[pv.BaseReader[Any] | None, ...] | None:
122 """Return the base file reader(s) used to read the files."""
123
124 @property
125 def unique_reader_type(
126 self,
127 ) -> type[pv.BaseReader[Any]] | tuple[type[pv.BaseReader[Any]], ...] | None:
128 """Return unique reader type(s) from all file readers."""
129 return _get_unique_reader_type(self._reader)
130
131
132 class _SingleFilePropsProtocol(_BaseFilePropsProtocol[str, int]):
133 """Define file properties of a single file."""
134
135
136 class _MultiFilePropsProtocol(
137 _BaseFilePropsProtocol[tuple[str, ...], tuple[int, ...]],
138 ):
139 """Define file properties of multiple files."""
140
141
142 @runtime_checkable
143 class _Downloadable(Protocol[_FilePropStrType_co]):
144 """Class which downloads file(s) from a source."""
145
146 @property
147 @abstractmethod
148 def source_name(self) -> _FilePropStrType_co:
149 """Return the name of the download relative to the base url."""
150
151 @property
152 @abstractmethod
153 def base_url(self) -> _FilePropStrType_co:
154 """Return the base url of the download."""
155
156 @property
157 def source_url_raw(self) -> _FilePropStrType_co:
158 """Return the raw source of the download.
159
160 This is the full URL used to download the data directly.
161 """
162 name = self.source_name
163 name_iter = [name] if isinstance(name, str) else name
164 url = self.base_url
165 base_url_iter = [url] if isinstance(url, str) else url
166 url_raw = list(map(os.path.join, base_url_iter, name_iter))
167 return url_raw[0] if isinstance(name, str) else tuple(url_raw)
168
169 @property
170 def source_url_blob(self) -> _FilePropStrType_co:
171 """Return the blob source of the download.
172
173 This URL is useful for linking to the source webpage for
174 a human to open on a browser.
175 """
176 # Make single urls iterable and replace 'raw' with 'blob'
177 url_raw = self.source_url_raw
178 url_iter = [url_raw] if isinstance(url_raw, str) else url_raw
179 url_blob = [url.replace('/raw/', '/blob/') for url in url_iter]
180 return url_blob[0] if isinstance(url_raw, str) else tuple(url_blob)
181
182 @property
183 @abstractmethod
184 def path(self) -> _FilePropStrType_co:
185 """Return the file path of downloaded file."""
186
187 @abstractmethod
188 def download(self) -> _FilePropStrType_co:
189 """Download and return the file path(s)."""
190
191
192 class _DatasetLoader:
193 """Load a dataset."""
194
195 def __init__(self, load_func: Callable[..., DatasetObject]):
196 self._load_func = load_func
197 self._dataset: DatasetObject | None = None
198
199 @property
200 @final
201 def dataset(self) -> DatasetObject | None:
202 """Return the loaded dataset object(s)."""
203 return self._dataset
204
205 def load(self, *args, **kwargs) -> DatasetObject:
206 """Load and return the dataset."""
207 # Subclasses should override this as needed
208 return self._load_func(*args, **kwargs)
209
210 @final
211 def load_and_store_dataset(self) -> DatasetObject:
212 """Load the dataset and store it."""
213 dataset = self.load()
214 self._dataset = dataset
215 return dataset
216
217 @final
218 def clear_dataset(self):
219 """Clear the stored dataset object from memory."""
220 del self._dataset
221
222 @property
223 @final
224 def dataset_iterable(self) -> tuple[DatasetObject, ...]:
225 """Return a tuple of all dataset object(s), including any nested objects.
226
227 If the dataset is a MultiBlock, the MultiBlock itself is also returned as the first
228 item. Any nested MultiBlocks are not included, only their datasets.
229
230 E.g. for a composite dataset:
231 MultiBlock -> (MultiBlock, Block0, Block1, ...)
232 """
233 dataset = self.dataset
234
235 def _flat(obj):
236 if isinstance(obj, Sequence):
237 output_list = [] # type: ignore[var-annotated]
238 for item in obj:
239 (
240 output_list.extend(item)
241 if isinstance(item, Sequence)
242 else output_list.append(item)
243 )
244 if any(isinstance(item, Sequence) for item in output_list):
245 return _flat(output_list)
246 return output_list
247 else:
248 return [obj]
249
250 flat = _flat(dataset)
251 if isinstance(dataset, pv.MultiBlock):
252 flat.insert(0, dataset)
253 return tuple(flat)
254
255 @property
256 @final
257 def unique_dataset_type(
258 self,
259 ) -> DatasetType | tuple[DatasetType, ...] | None:
260 """Return unique dataset type(s) from all datasets."""
261 return _get_unique_dataset_type(self.dataset_iterable)
262
263 @property
264 @final
265 def unique_cell_types(
266 self,
267 ) -> tuple[pv.CellType, ...]:
268 """Return unique cell types from all datasets."""
269 cell_types: dict[pv.CellType, None] = {}
270 for data in self.dataset_iterable:
271 # Get the underlying dataset for the texture
272 dataset = (
273 cast('pv.ImageData', pv.wrap(data.GetInput()))
274 if isinstance(data, pv.Texture)
275 else data
276 )
277 try:
278 if isinstance(dataset, pv.ExplicitStructuredGrid):
279 # extract_cells_by_type does not support this datatype
280 # so get cells manually
281 cells = (c.type for c in dataset.cell)
282 [cell_types.update({cell_type: None}) for cell_type in cells]
283 else:
284 for cell_type in pv.CellType:
285 extracted = dataset.extract_cells_by_type(cell_type) # type: ignore[union-attr]
286 if extracted.n_cells > 0:
287 cell_types[cell_type] = None
288 except AttributeError:
289 continue
290 return tuple(sorted(cell_types.keys()))
291
292
293 class _SingleFile(_SingleFilePropsProtocol):
294 """Wrap a single file."""
295
296 def __init__(self, path):
297 from pyvista.examples.downloads import USER_DATA_PATH # noqa: PLC0415
298
299 self._path = (
300 path if path is None or os.path.isabs(path) else os.path.join(USER_DATA_PATH, path)
301 )
302
303 @property
304 def path(self) -> str:
305 return self._path
306
307 @property
308 def _filesize_bytes(self) -> int:
309 return _get_file_or_folder_size(self.path)
310
311 @property
312 def _filesize_format(self) -> str:
313 return _format_file_size(self._filesize_bytes)
314
315 @property
316 def _total_size_bytes(self) -> int:
317 return self._filesize_bytes
318
319 @property
320 def total_size(self) -> str:
321 return self._filesize_format
322
323 @property
324 def _reader(self) -> pv.BaseReader[Any] | None:
325 return None
326
327
328 class _SingleFileDatasetLoader(_SingleFile, _DatasetLoader):
329 """Wrap a single file for loading.
330
331 Specify the read function and/or load functions for reading and processing the
332 dataset. The read function is called on the file path first, then, if a load
333 function is specified, the load function is called on the output from the read
334 function.
335
336 Parameters
337 ----------
338 path
339 Path of the file to be loaded.
340
341 read_func
342 Specify the function used to read the file. Defaults to :func:`pyvista.read`.
343 This can be used for customizing the reader's properties, or using another
344 read function (e.g. :func:`pyvista.read_texture` for textures). The function
345 must have the file path as the first argument and should return a dataset.
346 If default arguments are required by your desired read function, consider
347 using :class:`functools.partial` to pre-set the arguments before passing it
348 as an argument to the loader.
349
350 load_func
351 Specify the function used to load the file. Defaults to `None`. This is typically
352 used to specify any processing of the dataset after reading. The load function
353 typically will accept a dataset as an input and return a dataset.
354
355 """
356
357 def __init__(
358 self,
359 path: str,
360 read_func: Callable[[str], DatasetObject] | None = None,
361 load_func: Callable[[DatasetObject], Any] | None = None,
362 ):
363 _SingleFile.__init__(self, path)
364 # _DatasetLoader.__init__ is typed as requiring a non-None
365 # ``Callable[..., DatasetObject]`` but this subclass treats
366 # load_func as optional (see ``load`` which falls back to
367 # ``read_func`` when load_func is None). Fixing the base
368 # signature is out of scope for the typing refactor.
369 _DatasetLoader.__init__(self, load_func) # type: ignore[arg-type]
370 self._read_func = pv.read if path and read_func is None else read_func
371
372 @property
373 def _reader(self) -> pv.BaseReader[Any] | None:
374 # TODO: return the actual reader used, and not just a lookup
375 # (this will require an update to the 'read_func' API)
376 try:
377 return pv.get_reader(self.path)
378 except ValueError:
379 # Cannot be read directly (requires custom reader)
380 return None
381
382 @property
383 def path_loadable(self) -> str:
384 return self.path
385
386 def load(self):
387 path = self.path
388 read_func = self._read_func
389 load_func = self._load_func
390 try:
391 # Read and load normally
392 return read_func(path) if load_func is None else load_func(read_func(path)) # type: ignore[misc]
393 except OSError:
394 # Handle error generated by pv.read if reading a directory
395 if read_func is pv.read and Path(path).is_dir():
396 # Re-define read function to read all files in a directory as a multiblock
397 read_func = lambda path: _load_as_multiblock(
398 [
399 _SingleFileDatasetLoader(str(Path(path, fname)))
400 for fname in sorted(os.listdir(path)) # noqa: PTH208
401 ],
402 )
403 return read_func(path) if load_func is None else load_func(read_func(path))
404 else:
405 msg = f'Error loading dataset from path:\n\t{self.path}'
406 raise RuntimeError(msg)
407
408
409 class _DownloadableFile(_SingleFile, _Downloadable[str]):
410 """Wrap a single file which must be downloaded.
411
412 If downloading a file from an archive, set the filepath of the zip as
413 ``path`` and set ``target_file`` as the file to extract. If the path is
414 a zip file and no target file is specified, the entire archive is downloaded
415 and extracted and the root directory of the path is returned.
416
417 """
418
419 def __init__(
420 self,
421 path: str,
422 target_file: str | None = None,
423 ):
424 _SingleFile.__init__(self, path)
425
426 from pyvista.examples.downloads import SOURCE # noqa: PLC0415
427 from pyvista.examples.downloads import USER_DATA_PATH # noqa: PLC0415
428 from pyvista.examples.downloads import _download_archive_file_or_folder # noqa: PLC0415
429 from pyvista.examples.downloads import download_file # noqa: PLC0415
430 from pyvista.examples.downloads import file_from_files # noqa: PLC0415
431 from pyvista.examples.examples import dir_path # noqa: PLC0415
432
433 if Path(path).is_absolute():
434 # Absolute path must point to a built-in dataset
435 assert Path(path).parent == Path(
436 dir_path,
437 ), 'Absolute path must point to a built-in dataset.'
438 self._base_url = 'https://github.com/pyvista/pyvista/raw/main/pyvista/examples/'
439 self._source_name = Path(path).name
440 # the dataset is already downloaded (it's built-in)
441 # so make download() simply return the local filepath
442 self._download_func = lambda _: path
443 else:
444 # Relative path, use vars from downloads.py
445 self._base_url = SOURCE
446 self._download_func = download_file
447 self._source_name = Path(path).name if Path(path).is_absolute() else path
448
449 target_file = '' if target_file is None and (get_ext(path) == '.zip') else target_file
450 if target_file is not None:
451 # download from archive
452 self._download_func = functools.partial(
453 _download_archive_file_or_folder,
454 target_file=target_file,
455 )
456 # The file path currently points to the archive, not the target file itself
457 # Try to resolve the full path to the target file (without downloading) if
458 # the archive already exists in the cache
459 fullpath = None
460 if os.path.isfile(self.path):
461 try:
462 # Get file path
463 fullpath = file_from_files(target_file, self.path)
464 except (FileNotFoundError, RuntimeError):
465 # Get folder path
466 fullpath = os.path.join(USER_DATA_PATH, path + '.unzip', target_file)
467 fullpath = fullpath if os.path.isdir(fullpath) else None
468 # set the file path as the relative path of the target file if
469 # the fullpath could not be resolved (i.e. not yet downloaded)
470 self._path = target_file if fullpath is None else fullpath
471
472 @property
473 def source_name(self) -> str:
474 return self._source_name
475
476 @property
477 def base_url(self) -> str:
478 return self._base_url
479
480 def download(self) -> str:
481 path = self._download_func(self._source_name)
482 assert os.path.isfile(path) or os.path.isdir(path)
483 # Reset the path since the full path for archive files
484 # isn't known until after downloading
485 self._path = path
486 return path
487
488
489 class _SingleFileDownloadableDatasetLoader(_SingleFileDatasetLoader, _DownloadableFile):
490 """Wrap a single file which must first be downloaded and which can also be loaded.
491
492 .. warning::
493
494 ``download()`` should be called before accessing other attributes. Otherwise,
495 calling ``load()`` or ``path`` may fail or produce unexpected results.
496
497 """
498
499 def __init__( # noqa: PLR0917
500 self,
501 path: str,
502 read_func: Callable[[str], DatasetObject] | None = None,
503 load_func: Callable[[DatasetObject], DatasetObject] | None = None,
504 target_file: str | None = None,
505 ):
506 _SingleFileDatasetLoader.__init__(self, path, read_func=read_func, load_func=load_func)
507 _DownloadableFile.__init__(self, path, target_file=target_file)
508
509
510 class _MultiFileDatasetLoader(_DatasetLoader, _MultiFilePropsProtocol):
511 """Wrap multiple files for loading.
512
513 Some use cases for loading multi-file examples include:
514
515 1. Multiple input files, and each file is read/loaded independently
516 E.g.: loading two separate datasets for the example
517 See ``download_bolt_nut`` for a reference implementation.
518
519 2. Multiple input files, but only one is read or loaded directly
520 E.g.: loading a single dataset from a file format where data and metadata are
521 stored in separate files, such as ``.raw`` and ``.mhd``.
522 See ``download_head`` for a reference implementation.
523
524 3. Multiple input files, all of which make up part of the loaded dataset
525 E.g.: loading six separate image files for cubemaps
526 See ``download_sky_box_cube_map`` for a reference implementation.
527
528 Parameters
529 ----------
530 files_func
531 Specify the function which will return a sequence of :class:`_SingleFile`
532 objects required for loading the dataset. Alternatively, a directory can be
533 specified, in which case a separate single-file dataset loader is created
534 for each file with a default reader.
535
536 load_func
537 Specify the function used to load the files. By default, :meth:`load()` is called
538 on all the files (if loadable) and a tuple containing the loaded datasets is returned.
539
540 """
541
542 def __init__(
543 self,
544 files_func: str | Callable[[], Sequence[_SingleFileDatasetLoader | _DownloadableFile]],
545 load_func: Callable[[Sequence[_SingleFileDatasetLoader]], Any] | None = None,
546 ):
547 self._files_func = files_func
548 self._file_loaders_ = None
549 if load_func is None:
550 load_func = _load_as_dataset_or_multiblock
551
552 _DatasetLoader.__init__(self, load_func)
553
554 @property
555 def _file_objects(self):
556 if self._file_loaders_ is None and not isinstance(self._files_func, str):
557 self._file_loaders_ = self._files_func() # type: ignore[assignment]
558 return self._file_loaders_
559
560 @property
561 def path(self) -> tuple[str, ...]:
562 return tuple(_flatten_nested_sequence([file.path for file in self._file_objects]))
563
564 @property
565 def path_loadable(self) -> tuple[str, ...]:
566 return tuple(
567 file.path for file in self._file_objects if isinstance(file, _SingleFileDatasetLoader)
568 )
569
570 @property
571 def _filesize_bytes(self) -> tuple[int, ...]:
572 return tuple(
573 _flatten_nested_sequence([file._filesize_bytes for file in self._file_objects]),
574 )
575
576 @property
577 def _filesize_format(self) -> tuple[str, ...]:
578 return tuple(_format_file_size(size) for size in self._filesize_bytes)
579
580 @property
581 def _total_size_bytes(self) -> int:
582 return sum(file._total_size_bytes for file in self._file_objects)
583
584 @property
585 def total_size(self) -> str:
586 return _format_file_size(self._total_size_bytes)
587
588 @property
589 def _reader(
590 self,
591 ) -> pv.BaseReader[Any] | tuple[pv.BaseReader[Any] | None, ...] | None:
592 # TODO: return the actual reader used, and not just a lookup
593 # (this will require an update to the 'read_func' API)
594 reader = _flatten_nested_sequence([file._reader for file in self._file_objects])
595 # flatten in case any file objects themselves are multifiles
596 reader_out: list[pv.BaseReader[Any]] = []
597 for r in reader:
598 reader_out.extend(r) if isinstance(r, Sequence) else reader_out.append(r)
599 return tuple(reader_out)
600
601 def load(self):
602 return self._load_func(self._file_objects)
603
604
605 class _MultiFileDownloadableDatasetLoader(
606 _MultiFileDatasetLoader,
607 _Downloadable[tuple[str, ...]],
608 ):
609 """Wrap multiple files for downloading and loading."""
610
611 @property
612 def source_name(self) -> tuple[str, ...]:
613 name = [file.source_name for file in self._file_objects if isinstance(file, _Downloadable)]
614 return tuple(_flatten_nested_sequence(name))
615
616 @property
617 def base_url(self) -> tuple[str, ...]:
618 url = [file.base_url for file in self._file_objects if isinstance(file, _Downloadable)]
619 return tuple(_flatten_nested_sequence(url))
620
621 def download(self) -> tuple[str, ...]:
622 path = [file.download() for file in self._file_objects if isinstance(file, _Downloadable)]
623 # flatten paths in case any loaders have multiple files
624 path_out = _flatten_nested_sequence(path)
625 assert all(os.path.isfile(p) or os.path.isdir(p) for p in path_out)
626 return tuple(path_out)
627
628
629 _ScalarType = TypeVar('_ScalarType', int, str, pv.BaseReader[Any])
630
631
632 def _flatten_nested_sequence(nested: Sequence[_ScalarType | Sequence[_ScalarType]]):
633 """Flatten nested sequences of objects."""
634 flat: list[_ScalarType] = []
635 for item in nested:
636 if isinstance(item, Sequence) and not isinstance(item, str):
637 flat.extend(item)
638 else:
639 flat.append(item)
640 return flat
641
642
643 def _download_dataset(
644 dataset_loader: _SingleFileDownloadableDatasetLoader | _MultiFileDownloadableDatasetLoader,
645 *,
646 load: bool = True,
647 metafiles: bool = False,
648 ):
649 """Download and load a dataset file or files.
650
651 Parameters
652 ----------
653 dataset_loader
654 SingleFile or MultiFile object(s) of the dataset(s) to download or load.
655
656 load
657 Read and load the file after downloading. When ``False``,
658 return the path or paths to the example's file(s).
659
660 metafiles
661 When ``load`` is ``False``, set this value to ``True`` to
662 return all files required to load the example, including any metafiles.
663 If ``False``, only the paths of files which are explicitly loaded are
664 returned. E.g if a file format uses two files to specify the header info
665 and file data separately, setting ``metafiles=True`` will return a tuple
666 with both file paths, whereas setting ``metafiles=False`` will only return
667 the single path of the header file as a string.
668
669 Returns
670 -------
671 Any
672 Loaded dataset or path(s) to the example's files depending on the ``load``
673 parameter. Dataset may be a texture, mesh, multiblock, array, tuple of meshes,
674 or any other output loaded by the example.
675
676 """
677 # Download all files for the dataset, include any metafiles
678 path = dataset_loader.download()
679
680 # Exclude non-loadable metafiles from result (if any)
681 if not metafiles and isinstance(dataset_loader, _MultiFileDownloadableDatasetLoader):
682 path = dataset_loader.path_loadable
683 # Return scalar if only one loadable file
684 path = path[0] if len(path) == 1 else path
685
686 return dataset_loader.load() if load else path
687
688
689 def _load_as_multiblock(
690 files: Sequence[_SingleFileDatasetLoader | _MultiFileDatasetLoader],
691 names: Sequence[str] | None = None,
692 ) -> pv.MultiBlock:
693 """Load multiple files as a MultiBlock.
694
695 This function can be used as a loading function for :class:`MultiFileLoadable`
696 If the use of the ``names`` parameter is needed, use :class:`functools.partial`
697 to partially specify the names parameter before passing it as loading function.
698 """
699 multi = pv.MultiBlock()
700 if names is None:
701 # set names, use filename without ext by default or dirname
702 paths = _flatten_nested_sequence(
703 [file.path_loadable for file in files if isinstance(file, _DatasetLoader)],
704 )
705 paths = [Path(path) for path in paths]
706 names = [
707 path.name[: -len(get_ext(path.name))] if path.is_file() else path.name
708 for path in paths
709 ]
710
711 for file, name in zip(files, names, strict=False):
712 if not isinstance(file, _DatasetLoader):
713 continue # type: ignore[unreachable]
714 loaded = file.load()
715 assert isinstance(
716 loaded,
717 (pv.MultiBlock, pv.DataSet),
718 ), (
719 f'Only MultiBlock or DataSet objects can be loaded as a MultiBlock. '
720 f"Got {type(loaded)}.'"
721 )
722 multi.append(loaded, name)
723 return multi
724
725
726 def _load_as_cubemap(files: str | _SingleFile | Sequence[_SingleFile]) -> pv.Texture:
727 """Load multiple files as a cubemap.
728
729 Input may be a single directory with 6 cubemap files, or a sequence
730 of 6 files
731 """
732 path = (
733 files
734 if isinstance(files, str)
735 else (files.path if isinstance(files, _SingleFile) else [file.path for file in files])
736 )
737
738 return (
739 pv.cubemap(path)
740 if isinstance(files, str) and os.path.isdir(files)
741 else pv.cubemap_from_filenames(path)
742 )
743
744
745 def _load_as_dataset_or_multiblock(files):
746 multiblock = _load_as_multiblock(files)
747 return multiblock[0] if len(multiblock) == 1 else multiblock
748
749
750 def _load_and_merge(files: Sequence[_SingleFile]):
751 """Load all loadable files as separate datasets and merge them."""
752 loaded = [file.load() for file in files if isinstance(file, _DatasetLoader)]
753 assert len(loaded) > 0
754 return pv.merge(loaded)
755
756
757 def _get_file_or_folder_size(filepath) -> int:
758 if os.path.isfile(filepath):
759 return os.path.getsize(filepath)
760 assert os.path.isdir(filepath), 'Expected a file or folder path.'
761 all_filepaths = _get_all_nested_filepaths(filepath)
762 return sum(os.path.getsize(file) for file in all_filepaths)
763
764
765 def _format_file_size(size: int) -> str:
766 size_flt = float(size)
767 for unit in ('B', 'KB', 'MB'):
768 if round(size_flt * 10) / 10 < 1000.0:
769 return f'{int(size_flt)} {unit}' if unit == 'B' else f'{size_flt:3.1f} {unit}'
770 size_flt /= 1000.0
771 return f'{size_flt:.1f} GB'
772
773
774 def _get_file_or_folder_ext(path: str):
775 """Wrap the `get_ext` function to handle special cases for directories."""
776 if os.path.isfile(path):
777 return get_ext(path)
778 assert os.path.isdir(path), 'Expected a file or folder path.'
779 all_paths = _get_all_nested_filepaths(path)
780 ext = [get_ext(file) for file in all_paths]
781 assert len(ext) != 0, f'No files with extensions were found in"\n\t{path}'
782 return ext
783
784
785 def _get_all_nested_filepaths(filepath, *, exclude_readme=True):
786 """Walk through directory and get all file paths.
787
788 Optionally exclude any readme files (if any).
789 """
790 assert os.path.isfile(filepath) or os.path.isdir(filepath)
791 condition = lambda name: True if not exclude_readme else not name.lower().startswith('readme')
792 return next(
793 [os.path.join(path, name) for name in files if condition(name)]
794 for path, _, files in os.walk(filepath)
795 )
796
797
798 def _get_unique_extension(path: str | Sequence[str]):
799 """Return a file extension or unique set of file extensions from a path or paths."""
800 ext_set = set()
801 fname_sequence = [path] if isinstance(path, str) else path
802
803 # Add all file extensions to the set
804 for file in fname_sequence:
805 ext = _get_file_or_folder_ext(file)
806 ext_set.add(ext) if isinstance(ext, str) else ext_set.update(ext)
807
808 # Format output
809 ext_output = tuple(ext_set)
810 return ext_output[0] if len(ext_output) == 1 else tuple(sorted(ext_output))
811
812
813 def _get_unique_reader_type(
814 reader: pv.BaseReader[Any] | tuple[pv.BaseReader[Any] | None, ...] | None,
815 ) -> type[pv.BaseReader[Any]] | tuple[type[pv.BaseReader[Any]], ...] | None:
816 """Return a reader type or tuple of unique reader types."""
817 if reader is None or (isinstance(reader, Sequence) and all(r is None for r in reader)):
818 return None
819 reader_set: set[type[pv.BaseReader[Any]]] = set()
820 reader_type = (
821 [type(reader)]
822 if not isinstance(reader, Sequence)
823 else [type(r) for r in reader if r is not None]
824 )
825
826 # Add all reader types to the set
827 reader_set.update(reader_type)
828
829 # Format output
830 reader_output = tuple(reader_set)
831 return reader_output[0] if len(reader_output) == 1 else tuple(reader_output)
832
833
834 def _get_unique_dataset_type(
835 dataset_iterable: tuple[DatasetObject, ...],
836 ) -> DatasetType | tuple[DatasetType, ...]:
837 """Return a dataset type or tuple of unique dataset types."""
838 dataset_types: dict[DatasetType, None] = {} # use dict as an ordered set
839 for dataset in dataset_iterable:
840 dataset_types[type(dataset)] = None
841 output = tuple(dataset_types.keys())
842 return output[0] if len(output) == 1 else output