| 1 |
"""Abstraction layer for downloading, reading, and loading dataset files. |
| 2 |
|
| 3 |
The classes and methods in this module define an API for working with either |
| 4 |
a single file or multiple files which may be downloaded and/or loaded as an |
| 5 |
example dataset. |
| 6 |
|
| 7 |
Many datasets have a straightforward input to output mapping: |
| 8 |
file -> read -> dataset |
| 9 |
|
| 10 |
However, some file formats require multiple input files for reading (e.g. |
| 11 |
separate data and header files): |
| 12 |
(file1, file1) -> read -> dataset |
| 13 |
|
| 14 |
Or, a dataset may be combination of two separate datasets: |
| 15 |
file1 -> read -> dataset1 ┬─> combined_dataset |
| 16 |
file2 -> read -> dataset2 ┘ |
| 17 |
|
| 18 |
In some cases, the input may be a folder instead of a file (e.g. DICOM): |
| 19 |
folder -> read -> dataset |
| 20 |
|
| 21 |
In addition, there may be a need to customize the reading function to read |
| 22 |
files with specific options enabled (e.g. set a time value), or perform |
| 23 |
post-read processing to modify the dataset (e.g. set active scalars). |
| 24 |
|
| 25 |
This module aims to serve these use cases and provide a flexible way of |
| 26 |
downloading, reading, and processing files with a generic mapping: |
| 27 |
file or files or folder -> fully processed dataset(s) in any form |
| 28 |
|
| 29 |
""" |
| 30 |
|
| 31 |
# mypy: disable-error-code="redundant-expr" |
| 32 |
from __future__ import annotations |
| 33 |
|
| 34 |
from abc import abstractmethod |
| 35 |
from collections.abc import Sequence |
| 36 |
import functools |
| 37 |
import os |
| 38 |
from pathlib import Path |
| 39 |
from typing import TYPE_CHECKING |
| 40 |
from typing import Any |
| 41 |
from typing import Generic |
| 42 |
from typing import Protocol |
| 43 |
from typing import TypeVar |
| 44 |
from typing import cast |
| 45 |
from typing import final |
| 46 |
from typing import runtime_checkable |
| 47 |
|
| 48 |
import pyvista as pv |
| 49 |
from pyvista.core._typing_core import NumpyArray |
| 50 |
from pyvista.core.utilities.fileio import get_ext |
| 51 |
|
| 52 |
if TYPE_CHECKING: |
| 53 |
from collections.abc import Callable |
| 54 |
|
| 55 |
# Define TypeVars for two main class definitions used by this module: |
| 56 |
# 1. classes for single file inputs: T -> T |
| 57 |
# 2. classes for multi-file inputs: (T, ...) -> (T, ...) |
| 58 |
# Any properties with these typevars should have a one-to-one mapping for all files |
| 59 |
_FilePropStrType_co = TypeVar( |
| 60 |
'_FilePropStrType_co', |
| 61 |
str, |
| 62 |
tuple[str, ...], |
| 63 |
covariant=True, |
| 64 |
) |
| 65 |
_FilePropIntType_co = TypeVar( |
| 66 |
'_FilePropIntType_co', |
| 67 |
int, |
| 68 |
tuple[int, ...], |
| 69 |
covariant=True, |
| 70 |
) |
| 71 |
|
| 72 |
DatasetObject = pv.DataSet | pv.Texture | NumpyArray[Any] | pv.MultiBlock |
| 73 |
DatasetType = type[pv.DataSet] | type[pv.Texture] | type[NumpyArray[Any]] | type[pv.MultiBlock] |
| 74 |
|
| 75 |
|
| 76 |
class _BaseFilePropsProtocol(Generic[_FilePropStrType_co, _FilePropIntType_co]): |
| 77 |
@property |
| 78 |
@abstractmethod |
| 79 |
def path(self) -> _FilePropStrType_co: |
| 80 |
"""Return the path(s) of all files.""" |
| 81 |
|
| 82 |
@property |
| 83 |
def num_files(self) -> int: |
| 84 |
"""Return the number of files from path or paths. |
| 85 |
|
| 86 |
If a path is a folder, the number of files contained in the folder is returned. |
| 87 |
""" |
| 88 |
path = self.path |
| 89 |
paths = [path] if isinstance(path, str) else path |
| 90 |
return sum(1 if os.path.isfile(p) else len(_get_all_nested_filepaths(p)) for p in paths) |
| 91 |
|
| 92 |
@property |
| 93 |
def unique_extension(self) -> str | tuple[str, ...]: |
| 94 |
"""Return the unique file extension(s) from all files.""" |
| 95 |
return _get_unique_extension(self.path) |
| 96 |
|
| 97 |
@property |
| 98 |
@abstractmethod |
| 99 |
def _filesize_bytes(self) -> _FilePropIntType_co: |
| 100 |
"""Return the file size(s) of all files in bytes.""" |
| 101 |
|
| 102 |
@property |
| 103 |
@abstractmethod |
| 104 |
def _filesize_format(self) -> _FilePropStrType_co: |
| 105 |
"""Return the formatted size of all file(s).""" |
| 106 |
|
| 107 |
@property |
| 108 |
@abstractmethod |
| 109 |
def _total_size_bytes(self) -> int: |
| 110 |
"""Return the total size of all files in bytes.""" |
| 111 |
|
| 112 |
@property |
| 113 |
@abstractmethod |
| 114 |
def total_size(self) -> str: |
| 115 |
"""Return the total size of all files formatted as a string.""" |
| 116 |
|
| 117 |
@property |
| 118 |
@abstractmethod |
| 119 |
def _reader( |
| 120 |
self, |
| 121 |
) -> pv.BaseReader[Any] | tuple[pv.BaseReader[Any] | None, ...] | None: |
| 122 |
"""Return the base file reader(s) used to read the files.""" |
| 123 |
|
| 124 |
@property |
| 125 |
def unique_reader_type( |
| 126 |
self, |
| 127 |
) -> type[pv.BaseReader[Any]] | tuple[type[pv.BaseReader[Any]], ...] | None: |
| 128 |
"""Return unique reader type(s) from all file readers.""" |
| 129 |
return _get_unique_reader_type(self._reader) |
| 130 |
|
| 131 |
|
| 132 |
class _SingleFilePropsProtocol(_BaseFilePropsProtocol[str, int]): |
| 133 |
"""Define file properties of a single file.""" |
| 134 |
|
| 135 |
|
| 136 |
class _MultiFilePropsProtocol( |
| 137 |
_BaseFilePropsProtocol[tuple[str, ...], tuple[int, ...]], |
| 138 |
): |
| 139 |
"""Define file properties of multiple files.""" |
| 140 |
|
| 141 |
|
| 142 |
@runtime_checkable |
| 143 |
class _Downloadable(Protocol[_FilePropStrType_co]): |
| 144 |
"""Class which downloads file(s) from a source.""" |
| 145 |
|
| 146 |
@property |
| 147 |
@abstractmethod |
| 148 |
def source_name(self) -> _FilePropStrType_co: |
| 149 |
"""Return the name of the download relative to the base url.""" |
| 150 |
|
| 151 |
@property |
| 152 |
@abstractmethod |
| 153 |
def base_url(self) -> _FilePropStrType_co: |
| 154 |
"""Return the base url of the download.""" |
| 155 |
|
| 156 |
@property |
| 157 |
def source_url_raw(self) -> _FilePropStrType_co: |
| 158 |
"""Return the raw source of the download. |
| 159 |
|
| 160 |
This is the full URL used to download the data directly. |
| 161 |
""" |
| 162 |
name = self.source_name |
| 163 |
name_iter = [name] if isinstance(name, str) else name |
| 164 |
url = self.base_url |
| 165 |
base_url_iter = [url] if isinstance(url, str) else url |
| 166 |
url_raw = list(map(os.path.join, base_url_iter, name_iter)) |
| 167 |
return url_raw[0] if isinstance(name, str) else tuple(url_raw) |
| 168 |
|
| 169 |
@property |
| 170 |
def source_url_blob(self) -> _FilePropStrType_co: |
| 171 |
"""Return the blob source of the download. |
| 172 |
|
| 173 |
This URL is useful for linking to the source webpage for |
| 174 |
a human to open on a browser. |
| 175 |
""" |
| 176 |
# Make single urls iterable and replace 'raw' with 'blob' |
| 177 |
url_raw = self.source_url_raw |
| 178 |
url_iter = [url_raw] if isinstance(url_raw, str) else url_raw |
| 179 |
url_blob = [url.replace('/raw/', '/blob/') for url in url_iter] |
| 180 |
return url_blob[0] if isinstance(url_raw, str) else tuple(url_blob) |
| 181 |
|
| 182 |
@property |
| 183 |
@abstractmethod |
| 184 |
def path(self) -> _FilePropStrType_co: |
| 185 |
"""Return the file path of downloaded file.""" |
| 186 |
|
| 187 |
@abstractmethod |
| 188 |
def download(self) -> _FilePropStrType_co: |
| 189 |
"""Download and return the file path(s).""" |
| 190 |
|
| 191 |
|
| 192 |
class _DatasetLoader: |
| 193 |
"""Load a dataset.""" |
| 194 |
|
| 195 |
def __init__(self, load_func: Callable[..., DatasetObject]): |
| 196 |
self._load_func = load_func |
| 197 |
self._dataset: DatasetObject | None = None |
| 198 |
|
| 199 |
@property |
| 200 |
@final |
| 201 |
def dataset(self) -> DatasetObject | None: |
| 202 |
"""Return the loaded dataset object(s).""" |
| 203 |
return self._dataset |
| 204 |
|
| 205 |
def load(self, *args, **kwargs) -> DatasetObject: |
| 206 |
"""Load and return the dataset.""" |
| 207 |
# Subclasses should override this as needed |
| 208 |
return self._load_func(*args, **kwargs) |
| 209 |
|
| 210 |
@final |
| 211 |
def load_and_store_dataset(self) -> DatasetObject: |
| 212 |
"""Load the dataset and store it.""" |
| 213 |
dataset = self.load() |
| 214 |
self._dataset = dataset |
| 215 |
return dataset |
| 216 |
|
| 217 |
@final |
| 218 |
def clear_dataset(self): |
| 219 |
"""Clear the stored dataset object from memory.""" |
| 220 |
del self._dataset |
| 221 |
|
| 222 |
@property |
| 223 |
@final |
| 224 |
def dataset_iterable(self) -> tuple[DatasetObject, ...]: |
| 225 |
"""Return a tuple of all dataset object(s), including any nested objects. |
| 226 |
|
| 227 |
If the dataset is a MultiBlock, the MultiBlock itself is also returned as the first |
| 228 |
item. Any nested MultiBlocks are not included, only their datasets. |
| 229 |
|
| 230 |
E.g. for a composite dataset: |
| 231 |
MultiBlock -> (MultiBlock, Block0, Block1, ...) |
| 232 |
""" |
| 233 |
dataset = self.dataset |
| 234 |
|
| 235 |
def _flat(obj): |
| 236 |
if isinstance(obj, Sequence): |
| 237 |
output_list = [] # type: ignore[var-annotated] |
| 238 |
for item in obj: |
| 239 |
( |
| 240 |
output_list.extend(item) |
| 241 |
if isinstance(item, Sequence) |
| 242 |
else output_list.append(item) |
| 243 |
) |
| 244 |
if any(isinstance(item, Sequence) for item in output_list): |
| 245 |
return _flat(output_list) |
| 246 |
return output_list |
| 247 |
else: |
| 248 |
return [obj] |
| 249 |
|
| 250 |
flat = _flat(dataset) |
| 251 |
if isinstance(dataset, pv.MultiBlock): |
| 252 |
flat.insert(0, dataset) |
| 253 |
return tuple(flat) |
| 254 |
|
| 255 |
@property |
| 256 |
@final |
| 257 |
def unique_dataset_type( |
| 258 |
self, |
| 259 |
) -> DatasetType | tuple[DatasetType, ...] | None: |
| 260 |
"""Return unique dataset type(s) from all datasets.""" |
| 261 |
return _get_unique_dataset_type(self.dataset_iterable) |
| 262 |
|
| 263 |
@property |
| 264 |
@final |
| 265 |
def unique_cell_types( |
| 266 |
self, |
| 267 |
) -> tuple[pv.CellType, ...]: |
| 268 |
"""Return unique cell types from all datasets.""" |
| 269 |
cell_types: dict[pv.CellType, None] = {} |
| 270 |
for data in self.dataset_iterable: |
| 271 |
# Get the underlying dataset for the texture |
| 272 |
dataset = ( |
| 273 |
cast('pv.ImageData', pv.wrap(data.GetInput())) |
| 274 |
if isinstance(data, pv.Texture) |
| 275 |
else data |
| 276 |
) |
| 277 |
try: |
| 278 |
if isinstance(dataset, pv.ExplicitStructuredGrid): |
| 279 |
# extract_cells_by_type does not support this datatype |
| 280 |
# so get cells manually |
| 281 |
cells = (c.type for c in dataset.cell) |
| 282 |
[cell_types.update({cell_type: None}) for cell_type in cells] |
| 283 |
else: |
| 284 |
for cell_type in pv.CellType: |
| 285 |
extracted = dataset.extract_cells_by_type(cell_type) # type: ignore[union-attr] |
| 286 |
if extracted.n_cells > 0: |
| 287 |
cell_types[cell_type] = None |
| 288 |
except AttributeError: |
| 289 |
continue |
| 290 |
return tuple(sorted(cell_types.keys())) |
| 291 |
|
| 292 |
|
| 293 |
class _SingleFile(_SingleFilePropsProtocol): |
| 294 |
"""Wrap a single file.""" |
| 295 |
|
| 296 |
def __init__(self, path): |
| 297 |
from pyvista.examples.downloads import USER_DATA_PATH # noqa: PLC0415 |
| 298 |
|
| 299 |
self._path = ( |
| 300 |
path if path is None or os.path.isabs(path) else os.path.join(USER_DATA_PATH, path) |
| 301 |
) |
| 302 |
|
| 303 |
@property |
| 304 |
def path(self) -> str: |
| 305 |
return self._path |
| 306 |
|
| 307 |
@property |
| 308 |
def _filesize_bytes(self) -> int: |
| 309 |
return _get_file_or_folder_size(self.path) |
| 310 |
|
| 311 |
@property |
| 312 |
def _filesize_format(self) -> str: |
| 313 |
return _format_file_size(self._filesize_bytes) |
| 314 |
|
| 315 |
@property |
| 316 |
def _total_size_bytes(self) -> int: |
| 317 |
return self._filesize_bytes |
| 318 |
|
| 319 |
@property |
| 320 |
def total_size(self) -> str: |
| 321 |
return self._filesize_format |
| 322 |
|
| 323 |
@property |
| 324 |
def _reader(self) -> pv.BaseReader[Any] | None: |
| 325 |
return None |
| 326 |
|
| 327 |
|
| 328 |
class _SingleFileDatasetLoader(_SingleFile, _DatasetLoader): |
| 329 |
"""Wrap a single file for loading. |
| 330 |
|
| 331 |
Specify the read function and/or load functions for reading and processing the |
| 332 |
dataset. The read function is called on the file path first, then, if a load |
| 333 |
function is specified, the load function is called on the output from the read |
| 334 |
function. |
| 335 |
|
| 336 |
Parameters |
| 337 |
---------- |
| 338 |
path |
| 339 |
Path of the file to be loaded. |
| 340 |
|
| 341 |
read_func |
| 342 |
Specify the function used to read the file. Defaults to :func:`pyvista.read`. |
| 343 |
This can be used for customizing the reader's properties, or using another |
| 344 |
read function (e.g. :func:`pyvista.read_texture` for textures). The function |
| 345 |
must have the file path as the first argument and should return a dataset. |
| 346 |
If default arguments are required by your desired read function, consider |
| 347 |
using :class:`functools.partial` to pre-set the arguments before passing it |
| 348 |
as an argument to the loader. |
| 349 |
|
| 350 |
load_func |
| 351 |
Specify the function used to load the file. Defaults to `None`. This is typically |
| 352 |
used to specify any processing of the dataset after reading. The load function |
| 353 |
typically will accept a dataset as an input and return a dataset. |
| 354 |
|
| 355 |
""" |
| 356 |
|
| 357 |
def __init__( |
| 358 |
self, |
| 359 |
path: str, |
| 360 |
read_func: Callable[[str], DatasetObject] | None = None, |
| 361 |
load_func: Callable[[DatasetObject], Any] | None = None, |
| 362 |
): |
| 363 |
_SingleFile.__init__(self, path) |
| 364 |
# _DatasetLoader.__init__ is typed as requiring a non-None |
| 365 |
# ``Callable[..., DatasetObject]`` but this subclass treats |
| 366 |
# load_func as optional (see ``load`` which falls back to |
| 367 |
# ``read_func`` when load_func is None). Fixing the base |
| 368 |
# signature is out of scope for the typing refactor. |
| 369 |
_DatasetLoader.__init__(self, load_func) # type: ignore[arg-type] |
| 370 |
self._read_func = pv.read if path and read_func is None else read_func |
| 371 |
|
| 372 |
@property |
| 373 |
def _reader(self) -> pv.BaseReader[Any] | None: |
| 374 |
# TODO: return the actual reader used, and not just a lookup |
| 375 |
# (this will require an update to the 'read_func' API) |
| 376 |
try: |
| 377 |
return pv.get_reader(self.path) |
| 378 |
except ValueError: |
| 379 |
# Cannot be read directly (requires custom reader) |
| 380 |
return None |
| 381 |
|
| 382 |
@property |
| 383 |
def path_loadable(self) -> str: |
| 384 |
return self.path |
| 385 |
|
| 386 |
def load(self): |
| 387 |
path = self.path |
| 388 |
read_func = self._read_func |
| 389 |
load_func = self._load_func |
| 390 |
try: |
| 391 |
# Read and load normally |
| 392 |
return read_func(path) if load_func is None else load_func(read_func(path)) # type: ignore[misc] |
| 393 |
except OSError: |
| 394 |
# Handle error generated by pv.read if reading a directory |
| 395 |
if read_func is pv.read and Path(path).is_dir(): |
| 396 |
# Re-define read function to read all files in a directory as a multiblock |
| 397 |
read_func = lambda path: _load_as_multiblock( |
| 398 |
[ |
| 399 |
_SingleFileDatasetLoader(str(Path(path, fname))) |
| 400 |
for fname in sorted(os.listdir(path)) # noqa: PTH208 |
| 401 |
], |
| 402 |
) |
| 403 |
return read_func(path) if load_func is None else load_func(read_func(path)) |
| 404 |
else: |
| 405 |
msg = f'Error loading dataset from path:\n\t{self.path}' |
| 406 |
raise RuntimeError(msg) |
| 407 |
|
| 408 |
|
| 409 |
class _DownloadableFile(_SingleFile, _Downloadable[str]): |
| 410 |
"""Wrap a single file which must be downloaded. |
| 411 |
|
| 412 |
If downloading a file from an archive, set the filepath of the zip as |
| 413 |
``path`` and set ``target_file`` as the file to extract. If the path is |
| 414 |
a zip file and no target file is specified, the entire archive is downloaded |
| 415 |
and extracted and the root directory of the path is returned. |
| 416 |
|
| 417 |
""" |
| 418 |
|
| 419 |
def __init__( |
| 420 |
self, |
| 421 |
path: str, |
| 422 |
target_file: str | None = None, |
| 423 |
): |
| 424 |
_SingleFile.__init__(self, path) |
| 425 |
|
| 426 |
from pyvista.examples.downloads import SOURCE # noqa: PLC0415 |
| 427 |
from pyvista.examples.downloads import USER_DATA_PATH # noqa: PLC0415 |
| 428 |
from pyvista.examples.downloads import _download_archive_file_or_folder # noqa: PLC0415 |
| 429 |
from pyvista.examples.downloads import download_file # noqa: PLC0415 |
| 430 |
from pyvista.examples.downloads import file_from_files # noqa: PLC0415 |
| 431 |
from pyvista.examples.examples import dir_path # noqa: PLC0415 |
| 432 |
|
| 433 |
if Path(path).is_absolute(): |
| 434 |
# Absolute path must point to a built-in dataset |
| 435 |
assert Path(path).parent == Path( |
| 436 |
dir_path, |
| 437 |
), 'Absolute path must point to a built-in dataset.' |
| 438 |
self._base_url = 'https://github.com/pyvista/pyvista/raw/main/pyvista/examples/' |
| 439 |
self._source_name = Path(path).name |
| 440 |
# the dataset is already downloaded (it's built-in) |
| 441 |
# so make download() simply return the local filepath |
| 442 |
self._download_func = lambda _: path |
| 443 |
else: |
| 444 |
# Relative path, use vars from downloads.py |
| 445 |
self._base_url = SOURCE |
| 446 |
self._download_func = download_file |
| 447 |
self._source_name = Path(path).name if Path(path).is_absolute() else path |
| 448 |
|
| 449 |
target_file = '' if target_file is None and (get_ext(path) == '.zip') else target_file |
| 450 |
if target_file is not None: |
| 451 |
# download from archive |
| 452 |
self._download_func = functools.partial( |
| 453 |
_download_archive_file_or_folder, |
| 454 |
target_file=target_file, |
| 455 |
) |
| 456 |
# The file path currently points to the archive, not the target file itself |
| 457 |
# Try to resolve the full path to the target file (without downloading) if |
| 458 |
# the archive already exists in the cache |
| 459 |
fullpath = None |
| 460 |
if os.path.isfile(self.path): |
| 461 |
try: |
| 462 |
# Get file path |
| 463 |
fullpath = file_from_files(target_file, self.path) |
| 464 |
except (FileNotFoundError, RuntimeError): |
| 465 |
# Get folder path |
| 466 |
fullpath = os.path.join(USER_DATA_PATH, path + '.unzip', target_file) |
| 467 |
fullpath = fullpath if os.path.isdir(fullpath) else None |
| 468 |
# set the file path as the relative path of the target file if |
| 469 |
# the fullpath could not be resolved (i.e. not yet downloaded) |
| 470 |
self._path = target_file if fullpath is None else fullpath |
| 471 |
|
| 472 |
@property |
| 473 |
def source_name(self) -> str: |
| 474 |
return self._source_name |
| 475 |
|
| 476 |
@property |
| 477 |
def base_url(self) -> str: |
| 478 |
return self._base_url |
| 479 |
|
| 480 |
def download(self) -> str: |
| 481 |
path = self._download_func(self._source_name) |
| 482 |
assert os.path.isfile(path) or os.path.isdir(path) |
| 483 |
# Reset the path since the full path for archive files |
| 484 |
# isn't known until after downloading |
| 485 |
self._path = path |
| 486 |
return path |
| 487 |
|
| 488 |
|
| 489 |
class _SingleFileDownloadableDatasetLoader(_SingleFileDatasetLoader, _DownloadableFile): |
| 490 |
"""Wrap a single file which must first be downloaded and which can also be loaded. |
| 491 |
|
| 492 |
.. warning:: |
| 493 |
|
| 494 |
``download()`` should be called before accessing other attributes. Otherwise, |
| 495 |
calling ``load()`` or ``path`` may fail or produce unexpected results. |
| 496 |
|
| 497 |
""" |
| 498 |
|
| 499 |
def __init__( # noqa: PLR0917 |
| 500 |
self, |
| 501 |
path: str, |
| 502 |
read_func: Callable[[str], DatasetObject] | None = None, |
| 503 |
load_func: Callable[[DatasetObject], DatasetObject] | None = None, |
| 504 |
target_file: str | None = None, |
| 505 |
): |
| 506 |
_SingleFileDatasetLoader.__init__(self, path, read_func=read_func, load_func=load_func) |
| 507 |
_DownloadableFile.__init__(self, path, target_file=target_file) |
| 508 |
|
| 509 |
|
| 510 |
class _MultiFileDatasetLoader(_DatasetLoader, _MultiFilePropsProtocol): |
| 511 |
"""Wrap multiple files for loading. |
| 512 |
|
| 513 |
Some use cases for loading multi-file examples include: |
| 514 |
|
| 515 |
1. Multiple input files, and each file is read/loaded independently |
| 516 |
E.g.: loading two separate datasets for the example |
| 517 |
See ``download_bolt_nut`` for a reference implementation. |
| 518 |
|
| 519 |
2. Multiple input files, but only one is read or loaded directly |
| 520 |
E.g.: loading a single dataset from a file format where data and metadata are |
| 521 |
stored in separate files, such as ``.raw`` and ``.mhd``. |
| 522 |
See ``download_head`` for a reference implementation. |
| 523 |
|
| 524 |
3. Multiple input files, all of which make up part of the loaded dataset |
| 525 |
E.g.: loading six separate image files for cubemaps |
| 526 |
See ``download_sky_box_cube_map`` for a reference implementation. |
| 527 |
|
| 528 |
Parameters |
| 529 |
---------- |
| 530 |
files_func |
| 531 |
Specify the function which will return a sequence of :class:`_SingleFile` |
| 532 |
objects required for loading the dataset. Alternatively, a directory can be |
| 533 |
specified, in which case a separate single-file dataset loader is created |
| 534 |
for each file with a default reader. |
| 535 |
|
| 536 |
load_func |
| 537 |
Specify the function used to load the files. By default, :meth:`load()` is called |
| 538 |
on all the files (if loadable) and a tuple containing the loaded datasets is returned. |
| 539 |
|
| 540 |
""" |
| 541 |
|
| 542 |
def __init__( |
| 543 |
self, |
| 544 |
files_func: str | Callable[[], Sequence[_SingleFileDatasetLoader | _DownloadableFile]], |
| 545 |
load_func: Callable[[Sequence[_SingleFileDatasetLoader]], Any] | None = None, |
| 546 |
): |
| 547 |
self._files_func = files_func |
| 548 |
self._file_loaders_ = None |
| 549 |
if load_func is None: |
| 550 |
load_func = _load_as_dataset_or_multiblock |
| 551 |
|
| 552 |
_DatasetLoader.__init__(self, load_func) |
| 553 |
|
| 554 |
@property |
| 555 |
def _file_objects(self): |
| 556 |
if self._file_loaders_ is None and not isinstance(self._files_func, str): |
| 557 |
self._file_loaders_ = self._files_func() # type: ignore[assignment] |
| 558 |
return self._file_loaders_ |
| 559 |
|
| 560 |
@property |
| 561 |
def path(self) -> tuple[str, ...]: |
| 562 |
return tuple(_flatten_nested_sequence([file.path for file in self._file_objects])) |
| 563 |
|
| 564 |
@property |
| 565 |
def path_loadable(self) -> tuple[str, ...]: |
| 566 |
return tuple( |
| 567 |
file.path for file in self._file_objects if isinstance(file, _SingleFileDatasetLoader) |
| 568 |
) |
| 569 |
|
| 570 |
@property |
| 571 |
def _filesize_bytes(self) -> tuple[int, ...]: |
| 572 |
return tuple( |
| 573 |
_flatten_nested_sequence([file._filesize_bytes for file in self._file_objects]), |
| 574 |
) |
| 575 |
|
| 576 |
@property |
| 577 |
def _filesize_format(self) -> tuple[str, ...]: |
| 578 |
return tuple(_format_file_size(size) for size in self._filesize_bytes) |
| 579 |
|
| 580 |
@property |
| 581 |
def _total_size_bytes(self) -> int: |
| 582 |
return sum(file._total_size_bytes for file in self._file_objects) |
| 583 |
|
| 584 |
@property |
| 585 |
def total_size(self) -> str: |
| 586 |
return _format_file_size(self._total_size_bytes) |
| 587 |
|
| 588 |
@property |
| 589 |
def _reader( |
| 590 |
self, |
| 591 |
) -> pv.BaseReader[Any] | tuple[pv.BaseReader[Any] | None, ...] | None: |
| 592 |
# TODO: return the actual reader used, and not just a lookup |
| 593 |
# (this will require an update to the 'read_func' API) |
| 594 |
reader = _flatten_nested_sequence([file._reader for file in self._file_objects]) |
| 595 |
# flatten in case any file objects themselves are multifiles |
| 596 |
reader_out: list[pv.BaseReader[Any]] = [] |
| 597 |
for r in reader: |
| 598 |
reader_out.extend(r) if isinstance(r, Sequence) else reader_out.append(r) |
| 599 |
return tuple(reader_out) |
| 600 |
|
| 601 |
def load(self): |
| 602 |
return self._load_func(self._file_objects) |
| 603 |
|
| 604 |
|
| 605 |
class _MultiFileDownloadableDatasetLoader( |
| 606 |
_MultiFileDatasetLoader, |
| 607 |
_Downloadable[tuple[str, ...]], |
| 608 |
): |
| 609 |
"""Wrap multiple files for downloading and loading.""" |
| 610 |
|
| 611 |
@property |
| 612 |
def source_name(self) -> tuple[str, ...]: |
| 613 |
name = [file.source_name for file in self._file_objects if isinstance(file, _Downloadable)] |
| 614 |
return tuple(_flatten_nested_sequence(name)) |
| 615 |
|
| 616 |
@property |
| 617 |
def base_url(self) -> tuple[str, ...]: |
| 618 |
url = [file.base_url for file in self._file_objects if isinstance(file, _Downloadable)] |
| 619 |
return tuple(_flatten_nested_sequence(url)) |
| 620 |
|
| 621 |
def download(self) -> tuple[str, ...]: |
| 622 |
path = [file.download() for file in self._file_objects if isinstance(file, _Downloadable)] |
| 623 |
# flatten paths in case any loaders have multiple files |
| 624 |
path_out = _flatten_nested_sequence(path) |
| 625 |
assert all(os.path.isfile(p) or os.path.isdir(p) for p in path_out) |
| 626 |
return tuple(path_out) |
| 627 |
|
| 628 |
|
| 629 |
_ScalarType = TypeVar('_ScalarType', int, str, pv.BaseReader[Any]) |
| 630 |
|
| 631 |
|
| 632 |
def _flatten_nested_sequence(nested: Sequence[_ScalarType | Sequence[_ScalarType]]): |
| 633 |
"""Flatten nested sequences of objects.""" |
| 634 |
flat: list[_ScalarType] = [] |
| 635 |
for item in nested: |
| 636 |
if isinstance(item, Sequence) and not isinstance(item, str): |
| 637 |
flat.extend(item) |
| 638 |
else: |
| 639 |
flat.append(item) |
| 640 |
return flat |
| 641 |
|
| 642 |
|
| 643 |
def _download_dataset( |
| 644 |
dataset_loader: _SingleFileDownloadableDatasetLoader | _MultiFileDownloadableDatasetLoader, |
| 645 |
*, |
| 646 |
load: bool = True, |
| 647 |
metafiles: bool = False, |
| 648 |
): |
| 649 |
"""Download and load a dataset file or files. |
| 650 |
|
| 651 |
Parameters |
| 652 |
---------- |
| 653 |
dataset_loader |
| 654 |
SingleFile or MultiFile object(s) of the dataset(s) to download or load. |
| 655 |
|
| 656 |
load |
| 657 |
Read and load the file after downloading. When ``False``, |
| 658 |
return the path or paths to the example's file(s). |
| 659 |
|
| 660 |
metafiles |
| 661 |
When ``load`` is ``False``, set this value to ``True`` to |
| 662 |
return all files required to load the example, including any metafiles. |
| 663 |
If ``False``, only the paths of files which are explicitly loaded are |
| 664 |
returned. E.g if a file format uses two files to specify the header info |
| 665 |
and file data separately, setting ``metafiles=True`` will return a tuple |
| 666 |
with both file paths, whereas setting ``metafiles=False`` will only return |
| 667 |
the single path of the header file as a string. |
| 668 |
|
| 669 |
Returns |
| 670 |
------- |
| 671 |
Any |
| 672 |
Loaded dataset or path(s) to the example's files depending on the ``load`` |
| 673 |
parameter. Dataset may be a texture, mesh, multiblock, array, tuple of meshes, |
| 674 |
or any other output loaded by the example. |
| 675 |
|
| 676 |
""" |
| 677 |
# Download all files for the dataset, include any metafiles |
| 678 |
path = dataset_loader.download() |
| 679 |
|
| 680 |
# Exclude non-loadable metafiles from result (if any) |
| 681 |
if not metafiles and isinstance(dataset_loader, _MultiFileDownloadableDatasetLoader): |
| 682 |
path = dataset_loader.path_loadable |
| 683 |
# Return scalar if only one loadable file |
| 684 |
path = path[0] if len(path) == 1 else path |
| 685 |
|
| 686 |
return dataset_loader.load() if load else path |
| 687 |
|
| 688 |
|
| 689 |
def _load_as_multiblock( |
| 690 |
files: Sequence[_SingleFileDatasetLoader | _MultiFileDatasetLoader], |
| 691 |
names: Sequence[str] | None = None, |
| 692 |
) -> pv.MultiBlock: |
| 693 |
"""Load multiple files as a MultiBlock. |
| 694 |
|
| 695 |
This function can be used as a loading function for :class:`MultiFileLoadable` |
| 696 |
If the use of the ``names`` parameter is needed, use :class:`functools.partial` |
| 697 |
to partially specify the names parameter before passing it as loading function. |
| 698 |
""" |
| 699 |
multi = pv.MultiBlock() |
| 700 |
if names is None: |
| 701 |
# set names, use filename without ext by default or dirname |
| 702 |
paths = _flatten_nested_sequence( |
| 703 |
[file.path_loadable for file in files if isinstance(file, _DatasetLoader)], |
| 704 |
) |
| 705 |
paths = [Path(path) for path in paths] |
| 706 |
names = [ |
| 707 |
path.name[: -len(get_ext(path.name))] if path.is_file() else path.name |
| 708 |
for path in paths |
| 709 |
] |
| 710 |
|
| 711 |
for file, name in zip(files, names, strict=False): |
| 712 |
if not isinstance(file, _DatasetLoader): |
| 713 |
continue # type: ignore[unreachable] |
| 714 |
loaded = file.load() |
| 715 |
assert isinstance( |
| 716 |
loaded, |
| 717 |
(pv.MultiBlock, pv.DataSet), |
| 718 |
), ( |
| 719 |
f'Only MultiBlock or DataSet objects can be loaded as a MultiBlock. ' |
| 720 |
f"Got {type(loaded)}.'" |
| 721 |
) |
| 722 |
multi.append(loaded, name) |
| 723 |
return multi |
| 724 |
|
| 725 |
|
| 726 |
def _load_as_cubemap(files: str | _SingleFile | Sequence[_SingleFile]) -> pv.Texture: |
| 727 |
"""Load multiple files as a cubemap. |
| 728 |
|
| 729 |
Input may be a single directory with 6 cubemap files, or a sequence |
| 730 |
of 6 files |
| 731 |
""" |
| 732 |
path = ( |
| 733 |
files |
| 734 |
if isinstance(files, str) |
| 735 |
else (files.path if isinstance(files, _SingleFile) else [file.path for file in files]) |
| 736 |
) |
| 737 |
|
| 738 |
return ( |
| 739 |
pv.cubemap(path) |
| 740 |
if isinstance(files, str) and os.path.isdir(files) |
| 741 |
else pv.cubemap_from_filenames(path) |
| 742 |
) |
| 743 |
|
| 744 |
|
| 745 |
def _load_as_dataset_or_multiblock(files): |
| 746 |
multiblock = _load_as_multiblock(files) |
| 747 |
return multiblock[0] if len(multiblock) == 1 else multiblock |
| 748 |
|
| 749 |
|
| 750 |
def _load_and_merge(files: Sequence[_SingleFile]): |
| 751 |
"""Load all loadable files as separate datasets and merge them.""" |
| 752 |
loaded = [file.load() for file in files if isinstance(file, _DatasetLoader)] |
| 753 |
assert len(loaded) > 0 |
| 754 |
return pv.merge(loaded) |
| 755 |
|
| 756 |
|
| 757 |
def _get_file_or_folder_size(filepath) -> int: |
| 758 |
if os.path.isfile(filepath): |
| 759 |
return os.path.getsize(filepath) |
| 760 |
assert os.path.isdir(filepath), 'Expected a file or folder path.' |
| 761 |
all_filepaths = _get_all_nested_filepaths(filepath) |
| 762 |
return sum(os.path.getsize(file) for file in all_filepaths) |
| 763 |
|
| 764 |
|
| 765 |
def _format_file_size(size: int) -> str: |
| 766 |
size_flt = float(size) |
| 767 |
for unit in ('B', 'KB', 'MB'): |
| 768 |
if round(size_flt * 10) / 10 < 1000.0: |
| 769 |
return f'{int(size_flt)} {unit}' if unit == 'B' else f'{size_flt:3.1f} {unit}' |
| 770 |
size_flt /= 1000.0 |
| 771 |
return f'{size_flt:.1f} GB' |
| 772 |
|
| 773 |
|
| 774 |
def _get_file_or_folder_ext(path: str): |
| 775 |
"""Wrap the `get_ext` function to handle special cases for directories.""" |
| 776 |
if os.path.isfile(path): |
| 777 |
return get_ext(path) |
| 778 |
assert os.path.isdir(path), 'Expected a file or folder path.' |
| 779 |
all_paths = _get_all_nested_filepaths(path) |
| 780 |
ext = [get_ext(file) for file in all_paths] |
| 781 |
assert len(ext) != 0, f'No files with extensions were found in"\n\t{path}' |
| 782 |
return ext |
| 783 |
|
| 784 |
|
| 785 |
def _get_all_nested_filepaths(filepath, *, exclude_readme=True): |
| 786 |
"""Walk through directory and get all file paths. |
| 787 |
|
| 788 |
Optionally exclude any readme files (if any). |
| 789 |
""" |
| 790 |
assert os.path.isfile(filepath) or os.path.isdir(filepath) |
| 791 |
condition = lambda name: True if not exclude_readme else not name.lower().startswith('readme') |
| 792 |
return next( |
| 793 |
[os.path.join(path, name) for name in files if condition(name)] |
| 794 |
for path, _, files in os.walk(filepath) |
| 795 |
) |
| 796 |
|
| 797 |
|
| 798 |
def _get_unique_extension(path: str | Sequence[str]): |
| 799 |
"""Return a file extension or unique set of file extensions from a path or paths.""" |
| 800 |
ext_set = set() |
| 801 |
fname_sequence = [path] if isinstance(path, str) else path |
| 802 |
|
| 803 |
# Add all file extensions to the set |
| 804 |
for file in fname_sequence: |
| 805 |
ext = _get_file_or_folder_ext(file) |
| 806 |
ext_set.add(ext) if isinstance(ext, str) else ext_set.update(ext) |
| 807 |
|
| 808 |
# Format output |
| 809 |
ext_output = tuple(ext_set) |
| 810 |
return ext_output[0] if len(ext_output) == 1 else tuple(sorted(ext_output)) |
| 811 |
|
| 812 |
|
| 813 |
def _get_unique_reader_type( |
| 814 |
reader: pv.BaseReader[Any] | tuple[pv.BaseReader[Any] | None, ...] | None, |
| 815 |
) -> type[pv.BaseReader[Any]] | tuple[type[pv.BaseReader[Any]], ...] | None: |
| 816 |
"""Return a reader type or tuple of unique reader types.""" |
| 817 |
if reader is None or (isinstance(reader, Sequence) and all(r is None for r in reader)): |
| 818 |
return None |
| 819 |
reader_set: set[type[pv.BaseReader[Any]]] = set() |
| 820 |
reader_type = ( |
| 821 |
[type(reader)] |
| 822 |
if not isinstance(reader, Sequence) |
| 823 |
else [type(r) for r in reader if r is not None] |
| 824 |
) |
| 825 |
|
| 826 |
# Add all reader types to the set |
| 827 |
reader_set.update(reader_type) |
| 828 |
|
| 829 |
# Format output |
| 830 |
reader_output = tuple(reader_set) |
| 831 |
return reader_output[0] if len(reader_output) == 1 else tuple(reader_output) |
| 832 |
|
| 833 |
|
| 834 |
def _get_unique_dataset_type( |
| 835 |
dataset_iterable: tuple[DatasetObject, ...], |
| 836 |
) -> DatasetType | tuple[DatasetType, ...]: |
| 837 |
"""Return a dataset type or tuple of unique dataset types.""" |
| 838 |
dataset_types: dict[DatasetType, None] = {} # use dict as an ordered set |
| 839 |
for dataset in dataset_iterable: |
| 840 |
dataset_types[type(dataset)] = None |
| 841 |
output = tuple(dataset_types.keys()) |
| 842 |
return output[0] if len(output) == 1 else output |