| 1 |
# Copyright (c) 2023 European Synchrotron Radiation Facility |
| 2 |
# License: MIT https://opensource.org/license/mit/ |
| 3 |
from __future__ import annotations |
| 4 |
|
| 5 |
import json |
| 6 |
import os |
| 7 |
import sys |
| 8 |
import tempfile |
| 9 |
import time |
| 10 |
from typing import Optional, NamedTuple |
| 11 |
|
| 12 |
import h5py |
| 13 |
import numpy |
| 14 |
|
| 15 |
|
| 16 |
# Set affinity and env. var. before importing hdf5plugin |
| 17 |
|
| 18 |
if len(sys.argv) >= 2: |
| 19 |
NCPU = int(sys.argv[1]) |
| 20 |
print(f"Set affinity: Using {NCPU} CPU(s)") |
| 21 |
os.sched_setaffinity(0, list(range(0, NCPU, 4))) |
| 22 |
os.environ["OPENMP_NUM_THREADS"] = str(NCPU) |
| 23 |
os.environ["BLOSC_NTHREADS"] = str(NCPU) |
| 24 |
else: |
| 25 |
NCPU = len(os.sched_getaffinity(0)) |
| 26 |
print(f"Default affinity: Using {NCPU} CPU(s)") |
| 27 |
|
| 28 |
# Directory where to run read/write benchmarks |
| 29 |
DIRECTORY = "/dev/shm" |
| 30 |
|
| 31 |
# Function to get data to use for tests |
| 32 |
def get_data(): |
| 33 |
with h5py.File("/dev/shm/kevlar.h5") as h5file: |
| 34 |
return h5file["/entry/data/data"][::10] # Take 100 frames |
| 35 |
|
| 36 |
|
| 37 |
import hdf5plugin |
| 38 |
|
| 39 |
|
| 40 |
class Result(NamedTuple): |
| 41 |
"""Store benchmark result""" |
| 42 |
|
| 43 |
raw_nbytes: int |
| 44 |
compressed_nbytes: int |
| 45 |
write_duration: float |
| 46 |
read_duration: float |
| 47 |
chunks: tuple[int] |
| 48 |
|
| 49 |
compression_rate = property( |
| 50 |
lambda self: self.raw_nbytes / self.compressed_nbytes) |
| 51 |
write_speed = property( |
| 52 |
lambda self: (self.raw_nbytes / 1024**2) / self.write_duration, |
| 53 |
doc="Unit: MB/sec") |
| 54 |
read_speed = property( |
| 55 |
lambda self: (self.raw_nbytes / 1024**2) / self.read_duration, |
| 56 |
doc="Unit: MB/sec") |
| 57 |
|
| 58 |
|
| 59 |
def benchmark( |
| 60 |
data: numpy.ndarray, |
| 61 |
directory: Optional[str] = None, |
| 62 |
**kwargs |
| 63 |
) -> Result: |
| 64 |
"""Run benchmark for given conditions |
| 65 |
|
| 66 |
:param data: Dataset to use |
| 67 |
:param directory: Directory where to write HDF5 file for benchmark |
| 68 |
:param kwargs: Arguments passed to h5py.Group.create_dataset |
| 69 |
""" |
| 70 |
if directory is None: |
| 71 |
tmpdir = tempfile.TemporaryDirectory() |
| 72 |
dirname = tmpdir.name |
| 73 |
else: |
| 74 |
dirname = directory |
| 75 |
|
| 76 |
filename = os.path.join(dirname, 'hdf5plugin_benchmark.h5') |
| 77 |
|
| 78 |
if os.path.exists(filename): |
| 79 |
os.remove(filename) |
| 80 |
|
| 81 |
# Compression |
| 82 |
with h5py.File(filename, "w") as h5file: |
| 83 |
dataset = h5file.create_dataset( |
| 84 |
"data", shape=data.shape, dtype=data.dtype, **kwargs) |
| 85 |
start_write_time = time.perf_counter() |
| 86 |
dataset[:] = data |
| 87 |
dataset.flush() |
| 88 |
write_duration = time.perf_counter() - start_write_time |
| 89 |
|
| 90 |
# Decompression |
| 91 |
with h5py.File(filename, "r") as h5file: |
| 92 |
dataset = h5file["data"] |
| 93 |
start_time = time.perf_counter() |
| 94 |
read_data = dataset[()] |
| 95 |
read_duration = time.perf_counter() - start_time |
| 96 |
storage_size = dataset.id.get_storage_size() |
| 97 |
chunks = dataset.chunks |
| 98 |
|
| 99 |
os.remove(filename) |
| 100 |
|
| 101 |
return Result(data.nbytes, storage_size, write_duration, read_duration, chunks=chunks) |
| 102 |
|
| 103 |
|
| 104 |
DEFAULT_FILTERS = { # Filters available with h5py/libhdf5 |
| 105 |
"Raw": None, |
| 106 |
"GZip": "gzip", |
| 107 |
"LZF": "lzf", |
| 108 |
} |
| 109 |
|
| 110 |
LOSSLESS_FILTERS = { |
| 111 |
"BZip2": hdf5plugin.BZip2(), |
| 112 |
"LZ4": hdf5plugin.LZ4(), |
| 113 |
"ZStd": hdf5plugin.Zstd(), |
| 114 |
} |
| 115 |
|
| 116 |
BITSHUFFLE_FILTERS = { |
| 117 |
"Bitshuffle-lz4": hdf5plugin.Bitshuffle(cname='lz4'), |
| 118 |
"Bitshuffle-zstd": hdf5plugin.Bitshuffle(cname='zstd'), |
| 119 |
} |
| 120 |
|
| 121 |
BLOSC_FILTERS = {} |
| 122 |
for cname in ('lz4', 'blosclz', 'lz4', 'lz4hc', 'snappy', 'zlib', 'zstd'): |
| 123 |
for shuffle_name, shuffle in [('NoShuffle', hdf5plugin.Blosc.NOSHUFFLE), |
| 124 |
('Shuffle', hdf5plugin.Blosc.SHUFFLE), |
| 125 |
('BitShuffle', hdf5plugin.Blosc.BITSHUFFLE)]: |
| 126 |
for clevel in [5]: #(1, 3, 5, 9): |
| 127 |
BLOSC_FILTERS[f"Blosc-{cname}-{shuffle_name}-{clevel}"] = hdf5plugin.Blosc( |
| 128 |
cname=cname, clevel=clevel, shuffle=shuffle) |
| 129 |
|
| 130 |
|
| 131 |
BLOSC2_FILTERS = {} |
| 132 |
for cname in ('lz4', 'blosclz', 'lz4', 'lz4hc', 'zlib', 'zstd'): |
| 133 |
for filters_name, filters in [('NoFilter', hdf5plugin.Blosc2.NOFILTER), |
| 134 |
('Shuffle', hdf5plugin.Blosc2.SHUFFLE), |
| 135 |
('BitShuffle', hdf5plugin.Blosc2.BITSHUFFLE)]: |
| 136 |
for clevel in [5]: # (1, 3, 5, 9): |
| 137 |
BLOSC2_FILTERS[f"Blosc2-{cname}-{filters_name}-{clevel}"] = hdf5plugin.Blosc2( |
| 138 |
cname=cname, clevel=clevel, filters=filters) |
| 139 |
|
| 140 |
|
| 141 |
FILTERS = { |
| 142 |
**DEFAULT_FILTERS, |
| 143 |
**LOSSLESS_FILTERS, |
| 144 |
**BITSHUFFLE_FILTERS, |
| 145 |
**BLOSC_FILTERS, |
| 146 |
**BLOSC2_FILTERS, |
| 147 |
} |
| 148 |
|
| 149 |
|
| 150 |
print("Read benchmark data...") |
| 151 |
data = get_data() |
| 152 |
print("Run benchmarks:") |
| 153 |
|
| 154 |
chunks = (1,) + data.shape[1:] |
| 155 |
|
| 156 |
results = {} |
| 157 |
for name, compression in FILTERS.items(): |
| 158 |
print(f"- {name}") |
| 159 |
results[name] = benchmark( |
| 160 |
data, |
| 161 |
directory=DIRECTORY, |
| 162 |
chunks=chunks, |
| 163 |
compression=compression, |
| 164 |
)._asdict() |
| 165 |
|
| 166 |
|
| 167 |
print("Write results") |
| 168 |
with open("benchmark.json", "w") as f: |
| 169 |
json.dump( |
| 170 |
dict( |
| 171 |
config=dict( |
| 172 |
ncpu=NCPU, |
| 173 |
affinity=list(os.sched_getaffinity(0)), |
| 174 |
openmp_num_threads=os.environ.get("OPENMP_NUM_THREADS", "unset"), |
| 175 |
blosc_nthreads=os.environ.get("BLOSC_NTHREADS", "unset"), |
| 176 |
hdf5plugin=hdf5plugin.get_config().build_config._asdict(), |
| 177 |
chunks=chunks, |
| 178 |
), |
| 179 |
results=results, |
| 180 |
), |
| 181 |
f, |
| 182 |
indent=2, |
| 183 |
default=lambda obj: type(obj).__name__, |
| 184 |
) |