Debian Codegraph @

doc/hdf5plugin_EuropeanHUG2023/hdf5_compressed_chunk_direct_read.ipynb

raw
1 {
2 "cells": [
3 {
4 "cell_type": "markdown",
5 "id": "f7952927-1f3f-49cd-9f1c-d6dad0835ae9",
6 "metadata": {},
7 "source": [
8 "# HDF5 compressed chunk direct read\n",
9 "\n",
10 "This notebooks illustrate how to read compressed chunks directly and decompress them from Python for `Blosc2` and `Bitshuffle` filters.\n",
11 "\n",
12 "It compares this approach with reading compressed chunks with `h5py` and `hdf5plugin`."
13 ]
14 },
15 {
16 "cell_type": "markdown",
17 "id": "d452efbf-8c70-45ab-bb3a-885491601785",
18 "metadata": {
19 "tags": []
20 },
21 "source": [
22 "## hdf5plugin config\n",
23 "\n",
24 "Performance changes depending on `hdf5plugin` build config, environment variables (`OPENMP_NUM_THREADS` and `BLOSC_NTHREADS`) and available CPU cores."
25 ]
26 },
27 {
28 "cell_type": "code",
29 "execution_count": 1,
30 "id": "fc1c65ce-75b9-460f-9065-be975e2ab6c8",
31 "metadata": {
32 "tags": []
33 },
34 "outputs": [
35 {
36 "name": "stdout",
37 "output_type": "stream",
38 "text": [
39 "Number of CPU: 1; Affinity: {0}\n",
40 "env:\n",
41 " OPENMP_NUM_THREADS: 1\n",
42 " BLOSC_NTHREADS: 1\n",
43 "\n",
44 "hdf5plugin:\n",
45 " Version: 4.2.0\n",
46 " Build config:\n",
47 " openmp: True\n",
48 " native: True\n",
49 " bmi2: True\n",
50 " sse2: True\n",
51 " avx2: True\n",
52 " avx512: False\n",
53 " cpp11: True\n",
54 " cpp14: True\n",
55 " ipp: False\n",
56 " filter_file_extension: .so\n",
57 " embedded_filters: ('blosc', 'blosc2', 'bshuf', 'bzip2', 'fcidecomp', 'lz4', 'sz', 'sz3', 'zfp', 'zstd')\n",
58 "\n"
59 ]
60 }
61 ],
62 "source": [
63 "#Set affinity and multithreading env. var. before any import\n",
64 "import os\n",
65 "\n",
66 "os.sched_setaffinity(0, [0])\n",
67 "\n",
68 "AFFINITY = os.sched_getaffinity(0)\n",
69 "NCPU = len(AFFINITY)\n",
70 "\n",
71 "print(f\"Number of CPU: {NCPU}; Affinity: {AFFINITY}\")\n",
72 "\n",
73 "os.environ[\"OPENMP_NUM_THREADS\"] = str(NCPU)\n",
74 "os.environ[\"BLOSC_NTHREADS\"] = str(NCPU)\n",
75 "print(f\"\"\"env:\n",
76 " OPENMP_NUM_THREADS: {os.environ.get(\"OPENMP_NUM_THREADS\", \"unset\")}\n",
77 " BLOSC_NTHREADS: {os.environ.get(\"BLOSC_NTHREADS\", \"unset\")}\n",
78 "\"\"\")\n",
79 "\n",
80 "import h5py\n",
81 "import hdf5plugin\n",
82 "\n",
83 "config = hdf5plugin.get_config()\n",
84 "\n",
85 "print(f\"\"\"hdf5plugin:\n",
86 " Version: {hdf5plugin.version}\n",
87 " Build config:\n",
88 "{'''\n",
89 "'''.join(f' {k}: {v}' for k, v in config.build_config._asdict().items())}\n",
90 "\"\"\")"
91 ]
92 },
93 {
94 "cell_type": "markdown",
95 "id": "89f06f93-9538-43cb-b1d9-f8c4cca8ada8",
96 "metadata": {
97 "tags": []
98 },
99 "source": [
100 "## Demo data\n",
101 "\n",
102 "Data file is available here: http://www.silx.org/pub/pyFAI/pyFAI_UM_2020/data_ID13/kevlar.h5\n",
103 "\n",
104 "Prepare 2 files with 2 compressed datasets with `Blosc2` and `Bitshuffle` filters."
105 ]
106 },
107 {
108 "cell_type": "code",
109 "execution_count": 2,
110 "id": "e06d1bbe-c23b-4c5f-a3d6-51c98018afa9",
111 "metadata": {
112 "tags": []
113 },
114 "outputs": [],
115 "source": [
116 "# Download dataset\n",
117 "!wget -O /dev/shm/kevlar.h5 http://www.silx.org/pub/pyFAI/pyFAI_UM_2020/data_ID13/kevlar.h5"
118 ]
119 },
120 {
121 "cell_type": "code",
122 "execution_count": 3,
123 "id": "3d62a294-0016-40e2-83d3-d0829be0b81f",
124 "metadata": {
125 "tags": []
126 },
127 "outputs": [],
128 "source": [
129 "import h5py\n",
130 "import hdf5plugin\n",
131 "\n",
132 "with h5py.File(\"/dev/shm/kevlar.h5\", \"r\") as h:\n",
133 " data_ref = h[\"/entry/data/data\"][500]\n",
134 "\n",
135 "with h5py.File(\"/dev/shm/kevlar_blosc2.h5\", \"w\") as h:\n",
136 " h.create_dataset(\n",
137 " \"data\",\n",
138 " data=data_ref,\n",
139 " chunks=data_ref.shape,\n",
140 " compression=hdf5plugin.Blosc2(\n",
141 " cname='lz4',\n",
142 " clevel=5,\n",
143 " filters=hdf5plugin.Blosc2.BITSHUFFLE,\n",
144 " ),\n",
145 " )\n",
146 "with h5py.File(\"/dev/shm/kevlar_bitshuffle.h5\", \"w\") as h:\n",
147 " h.create_dataset(\n",
148 " \"data\",\n",
149 " data=data_ref,\n",
150 " chunks=data_ref.shape,\n",
151 " compression=hdf5plugin.Bitshuffle(),\n",
152 " )"
153 ]
154 },
155 {
156 "cell_type": "markdown",
157 "id": "da76a266-89ab-45aa-9dd4-b9117f9e5c36",
158 "metadata": {},
159 "source": [
160 "## With Blosc2\n",
161 "\n",
162 "Read compressed chunk with `read_direct_chunk` and decompress it with `blosc2`."
163 ]
164 },
165 {
166 "cell_type": "code",
167 "execution_count": 4,
168 "id": "abef1f80-011a-489b-8dc7-6e5a84d4be4a",
169 "metadata": {
170 "tags": []
171 },
172 "outputs": [],
173 "source": [
174 "import blosc2\n",
175 "import numpy\n",
176 "\n",
177 "def decompress_blosc2_chunk(chunk: bytes, array: numpy.ndarray):\n",
178 " \"\"\"Decompress chunk data to provided array\"\"\"\n",
179 " blosc2.schunk_from_cframe(chunk).get_slice(out=array)"
180 ]
181 },
182 {
183 "cell_type": "code",
184 "execution_count": 5,
185 "id": "1766ae0b-1130-48cb-9378-47baef27b68b",
186 "metadata": {
187 "tags": []
188 },
189 "outputs": [],
190 "source": [
191 "# Allocate array\n",
192 "with h5py.File(\"/dev/shm/kevlar_blosc2.h5\", \"r\") as h:\n",
193 " ds = h[\"data\"]\n",
194 " array = numpy.empty(ds.shape, dtype=ds.dtype)"
195 ]
196 },
197 {
198 "cell_type": "code",
199 "execution_count": 6,
200 "id": "228421e8-60d3-4ae8-b8b2-86e3b1d88c7d",
201 "metadata": {
202 "tags": []
203 },
204 "outputs": [
205 {
206 "data": {
207 "text/plain": [
208 "<TimeitResult : 3.7 ms ± 209 µs per loop (mean ± std. dev. of 10 runs, 10 loops each)>"
209 ]
210 },
211 "execution_count": 6,
212 "metadata": {},
213 "output_type": "execute_result"
214 }
215 ],
216 "source": [
217 "%%timeit -r10 -n10 -o -q\n",
218 "\n",
219 "# Read compressed chunk and decompress into array\n",
220 "with h5py.File(\"/dev/shm/kevlar_blosc2.h5\", \"r\") as h5file:\n",
221 " ds = h5file[\"data\"]\n",
222 " filter_mask, chunk = ds.id.read_direct_chunk(ds.id.get_chunk_info(0).chunk_offset)\n",
223 " decompress_blosc2_chunk(chunk, array)"
224 ]
225 },
226 {
227 "cell_type": "code",
228 "execution_count": 7,
229 "id": "515244ea-3fe7-4065-9f9b-85e61a58f778",
230 "metadata": {
231 "tags": []
232 },
233 "outputs": [
234 {
235 "data": {
236 "text/plain": [
237 "<TimeitResult : 8.92 ms ± 116 µs per loop (mean ± std. dev. of 10 runs, 10 loops each)>"
238 ]
239 },
240 "execution_count": 7,
241 "metadata": {},
242 "output_type": "execute_result"
243 }
244 ],
245 "source": [
246 "%%timeit -r10 -n10 -o -q\n",
247 "\n",
248 "# Read data through h5py and libhdf5\n",
249 "with h5py.File(\"/dev/shm/kevlar_blosc2.h5\", \"r\") as h5file:\n",
250 " data = h5file[\"data\"][()]"
251 ]
252 },
253 {
254 "cell_type": "code",
255 "execution_count": 8,
256 "id": "4075e042-f8bd-46f9-bcb9-000a6e20dbca",
257 "metadata": {
258 "tags": []
259 },
260 "outputs": [
261 {
262 "data": {
263 "text/plain": [
264 "<TimeitResult : 5.18 ms ± 110 µs per loop (mean ± std. dev. of 10 runs, 10 loops each)>"
265 ]
266 },
267 "execution_count": 8,
268 "metadata": {},
269 "output_type": "execute_result"
270 }
271 ],
272 "source": [
273 "%%timeit -r10 -n10 -o -q\n",
274 "\n",
275 "# Use read_direct\n",
276 "with h5py.File(\"/dev/shm/kevlar_blosc2.h5\", \"r\") as h:\n",
277 " h[\"data\"].read_direct(array)"
278 ]
279 },
280 {
281 "cell_type": "markdown",
282 "id": "b3fb49a0-c3d6-4f73-a553-26bae5fa7362",
283 "metadata": {
284 "tags": []
285 },
286 "source": [
287 "## With Bitshuffle\n",
288 "\n",
289 "Read compressed chunk with `read_direct_chunk` and decompress it with `bitshuffle`."
290 ]
291 },
292 {
293 "cell_type": "code",
294 "execution_count": 9,
295 "id": "9fd29f2b-d5a1-4504-bc60-25aab2f0732b",
296 "metadata": {
297 "tags": []
298 },
299 "outputs": [],
300 "source": [
301 "import struct\n",
302 "import bitshuffle\n",
303 "import numpy\n",
304 "\n",
305 "def decompress_bslz4_chunk(payload, dtype, chunk_shape):\n",
306 " \"\"\"This function decompresses ONE chunk with bitshuffle-LZ4. \n",
307 " The library needs to be compiled without OpenMP when using threads !\n",
308 " \n",
309 " :param payload: string with the compressed data as read by h5py.\n",
310 " :param dtype: data type of the stored content\n",
311 " :param chunk_shape: shape of one chunk\n",
312 " :return: decompressed chunk\"\"\"\n",
313 " total_nbytes, block_nbytes = struct.unpack(\">QI\", payload[:12])\n",
314 " block_size = block_nbytes // dtype.itemsize\n",
315 "\n",
316 " arr = numpy.frombuffer(payload, dtype=numpy.uint8, offset=12) # No copy here\n",
317 " chunk_data = bitshuffle.decompress_lz4(arr, chunk_shape, dtype, block_size)\n",
318 " return chunk_data"
319 ]
320 },
321 {
322 "cell_type": "code",
323 "execution_count": 10,
324 "id": "5324783e-7726-4e22-98a2-9319776eae76",
325 "metadata": {
326 "tags": []
327 },
328 "outputs": [
329 {
330 "data": {
331 "text/plain": [
332 "<TimeitResult : 4.35 ms ± 104 µs per loop (mean ± std. dev. of 10 runs, 10 loops each)>"
333 ]
334 },
335 "execution_count": 10,
336 "metadata": {},
337 "output_type": "execute_result"
338 }
339 ],
340 "source": [
341 "%%timeit -r10 -n10 -o -q\n",
342 "\n",
343 "# Read compressed chunk and decompress it\n",
344 "with h5py.File(\"/dev/shm/kevlar_bitshuffle.h5\", \"r\") as h:\n",
345 " ds = h[\"data\"]\n",
346 " filter_mask, chunk = ds.id.read_direct_chunk(ds.id.get_chunk_info(0).chunk_offset)\n",
347 " array = decompress_bslz4_chunk(chunk, ds.dtype, ds.chunks)"
348 ]
349 },
350 {
351 "cell_type": "code",
352 "execution_count": 11,
353 "id": "07581996-beb6-4921-b88d-84c988fc38ef",
354 "metadata": {
355 "tags": []
356 },
357 "outputs": [
358 {
359 "data": {
360 "text/plain": [
361 "<TimeitResult : 9.24 ms ± 154 µs per loop (mean ± std. dev. of 10 runs, 10 loops each)>"
362 ]
363 },
364 "execution_count": 11,
365 "metadata": {},
366 "output_type": "execute_result"
367 }
368 ],
369 "source": [
370 "%%timeit -r10 -n10 -o -q\n",
371 "\n",
372 "# Read data through h5py and libhdf5\n",
373 "with h5py.File(\"/dev/shm/kevlar_bitshuffle.h5\", \"r\") as h:\n",
374 " data = h[\"data\"][()]"
375 ]
376 },
377 {
378 "cell_type": "code",
379 "execution_count": 12,
380 "id": "e1e326eb-b3d7-4cf4-92b1-3946bd8b8d7e",
381 "metadata": {
382 "tags": []
383 },
384 "outputs": [
385 {
386 "data": {
387 "text/plain": [
388 "<TimeitResult : 5.45 ms ± 175 µs per loop (mean ± std. dev. of 10 runs, 10 loops each)>"
389 ]
390 },
391 "execution_count": 12,
392 "metadata": {},
393 "output_type": "execute_result"
394 }
395 ],
396 "source": [
397 "%%timeit -r10 -n10 -o -q\n",
398 "\n",
399 "# Use read_direct\n",
400 "with h5py.File(\"/dev/shm/kevlar_bitshuffle.h5\", \"r\") as h:\n",
401 " h[\"data\"].read_direct(array)"
402 ]
403 },
404 {
405 "cell_type": "code",
406 "execution_count": null,
407 "id": "bd2e177b-e624-4e0c-89f4-62f5b86a4d29",
408 "metadata": {},
409 "outputs": [],
410 "source": []
411 }
412 ],
413 "metadata": {
414 "kernelspec": {
415 "display_name": "Python 3 (ipykernel)",
416 "language": "python",
417 "name": "python3"
418 },
419 "language_info": {
420 "codemirror_mode": {
421 "name": "ipython",
422 "version": 3
423 },
424 "file_extension": ".py",
425 "mimetype": "text/x-python",
426 "name": "python",
427 "nbconvert_exporter": "python",
428 "pygments_lexer": "ipython3",
429 "version": "3.10.0"
430 }
431 },
432 "nbformat": 4,
433 "nbformat_minor": 5
434 }