Debian Codegraph @

src/c-blosc2/bench/b2nd/bench_zfp_getitem.c

raw
1 /*********************************************************************
2 Blosc - Blocked Shuffling and Compression Library
3
4 Copyright (c) 2021 Blosc Development Team <[email protected]>
5 https://blosc.org
6 License: BSD 3-Clause (see LICENSE.txt)
7
8 See LICENSE.txt for details about copyright and rights to use.
9 **********************************************************************/
10
11 /*
12 * Benchmark to measure the retrieval time of a number of elements in random positions
13 * in b2nd arrays. To get the necessary arrays (air1.cat, precip1.cat, snow1.cat...)
14 * you can use the following script, changing the dataset by your preference (see
15 * https://docs.digitalearthafrica.org/fr/latest/sandbox/notebooks/Datasets/Climate_Data_ERA5_AWS.html):
16
17 8<---snip---- "fetch_data.py"
18 #!/usr/bin/env python
19 import numpy as np
20 import s3fs
21 import xarray as xr
22 import blosc2
23
24 def open_zarr(year, month, datestart, dateend):
25 fs = s3fs.S3FileSystem(anon=True)
26 datestring = "era5-pds/zarr/{year}/{month:02d}/data/".format(year=year, month=month)
27 s3map = s3fs.S3Map(datestring + "precipitation_amount_1hour_Accumulation.zarr/", s3=fs)
28 precip_zarr = xr.open_dataset(s3map, engine="zarr")
29 precip_zarr = precip_zarr.sel(time1=slice(np.datetime64(datestart), np.datetime64(dateend)))
30 return precip_zarr.precipitation_amount_1hour_Accumulation
31
32 print("Fetching data from S3 (era5-pds)...")
33 precip_m0 = open_zarr(1987, 10, "1987-10-01", "1987-10-30 23:59")
34 precip0 = blosc2.empty(shape=precip_m0.shape, dtype=precip_m0.dtype, urlpath="precip1.b2nd")
35 print("Fetching and storing 1st month...")
36 values = precip_m0.values
37 precip0[:] = values
38 8<---snip----
39
40 * To call this script, you can run the following commands:
41 * $ pip install blosc2
42 * $ python fetch_data.py
43 *
44 */
45
46 #include "../plugins/codecs/zfp/zfp-private.h"
47 #include "../../plugins/codecs/zfp/blosc2-zfp.h"
48 #include "context.h"
49 #include "blosc2/codecs-registry.h"
50 #include "b2nd.h"
51 #include "blosc2.h"
52
53 int comp(const char *urlpath) {
54 blosc2_init();
55
56 blosc2_schunk *schunk = blosc2_schunk_open(urlpath);
57
58 if (schunk->typesize != 4) {
59 printf("Error: This test is only for floats.\n");
60 return -1;
61 }
62
63 blosc2_remove_urlpath("schunk_rate.cat");
64 blosc2_remove_urlpath("schunk.cat");
65
66 // Get multidimensional parameters and configure Blosc2 NDim array
67 int8_t ndim;
68 int64_t shape[4];
69 int64_t shape_aux[4];
70 int32_t chunkshape[4];
71 int32_t blockshape[4];
72 uint8_t *smeta;
73 int32_t smeta_len;
74 if (blosc2_meta_get(schunk, "b2nd", &smeta, &smeta_len) < 0) {
75 printf("This benchmark only supports b2nd arrays");
76 return -1;
77 }
78 char *dtype;
79 int8_t dtype_format;
80 b2nd_deserialize_meta(smeta, smeta_len, &ndim, shape_aux, chunkshape, blockshape, &dtype, &dtype_format);
81 free(smeta);
82 free(dtype);
83
84 blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
85 cparams.nthreads = 6;
86
87 blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
88 blosc2_storage b2_storage = {.cparams=&cparams, .dparams=&dparams};
89 b2_storage.urlpath = "schunk_rate.cat";
90
91 b2nd_context_t *ctx = b2nd_create_ctx(&b2_storage, ndim, shape_aux, chunkshape, blockshape, NULL, 0,
92 NULL, 0);
93
94 b2nd_array_t *arr;
95 b2nd_from_schunk(schunk, &arr);
96 int copied;
97 printf("LZ4 comp ratio: %f \n", (float) arr->sc->nbytes / (float) arr->sc->cbytes);
98
99 /* Use BLOSC_CODEC_ZFP_FIXED_RATE */
100 b2nd_array_t *arr_rate;
101 blosc2_context *ctx_zfp = blosc2_create_cctx(cparams);
102 ctx_zfp->compcode = BLOSC_CODEC_ZFP_FIXED_RATE;
103 ctx_zfp->splitmode = BLOSC_NEVER_SPLIT;
104 ctx_zfp->compcode_meta = (uint8_t) (100.0 * (float) arr->sc->cbytes / (float) arr->sc->nbytes);
105 ctx_zfp->filters[BLOSC2_MAX_FILTERS - 1] = 0;
106 ctx_zfp->filters_meta[BLOSC2_MAX_FILTERS - 1] = 0;
107 copied = b2nd_copy(ctx, arr, &arr_rate);
108 if (copied != 0) {
109 printf("Error BLOSC_CODEC_ZFP_FIXED_RATE \n");
110 b2nd_free(arr);
111 return -1;
112 }
113 printf("ZFP_FIXED_RATE comp ratio: %f \n", (float) arr_rate->sc->nbytes / (float) arr_rate->sc->cbytes);
114
115 int64_t nelems = arr_rate->nitems;
116 int dsize_zfp, dsize_blosc;
117 int64_t index;
118 float item_zfp, item_blosc;
119 blosc_timestamp_t t0, t1;
120 double zfp_time, blosc_time;
121 zfp_time = blosc_time = 0;
122 int64_t index_ndim[ZFP_MAX_DIM];
123 int64_t index_chunk_ndim[ZFP_MAX_DIM];
124 int64_t ind_ndim[ZFP_MAX_DIM];
125 int32_t stride_chunk, ind_chunk;
126 int64_t nchunk;
127 bool needs_free_blosc, needs_free_zfp;
128 uint8_t *chunk_blosc, *chunk_zfp;
129 int32_t chunk_nbytes_zfp, chunk_cbytes_zfp, chunk_nbytes_lossy, chunk_cbytes_lossy;
130 double ntests = 500.0;
131 for (int i = 0; i < ntests; ++i) {
132 srand(i);
133 index = rand() % nelems;
134 blosc2_unidim_to_multidim(ndim, shape, index, index_ndim);
135 for (int j = 0; j < ndim; ++j) {
136 index_chunk_ndim[j] = index_ndim[j] / chunkshape[j];
137 ind_ndim[j] = index_ndim[j] % chunkshape[j];
138 }
139 stride_chunk = (int32_t)(shape[1] - 1) / chunkshape[1] + 1;
140 nchunk = index_chunk_ndim[0] * stride_chunk + index_chunk_ndim[1];
141 ind_chunk = (int32_t) (ind_ndim[0] * chunkshape[1] + ind_ndim[1]);
142 blosc2_schunk_get_lazychunk(arr->sc, nchunk, &chunk_blosc, &needs_free_blosc);
143 blosc2_cbuffer_sizes(chunk_blosc, &chunk_nbytes_lossy, &chunk_cbytes_lossy, NULL);
144 blosc_set_timestamp(&t0);
145 dsize_blosc = blosc2_getitem_ctx(arr->sc->dctx, chunk_blosc, chunk_cbytes_lossy,
146 ind_chunk, 1, &item_blosc, sizeof(item_blosc));
147 blosc_set_timestamp(&t1);
148 blosc_time += blosc_elapsed_secs(t0, t1);
149 blosc2_schunk_get_lazychunk(arr_rate->sc, nchunk, &chunk_zfp, &needs_free_zfp);
150 blosc2_cbuffer_sizes(chunk_zfp, &chunk_nbytes_zfp, &chunk_cbytes_zfp, NULL);
151 blosc_set_timestamp(&t0);
152 dsize_zfp = blosc2_getitem_ctx(arr_rate->sc->dctx, chunk_zfp, chunk_cbytes_zfp,
153 ind_chunk, 1, &item_zfp, sizeof(item_zfp));
154 blosc_set_timestamp(&t1);
155 zfp_time += blosc_elapsed_secs(t0, t1);
156 if (dsize_blosc != dsize_zfp) {
157 printf("Different amount of items gotten");
158 return -1;
159 }
160 }
161 printf("ZFP_FIXED_RATE time: %.5f microseconds\n", (zfp_time * 1000000.0 / ntests));
162 printf("Blosc2 time: %.5f microseconds\n", (blosc_time * 1000000.0 / ntests));
163
164 b2nd_free(arr);
165 b2nd_free(arr_rate);
166 BLOSC_ERROR(b2nd_free_ctx(ctx));
167 blosc2_free_ctx(ctx_zfp);
168 if (needs_free_blosc) {
169 free(chunk_blosc);
170 }
171 if (needs_free_zfp) {
172 free(chunk_zfp);
173 }
174 blosc2_destroy();
175
176 return BLOSC2_ERROR_SUCCESS;
177 }
178
179 int solar1(void) {
180 const char *urlpath = "../../bench/solar1.cat";
181
182 int result = comp(urlpath);
183 return result;
184 }
185
186 int air1(void) {
187 const char *urlpath = "../../bench/air1.cat";
188
189 int result = comp(urlpath);
190 return result;
191 }
192
193 int snow1(void) {
194 const char *urlpath = "../../bench/snow1.cat";
195
196 int result = comp(urlpath);
197 return result;
198 }
199
200 int wind1(void) {
201 const char *urlpath = "../../bench/wind1.cat";
202
203 int result = comp(urlpath);
204 return result;
205 }
206
207 int precip1(void) {
208 const char *urlpath = "../../bench/precip1.cat";
209
210 int result = comp(urlpath);
211 return result;
212 }
213
214 int precip2(void) {
215 const char *urlpath = "../../bench/precip2.cat";
216
217 int result = comp(urlpath);
218 return result;
219 }
220
221 int precip3(void) {
222 const char *urlpath = "../../bench/precip3.cat";
223
224 int result = comp(urlpath);
225 return result;
226 }
227
228 int precip3m(void) {
229 const char *urlpath = "../../bench/precip-3m.cat";
230
231 int result = comp(urlpath);
232 return result;
233 }
234
235
236 int main() {
237
238 printf("wind1 \n");
239 BLOSC_ERROR(wind1());
240 printf("air1 \n");
241 BLOSC_ERROR(air1());
242 printf("solar1 \n");
243 BLOSC_ERROR(solar1());
244 printf("snow1 \n");
245 BLOSC_ERROR(snow1());
246 printf("precip1 \n");
247 BLOSC_ERROR(precip1());
248 printf("precip2 \n");
249 BLOSC_ERROR(precip2());
250 printf("precip3 \n");
251 BLOSC_ERROR(precip3());
252 // printf("precip3m \n");
253 // BLOSC_ERROR(precip3m());
254 return BLOSC2_ERROR_SUCCESS;
255
256 }