Debian Codegraph @

src/c-blosc2/bench/sframe_bench.c

raw
1 /*********************************************************************
2 Copyright (c) 2021 Blosc Development Team <[email protected]>
3 https://blosc.org
4 License: BSD 3-Clause (see LICENSE.txt)
5
6 Benchmark for testing sframe vs frame.
7 For usage instructions of this benchmark, please see:
8 https://www.blosc.org/pages/synthetic-benchmarks/
9 We are collecting speeds for different machines, so the output of your
10 benchmarks and your processor specifications are welcome!
11
12 See LICENSE.txt for details about copyright and rights to use.
13 **********************************************************************/
14
15 #include "blosc2.h"
16
17 #if defined(_WIN32)
18 #include <malloc.h>
19 #endif
20
21 #if defined(_WIN32) && !defined(__MINGW32__)
22 #include <time.h>
23 #else
24 #include <sys/time.h>
25 #endif
26
27 #include <stdlib.h>
28 #include <stdio.h>
29 #include <string.h>
30 #include <stdint.h>
31 #include <assert.h>
32 #include <inttypes.h>
33
34 #define KB 1024
35 #define MB (1024*KB)
36 #define GB (1024*MB)
37
38 #define NCHUNKS 1000 /* number of chunks */
39 #define CHUNKSIZE (2000 * 1000)
40
41 int nchunks = NCHUNKS;
42 int iterations = 5;
43 int io_type = BLOSC2_IO_FILESYSTEM;
44
45
46
47 void test_update(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) {
48 blosc_timestamp_t last, current;
49 double cframe_update_time, sframe_update_time;
50
51 size_t isize = CHUNKSIZE * sizeof(int32_t);
52 int32_t* data = malloc(isize);
53
54 // Random update list
55 int64_t* update_chunks = malloc(sizeof(int64_t) * iterations);
56 srand(time(NULL));
57 for (int i = 0; i < iterations; i++) {
58 update_chunks[i] = rand() % schunk_sframe->nchunks;
59 }
60
61 printf("*******************************************************\n");
62 printf("******************* Updating %d chunks ******************\n", iterations);
63 printf("*******************************************************\n");
64
65 // Update the sframe chunks
66 sframe_update_time = 0.0;
67 cframe_update_time = 0.0;
68 int32_t datasize = sizeof(int32_t) * CHUNKSIZE;
69 int32_t chunksize = sizeof(int32_t) * CHUNKSIZE + BLOSC2_MAX_OVERHEAD;
70 uint8_t* chunk;
71 int csize;
72 int64_t _nchunks;
73 for (int i = 0; i < iterations; i++) {
74 // Generate data
75 for (int j = 0; j < CHUNKSIZE; j++) {
76 data[j] = i * CHUNKSIZE;
77 }
78 chunk = malloc(chunksize);
79 csize = blosc2_compress_ctx(schunk_sframe->cctx, data, datasize, chunk, chunksize);
80 if (csize < 0) {
81 printf("ERROR: chunk cannot be compressed\n");
82 }
83 // Sframe
84 blosc_set_timestamp(&current);
85 _nchunks = blosc2_schunk_update_chunk(schunk_sframe, update_chunks[i], chunk, true);
86 blosc_set_timestamp(&last);
87 if (_nchunks < 0){
88 printf("ERROR: chunk cannot be updated correctly\n");
89 }
90 sframe_update_time += blosc_elapsed_secs(current, last);
91
92 // Frame
93 blosc_set_timestamp(&current);
94 _nchunks = blosc2_schunk_update_chunk(schunk_cframe, update_chunks[i], chunk, true);
95 blosc_set_timestamp(&last);
96 free(chunk);
97 if (_nchunks <= 0){
98 printf("ERROR: chunk cannot be updated correctly\n");
99 }
100 cframe_update_time += blosc_elapsed_secs(current, last);
101 }
102
103 printf("[Sframe Update] Elapsed time:\t %6.3f s. Total sframe size: %.3" PRId64 " bytes\n",
104 sframe_update_time, schunk_sframe->cbytes);
105 printf("[Cframe Update] Elapsed time:\t %6.3f s. Total cframe size: %.3" PRId64 " bytes\n",
106 cframe_update_time, schunk_cframe->cbytes);
107
108 /* Free resources */
109 free(update_chunks);
110 free(data);
111 }
112
113 void test_insert(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) {
114 blosc_timestamp_t last, current;
115 double cframe_insert_time, sframe_insert_time;
116
117 size_t isize = CHUNKSIZE * sizeof(int32_t);
118 int32_t* data = malloc(isize);
119
120 // Random insert list
121 int64_t* insert_chunks = malloc(sizeof(int64_t) * iterations);
122 srand(time(NULL));
123 for (int i = 0; i < iterations; i++) {
124 insert_chunks[i] = rand() % schunk_sframe->nchunks;
125 }
126
127 printf("*******************************************************\n");
128 printf("****************** Inserting %d chunks *****************\n", iterations);
129 printf("*******************************************************\n");
130
131 // Update the sframe chunks
132 sframe_insert_time = 0.0;
133 cframe_insert_time = 0.0;
134 int32_t datasize = sizeof(int32_t) * CHUNKSIZE;
135 int32_t chunksize = sizeof(int32_t) * CHUNKSIZE + BLOSC2_MAX_OVERHEAD;
136 uint8_t* chunk;
137 int csize;
138 int64_t _nchunks;
139 for (int i = 0; i < iterations; i++) {
140 // Generate data
141 for (int j = 0; j < CHUNKSIZE; j++) {
142 data[j] = j + i * CHUNKSIZE;
143 }
144 chunk = malloc(chunksize);
145 csize = blosc2_compress_ctx(schunk_sframe->cctx, data, datasize, chunk, chunksize);
146 if (csize < 0) {
147 printf("ERROR: chunk cannot be compressed\n");
148 }
149 // Sframe
150 blosc_set_timestamp(&current);
151 _nchunks = blosc2_schunk_insert_chunk(schunk_sframe, insert_chunks[i], chunk, true);
152 blosc_set_timestamp(&last);
153 if (_nchunks < 0){
154 printf("ERROR: chunk cannot be updated correctly\n");
155 }
156 sframe_insert_time += blosc_elapsed_secs(current, last);
157
158 // Frame
159 blosc_set_timestamp(&current);
160 _nchunks = blosc2_schunk_update_chunk(schunk_cframe, insert_chunks[i], chunk, true);
161 blosc_set_timestamp(&last);
162 free(chunk);
163 if (_nchunks <= 0){
164 printf("ERROR: chunk cannot be updated correctly\n");
165 }
166 cframe_insert_time += blosc_elapsed_secs(current, last);
167 }
168
169 printf("[Sframe Insert] Elapsed time:\t %6.3f s. Total sframe size: %.3" PRId64 " bytes\n",
170 sframe_insert_time, schunk_sframe->cbytes);
171 printf("[Cframe Insert] Elapsed time:\t %6.3f s. Total cframe size: %.3" PRId64 " bytes\n",
172 cframe_insert_time, schunk_cframe->cbytes);
173
174
175 /* Free resources */
176 free(insert_chunks);
177 free(data);
178 }
179
180 void test_reorder(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) {
181 blosc_timestamp_t last, current;
182 double cframe_reorder_time, sframe_reorder_time;
183
184 // Reorder list
185 int64_t *offsets_order = malloc(sizeof(int64_t) * schunk_sframe->nchunks);
186 for (int i = 0; i < schunk_sframe->nchunks; ++i) {
187 offsets_order[i] = (i + 3) % schunk_sframe->nchunks;
188 }
189
190 printf("*******************************************************\n");
191 printf("****************** Reordering chunks ******************\n");
192 printf("*******************************************************\n");
193
194 // Reorder sframe
195 blosc_set_timestamp(&current);
196 int err = blosc2_schunk_reorder_offsets(schunk_sframe, offsets_order);
197 blosc_set_timestamp(&last);
198 if (err < 0) {
199 printf("ERROR: cannot reorder chunks\n");
200 }
201 sframe_reorder_time = blosc_elapsed_secs(current, last);
202
203 // Reorder frame
204 blosc_set_timestamp(&current);
205 err = blosc2_schunk_reorder_offsets(schunk_cframe, offsets_order);
206 blosc_set_timestamp(&last);
207 if (err < 0) {
208 printf("ERROR: cannot reorder chunks\n");
209 }
210 cframe_reorder_time = blosc_elapsed_secs(current, last);
211
212 printf("[Sframe Update] Elapsed time:\t %f s. Total sframe size: %.3" PRId64 " bytes\n",
213 sframe_reorder_time, schunk_sframe->cbytes);
214 printf("[Cframe Update] Elapsed time:\t %f s. Total cframe size: %.3" PRId64 " bytes\n",
215 cframe_reorder_time, schunk_cframe->cbytes);
216
217 /* Free resources */
218 free(offsets_order);
219 }
220
221 void test_create_sframe_frame(char* operation) {
222 blosc_timestamp_t last, current;
223 double cframe_append_time, sframe_append_time, cframe_decompress_time, sframe_decompress_time;
224
225 int64_t nbytes, cbytes;
226 int32_t isize = CHUNKSIZE * sizeof(int32_t);
227 int dsize;
228 float totalsize = (float)isize * nchunks;
229 int32_t* data = malloc(isize);
230 int32_t* data_dest = malloc(isize);
231
232 blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
233 blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
234
235 blosc2_schunk* schunk_sframe;
236 blosc2_schunk* schunk_cframe;
237
238 /* Initialize the Blosc compressor */
239 blosc2_init();
240 printf("*******************************************************\n");
241 printf("***** Creating the frame and sframe with %d chunks ****\n", nchunks);
242 printf("*******************************************************\n");
243 /* Create a frame container */
244 cparams.typesize = sizeof(int32_t);
245
246 cparams.nthreads = 2;
247 dparams.nthreads = 2;
248
249 blosc2_storage storage = {.contiguous=false, .urlpath="dir.b2frame", .cparams=&cparams, .dparams=&dparams};
250 blosc2_remove_urlpath(storage.urlpath);
251 schunk_sframe = blosc2_schunk_new(&storage);
252
253 blosc2_stdio_mmap mmap_file = BLOSC2_STDIO_MMAP_DEFAULTS;
254 mmap_file.mode = "w+";
255 blosc2_io io_mmap = {.id = BLOSC2_IO_FILESYSTEM_MMAP, .name = "filesystem_mmap", .params = &mmap_file};
256
257 blosc2_storage storage2 = {.contiguous=true, .urlpath="test_cframe.b2frame", .cparams=&cparams, .dparams=&dparams};
258 if (io_type == BLOSC2_IO_FILESYSTEM) {
259 storage2.io = (blosc2_io*)&BLOSC2_IO_DEFAULTS;
260 }
261 else if (io_type == BLOSC2_IO_FILESYSTEM_MMAP) {
262 storage2.io = &io_mmap;
263 }
264
265 blosc2_remove_urlpath(storage2.urlpath);
266 schunk_cframe = blosc2_schunk_new(&storage2);
267
268 printf("Test comparison frame vs sframe with %d chunks.\n", nchunks);
269
270 // Feed it with data
271 sframe_append_time=0.0;
272 cframe_append_time=0.0;
273 blosc_set_timestamp(&current);
274 for (int nchunk = 0; nchunk < nchunks; nchunk++) {
275 for (int i = 0; i < CHUNKSIZE; i++) {
276 data[i] = i + nchunk * CHUNKSIZE;
277 }
278 blosc_set_timestamp(&current);
279 blosc2_schunk_append_buffer(schunk_sframe, data, isize);
280 blosc_set_timestamp(&last);
281 sframe_append_time += blosc_elapsed_secs(current, last);
282
283 blosc_set_timestamp(&current);
284 blosc2_schunk_append_buffer(schunk_cframe, data, isize);
285 blosc_set_timestamp(&last);
286 cframe_append_time += blosc_elapsed_secs(current, last);
287 }
288 printf("[Sframe Compr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n",
289 sframe_append_time, totalsize / GB, totalsize / (GB * sframe_append_time));
290 printf("[Cframe Compr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n",
291 cframe_append_time, totalsize / GB, totalsize / (GB * cframe_append_time));
292
293 /* Gather some info */
294 nbytes = schunk_sframe->nbytes;
295 cbytes = schunk_sframe->cbytes;
296 printf("Compression super-chunk-sframe: %ld -> %ld (%.1fx)\n",
297 (long)nbytes, (long)cbytes, (1. * (double)nbytes) / (double)cbytes);
298 nbytes = schunk_cframe->nbytes;
299 cbytes = schunk_cframe->cbytes;
300 printf("Compression super-chunk-cframe: %ld -> %ld (%.1fx)\n",
301 (long)nbytes, (long)cbytes, (1. * (double)nbytes) / (double)cbytes);
302
303 // Decompress the data
304 sframe_decompress_time = 0;
305 cframe_decompress_time = 0;
306 for (int nchunk = 0; nchunk < nchunks; nchunk++) {
307 // Sframe
308 blosc_set_timestamp(&current);
309 dsize = blosc2_schunk_decompress_chunk(schunk_sframe, nchunk, (void *) data_dest, isize);
310 blosc_set_timestamp(&last);
311 if (dsize < 0) {
312 printf("Decompression error sframe. Error code: %d\n", dsize);
313 }
314 assert (dsize == (int)isize);
315 sframe_decompress_time += blosc_elapsed_secs(current, last);
316 // Frame
317 blosc_set_timestamp(&current);
318 dsize = blosc2_schunk_decompress_chunk(schunk_cframe, nchunk, (void *) data_dest, isize);
319 blosc_set_timestamp(&last);
320 if (dsize < 0) {
321 printf("Decompression error cframe. Error code: %d\n", dsize);
322 }
323 assert (dsize == (int)isize);
324 cframe_decompress_time += blosc_elapsed_secs(current, last);
325 }
326
327 printf("[Sframe Decompr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n",
328 sframe_decompress_time, totalsize / GB, totalsize / (GB * sframe_decompress_time));
329 printf("[Cframe Decompr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n",
330 cframe_decompress_time, totalsize / GB, totalsize / (GB * cframe_decompress_time));
331
332 printf("Decompression successful!\n");
333
334 printf("Successful roundtrip!\n");
335
336 /* Free resources */
337 free(data_dest);
338 free(data);
339
340 if (operation != NULL) {
341 if (strcmp(operation, "insert") == 0) {
342 test_insert(schunk_sframe, schunk_cframe);
343 }
344 else if (strcmp(operation, "update") == 0) {
345 test_update(schunk_sframe, schunk_cframe);
346 }
347 else if (strcmp(operation, "reorder") == 0) {
348 test_reorder(schunk_sframe, schunk_cframe);
349 }
350 }
351
352
353 blosc2_remove_urlpath(schunk_sframe->storage->urlpath);
354 blosc2_remove_urlpath(schunk_cframe->storage->urlpath);
355 blosc2_schunk_free(schunk_sframe);
356 blosc2_schunk_free(schunk_cframe);
357 /* Destroy the Blosc environment */
358 blosc2_destroy();
359
360 }
361
362 int main(int argc, char* argv[]) {
363 char* operation = NULL;
364
365 if (argc >= 6) {
366 printf("Usage: ./sframe_bench [nchunks] [insert | update | reorder] [num operations] [io_file | io_mmap]\n");
367 exit(1);
368 }
369 else if (argc >= 2) {
370 nchunks = (int)strtol(argv[1], NULL, 10);
371 }
372 if (argc >= 3) {
373 operation = argv[2];
374 }
375 if (argc >= 4) {
376 iterations = (int)strtol(argv[3], NULL, 10);
377 }
378 if (argc == 5) {
379 if (strcmp(argv[4], "io_file") == 0) {
380 io_type = BLOSC2_IO_FILESYSTEM;
381 } else if (strcmp(argv[4], "io_mmap") == 0) {
382 io_type = BLOSC2_IO_FILESYSTEM_MMAP;
383 } else {
384 printf("Invalid io type. Use io_file or io_mmap\n");
385 exit(1);
386 }
387 }
388
389 test_create_sframe_frame(operation);
390
391 return 0;
392 }