| 1 |
/********************************************************************* |
| 2 |
Copyright (c) 2021 Blosc Development Team <[email protected]> |
| 3 |
https://blosc.org |
| 4 |
License: BSD 3-Clause (see LICENSE.txt) |
| 5 |
|
| 6 |
Benchmark for testing sframe vs frame. |
| 7 |
For usage instructions of this benchmark, please see: |
| 8 |
https://www.blosc.org/pages/synthetic-benchmarks/ |
| 9 |
We are collecting speeds for different machines, so the output of your |
| 10 |
benchmarks and your processor specifications are welcome! |
| 11 |
|
| 12 |
See LICENSE.txt for details about copyright and rights to use. |
| 13 |
**********************************************************************/ |
| 14 |
|
| 15 |
#include "blosc2.h" |
| 16 |
|
| 17 |
#if defined(_WIN32) |
| 18 |
#include <malloc.h> |
| 19 |
#endif |
| 20 |
|
| 21 |
#if defined(_WIN32) && !defined(__MINGW32__) |
| 22 |
#include <time.h> |
| 23 |
#else |
| 24 |
#include <sys/time.h> |
| 25 |
#endif |
| 26 |
|
| 27 |
#include <stdlib.h> |
| 28 |
#include <stdio.h> |
| 29 |
#include <string.h> |
| 30 |
#include <stdint.h> |
| 31 |
#include <assert.h> |
| 32 |
#include <inttypes.h> |
| 33 |
|
| 34 |
#define KB 1024 |
| 35 |
#define MB (1024*KB) |
| 36 |
#define GB (1024*MB) |
| 37 |
|
| 38 |
#define NCHUNKS 1000 /* number of chunks */ |
| 39 |
#define CHUNKSIZE (2000 * 1000) |
| 40 |
|
| 41 |
int nchunks = NCHUNKS; |
| 42 |
int iterations = 5; |
| 43 |
int io_type = BLOSC2_IO_FILESYSTEM; |
| 44 |
|
| 45 |
|
| 46 |
|
| 47 |
void test_update(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) { |
| 48 |
blosc_timestamp_t last, current; |
| 49 |
double cframe_update_time, sframe_update_time; |
| 50 |
|
| 51 |
size_t isize = CHUNKSIZE * sizeof(int32_t); |
| 52 |
int32_t* data = malloc(isize); |
| 53 |
|
| 54 |
// Random update list |
| 55 |
int64_t* update_chunks = malloc(sizeof(int64_t) * iterations); |
| 56 |
srand(time(NULL)); |
| 57 |
for (int i = 0; i < iterations; i++) { |
| 58 |
update_chunks[i] = rand() % schunk_sframe->nchunks; |
| 59 |
} |
| 60 |
|
| 61 |
printf("*******************************************************\n"); |
| 62 |
printf("******************* Updating %d chunks ******************\n", iterations); |
| 63 |
printf("*******************************************************\n"); |
| 64 |
|
| 65 |
// Update the sframe chunks |
| 66 |
sframe_update_time = 0.0; |
| 67 |
cframe_update_time = 0.0; |
| 68 |
int32_t datasize = sizeof(int32_t) * CHUNKSIZE; |
| 69 |
int32_t chunksize = sizeof(int32_t) * CHUNKSIZE + BLOSC2_MAX_OVERHEAD; |
| 70 |
uint8_t* chunk; |
| 71 |
int csize; |
| 72 |
int64_t _nchunks; |
| 73 |
for (int i = 0; i < iterations; i++) { |
| 74 |
// Generate data |
| 75 |
for (int j = 0; j < CHUNKSIZE; j++) { |
| 76 |
data[j] = i * CHUNKSIZE; |
| 77 |
} |
| 78 |
chunk = malloc(chunksize); |
| 79 |
csize = blosc2_compress_ctx(schunk_sframe->cctx, data, datasize, chunk, chunksize); |
| 80 |
if (csize < 0) { |
| 81 |
printf("ERROR: chunk cannot be compressed\n"); |
| 82 |
} |
| 83 |
// Sframe |
| 84 |
blosc_set_timestamp(¤t); |
| 85 |
_nchunks = blosc2_schunk_update_chunk(schunk_sframe, update_chunks[i], chunk, true); |
| 86 |
blosc_set_timestamp(&last); |
| 87 |
if (_nchunks < 0){ |
| 88 |
printf("ERROR: chunk cannot be updated correctly\n"); |
| 89 |
} |
| 90 |
sframe_update_time += blosc_elapsed_secs(current, last); |
| 91 |
|
| 92 |
// Frame |
| 93 |
blosc_set_timestamp(¤t); |
| 94 |
_nchunks = blosc2_schunk_update_chunk(schunk_cframe, update_chunks[i], chunk, true); |
| 95 |
blosc_set_timestamp(&last); |
| 96 |
free(chunk); |
| 97 |
if (_nchunks <= 0){ |
| 98 |
printf("ERROR: chunk cannot be updated correctly\n"); |
| 99 |
} |
| 100 |
cframe_update_time += blosc_elapsed_secs(current, last); |
| 101 |
} |
| 102 |
|
| 103 |
printf("[Sframe Update] Elapsed time:\t %6.3f s. Total sframe size: %.3" PRId64 " bytes\n", |
| 104 |
sframe_update_time, schunk_sframe->cbytes); |
| 105 |
printf("[Cframe Update] Elapsed time:\t %6.3f s. Total cframe size: %.3" PRId64 " bytes\n", |
| 106 |
cframe_update_time, schunk_cframe->cbytes); |
| 107 |
|
| 108 |
/* Free resources */ |
| 109 |
free(update_chunks); |
| 110 |
free(data); |
| 111 |
} |
| 112 |
|
| 113 |
void test_insert(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) { |
| 114 |
blosc_timestamp_t last, current; |
| 115 |
double cframe_insert_time, sframe_insert_time; |
| 116 |
|
| 117 |
size_t isize = CHUNKSIZE * sizeof(int32_t); |
| 118 |
int32_t* data = malloc(isize); |
| 119 |
|
| 120 |
// Random insert list |
| 121 |
int64_t* insert_chunks = malloc(sizeof(int64_t) * iterations); |
| 122 |
srand(time(NULL)); |
| 123 |
for (int i = 0; i < iterations; i++) { |
| 124 |
insert_chunks[i] = rand() % schunk_sframe->nchunks; |
| 125 |
} |
| 126 |
|
| 127 |
printf("*******************************************************\n"); |
| 128 |
printf("****************** Inserting %d chunks *****************\n", iterations); |
| 129 |
printf("*******************************************************\n"); |
| 130 |
|
| 131 |
// Update the sframe chunks |
| 132 |
sframe_insert_time = 0.0; |
| 133 |
cframe_insert_time = 0.0; |
| 134 |
int32_t datasize = sizeof(int32_t) * CHUNKSIZE; |
| 135 |
int32_t chunksize = sizeof(int32_t) * CHUNKSIZE + BLOSC2_MAX_OVERHEAD; |
| 136 |
uint8_t* chunk; |
| 137 |
int csize; |
| 138 |
int64_t _nchunks; |
| 139 |
for (int i = 0; i < iterations; i++) { |
| 140 |
// Generate data |
| 141 |
for (int j = 0; j < CHUNKSIZE; j++) { |
| 142 |
data[j] = j + i * CHUNKSIZE; |
| 143 |
} |
| 144 |
chunk = malloc(chunksize); |
| 145 |
csize = blosc2_compress_ctx(schunk_sframe->cctx, data, datasize, chunk, chunksize); |
| 146 |
if (csize < 0) { |
| 147 |
printf("ERROR: chunk cannot be compressed\n"); |
| 148 |
} |
| 149 |
// Sframe |
| 150 |
blosc_set_timestamp(¤t); |
| 151 |
_nchunks = blosc2_schunk_insert_chunk(schunk_sframe, insert_chunks[i], chunk, true); |
| 152 |
blosc_set_timestamp(&last); |
| 153 |
if (_nchunks < 0){ |
| 154 |
printf("ERROR: chunk cannot be updated correctly\n"); |
| 155 |
} |
| 156 |
sframe_insert_time += blosc_elapsed_secs(current, last); |
| 157 |
|
| 158 |
// Frame |
| 159 |
blosc_set_timestamp(¤t); |
| 160 |
_nchunks = blosc2_schunk_update_chunk(schunk_cframe, insert_chunks[i], chunk, true); |
| 161 |
blosc_set_timestamp(&last); |
| 162 |
free(chunk); |
| 163 |
if (_nchunks <= 0){ |
| 164 |
printf("ERROR: chunk cannot be updated correctly\n"); |
| 165 |
} |
| 166 |
cframe_insert_time += blosc_elapsed_secs(current, last); |
| 167 |
} |
| 168 |
|
| 169 |
printf("[Sframe Insert] Elapsed time:\t %6.3f s. Total sframe size: %.3" PRId64 " bytes\n", |
| 170 |
sframe_insert_time, schunk_sframe->cbytes); |
| 171 |
printf("[Cframe Insert] Elapsed time:\t %6.3f s. Total cframe size: %.3" PRId64 " bytes\n", |
| 172 |
cframe_insert_time, schunk_cframe->cbytes); |
| 173 |
|
| 174 |
|
| 175 |
/* Free resources */ |
| 176 |
free(insert_chunks); |
| 177 |
free(data); |
| 178 |
} |
| 179 |
|
| 180 |
void test_reorder(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) { |
| 181 |
blosc_timestamp_t last, current; |
| 182 |
double cframe_reorder_time, sframe_reorder_time; |
| 183 |
|
| 184 |
// Reorder list |
| 185 |
int64_t *offsets_order = malloc(sizeof(int64_t) * schunk_sframe->nchunks); |
| 186 |
for (int i = 0; i < schunk_sframe->nchunks; ++i) { |
| 187 |
offsets_order[i] = (i + 3) % schunk_sframe->nchunks; |
| 188 |
} |
| 189 |
|
| 190 |
printf("*******************************************************\n"); |
| 191 |
printf("****************** Reordering chunks ******************\n"); |
| 192 |
printf("*******************************************************\n"); |
| 193 |
|
| 194 |
// Reorder sframe |
| 195 |
blosc_set_timestamp(¤t); |
| 196 |
int err = blosc2_schunk_reorder_offsets(schunk_sframe, offsets_order); |
| 197 |
blosc_set_timestamp(&last); |
| 198 |
if (err < 0) { |
| 199 |
printf("ERROR: cannot reorder chunks\n"); |
| 200 |
} |
| 201 |
sframe_reorder_time = blosc_elapsed_secs(current, last); |
| 202 |
|
| 203 |
// Reorder frame |
| 204 |
blosc_set_timestamp(¤t); |
| 205 |
err = blosc2_schunk_reorder_offsets(schunk_cframe, offsets_order); |
| 206 |
blosc_set_timestamp(&last); |
| 207 |
if (err < 0) { |
| 208 |
printf("ERROR: cannot reorder chunks\n"); |
| 209 |
} |
| 210 |
cframe_reorder_time = blosc_elapsed_secs(current, last); |
| 211 |
|
| 212 |
printf("[Sframe Update] Elapsed time:\t %f s. Total sframe size: %.3" PRId64 " bytes\n", |
| 213 |
sframe_reorder_time, schunk_sframe->cbytes); |
| 214 |
printf("[Cframe Update] Elapsed time:\t %f s. Total cframe size: %.3" PRId64 " bytes\n", |
| 215 |
cframe_reorder_time, schunk_cframe->cbytes); |
| 216 |
|
| 217 |
/* Free resources */ |
| 218 |
free(offsets_order); |
| 219 |
} |
| 220 |
|
| 221 |
void test_create_sframe_frame(char* operation) { |
| 222 |
blosc_timestamp_t last, current; |
| 223 |
double cframe_append_time, sframe_append_time, cframe_decompress_time, sframe_decompress_time; |
| 224 |
|
| 225 |
int64_t nbytes, cbytes; |
| 226 |
int32_t isize = CHUNKSIZE * sizeof(int32_t); |
| 227 |
int dsize; |
| 228 |
float totalsize = (float)isize * nchunks; |
| 229 |
int32_t* data = malloc(isize); |
| 230 |
int32_t* data_dest = malloc(isize); |
| 231 |
|
| 232 |
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; |
| 233 |
blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; |
| 234 |
|
| 235 |
blosc2_schunk* schunk_sframe; |
| 236 |
blosc2_schunk* schunk_cframe; |
| 237 |
|
| 238 |
/* Initialize the Blosc compressor */ |
| 239 |
blosc2_init(); |
| 240 |
printf("*******************************************************\n"); |
| 241 |
printf("***** Creating the frame and sframe with %d chunks ****\n", nchunks); |
| 242 |
printf("*******************************************************\n"); |
| 243 |
/* Create a frame container */ |
| 244 |
cparams.typesize = sizeof(int32_t); |
| 245 |
|
| 246 |
cparams.nthreads = 2; |
| 247 |
dparams.nthreads = 2; |
| 248 |
|
| 249 |
blosc2_storage storage = {.contiguous=false, .urlpath="dir.b2frame", .cparams=&cparams, .dparams=&dparams}; |
| 250 |
blosc2_remove_urlpath(storage.urlpath); |
| 251 |
schunk_sframe = blosc2_schunk_new(&storage); |
| 252 |
|
| 253 |
blosc2_stdio_mmap mmap_file = BLOSC2_STDIO_MMAP_DEFAULTS; |
| 254 |
mmap_file.mode = "w+"; |
| 255 |
blosc2_io io_mmap = {.id = BLOSC2_IO_FILESYSTEM_MMAP, .name = "filesystem_mmap", .params = &mmap_file}; |
| 256 |
|
| 257 |
blosc2_storage storage2 = {.contiguous=true, .urlpath="test_cframe.b2frame", .cparams=&cparams, .dparams=&dparams}; |
| 258 |
if (io_type == BLOSC2_IO_FILESYSTEM) { |
| 259 |
storage2.io = (blosc2_io*)&BLOSC2_IO_DEFAULTS; |
| 260 |
} |
| 261 |
else if (io_type == BLOSC2_IO_FILESYSTEM_MMAP) { |
| 262 |
storage2.io = &io_mmap; |
| 263 |
} |
| 264 |
|
| 265 |
blosc2_remove_urlpath(storage2.urlpath); |
| 266 |
schunk_cframe = blosc2_schunk_new(&storage2); |
| 267 |
|
| 268 |
printf("Test comparison frame vs sframe with %d chunks.\n", nchunks); |
| 269 |
|
| 270 |
// Feed it with data |
| 271 |
sframe_append_time=0.0; |
| 272 |
cframe_append_time=0.0; |
| 273 |
blosc_set_timestamp(¤t); |
| 274 |
for (int nchunk = 0; nchunk < nchunks; nchunk++) { |
| 275 |
for (int i = 0; i < CHUNKSIZE; i++) { |
| 276 |
data[i] = i + nchunk * CHUNKSIZE; |
| 277 |
} |
| 278 |
blosc_set_timestamp(¤t); |
| 279 |
blosc2_schunk_append_buffer(schunk_sframe, data, isize); |
| 280 |
blosc_set_timestamp(&last); |
| 281 |
sframe_append_time += blosc_elapsed_secs(current, last); |
| 282 |
|
| 283 |
blosc_set_timestamp(¤t); |
| 284 |
blosc2_schunk_append_buffer(schunk_cframe, data, isize); |
| 285 |
blosc_set_timestamp(&last); |
| 286 |
cframe_append_time += blosc_elapsed_secs(current, last); |
| 287 |
} |
| 288 |
printf("[Sframe Compr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n", |
| 289 |
sframe_append_time, totalsize / GB, totalsize / (GB * sframe_append_time)); |
| 290 |
printf("[Cframe Compr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n", |
| 291 |
cframe_append_time, totalsize / GB, totalsize / (GB * cframe_append_time)); |
| 292 |
|
| 293 |
/* Gather some info */ |
| 294 |
nbytes = schunk_sframe->nbytes; |
| 295 |
cbytes = schunk_sframe->cbytes; |
| 296 |
printf("Compression super-chunk-sframe: %ld -> %ld (%.1fx)\n", |
| 297 |
(long)nbytes, (long)cbytes, (1. * (double)nbytes) / (double)cbytes); |
| 298 |
nbytes = schunk_cframe->nbytes; |
| 299 |
cbytes = schunk_cframe->cbytes; |
| 300 |
printf("Compression super-chunk-cframe: %ld -> %ld (%.1fx)\n", |
| 301 |
(long)nbytes, (long)cbytes, (1. * (double)nbytes) / (double)cbytes); |
| 302 |
|
| 303 |
// Decompress the data |
| 304 |
sframe_decompress_time = 0; |
| 305 |
cframe_decompress_time = 0; |
| 306 |
for (int nchunk = 0; nchunk < nchunks; nchunk++) { |
| 307 |
// Sframe |
| 308 |
blosc_set_timestamp(¤t); |
| 309 |
dsize = blosc2_schunk_decompress_chunk(schunk_sframe, nchunk, (void *) data_dest, isize); |
| 310 |
blosc_set_timestamp(&last); |
| 311 |
if (dsize < 0) { |
| 312 |
printf("Decompression error sframe. Error code: %d\n", dsize); |
| 313 |
} |
| 314 |
assert (dsize == (int)isize); |
| 315 |
sframe_decompress_time += blosc_elapsed_secs(current, last); |
| 316 |
// Frame |
| 317 |
blosc_set_timestamp(¤t); |
| 318 |
dsize = blosc2_schunk_decompress_chunk(schunk_cframe, nchunk, (void *) data_dest, isize); |
| 319 |
blosc_set_timestamp(&last); |
| 320 |
if (dsize < 0) { |
| 321 |
printf("Decompression error cframe. Error code: %d\n", dsize); |
| 322 |
} |
| 323 |
assert (dsize == (int)isize); |
| 324 |
cframe_decompress_time += blosc_elapsed_secs(current, last); |
| 325 |
} |
| 326 |
|
| 327 |
printf("[Sframe Decompr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n", |
| 328 |
sframe_decompress_time, totalsize / GB, totalsize / (GB * sframe_decompress_time)); |
| 329 |
printf("[Cframe Decompr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n", |
| 330 |
cframe_decompress_time, totalsize / GB, totalsize / (GB * cframe_decompress_time)); |
| 331 |
|
| 332 |
printf("Decompression successful!\n"); |
| 333 |
|
| 334 |
printf("Successful roundtrip!\n"); |
| 335 |
|
| 336 |
/* Free resources */ |
| 337 |
free(data_dest); |
| 338 |
free(data); |
| 339 |
|
| 340 |
if (operation != NULL) { |
| 341 |
if (strcmp(operation, "insert") == 0) { |
| 342 |
test_insert(schunk_sframe, schunk_cframe); |
| 343 |
} |
| 344 |
else if (strcmp(operation, "update") == 0) { |
| 345 |
test_update(schunk_sframe, schunk_cframe); |
| 346 |
} |
| 347 |
else if (strcmp(operation, "reorder") == 0) { |
| 348 |
test_reorder(schunk_sframe, schunk_cframe); |
| 349 |
} |
| 350 |
} |
| 351 |
|
| 352 |
|
| 353 |
blosc2_remove_urlpath(schunk_sframe->storage->urlpath); |
| 354 |
blosc2_remove_urlpath(schunk_cframe->storage->urlpath); |
| 355 |
blosc2_schunk_free(schunk_sframe); |
| 356 |
blosc2_schunk_free(schunk_cframe); |
| 357 |
/* Destroy the Blosc environment */ |
| 358 |
blosc2_destroy(); |
| 359 |
|
| 360 |
} |
| 361 |
|
| 362 |
int main(int argc, char* argv[]) { |
| 363 |
char* operation = NULL; |
| 364 |
|
| 365 |
if (argc >= 6) { |
| 366 |
printf("Usage: ./sframe_bench [nchunks] [insert | update | reorder] [num operations] [io_file | io_mmap]\n"); |
| 367 |
exit(1); |
| 368 |
} |
| 369 |
else if (argc >= 2) { |
| 370 |
nchunks = (int)strtol(argv[1], NULL, 10); |
| 371 |
} |
| 372 |
if (argc >= 3) { |
| 373 |
operation = argv[2]; |
| 374 |
} |
| 375 |
if (argc >= 4) { |
| 376 |
iterations = (int)strtol(argv[3], NULL, 10); |
| 377 |
} |
| 378 |
if (argc == 5) { |
| 379 |
if (strcmp(argv[4], "io_file") == 0) { |
| 380 |
io_type = BLOSC2_IO_FILESYSTEM; |
| 381 |
} else if (strcmp(argv[4], "io_mmap") == 0) { |
| 382 |
io_type = BLOSC2_IO_FILESYSTEM_MMAP; |
| 383 |
} else { |
| 384 |
printf("Invalid io type. Use io_file or io_mmap\n"); |
| 385 |
exit(1); |
| 386 |
} |
| 387 |
} |
| 388 |
|
| 389 |
test_create_sframe_frame(operation); |
| 390 |
|
| 391 |
return 0; |
| 392 |
} |