| 1 |
/* |
| 2 |
Copyright (c) 2021 Blosc Development Team <[email protected]> |
| 3 |
https://blosc.org |
| 4 |
License: BSD 3-Clause (see LICENSE.txt) |
| 5 |
|
| 6 |
Benchmark showing Blosc zero detection capabilities via run-length. |
| 7 |
|
| 8 |
*/ |
| 9 |
|
| 10 |
#include <stdio.h> |
| 11 |
#include <stdint.h> |
| 12 |
#include <assert.h> |
| 13 |
#include <math.h> |
| 14 |
#include <inttypes.h> |
| 15 |
|
| 16 |
#include "blosc2.h" |
| 17 |
|
| 18 |
|
| 19 |
#define KB 1024 |
| 20 |
#define MB (1024*KB) |
| 21 |
#define GB (1024*MB) |
| 22 |
|
| 23 |
#define NCHUNKS (2000) |
| 24 |
#define CHUNKSIZE (500 * 1000) // > NCHUNKS for the bench purposes |
| 25 |
#define NTHREADS 8 |
| 26 |
|
| 27 |
enum { |
| 28 |
ZERO_DETECTION = 0, |
| 29 |
CHECK_ZEROS = 1, |
| 30 |
CHECK_NANS = 2, |
| 31 |
CHECK_VALUES = 3, |
| 32 |
CHECK_UNINIT = 4, |
| 33 |
}; |
| 34 |
#define REPEATED_VALUE 1 |
| 35 |
|
| 36 |
|
| 37 |
int check_special_values(int svalue) { |
| 38 |
blosc2_schunk *schunk; |
| 39 |
int32_t isize = CHUNKSIZE * sizeof(int32_t); |
| 40 |
int32_t osize = CHUNKSIZE * sizeof(int32_t) + BLOSC2_MAX_OVERHEAD; |
| 41 |
int dsize, csize; |
| 42 |
int64_t nbytes, frame_len; |
| 43 |
int nchunk; |
| 44 |
int64_t nchunks = 0; |
| 45 |
int rc; |
| 46 |
int32_t value = REPEATED_VALUE; |
| 47 |
float fvalue; |
| 48 |
blosc_timestamp_t last, current; |
| 49 |
double totaltime; |
| 50 |
double totalsize = (double)isize * NCHUNKS; |
| 51 |
int32_t *data_buffer = malloc(CHUNKSIZE * sizeof(int32_t)); |
| 52 |
int32_t *rec_buffer = malloc(CHUNKSIZE * sizeof(int32_t)); |
| 53 |
|
| 54 |
/* Initialize the Blosc compressor */ |
| 55 |
blosc2_init(); |
| 56 |
|
| 57 |
/* Create a super-chunk container */ |
| 58 |
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; |
| 59 |
cparams.typesize = sizeof(int32_t); |
| 60 |
cparams.compcode = BLOSC_BLOSCLZ; |
| 61 |
cparams.clevel = 9; |
| 62 |
cparams.nthreads = NTHREADS; |
| 63 |
blosc2_storage storage = {.cparams=&cparams, .contiguous=false}; |
| 64 |
schunk = blosc2_schunk_new(&storage); |
| 65 |
|
| 66 |
void* chunk = malloc(BLOSC_EXTENDED_HEADER_LENGTH + isize); |
| 67 |
|
| 68 |
// Cache the special chunks |
| 69 |
switch (svalue) { |
| 70 |
case ZERO_DETECTION: |
| 71 |
memset(data_buffer, 0, isize); |
| 72 |
csize = blosc2_compress(5, 1, sizeof(int32_t), data_buffer, isize, chunk, osize); |
| 73 |
break; |
| 74 |
case CHECK_ZEROS: |
| 75 |
csize = blosc2_chunk_zeros(cparams, isize, chunk, BLOSC_EXTENDED_HEADER_LENGTH); |
| 76 |
break; |
| 77 |
case CHECK_UNINIT: |
| 78 |
csize = blosc2_chunk_uninit(cparams, isize, chunk, BLOSC_EXTENDED_HEADER_LENGTH); |
| 79 |
break; |
| 80 |
case CHECK_NANS: |
| 81 |
csize = blosc2_chunk_nans(cparams, isize, chunk, BLOSC_EXTENDED_HEADER_LENGTH); |
| 82 |
break; |
| 83 |
case CHECK_VALUES: |
| 84 |
csize = blosc2_chunk_repeatval(cparams, isize, chunk, |
| 85 |
BLOSC_EXTENDED_HEADER_LENGTH + sizeof(int32_t), &value); |
| 86 |
break; |
| 87 |
default: |
| 88 |
printf("Unknown case\n"); |
| 89 |
exit(1); |
| 90 |
} |
| 91 |
if (csize < 0) { |
| 92 |
printf("Error creating chunk: %d\n", csize); |
| 93 |
exit(1); |
| 94 |
} |
| 95 |
|
| 96 |
/* Append the chunks */ |
| 97 |
blosc_set_timestamp(&last); |
| 98 |
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) { |
| 99 |
nchunks = blosc2_schunk_append_chunk(schunk, chunk, true); |
| 100 |
if (nchunks < 0) { |
| 101 |
printf("Error appending chunk: %" PRId64 "\n", nchunks); |
| 102 |
exit(1); |
| 103 |
} |
| 104 |
} |
| 105 |
blosc_set_timestamp(¤t); |
| 106 |
free(chunk); |
| 107 |
totaltime = blosc_elapsed_secs(last, current); |
| 108 |
printf("\n[Compr] Elapsed time:\t %6.3f s." |
| 109 |
" Processed data: %.3f GB (%.3f GB/s)\n", |
| 110 |
totaltime, totalsize / GB, totalsize / (GB * totaltime)); |
| 111 |
|
| 112 |
/* Gather some info */ |
| 113 |
nbytes = schunk->nbytes; |
| 114 |
frame_len = blosc2_schunk_frame_len(schunk); |
| 115 |
printf("Compression super-chunk: %ld -> %ld (%.1fx)\n", |
| 116 |
(long)nbytes, (long)frame_len, (1. * (double)nbytes) / (double)frame_len); |
| 117 |
|
| 118 |
/* Retrieve and decompress the chunks */ |
| 119 |
blosc_set_timestamp(&last); |
| 120 |
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) { |
| 121 |
dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, rec_buffer, isize); |
| 122 |
if (dsize < 0) { |
| 123 |
printf("Decompression error. Error code: %d\n", dsize); |
| 124 |
exit(dsize); |
| 125 |
} |
| 126 |
assert (dsize == (int)isize); |
| 127 |
} |
| 128 |
blosc_set_timestamp(¤t); |
| 129 |
totaltime = blosc_elapsed_secs(last, current); |
| 130 |
totalsize = (double)(isize) * (double)nchunks; |
| 131 |
printf("[Decompr] Elapsed time:\t %6.3f s." |
| 132 |
" Processed data: %.3f GB (%.3f GB/s)\n", |
| 133 |
totaltime, totalsize / GB, totalsize / (GB * totaltime)); |
| 134 |
|
| 135 |
/* Exercise the getitem */ |
| 136 |
blosc_set_timestamp(&last); |
| 137 |
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) { |
| 138 |
bool needs_free; |
| 139 |
uint8_t* chunk_; |
| 140 |
csize = blosc2_schunk_get_chunk(schunk, nchunk, &chunk_, &needs_free); |
| 141 |
if (csize < 0) { |
| 142 |
printf("blosc2_schunk_get_chunk error. Error code: %d\n", dsize); |
| 143 |
return csize; |
| 144 |
} |
| 145 |
switch (svalue) { |
| 146 |
case CHECK_VALUES: |
| 147 |
rc = blosc1_getitem(chunk_, nchunk, 1, &value); |
| 148 |
if (rc < 0) { |
| 149 |
printf("Error in getitem of a special value\n"); |
| 150 |
return rc; |
| 151 |
} |
| 152 |
if (value != REPEATED_VALUE) { |
| 153 |
printf("Wrong value!"); |
| 154 |
exit(1); |
| 155 |
} |
| 156 |
break; |
| 157 |
case CHECK_NANS: |
| 158 |
rc = blosc1_getitem(chunk_, nchunk, 1, &fvalue); |
| 159 |
if (rc < 0) { |
| 160 |
printf("Error in getitem of a special value\n"); |
| 161 |
return rc; |
| 162 |
} |
| 163 |
if (!isnan(fvalue)) { |
| 164 |
printf("Wrong value!"); |
| 165 |
exit(1); |
| 166 |
} |
| 167 |
break; |
| 168 |
case CHECK_ZEROS: |
| 169 |
rc = blosc1_getitem(chunk_, nchunk, 1, &value); |
| 170 |
if (rc < 0) { |
| 171 |
printf("Error in getitem of zeros value\n"); |
| 172 |
return rc; |
| 173 |
} |
| 174 |
if (value != 0) { |
| 175 |
printf("Wrong value!"); |
| 176 |
exit(1); |
| 177 |
} |
| 178 |
break; |
| 179 |
default: |
| 180 |
// It can only be non-initialized |
| 181 |
rc = blosc1_getitem(chunk_, nchunk, 1, &value); |
| 182 |
if (rc < 0) { |
| 183 |
printf("Error in getitem of an non-initialized value\n"); |
| 184 |
return rc; |
| 185 |
} |
| 186 |
} |
| 187 |
if (needs_free) { |
| 188 |
free(chunk_); |
| 189 |
} |
| 190 |
} |
| 191 |
blosc_set_timestamp(¤t); |
| 192 |
totaltime = blosc_elapsed_secs(last, current); |
| 193 |
printf("[getitem] Elapsed time:\t %6.3f s.\n", totaltime); |
| 194 |
|
| 195 |
// /* Check that all the values have a good roundtrip */ |
| 196 |
// blosc_set_timestamp(&last); |
| 197 |
// for (nchunk = 0; nchunk < NCHUNKS; nchunk++) { |
| 198 |
// dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, (void *) rec_buffer, isize); |
| 199 |
// if (dsize < 0) { |
| 200 |
// printf("Decompression error. Error code: %d\n", dsize); |
| 201 |
// return dsize; |
| 202 |
// } |
| 203 |
// assert (dsize == (int)isize); |
| 204 |
// if (CHECK_VALUE) { |
| 205 |
// int32_t* buffer = (int32_t*)rec_buffer; |
| 206 |
// for (int i = 0; i < CHUNKSIZE; i++) { |
| 207 |
// if (buffer[i] != REPEATED_VALUE) { |
| 208 |
// printf("Value is not correct in chunk %d, position: %d\n", nchunk, i); |
| 209 |
// return -1; |
| 210 |
// } |
| 211 |
// } |
| 212 |
// } |
| 213 |
// else if (CHECK_NAN) { |
| 214 |
// float* buffer = (float*)rec_buffer; |
| 215 |
// for (int i = 0; i < CHUNKSIZE; i++) { |
| 216 |
// if (!isnan(buffer[i])) { |
| 217 |
// printf("Value is not correct in chunk %d, position: %d\n", nchunk, i); |
| 218 |
// return -1; |
| 219 |
// } |
| 220 |
// } |
| 221 |
// } |
| 222 |
// else { |
| 223 |
// int32_t* buffer = (int32_t*)rec_buffer; |
| 224 |
// for (int i = 0; i < CHUNKSIZE; i++) { |
| 225 |
// if (buffer[i] != 0) { |
| 226 |
// printf("Value is not correct in chunk %d, position: %d\n", nchunk, i); |
| 227 |
// return -1; |
| 228 |
// } |
| 229 |
// } |
| 230 |
// } |
| 231 |
// } |
| 232 |
// printf("All data did a good roundtrip!\n"); |
| 233 |
|
| 234 |
/* Free resources */ |
| 235 |
free(data_buffer); |
| 236 |
free(rec_buffer); |
| 237 |
/* Destroy the super-chunk */ |
| 238 |
blosc2_schunk_free(schunk); |
| 239 |
/* Destroy the Blosc environment */ |
| 240 |
blosc2_destroy(); |
| 241 |
|
| 242 |
return 0; |
| 243 |
} |
| 244 |
|
| 245 |
|
| 246 |
int main(void) { |
| 247 |
int rc; |
| 248 |
printf("*** Testing special zeros..."); |
| 249 |
rc = check_special_values(CHECK_ZEROS); |
| 250 |
if (rc < 0) { |
| 251 |
return rc; |
| 252 |
} |
| 253 |
printf("*** Testing NaNs..."); |
| 254 |
rc = check_special_values(CHECK_NANS); |
| 255 |
if (rc < 0) { |
| 256 |
return rc; |
| 257 |
} |
| 258 |
printf("*** Testing repeated values..."); |
| 259 |
rc = check_special_values(CHECK_VALUES); |
| 260 |
if (rc < 0) { |
| 261 |
return rc; |
| 262 |
} |
| 263 |
printf("*** Testing non-initialized values..."); |
| 264 |
rc = check_special_values(CHECK_UNINIT); |
| 265 |
if (rc < 0) { |
| 266 |
return rc; |
| 267 |
} |
| 268 |
printf("Testing zero detection..."); |
| 269 |
rc = check_special_values(ZERO_DETECTION); |
| 270 |
if (rc < 0) { |
| 271 |
return rc; |
| 272 |
} |
| 273 |
} |