| 1 |
/* |
| 2 |
Copyright (c) 2021 Blosc Development Team <[email protected]> |
| 3 |
https://blosc.org |
| 4 |
License: BSD 3-Clause (see LICENSE.txt) |
| 5 |
|
| 6 |
Example program showing how to operate with compressed buffers. |
| 7 |
|
| 8 |
To compile this program for synthetic data (default): |
| 9 |
|
| 10 |
$ gcc -fopenmp -O3 sum_openmp.c -o sum_openmp -lblosc2 |
| 11 |
|
| 12 |
To run: |
| 13 |
|
| 14 |
$ OMP_PROC_BIND=spread OMP_NUM_THREADS=8 ./sum_openmp |
| 15 |
Blosc version info: 2.0.0a6.dev ($Date:: 2018-05-18 #$) |
| 16 |
Sum for uncompressed data: 199950000000 |
| 17 |
Sum time for uncompressed data: 0.0288 s, 26459.3 MB/s |
| 18 |
Compression ratio: 762.9 MB -> 14.0 MB (54.6x) |
| 19 |
Compression time: 0.288 s, 2653.5 MB/s |
| 20 |
Sum for *compressed* data: 199950000000 |
| 21 |
Sum time for *compressed* data: 0.0188 s, 40653.7 MB/s |
| 22 |
|
| 23 |
To use real (rainfall) data: |
| 24 |
|
| 25 |
$ gcc -DRAINFALL -fopenmp -Ofast sum_openmp.c -o sum_openmp |
| 26 |
|
| 27 |
And running it: |
| 28 |
|
| 29 |
$ OMP_PROC_BIND=spread OMP_NUM_THREADS=8 ./sum_openmp |
| 30 |
Blosc version info: 2.0.0a6.dev ($Date:: 2018-05-18 #$) |
| 31 |
Sum for uncompressed data: 29741012 |
| 32 |
Sum time for uncompressed data: 0.0149 s, 25627.4 MB/s |
| 33 |
Compression ratio: 381.5 MB -> 71.3 MB (5.3x) |
| 34 |
Compression time: 1.53 s, 249.1 MB/s |
| 35 |
Sum for *compressed* data: 29741012 |
| 36 |
Sum time for *compressed* data: 0.0247 s, 15467.5 MB/s |
| 37 |
|
| 38 |
*/ |
| 39 |
|
| 40 |
#include <stdio.h> |
| 41 |
#include <stdlib.h> |
| 42 |
#include <stdbool.h> |
| 43 |
#include <sys/stat.h> |
| 44 |
#include <errno.h> |
| 45 |
#include <assert.h> |
| 46 |
#include "blosc2.h" |
| 47 |
|
| 48 |
#define KB 1024. |
| 49 |
#define MB (1024*KB) |
| 50 |
#define GB (1024*MB) |
| 51 |
|
| 52 |
#define N (100 * 1000 * 1000) |
| 53 |
#define CHUNKSIZE (16 * 1000) |
| 54 |
#define NCHUNKS (N / CHUNKSIZE) |
| 55 |
#define NTHREADS 8 |
| 56 |
#define NITER 5 |
| 57 |
#ifdef RAINFALL |
| 58 |
#define SYNTHETIC false |
| 59 |
#else |
| 60 |
#define SYNTHETIC true |
| 61 |
#endif |
| 62 |
|
| 63 |
#if SYNTHETIC == true |
| 64 |
#define DTYPE int64_t |
| 65 |
#define CLEVEL 3 |
| 66 |
#define CODEC BLOSC_BLOSCLZ |
| 67 |
#else |
| 68 |
#define DTYPE float |
| 69 |
#define CLEVEL 1 |
| 70 |
#define CODEC BLOSC_LZ4 |
| 71 |
#endif |
| 72 |
|
| 73 |
|
| 74 |
int main(void) { |
| 75 |
blosc2_init(); |
| 76 |
|
| 77 |
static DTYPE udata[N]; |
| 78 |
DTYPE chunk_buf[CHUNKSIZE]; |
| 79 |
int32_t isize = CHUNKSIZE * sizeof(DTYPE); |
| 80 |
DTYPE sum, compressed_sum; |
| 81 |
int64_t nbytes, cbytes; |
| 82 |
blosc2_schunk* schunk; |
| 83 |
int i, j, nchunk; |
| 84 |
blosc_timestamp_t last, current; |
| 85 |
double ttotal, itotal; |
| 86 |
char* envvar = NULL; |
| 87 |
|
| 88 |
printf("Blosc version info: %s (%s)\n", |
| 89 |
BLOSC2_VERSION_STRING, BLOSC2_VERSION_DATE); |
| 90 |
|
| 91 |
// Fill the buffer for a chunk |
| 92 |
if (SYNTHETIC) { |
| 93 |
for (j = 0; j < CHUNKSIZE; j++) { |
| 94 |
chunk_buf[j] = j; |
| 95 |
} |
| 96 |
} |
| 97 |
else { |
| 98 |
struct stat info; |
| 99 |
const char *filegrid = "rainfall-grid-150x150.bin"; |
| 100 |
if (stat(filegrid, &info) != 0) { |
| 101 |
printf("Grid file %s not found!", filegrid); |
| 102 |
exit(1); |
| 103 |
} |
| 104 |
char *cdata = malloc(info.st_size); |
| 105 |
|
| 106 |
FILE *f = fopen(filegrid, "rb"); |
| 107 |
size_t blocks_read = fread(cdata, info.st_size, 1, f); |
| 108 |
if (blocks_read != 1) { |
| 109 |
printf("Error reading file!\n"); |
| 110 |
exit(1); |
| 111 |
} |
| 112 |
fclose(f); |
| 113 |
|
| 114 |
int dsize = blosc1_getitem(cdata, 0, CHUNKSIZE, chunk_buf); |
| 115 |
if (dsize < 0) { |
| 116 |
printf("blosc1_getitem() error. Error code: %d\n. Probably reading too much data?", dsize); |
| 117 |
exit(1); |
| 118 |
} |
| 119 |
free(cdata); |
| 120 |
} |
| 121 |
|
| 122 |
// Fill the uncompressed dataset with data chunks |
| 123 |
for (i = 0; i < N / CHUNKSIZE; i++) { |
| 124 |
for (j = 0; j < CHUNKSIZE; j++) { |
| 125 |
udata[i * CHUNKSIZE + j] = chunk_buf[j]; |
| 126 |
} |
| 127 |
} |
| 128 |
|
| 129 |
// Reduce uncompressed dataset |
| 130 |
ttotal = 1e10; |
| 131 |
sum = 0; |
| 132 |
for (int n = 0; n < NITER; n++) { |
| 133 |
sum = 0; |
| 134 |
blosc_set_timestamp(&last); |
| 135 |
#pragma omp parallel for reduction (+:sum) |
| 136 |
for (i = 0; i < N; i++) { |
| 137 |
sum += udata[i]; |
| 138 |
} |
| 139 |
blosc_set_timestamp(¤t); |
| 140 |
itotal = blosc_elapsed_secs(last, current); |
| 141 |
if (itotal < ttotal) ttotal = itotal; |
| 142 |
} |
| 143 |
printf("Sum for uncompressed data: %10.0f\n", (double)sum); |
| 144 |
printf("Sum time for uncompressed data: %.3g s, %.1f MB/s\n", |
| 145 |
ttotal, (double)(isize * NCHUNKS) / (double)(ttotal * MB)); |
| 146 |
|
| 147 |
// Create a super-chunk container for the compressed container |
| 148 |
long codec = CODEC; |
| 149 |
envvar = getenv("SUM_COMPRESSOR"); |
| 150 |
if (envvar != NULL) { |
| 151 |
codec = blosc2_compname_to_compcode(envvar); |
| 152 |
if (codec < 0) { |
| 153 |
printf("Unknown compresssor: %s\n", envvar); |
| 154 |
return 1; |
| 155 |
} |
| 156 |
} |
| 157 |
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS; |
| 158 |
cparams.compcode = (uint8_t)codec; |
| 159 |
|
| 160 |
long clevel = CLEVEL; |
| 161 |
envvar = getenv("SUM_CLEVEL"); |
| 162 |
if (envvar != NULL) { |
| 163 |
clevel = strtol(envvar, NULL, 10); |
| 164 |
} |
| 165 |
cparams.clevel = (uint8_t)clevel; |
| 166 |
|
| 167 |
cparams.typesize = sizeof(DTYPE); |
| 168 |
cparams.nthreads = 1; |
| 169 |
blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS; |
| 170 |
dparams.nthreads = 1; |
| 171 |
blosc_set_timestamp(&last); |
| 172 |
blosc2_storage storage = {.cparams=&cparams, .dparams=&dparams}; |
| 173 |
schunk = blosc2_schunk_new(&storage); |
| 174 |
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) { |
| 175 |
for (i = 0; i < CHUNKSIZE; i++) { |
| 176 |
chunk_buf[i] = udata[i + nchunk * CHUNKSIZE]; |
| 177 |
} |
| 178 |
blosc2_schunk_append_buffer(schunk, chunk_buf, isize); |
| 179 |
} |
| 180 |
blosc_set_timestamp(¤t); |
| 181 |
ttotal = blosc_elapsed_secs(last, current); |
| 182 |
nbytes = schunk->nbytes; |
| 183 |
cbytes = schunk->cbytes; |
| 184 |
printf("Compression ratio: %.1f MB -> %.1f MB (%.1fx)\n", |
| 185 |
(double)nbytes / MB, (double)cbytes / MB, (1. * (double)nbytes) / (double)cbytes); |
| 186 |
printf("Compression time: %.3g s, %.1f MB/s\n", |
| 187 |
ttotal, (double)nbytes / (ttotal * MB)); |
| 188 |
|
| 189 |
int nthreads = NTHREADS; |
| 190 |
envvar = getenv("OMP_NUM_THREADS"); |
| 191 |
if (envvar != NULL) { |
| 192 |
long value; |
| 193 |
value = strtol(envvar, NULL, 10); |
| 194 |
if ((value != EINVAL) && (value >= 0)) { |
| 195 |
nthreads = (int)value; |
| 196 |
} |
| 197 |
} |
| 198 |
// Build buffers and contexts for computations |
| 199 |
int nchunks_thread = NCHUNKS / nthreads; |
| 200 |
int remaining_chunks = NCHUNKS - nchunks_thread * nthreads; |
| 201 |
blosc2_context **dctx = malloc(nthreads * sizeof(void*)); |
| 202 |
DTYPE** chunk = malloc(nthreads * sizeof(void*)); |
| 203 |
for (j = 0; j < nthreads; j++) { |
| 204 |
chunk[j] = malloc(CHUNKSIZE * sizeof(DTYPE)); |
| 205 |
} |
| 206 |
|
| 207 |
// Reduce uncompressed dataset |
| 208 |
blosc_set_timestamp(&last); |
| 209 |
ttotal = 1e10; |
| 210 |
compressed_sum = 0; |
| 211 |
for (int n = 0; n < NITER; n++) { |
| 212 |
compressed_sum = 0; |
| 213 |
#pragma omp parallel for private(nchunk) reduction (+:compressed_sum) |
| 214 |
for (j = 0; j < nthreads; j++) { |
| 215 |
dctx[j] = blosc2_create_dctx(dparams); |
| 216 |
for (nchunk = 0; nchunk < nchunks_thread; nchunk++) { |
| 217 |
blosc2_decompress_ctx(dctx[j], schunk->data[j * nchunks_thread + nchunk], INT32_MAX, |
| 218 |
(void*)(chunk[j]), isize); |
| 219 |
for (i = 0; i < CHUNKSIZE; i++) { |
| 220 |
compressed_sum += chunk[j][i]; |
| 221 |
//compressed_sum += i + (j * nchunks_thread + nchunk) * CHUNKSIZE; |
| 222 |
} |
| 223 |
} |
| 224 |
} |
| 225 |
for (nchunk = NCHUNKS - remaining_chunks; nchunk < NCHUNKS; nchunk++) { |
| 226 |
blosc2_decompress_ctx(dctx[0], schunk->data[nchunk], INT32_MAX, (void*)(chunk[0]), isize); |
| 227 |
for (i = 0; i < CHUNKSIZE; i++) { |
| 228 |
compressed_sum += chunk[0][i]; |
| 229 |
//compressed_sum += i + nchunk * CHUNKSIZE; |
| 230 |
} |
| 231 |
} |
| 232 |
blosc_set_timestamp(¤t); |
| 233 |
itotal = blosc_elapsed_secs(last, current); |
| 234 |
if (itotal < ttotal) ttotal = itotal; |
| 235 |
} |
| 236 |
printf("Sum for *compressed* data: %10.0f\n", (double)compressed_sum); |
| 237 |
printf("Sum time for *compressed* data: %.3g s, %.1f MB/s\n", |
| 238 |
ttotal, (double)nbytes / (ttotal * MB)); |
| 239 |
//printf("sum, csum: %f, %f\n", sum, compressed_sum); |
| 240 |
if (SYNTHETIC) { |
| 241 |
// difficult to fulfill for single precision |
| 242 |
assert(sum == compressed_sum); |
| 243 |
} |
| 244 |
/* Free resources */ |
| 245 |
blosc2_schunk_free(schunk); |
| 246 |
|
| 247 |
blosc2_destroy(); |
| 248 |
|
| 249 |
return 0; |
| 250 |
} |