Debian Codegraph @

src/c-blosc2/bench/sum_openmp.c

raw
1 /*
2 Copyright (c) 2021 Blosc Development Team <[email protected]>
3 https://blosc.org
4 License: BSD 3-Clause (see LICENSE.txt)
5
6 Example program showing how to operate with compressed buffers.
7
8 To compile this program for synthetic data (default):
9
10 $ gcc -fopenmp -O3 sum_openmp.c -o sum_openmp -lblosc2
11
12 To run:
13
14 $ OMP_PROC_BIND=spread OMP_NUM_THREADS=8 ./sum_openmp
15 Blosc version info: 2.0.0a6.dev ($Date:: 2018-05-18 #$)
16 Sum for uncompressed data: 199950000000
17 Sum time for uncompressed data: 0.0288 s, 26459.3 MB/s
18 Compression ratio: 762.9 MB -> 14.0 MB (54.6x)
19 Compression time: 0.288 s, 2653.5 MB/s
20 Sum for *compressed* data: 199950000000
21 Sum time for *compressed* data: 0.0188 s, 40653.7 MB/s
22
23 To use real (rainfall) data:
24
25 $ gcc -DRAINFALL -fopenmp -Ofast sum_openmp.c -o sum_openmp
26
27 And running it:
28
29 $ OMP_PROC_BIND=spread OMP_NUM_THREADS=8 ./sum_openmp
30 Blosc version info: 2.0.0a6.dev ($Date:: 2018-05-18 #$)
31 Sum for uncompressed data: 29741012
32 Sum time for uncompressed data: 0.0149 s, 25627.4 MB/s
33 Compression ratio: 381.5 MB -> 71.3 MB (5.3x)
34 Compression time: 1.53 s, 249.1 MB/s
35 Sum for *compressed* data: 29741012
36 Sum time for *compressed* data: 0.0247 s, 15467.5 MB/s
37
38 */
39
40 #include <stdio.h>
41 #include <stdlib.h>
42 #include <stdbool.h>
43 #include <sys/stat.h>
44 #include <errno.h>
45 #include <assert.h>
46 #include "blosc2.h"
47
48 #define KB 1024.
49 #define MB (1024*KB)
50 #define GB (1024*MB)
51
52 #define N (100 * 1000 * 1000)
53 #define CHUNKSIZE (16 * 1000)
54 #define NCHUNKS (N / CHUNKSIZE)
55 #define NTHREADS 8
56 #define NITER 5
57 #ifdef RAINFALL
58 #define SYNTHETIC false
59 #else
60 #define SYNTHETIC true
61 #endif
62
63 #if SYNTHETIC == true
64 #define DTYPE int64_t
65 #define CLEVEL 3
66 #define CODEC BLOSC_BLOSCLZ
67 #else
68 #define DTYPE float
69 #define CLEVEL 1
70 #define CODEC BLOSC_LZ4
71 #endif
72
73
74 int main(void) {
75 blosc2_init();
76
77 static DTYPE udata[N];
78 DTYPE chunk_buf[CHUNKSIZE];
79 int32_t isize = CHUNKSIZE * sizeof(DTYPE);
80 DTYPE sum, compressed_sum;
81 int64_t nbytes, cbytes;
82 blosc2_schunk* schunk;
83 int i, j, nchunk;
84 blosc_timestamp_t last, current;
85 double ttotal, itotal;
86 char* envvar = NULL;
87
88 printf("Blosc version info: %s (%s)\n",
89 BLOSC2_VERSION_STRING, BLOSC2_VERSION_DATE);
90
91 // Fill the buffer for a chunk
92 if (SYNTHETIC) {
93 for (j = 0; j < CHUNKSIZE; j++) {
94 chunk_buf[j] = j;
95 }
96 }
97 else {
98 struct stat info;
99 const char *filegrid = "rainfall-grid-150x150.bin";
100 if (stat(filegrid, &info) != 0) {
101 printf("Grid file %s not found!", filegrid);
102 exit(1);
103 }
104 char *cdata = malloc(info.st_size);
105
106 FILE *f = fopen(filegrid, "rb");
107 size_t blocks_read = fread(cdata, info.st_size, 1, f);
108 if (blocks_read != 1) {
109 printf("Error reading file!\n");
110 exit(1);
111 }
112 fclose(f);
113
114 int dsize = blosc1_getitem(cdata, 0, CHUNKSIZE, chunk_buf);
115 if (dsize < 0) {
116 printf("blosc1_getitem() error. Error code: %d\n. Probably reading too much data?", dsize);
117 exit(1);
118 }
119 free(cdata);
120 }
121
122 // Fill the uncompressed dataset with data chunks
123 for (i = 0; i < N / CHUNKSIZE; i++) {
124 for (j = 0; j < CHUNKSIZE; j++) {
125 udata[i * CHUNKSIZE + j] = chunk_buf[j];
126 }
127 }
128
129 // Reduce uncompressed dataset
130 ttotal = 1e10;
131 sum = 0;
132 for (int n = 0; n < NITER; n++) {
133 sum = 0;
134 blosc_set_timestamp(&last);
135 #pragma omp parallel for reduction (+:sum)
136 for (i = 0; i < N; i++) {
137 sum += udata[i];
138 }
139 blosc_set_timestamp(&current);
140 itotal = blosc_elapsed_secs(last, current);
141 if (itotal < ttotal) ttotal = itotal;
142 }
143 printf("Sum for uncompressed data: %10.0f\n", (double)sum);
144 printf("Sum time for uncompressed data: %.3g s, %.1f MB/s\n",
145 ttotal, (double)(isize * NCHUNKS) / (double)(ttotal * MB));
146
147 // Create a super-chunk container for the compressed container
148 long codec = CODEC;
149 envvar = getenv("SUM_COMPRESSOR");
150 if (envvar != NULL) {
151 codec = blosc2_compname_to_compcode(envvar);
152 if (codec < 0) {
153 printf("Unknown compresssor: %s\n", envvar);
154 return 1;
155 }
156 }
157 blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
158 cparams.compcode = (uint8_t)codec;
159
160 long clevel = CLEVEL;
161 envvar = getenv("SUM_CLEVEL");
162 if (envvar != NULL) {
163 clevel = strtol(envvar, NULL, 10);
164 }
165 cparams.clevel = (uint8_t)clevel;
166
167 cparams.typesize = sizeof(DTYPE);
168 cparams.nthreads = 1;
169 blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
170 dparams.nthreads = 1;
171 blosc_set_timestamp(&last);
172 blosc2_storage storage = {.cparams=&cparams, .dparams=&dparams};
173 schunk = blosc2_schunk_new(&storage);
174 for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
175 for (i = 0; i < CHUNKSIZE; i++) {
176 chunk_buf[i] = udata[i + nchunk * CHUNKSIZE];
177 }
178 blosc2_schunk_append_buffer(schunk, chunk_buf, isize);
179 }
180 blosc_set_timestamp(&current);
181 ttotal = blosc_elapsed_secs(last, current);
182 nbytes = schunk->nbytes;
183 cbytes = schunk->cbytes;
184 printf("Compression ratio: %.1f MB -> %.1f MB (%.1fx)\n",
185 (double)nbytes / MB, (double)cbytes / MB, (1. * (double)nbytes) / (double)cbytes);
186 printf("Compression time: %.3g s, %.1f MB/s\n",
187 ttotal, (double)nbytes / (ttotal * MB));
188
189 int nthreads = NTHREADS;
190 envvar = getenv("OMP_NUM_THREADS");
191 if (envvar != NULL) {
192 long value;
193 value = strtol(envvar, NULL, 10);
194 if ((value != EINVAL) && (value >= 0)) {
195 nthreads = (int)value;
196 }
197 }
198 // Build buffers and contexts for computations
199 int nchunks_thread = NCHUNKS / nthreads;
200 int remaining_chunks = NCHUNKS - nchunks_thread * nthreads;
201 blosc2_context **dctx = malloc(nthreads * sizeof(void*));
202 DTYPE** chunk = malloc(nthreads * sizeof(void*));
203 for (j = 0; j < nthreads; j++) {
204 chunk[j] = malloc(CHUNKSIZE * sizeof(DTYPE));
205 }
206
207 // Reduce uncompressed dataset
208 blosc_set_timestamp(&last);
209 ttotal = 1e10;
210 compressed_sum = 0;
211 for (int n = 0; n < NITER; n++) {
212 compressed_sum = 0;
213 #pragma omp parallel for private(nchunk) reduction (+:compressed_sum)
214 for (j = 0; j < nthreads; j++) {
215 dctx[j] = blosc2_create_dctx(dparams);
216 for (nchunk = 0; nchunk < nchunks_thread; nchunk++) {
217 blosc2_decompress_ctx(dctx[j], schunk->data[j * nchunks_thread + nchunk], INT32_MAX,
218 (void*)(chunk[j]), isize);
219 for (i = 0; i < CHUNKSIZE; i++) {
220 compressed_sum += chunk[j][i];
221 //compressed_sum += i + (j * nchunks_thread + nchunk) * CHUNKSIZE;
222 }
223 }
224 }
225 for (nchunk = NCHUNKS - remaining_chunks; nchunk < NCHUNKS; nchunk++) {
226 blosc2_decompress_ctx(dctx[0], schunk->data[nchunk], INT32_MAX, (void*)(chunk[0]), isize);
227 for (i = 0; i < CHUNKSIZE; i++) {
228 compressed_sum += chunk[0][i];
229 //compressed_sum += i + nchunk * CHUNKSIZE;
230 }
231 }
232 blosc_set_timestamp(&current);
233 itotal = blosc_elapsed_secs(last, current);
234 if (itotal < ttotal) ttotal = itotal;
235 }
236 printf("Sum for *compressed* data: %10.0f\n", (double)compressed_sum);
237 printf("Sum time for *compressed* data: %.3g s, %.1f MB/s\n",
238 ttotal, (double)nbytes / (ttotal * MB));
239 //printf("sum, csum: %f, %f\n", sum, compressed_sum);
240 if (SYNTHETIC) {
241 // difficult to fulfill for single precision
242 assert(sum == compressed_sum);
243 }
244 /* Free resources */
245 blosc2_schunk_free(schunk);
246
247 blosc2_destroy();
248
249 return 0;
250 }