Debian Codegraph @

src/SZ3/tools/H5Z-SZ3/test/sz3ToHDF5.cpp

raw
1 /**
2 * @file szToHDF5.c
3 * @author Sheng Di
4 * @date July, 2017
5 * @brief This is an example of using compression interface (HDF5)
6 * (C) 2017 by Mathematics and Computer Science (MCS), Argonne National Laboratory.
7 * See COPYRIGHT in top-level directory.
8 */
9
10 #include <errno.h>
11 #include <stdio.h>
12 #include <stdlib.h>
13 #include <dlfcn.h>
14 #include "hdf5.h"
15 #include "H5Z_SZ3.hpp"
16 #include "SZ3/utils/FileUtil.hpp"
17
18 int sysEndianType = LITTLE_ENDIAN_SYSTEM;
19 int dataEndianType = LITTLE_ENDIAN_DATA;
20
21 #define DATASET "testdata_compressed"
22
23
24 using namespace SZ;
25
26 int main(int argc, char * argv[])
27 {
28
29 //(void) helper fn to detect system endian type
30 //detectSysEndianType();
31 //by default sysEndianType and dataEndianType are little endian, can set them manually here
32 //dataEndianType = BIG_ENDIAN_DATA;
33
34 size_t r5=0,r4=0,r3=0,r2=0,r1=0;
35 int cmp_algo, interp_algo; //select compression and interpolation for SZ3
36 char outDir[640], oriFilePath[640], outputFilePath[640];
37 size_t cd_nelmts, nbEle;
38 unsigned int *cd_values = NULL;
39 //unsigned int cd_values[7];
40
41 herr_t status;
42 htri_t avail;
43 unsigned filter_config;
44
45 hid_t sid, idsid, cpid, fid;
46
47 if(argc < 3)
48 {
49 printf("Test case: sz3ToHDF5 [dataType] [srcFilePath] [dimension sizes...]\n");
50 printf("Example1 : sz3ToHDF5 -f testdata/x86/testfloat_8_8_128.dat 8 8 128\n");
51 printf("Example 2: sz3ToHDF5 -i32 testdata/x86/testint32_8x8x8.dat 8 8 8\n");
52 exit(0);
53 }
54
55 //printf("config file = %s\n", argv[2]);
56
57 int dataType = 0;
58 if(strcmp(argv[1],"-f")==0)
59 dataType = SZ_FLOAT;
60 else if(strcmp(argv[1], "-d")==0)
61 dataType = SZ_DOUBLE;
62 else if(strcmp(argv[1], "-i8")==0)
63 dataType = SZ_INT8;
64 else if(strcmp(argv[1], "-u8")==0)
65 dataType = SZ_UINT8;
66 else if(strcmp(argv[1], "-i16")==0)
67 dataType = SZ_INT16;
68 else if(strcmp(argv[1], "-u16")==0)
69 dataType = SZ_UINT16;
70 else if(strcmp(argv[1], "-i32")==0)
71 dataType = SZ_INT32;
72 else if(strcmp(argv[1], "-u32")==0)
73 dataType = SZ_UINT32;
74 else if(strcmp(argv[1], "-i64")==0)
75 dataType = SZ_INT64;
76 else if(strcmp(argv[1], "-u64")==0)
77 dataType = SZ_UINT64;
78 else
79 {
80 printf("Error: unknown data type in sz3ToHDF5.c!\n");
81 exit(0);
82 }
83
84 printf("DTYPE: %i", dataType);
85 sprintf(oriFilePath, "%s", argv[2]);
86 if(argc>=4) {
87 r1 = atoi(argv[3]); //8
88 }
89 if(argc>=5) {
90 r2 = atoi(argv[4]); //8
91 }
92 if(argc>=6) {
93 r3 = atoi(argv[5]); //128
94 }
95 if(argc>=7) {
96 r4 = atoi(argv[6]);
97 }
98 if(argc>=8) {
99 r5 = atoi(argv[7]);
100 }
101
102 //read in compression and interp algo
103 //for testing set these here as defaults in config
104 cmp_algo = 1;
105 interp_algo = 1;
106
107 //printf("cfgFile=%s\n", cfgFile);
108 sprintf(outputFilePath, "%s.sz3.h5", oriFilePath);
109
110 // printf("argv[1]=%s, dataType=%d\n", argv[1], dataType);
111 nbEle = computeDataLength(r5, r4, r3, r2, r1);
112
113 // printf("nbEle=%u\n", nbEle);
114
115 //Create cd_values
116 printf("Dimension sizes: n5=%u, n4=%u, n3=%u, n2=%u, n1=%u\n", r5, r4, r3, r2, r1);
117 int mode = 1; //0: ABS, 1: REL, ...
118 SZ_errConfigToCdArray(&cd_nelmts, &cd_values, mode, 0.001, 0.001, 0, 0); //SZ_FLOAT or SZ_DOUBLE or SZ_INT 100x500x500 : 0, 0, 100, 500, 500, ABS, REL (0.01, 0.01*(max-min), PW_REL (0.01, 5, 6, 7, 8, 9 --> 5*0.01, 6*0.01, ...), PSNR (mean squared error)).
119 //load_conffile_flag = 0;
120 // REL
121 //SZ_metaDataErrToCdArray(&cd_nelmts, &cd_values, dataType, r5, r4, r3, r2, r1, 1, 0.01, 0.01, 0, 0); //SZ_FLOAT or SZ_DOUBLE or SZ_INT 100x500x500 : 0, 0, 100, 500, 500, ABS, REL (0.01, 0.01*(max-min), PW_REL (0.01, 5, 6, 7, 8, 9 --> 5*0.01, 6*0.01, ...), PSNR (mean squared error)).
122 /*cd_nelmts = 5;
123 cd_values[0] = 3;
124 cd_values[1] = 0;
125 cd_values[2] = 128;
126 cd_values[3] = 8;
127 cd_values[4] = 8;
128 cd_values[5] = 0;
129 cd_values[6] = 0;*/
130
131 int i = 0;
132 // for(i=0;i<cd_nelmts;i++)
133 // printf("cd_values[%d]=%u\n", i, cd_values[i]);
134
135 //compute dimension
136 int dim = computeDimension(r5, r4, r3, r2, r1);
137
138 hsize_t dims[5] = {0,0,0,0,0}, chunk[5] = {0,0,0,0,0};
139 init_dims_chunk(dim, dims, chunk, nbEle, r5, r4, r3, r2, r1);
140
141 /* create HDF5 file */
142 if (0 > (fid = H5Fcreate(outputFilePath, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) { printf("Error in H5Pcreate"); exit(0); }
143
144 /*Create dataspace. Setting maximum size */
145 if (0 > (sid = H5Screate_simple(dim, dims, NULL))) { printf("Error in H5Screate_simple"); exit(0); }
146
147 /* setup dataset creation properties */
148 if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) { printf("Error in H5Pcreate"); exit(0); }
149
150 /* Add the SZ compression filter and set the chunk size */
151 if (0 > H5Pset_filter(cpid, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values)) { printf("Error in H5Pcreate"); exit(0); }
152 avail = H5Zfilter_avail(H5Z_FILTER_SZ3);
153 if(avail)
154 {
155 status = H5Zget_filter_info(H5Z_FILTER_SZ3, &filter_config);
156
157 if(filter_config & H5Z_FILTER_CONFIG_ENCODE_ENABLED)
158 printf("sz filter is available for encoding and decoding.\n");
159 }
160 if (0 > H5Pset_chunk(cpid, dim, chunk)) { printf("Error in H5Pcreate"); exit(0); }
161
162 //Initialize the configuration for SZ
163 //You can also use the global variable conf_params to set the configuration for sz without cfgFile.
164 //Example of setting an absolute error bound:
165 // sz_params* params = H5Z_SZ_Init_Default();
166 // params->errorBoundMode = ABS;
167 // params->absErrBound = 1E-4;
168
169 //H5Z_SZ_Init(cfgFile);
170
171 printf("....Writing SZ compressed data.............\n");
172
173 if(dataType == SZ_FLOAT)
174 {
175 float *data = new float[nbEle];
176 readfile(oriFilePath, nbEle, data);
177
178 printf("original data = ");
179 for(i=0;i<20;i++)
180 printf("%f ", data[i]);
181 printf("....\n");
182
183 if(dataEndianType == LITTLE_ENDIAN_DATA)
184 {
185 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F32LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
186 if (0 > H5Dwrite(idsid, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
187 }
188 else //BIG_ENDIAN_DATA
189 {
190 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F32BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
191 if (0 > H5Dwrite(idsid, H5T_IEEE_F32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
192 }
193 delete[] data;
194 if (0 > H5Dclose(idsid)) { printf("Error in H5Dclose"); exit(0); };
195 }
196 else if(dataType == SZ_DOUBLE)
197 {
198 double *data = new double[nbEle];
199 readfile(oriFilePath, nbEle, data);
200
201 printf("original data = ");
202 for(i=0;i<20;i++)
203 printf("%f ", data[i]);
204 printf("....\n");
205
206 if(dataEndianType == LITTLE_ENDIAN_DATA)
207 {
208 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F64LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
209 if (0 > H5Dwrite(idsid, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
210 }
211 else //BIG_ENDIAN_DATA
212 {
213 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F64BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
214 if (0 > H5Dwrite(idsid, H5T_IEEE_F64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
215 }
216 delete[] data;
217 if (0 > H5Dclose(idsid)) { printf("Error in H5Dclose"); exit(0); };
218 }
219 else if(dataType == SZ_INT8)
220 {
221 int8_t *data = new int8_t[nbEle];
222 readfile(oriFilePath, nbEle, data);
223
224 printf("original data = ");
225 for(i=0;i<20;i++)
226 printf("%d ", data[i]);
227 printf("....\n");
228
229 if(dataEndianType == LITTLE_ENDIAN_DATA)
230 {
231 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I8LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
232 if (0 > H5Dwrite(idsid, H5T_STD_I8LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
233 }
234 else //BIG_ENDIAN_DATA
235 {
236 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I8BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
237 if (0 > H5Dwrite(idsid, H5T_STD_I8BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
238 }
239 delete[] data;
240 if (0 > H5Dclose(idsid)) { printf("Error in H5Dclose"); exit(0); }
241 }
242 else if(dataType == SZ_UINT8)
243 {
244 uint8_t *data = new uint8_t[nbEle];
245 readfile(oriFilePath, nbEle, data);
246
247 printf("original data = ");
248 for(i=0;i<20;i++)
249 printf("%d ", data[i]);
250 printf("....\n");
251
252 if(dataEndianType == LITTLE_ENDIAN_DATA)
253 {
254 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U8LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
255 if (0 > H5Dwrite(idsid, H5T_STD_U8LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
256 }
257 else //BIG_ENDIAN_DATA
258 {
259 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U8BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
260 if (0 > H5Dwrite(idsid, H5T_STD_U8BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
261 }
262 delete[] data;
263 if (0 > H5Dclose(idsid)) { printf("Error in H5Dclose"); exit(0); }
264 }
265 else if(dataType == SZ_INT16)
266 {
267
268 int16_t *data = new int16_t[nbEle];
269 readfile(oriFilePath, nbEle, data);
270
271 printf("original data = ");
272 for(i=0;i<20;i++)
273 printf("%d ", data[i]);
274 printf("....\n");
275
276 if(dataEndianType == LITTLE_ENDIAN_DATA)
277 {
278 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I16LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
279 if (0 > H5Dwrite(idsid, H5T_STD_I16LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
280 }
281 else //BIG_ENDIAN_DATA
282 {
283 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I16BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
284 if (0 > H5Dwrite(idsid, H5T_STD_I16BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
285 }
286 delete[] data;
287 if (0 > H5Dclose(idsid)) { printf("Error in H5Dclose"); exit(0); }
288 }
289 else if(dataType == SZ_UINT16)
290 {
291 uint16_t *data = new uint16_t[nbEle];
292 readfile(oriFilePath, nbEle, data);
293
294 printf("original data = ");
295 for(i=0;i<20;i++)
296 printf("%d ", data[i]);
297 printf("....\n");
298
299 if(dataEndianType == LITTLE_ENDIAN_DATA)
300 {
301 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U16LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
302 if (0 > H5Dwrite(idsid, H5T_STD_U16LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
303 }
304 else //BIG_ENDIAN_DATA
305 {
306 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U16BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
307 if (0 > H5Dwrite(idsid, H5T_STD_U16BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
308 }
309 delete[] data;
310 if (0 > H5Dclose(idsid)) { printf("Error in H5Dclose"); exit(0); }
311 }
312 else if(dataType == SZ_INT32)
313 {
314 //printf("%i \t %i\n", sizeof(int), sizeof(int32_t));
315 int32_t *data = new int32_t[nbEle];
316 readfile(oriFilePath, nbEle, data);
317
318 printf("original data = ");
319 for(i=0;i<20;i++)
320 printf("%d ", data[i]);
321 printf("....\n");
322
323 if(dataEndianType == LITTLE_ENDIAN_DATA)
324 {
325 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I32LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
326 if (0 > H5Dwrite(idsid, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
327 }
328 else //BIG_ENDIAN_DATA
329 {
330 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I32BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
331 if (0 > H5Dwrite(idsid, H5T_STD_I32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
332 }
333 delete[] data;
334 if (0 > H5Dclose(idsid)) { printf("Error in H5Dclose"); exit(0); }
335 }
336 else if(dataType == SZ_UINT32)
337 {
338 uint32_t *data = new uint32_t[nbEle];
339 readfile(oriFilePath, nbEle, data);
340
341 printf("original data = ");
342 for(i=0;i<20;i++)
343 printf("%d ", data[i]);
344 printf("....\n");
345
346 if(dataEndianType == LITTLE_ENDIAN_DATA)
347 {
348 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U32LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
349 if (0 > H5Dwrite(idsid, H5T_STD_U32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
350 }
351 else //BIG_ENDIAN_DATA
352 {
353 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U32BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
354 if (0 > H5Dwrite(idsid, H5T_STD_U32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
355 }
356 delete[] data;
357 if (0 > H5Dclose(idsid)) { printf("Error in H5Dclose"); exit(0); }
358 }
359 else if(dataType == SZ_INT64)
360 {
361 int64_t *data = new int64_t[nbEle];
362 readfile(oriFilePath, nbEle, data);
363
364 printf("original data = ");
365 for(i=0;i<20;i++)
366 printf("%ld ", data[i]);
367 printf("....\n");
368
369 if(dataEndianType == LITTLE_ENDIAN_DATA)
370 {
371 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I64LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
372 if (0 > H5Dwrite(idsid, H5T_STD_I64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
373 }
374 else //BIG_ENDIAN_DATA
375 {
376 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_I64BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
377 if (0 > H5Dwrite(idsid, H5T_STD_I64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
378 }
379 delete[] data;
380 if (0 > H5Dclose(idsid)) { printf("Error in H5Dclose"); exit(0); }
381 }
382 else if(dataType == SZ_UINT64)
383 {
384 uint64_t *data = new uint64_t[nbEle];
385 readfile(oriFilePath, nbEle, data);
386
387 printf("original data = ");
388 for(i=0;i<20;i++)
389 printf("%ld ", data[i]);
390 printf("....\n");
391
392 if(dataEndianType == LITTLE_ENDIAN_DATA)
393 {
394 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U64LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
395 if (0 > H5Dwrite(idsid, H5T_STD_U64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
396 }
397 else //BIG_ENDIAN_DATA
398 {
399 if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_STD_U64BE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) { printf("Error in H5Dcreate"); exit(0); }
400 if (0 > H5Dwrite(idsid, H5T_STD_U64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) { printf("Error in H5Dwrite"); exit(0); }
401 }
402 delete[] data;
403 if (0 > H5Dclose(idsid)) { printf("Error in H5Dclose"); exit(0); }
404 }
405 else
406 {
407 printf("Error: unknown data type in sz3ToHDF5.cpp!\n");
408 exit(0);
409 }
410
411 /*Close and release resources*/
412 if (0 > H5Sclose(sid)) { printf("Error in H5Sclose"); exit(0); }
413 if (0 > H5Pclose(cpid)) { printf("Error in H5Pclose"); exit(0); }
414 if (0 > H5Fclose(fid)) { printf("Error in H5Fclose"); exit(0); }
415 free(cd_values);
416 printf("Output hdf5 file: %s\n", outputFilePath);
417 herr_t ret = H5Zunregister(H5Z_FILTER_SZ3);
418 if (ret < 0) return -1;
419 H5close();
420 return 0;
421 }