Debian Codegraph @

src/SZ3/tools/mdz/include/mdz.hpp

raw
1 //
2 // Created by Kai Zhao on 7/1/21.
3 //
4
5 #ifndef SZ_MDZ_H
6 #define SZ_MDZ_H
7
8 #include <SZ3/compressor/SZGeneralCompressor.hpp>
9 #include <SZ3/predictor/Predictor.hpp>
10 #include <SZ3/predictor/LorenzoPredictor.hpp>
11 #include <SZ3/predictor/RegressionPredictor.hpp>
12 #include <SZ3/predictor/ComposedPredictor.hpp>
13 #include <SZ3/utils/FileUtil.hpp>
14 #include <SZ3/utils/Timer.hpp>
15 #include <SZ3/utils/Statistic.hpp>
16 #include <SZ3/def.hpp>
17 #include <SZ3/quantizer/IntegerQuantizer.hpp>
18 #include <SZ3/lossless/Lossless_zstd.hpp>
19 #include <SZ3/encoder/HuffmanEncoder.hpp>
20 #include <SZ3/frontend/SZGeneralFrontend.hpp>
21 #include <SZ3/utils/QuantOptimizatioin.hpp>
22 #include "KmeansUtil.hpp"
23 #include "TimeBasedFrontend.hpp"
24 #include "ExaaltCompressor.hpp"
25
26 double total_compress_time = 0;
27 double total_decompress_time = 0;
28 const char *compressor_names[] = {"VQ", "VQT", "MT", "LR", "TS"};
29
30
31 template<typename T, uint N, class Predictor>
32 SZ::concepts::CompressorInterface<T> *
33 make_sz_timebased2(const SZ::Config &conf, Predictor predictor, T *data_ts0) {
34 return new SZ::SZGeneralCompressor<T, N, SZ::TimeBasedFrontend<T, N, Predictor, SZ::LinearQuantizer<T>>,
35 SZ::HuffmanEncoder<int>, SZ::Lossless_zstd>(
36 SZ::TimeBasedFrontend<T, N, Predictor, SZ::LinearQuantizer<T>>(conf, predictor,
37 SZ::LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2),
38 data_ts0),
39 SZ::HuffmanEncoder<int>(),
40 SZ::Lossless_zstd());
41 }
42
43 template<typename T, uint N>
44 SZ::concepts::CompressorInterface<T> *
45 make_sz_timebased(const SZ::Config &conf, T *data_ts0) {
46 std::vector<std::shared_ptr<SZ::concepts::PredictorInterface<T, N - 1>>> predictors;
47
48 int use_single_predictor =
49 (conf.lorenzo + conf.lorenzo2 + conf.regression) == 1;
50 if (conf.lorenzo) {
51 if (use_single_predictor) {
52 return make_sz_timebased2<T, N>(conf, SZ::LorenzoPredictor<T, N - 1, 1>(conf.absErrorBound), data_ts0);
53 } else {
54 predictors.push_back(std::make_shared<SZ::LorenzoPredictor<T, N - 1, 1>>(conf.absErrorBound));
55 }
56 }
57 if (conf.lorenzo2) {
58 if (use_single_predictor) {
59 return make_sz_timebased2<T, N>(conf, SZ::LorenzoPredictor<T, N - 1, 2>(conf.absErrorBound), data_ts0);
60 } else {
61 predictors.push_back(std::make_shared<SZ::LorenzoPredictor<T, N - 1, 2>>(conf.absErrorBound));
62 }
63 }
64 if (conf.regression) {
65 if (use_single_predictor) {
66 return make_sz_timebased2<T, N>(conf, SZ::RegressionPredictor<T, N - 1>(conf.blockSize, conf.absErrorBound), data_ts0);
67 } else {
68 predictors.push_back(std::make_shared<SZ::RegressionPredictor<T, N - 1>>(conf.blockSize, conf.absErrorBound));
69 }
70 }
71
72 return make_sz_timebased2<T, N>(conf, SZ::ComposedPredictor<T, N - 1>(predictors), data_ts0);
73 }
74
75 template<typename T, uint N, class Predictor>
76 SZ::concepts::CompressorInterface<T> *
77 make_sz2(const SZ::Config &conf, Predictor predictor) {
78
79 return new SZ::SZGeneralCompressor<T, N, SZ::SZGeneralFrontend<T, N, Predictor, SZ::LinearQuantizer<T>>,
80 SZ::HuffmanEncoder<int>, SZ::Lossless_zstd>(
81 SZ::SZGeneralFrontend<T, N, Predictor, SZ::LinearQuantizer<T>>(conf, predictor,
82 SZ::LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2)),
83 SZ::HuffmanEncoder<int>(),
84 SZ::Lossless_zstd());
85 }
86
87 template<typename T, uint N>
88 SZ::concepts::CompressorInterface<T> *
89 make_sz(const SZ::Config &conf) {
90 std::vector<std::shared_ptr<SZ::concepts::PredictorInterface<T, N>>> predictors;
91
92 int use_single_predictor =
93 (conf.lorenzo + conf.lorenzo2 + conf.regression) == 1;
94 if (conf.lorenzo) {
95 if (use_single_predictor) {
96 return make_sz2<T, N>(conf, SZ::LorenzoPredictor<T, N, 1>(conf.absErrorBound));
97 } else {
98 predictors.push_back(std::make_shared<SZ::LorenzoPredictor<T, N, 1>>(conf.absErrorBound));
99 }
100 }
101 if (conf.lorenzo2) {
102 if (use_single_predictor) {
103 return make_sz2<T, N>(conf, SZ::LorenzoPredictor<T, N, 2>(conf.absErrorBound));
104 } else {
105 predictors.push_back(std::make_shared<SZ::LorenzoPredictor<T, N, 2>>(conf.absErrorBound));
106 }
107 }
108 if (conf.regression) {
109 if (use_single_predictor) {
110 return make_sz2<T, N>(conf, SZ::RegressionPredictor<T, N>(conf.blockSize, conf.absErrorBound));
111 } else {
112 predictors.push_back(std::make_shared<SZ::RegressionPredictor<T, N >>(conf.blockSize, conf.absErrorBound));
113 }
114 }
115
116 return make_sz2<T, N>(conf, SZ::ComposedPredictor<T, N>(predictors));
117 }
118
119
120 template<typename T, uint N>
121 float *
122 VQ(SZ::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom,
123 int method, float level_start, float level_offset, int level_num) {
124 if (level_num == 0) {
125 printf("VQ/VQT not availble on current dataset, please use ADP or MT\n");
126 exit(0);
127 }
128
129 auto sz = SZ::SZ_Exaalt_Compressor<float, N, SZ::LinearQuantizer<float>, SZ::HuffmanEncoder<int>,
130 SZ::Lossless_zstd>(conf, SZ::LinearQuantizer<float>(conf.absErrorBound, conf.quantbinCnt / 2),
131 SZ::HuffmanEncoder<int>(), SZ::Lossless_zstd(), method);
132 sz.set_level(level_start, level_offset, level_num);
133
134 SZ::Timer timer(true);
135 SZ::uchar *compressed;
136 compressed = sz.compress(data, compressed_size);
137 total_compress_time += timer.stop("Compression");
138 if (!decom) {
139 delete[]compressed;
140 return nullptr;
141 }
142 auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size;
143 std::cout << "Compression Ratio = " << ratio << std::endl;
144 std::cout << "Compressed size = " << compressed_size << std::endl;
145
146 timer.start();
147 auto ts_dec_data = sz.decompress(compressed, compressed_size);
148 total_decompress_time += timer.stop("Decompression");
149
150 delete[]compressed;
151 return ts_dec_data;
152 }
153
154
155 template<typename T, uint N>
156 float *
157 MT(SZ::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, T *ts0) {
158 // printf("eb=%.8f\n", conf.eb);
159 auto sz = make_sz_timebased<T, N>(conf, ts0);
160
161 SZ::uchar *compressed;
162 SZ::Timer timer(true);
163 compressed = sz->compress(conf, data, compressed_size);
164 total_compress_time += timer.stop("Compression");
165 if (!decom) {
166 delete[] compressed;
167 return nullptr;
168 }
169
170 auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size;
171 std::cout << "Compression Ratio = " << ratio << std::endl;
172 std::cout << "Compressed size = " << compressed_size << std::endl;
173
174
175 timer.start();
176 auto ts_dec_data = sz->decompress(compressed, compressed_size, conf.num);
177 total_decompress_time += timer.stop("Decompression");
178
179 delete sz;
180 delete[] compressed;
181 return ts_dec_data;
182 }
183
184 template<typename T, uint N>
185 float *
186 SZ2(SZ::Config conf, size_t ts, T *data, size_t &compressed_size, bool decom) {
187 auto sz = make_sz<T, N>(conf);
188 SZ::uchar *compressed;
189
190 SZ::Timer timer(true);
191 compressed = sz->compress(conf, data, compressed_size);
192 total_compress_time += timer.stop("Compression");
193 if (!decom) {
194 delete[] compressed;
195 return nullptr;
196 }
197
198 auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size;
199 std::cout << "Compression Ratio = " << ratio << std::endl;
200 std::cout << "Compressed size = " << compressed_size << std::endl;
201
202 timer.start();
203 auto ts_dec_data = sz->decompress(compressed, compressed_size, conf.num);
204 total_decompress_time += timer.stop("Decompression");
205
206 delete sz;
207 delete[] compressed;
208 return ts_dec_data;
209 }
210
211
212 template<typename T, uint N>
213 void select(SZ::Config conf, int &method, size_t ts, T *data_all,
214 float level_start, float level_offset, int level_num, T *data_ts0, size_t timestep_batch) {
215 // && (ts_last_select == -1 || t - ts_last_select >= conf.timestep_batch * 10)) {
216 std::cout << "****************** BEGIN Selection ****************" << std::endl;
217 // ts_last_select = ts;
218 std::vector<size_t> compressed_size(10, std::numeric_limits<size_t>::max());
219 std::vector<T> data1;
220 size_t t = ts;
221 if (ts == 0) {
222 t = conf.dims[0] / 2;
223 conf.dims[0] /= 2;
224 }
225 if (timestep_batch > 10) {
226 conf.dims[0] = 10;
227 }
228 conf.num = conf.dims[0] * conf.dims[1];
229
230 std::cout << conf.dims[0] << " " << conf.dims[1] << " " << t << std::endl;
231
232 if (level_num > 0) {
233 data1 = std::vector(&data_all[t * conf.dims[1]], &data_all[t * conf.dims[1]] + conf.num);
234 VQ<T, N>(conf, t, data1.data(), compressed_size[0], false, 0, level_start, level_offset, level_num);
235
236 data1 = std::vector(&data_all[t * conf.dims[1]], &data_all[t * conf.dims[1]] + conf.num);
237 VQ<T, N>(conf, t, data1.data(), compressed_size[1], false, 1, level_start, level_offset, level_num);
238 } else {
239 data1 = std::vector(&data_all[t * conf.dims[1]], &data_all[t * conf.dims[1]] + conf.num);
240 SZ2<T, N>(conf, t, data1.data(), compressed_size[3], false);
241 }
242
243 data1 = std::vector(&data_all[t * conf.dims[1]], &data_all[t * conf.dims[1]] + conf.num);
244 MT<T, N>(conf, t, data1.data(), compressed_size[2], false, data_ts0);
245
246 // data1 = std::vector(&data_all[t * conf.dims[1]], &data_all[t * conf.dims[1]] + conf.num);
247 // MT(conf, t, data1.data(), compressed_size[4], false, (T *) nullptr);
248
249 method = std::distance(compressed_size.begin(),
250 std::min_element(compressed_size.begin(), compressed_size.end()));
251 printf("Select %s as Compressor, timestep=%lu, method=%d\n",
252 compressor_names[method],
253 ts, method);
254 std::cout << "****************** END Selection ****************" << std::endl;
255 }
256
257 template<typename Type>
258 std::unique_ptr<Type[]> readfile(const char *file, size_t start, size_t num) {
259 std::ifstream fin(file, std::ios::binary);
260 if (!fin) {
261 std::cout << " Error, Couldn't find the file" << "\n";
262 return 0;
263 }
264 fin.seekg(0, std::ios::end);
265 const size_t total_num_elements = fin.tellg() / sizeof(Type);
266 assert(start + num <= total_num_elements);
267 fin.seekg(start * sizeof(Type), std::ios::beg);
268 auto data = std::make_unique<Type[]>(num);
269 fin.read(reinterpret_cast<char *>(&data[0]), num * sizeof(Type));
270 fin.close();
271 return data;
272 }
273
274
275 template<typename T, uint N>
276 SZ::uchar *LAMMPS_compress(SZ::Config conf, T *data, int method, size_t &compressed_size,
277 float level_start, float level_offset, int level_num, T *ts0) {
278 if ((method == 0 || method == 1) && level_num == 0) {
279 printf("VQ/VQT not available on current dataset, please use ADP or MT\n");
280 exit(0);
281 }
282 SZ::uchar *compressed_data;
283 if (method == 0 || method == 1) {
284 auto sz = SZ::SZ_Exaalt_Compressor<T, N, SZ::LinearQuantizer<float>, SZ::HuffmanEncoder<int>, SZ::Lossless_zstd>(
285 conf, SZ::LinearQuantizer<float>(conf.absErrorBound, conf.quantbinCnt / 2),
286 SZ::HuffmanEncoder<int>(), SZ::Lossless_zstd(), method);
287 sz.set_level(level_start, level_offset, level_num);
288 compressed_data = sz.compress(data, compressed_size);
289 } else if (method == 2 || method == 4) {
290 auto sz = make_sz_timebased<T, N>(conf, ts0);
291 compressed_data = sz->compress(conf, data, compressed_size);
292 } else {
293 auto sz = make_sz<T, N>(conf);
294 compressed_data = sz->compress(conf, data, compressed_size);
295 }
296 // auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size;
297 // std::cout << "Compression Ratio = " << ratio << std::endl;
298 // std::cout << "Compressed size = " << compressed_size << std::endl;
299 return compressed_data;
300 }
301
302 template<typename T, uint N>
303 int LAMMPS_select_compressor(SZ::Config conf, T *data, bool firsttime,
304 float level_start, float level_offset, int level_num, T *data_ts0) {
305 std::cout << "****************** BEGIN Selection ****************" << std::endl;
306
307 std::vector<size_t> compressed_size(10, std::numeric_limits<size_t>::max());
308
309 if (firsttime) {
310 conf.dims[0] /= 2;
311 conf.num = conf.dims[0] * conf.dims[1];
312 data += conf.num;
313 }
314 if (conf.dims[0] > 10) {
315 conf.dims[0] = 10;
316 conf.num = conf.dims[0] * conf.dims[1];
317 }
318
319 std::vector<T> data1;
320 SZ::uchar *cmpr;
321 if (level_num > 0) {
322
323 data1 = std::vector<T>(data, data + conf.num);
324 cmpr = LAMMPS_compress<T, N>(conf, data1.data(), 0, compressed_size[0], level_start, level_offset, level_num, data_ts0);
325 delete[] cmpr;
326
327 data1 = std::vector<T>(data, data + conf.num);
328 cmpr = LAMMPS_compress<T, N>(conf, data1.data(), 1, compressed_size[1], level_start, level_offset, level_num, data_ts0);
329 delete[] cmpr;
330 } else {
331 data1 = std::vector<T>(data, data + conf.num);
332 cmpr = LAMMPS_compress<T, N>(conf, data1.data(), 3, compressed_size[3], level_start, level_offset, level_num, data_ts0);
333 delete[] cmpr;
334 }
335
336 data1 = std::vector<T>(data, data + conf.num);
337 cmpr = LAMMPS_compress<T, N>(conf, data1.data(), 2, compressed_size[2], level_start, level_offset, level_num, data_ts0);
338 delete[] cmpr;
339
340 // data1 = std::vector(&data[t * conf.dims[1]], &data[t * conf.dims[1]] + conf.num);
341 // MT(conf, t, data1.data(), compressed_size[4], false, (T *) nullptr);
342
343 int method = std::distance(compressed_size.begin(),
344 std::min_element(compressed_size.begin(), compressed_size.end()));
345 printf("Select %s as Compressor, method=%d\n", compressor_names[method], method);
346 std::cout << "****************** END Selection ****************" << std::endl;
347 return method;
348 }
349
350 template<typename T, uint N>
351 size_t MDZ_Compress(SZ::Config conf, T *input_data, T *dec_data, size_t batch_size, int method = -1) {
352 if (N != 2) {
353 throw std::invalid_argument("dimension should be 2");
354 }
355 if (batch_size == 0) {
356 batch_size = conf.dims[0];
357 }
358 int method_batch = 0;
359 if (method == -1) {
360 method_batch = 50;
361 }
362 std::cout << "****************** Options ********************" << std::endl;
363 std::cout << "dimension = " << N
364 << ", error bound = " << conf.absErrorBound
365 << ", method = " << method
366 << ", method_update_batch = " << method_batch
367 << ", batch_size = " << batch_size
368 << ", quan_state_num = " << conf.quantbinCnt
369 // << ", encoder = " << conf.encoder_op
370 // << ", lossless = " << conf.lossless_op
371 << std::endl;
372
373 // auto data_all = readfile<T>(input_path.data(), 0, conf.num);
374 // auto data_all = input_data;
375 auto data_ts0 = std::vector<T>(input_data, input_data + conf.dims[1]);
376
377 float level_start, level_offset;
378 int level_num = 0;
379 if (method != 2 && method != 3 && method != 4) {
380 size_t sample_num = 0.1 * conf.dims[1];
381 sample_num = std::min(sample_num, (size_t) 20000);
382 sample_num = std::max(sample_num, std::min((size_t) 5000, conf.dims[1]));
383 SZ::get_cluster(input_data, conf.dims[1], level_start, level_offset, level_num,
384 sample_num);
385 if (level_num > conf.dims[1] * 0.25) {
386 level_num = 0;
387 }
388 if (level_num != 0) {
389 // printf("start = %.3f , level_offset = %.3f, nlevel=%d\n", level_start, level_offset, level_num);
390 }
391 }
392
393 auto dims = conf.dims;
394 auto total_num = conf.num;
395 // std::vector<T> dec_data(total_num);
396 size_t total_compressed_size = 0;
397 double compressed_size_pre = total_num * sizeof(T);
398 int current_method = method;
399
400 for (size_t ts = 0; ts < dims[0]; ts += batch_size) {
401 conf.dims[0] = (ts + batch_size > dims[0] ? dims[0] - ts : batch_size);
402 conf.num = conf.dims[0] * conf.dims[1];
403
404 // auto data = SZ::readfile<T>(conf.input_path.data(), ts * conf.dims[1], conf.num);
405 T *data = &input_data[ts * conf.dims[1]];
406
407 T max = *std::max_element(data, data + conf.num);
408 T min = *std::min_element(data, data + conf.num);
409 if (conf.errorBoundMode == SZ::EB_ABS) {
410 conf.relErrorBound = conf.absErrorBound / (max - min);
411 } else if (conf.errorBoundMode == SZ::EB_REL) {
412 conf.absErrorBound = conf.relErrorBound * (max - min);
413 }
414
415 std::cout << "****************** Compression From " << ts << " to " << ts + conf.dims[0] - 1
416 << " ******************" << std::endl;
417 // std::cout<<method_batch<<" "<<ts<<" "<<conf.batch_size<<" "<<method_batch<<std::endl;
418 if (method_batch > 0 && ts / batch_size % method_batch == 0) {
419 select<T, N>(conf, current_method, ts, input_data, level_start, level_offset, level_num, data_ts0.data(), batch_size);
420 }
421 printf("Compressor = %s\n", compressor_names[current_method]);
422
423 T *ts_dec_data;
424 size_t compressed_size;
425 if (current_method == 0) {
426 ts_dec_data = VQ<T, N>(conf, ts, data, compressed_size, true, current_method, level_start, level_offset,
427 level_num);
428 } else if (current_method == 1) {
429 ts_dec_data = VQ<T, N>(conf, ts, data, compressed_size, true, current_method, level_start, level_offset,
430 level_num);
431 } else if (current_method == 2) {
432 ts_dec_data = MT<T, N>(conf, ts, data, compressed_size, true, data_ts0.data());
433 } else if (current_method == 4) {
434 ts_dec_data = MT<T, N>(conf, ts, data, compressed_size, true, (T *) nullptr);
435 } else {
436 ts_dec_data = SZ2<T, N>(conf, ts, data, compressed_size, true);
437 }
438 total_compressed_size += compressed_size;
439 // if (compressed_size > 4.0 * compressed_size_pre) {
440 // select(conf, current_method, ts, data_all.get(), level_start, level_offset, level_num, data_ts0.get());
441 // }
442 compressed_size_pre = compressed_size;
443 memcpy(&dec_data[ts * conf.dims[1]], ts_dec_data, conf.num * sizeof(T));
444 }
445
446
447 return total_compressed_size;
448 }
449
450 #endif //SZ3_MDZ_H