Debian Codegraph @

src/SZ3/tools/sz3/deprecated/beta_sz_pw.cpp

raw
1 #include "SZ3/frontend/SZGeneralFrontend.hpp"
2 #include "SZ3/predictor/Predictor.hpp"
3 #include "SZ3/predictor/LorenzoPredictor.hpp"
4 #include "SZ3/predictor/RegressionPredictor.hpp"
5 #include "SZ3/predictor/ComposedPredictor.hpp"
6 #include "SZ3/quantizer/IntegerQuantizer.hpp"
7 #include "SZ3/utils/FileUtil.hpp"
8 #include "SZ3/utils/Config.hpp"
9 #include "SZ3/def.hpp"
10 #include "SZ3/compressor/SZGeneralCompressor.hpp"
11 #include "SZ3/encoder/HuffmanEncoder.hpp"
12 #include "SZ3/encoder/ArithmeticEncoder.hpp"
13 #include "SZ3/encoder/BypassEncoder.hpp"
14 #include "SZ3/lossless/Lossless_zstd.hpp"
15 #include "SZ3/lossless/Lossless_bypass.hpp"
16 #include "SZ3/utils/Statistic.hpp"
17 #include "SZ3/utils/Timer.hpp"
18 #include "SZ3/version.hpp"
19 #include <sstream>
20 #include <random>
21 #include <cstdio>
22 #include <iostream>
23 #include <memory>
24
25 std::string src_file_name;
26 float relative_error_bound = 0;
27 float compression_time = 0;
28
29
30 template<typename T, uint N, class Frontend, class Encoder, class Lossless>
31 float SZ_compress(std::unique_ptr<T[]> const &data,
32 SZ::Config &conf,
33 Frontend frontend, Encoder encoder, Lossless lossless) {
34
35 std::cout << "****************** Options ********************" << std::endl;
36 std::cout << "dimension = " << N
37 << ", error bound = " << conf.absErrorBound
38 << ", blockSize = " << conf.blockSize
39 << ", stride = " << conf.stride
40 << ", quan_state_num = " << conf.quantbinCnt
41 << std::endl
42 << "lorenzo = " << conf.lorenzo
43 << ", 2ndlorenzo = " << conf.lorenzo2
44 << ", regression = " << conf.regression
45 << ", encoder = " << conf.encoder
46 << ", lossless = " << conf.lossless
47 << std::endl;
48
49
50 std::vector<T> data_ = std::vector<T>(data.get(), data.get() + conf.num);
51 // preprocess
52 std::vector<unsigned char> signs(conf.num, 0);
53 float max_abs_log_data = 0;
54 float min_log_data = std::numeric_limits<float>::max();
55 bool positive = true;
56 for (size_t i = 0; i < conf.num; i++) {
57 if (data[i] == 0) {
58 signs[i] = 2;
59 }
60 if (data[i] < 0) {
61 signs[i] = 1;
62 data[i] = -data[i];
63 positive = false;
64 }
65 if (data[i] > 0) {
66 data[i] = log2(data[i]);
67 if (data[i] > max_abs_log_data) max_abs_log_data = data[i];
68 if (data[i] < min_log_data) min_log_data = data[i];
69 }
70
71 }
72
73 if (fabs(min_log_data) > max_abs_log_data) max_abs_log_data = fabs(min_log_data);
74 double realPrecision = log2(1.0 + conf.absErrorBound) - max_abs_log_data * 1.2e-7;
75 conf.absErrorBound = realPrecision;
76 for (size_t i = 0; i < conf.num; i++) {
77 if (signs[i] == 2) {
78 data[i] = min_log_data - 2.0001 * realPrecision;
79 signs[i] = 0;
80 }
81 }
82
83 double minLogValue = min_log_data - 1.0001 * realPrecision;
84 std::cout << "log eb=" << conf.absErrorBound
85 << ", minlogvalue=" << minLogValue << std::endl;
86 if (!positive) {
87 // compress signs
88 // uint64_t signSize = sz_lossless_compress(ZSTD_COMPRESSOR, 3, signs, dataLength, &comp_signs);
89 }
90
91 // std::vector<T> data1 = std::vector<T>(data.get(), data.get() + conf.num);
92 auto sz = SZ::make_sz_general_compressor<T, N>(frontend, encoder, lossless);
93
94 SZ::Timer timer;
95 timer.start();
96 std::cout << "****************** Compression ******************" << std::endl;
97
98
99 size_t compressed_size = 0;
100 std::unique_ptr<SZ::uchar[]> compressed;
101 compressed.reset(sz->compress(conf, data.get(), compressed_size));
102
103 compression_time = timer.stop("Compression");
104
105 auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size;
106 std::cout << "Compression Ratio = " << ratio << std::endl;
107 std::cout << "Compressed size = " << compressed_size << std::endl;
108
109 std::random_device rd; //Will be used to obtain a seed for the random number engine
110 std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
111 std::uniform_int_distribution<> dis(0, 10000);
112 std::stringstream ss;
113 ss << src_file_name.substr(src_file_name.rfind('/') + 1)
114 << "." << relative_error_bound << "." << dis(gen) << ".sz3";
115 auto compressed_file_name = ss.str();
116 SZ::writefile(compressed_file_name.c_str(), compressed.get(), compressed_size);
117 std::cout << "Compressed file = " << compressed_file_name << std::endl;
118
119 std::cout << "****************** Decompression ****************" << std::endl;
120 compressed = SZ::readfile<SZ::uchar>(compressed_file_name.c_str(), compressed_size);
121
122 timer.start();
123 T *dec_data = sz->decompress(compressed.get(), compressed_size, conf.num);
124 timer.stop("Decompression");
125 // SZ::verify<T>(data1.data(), dec_data, conf.num);
126
127 // float threshold = minLogValue;
128 if (!positive) {
129 // sz_lossless_decompress(ZSTD_COMPRESSOR, tdps->pwrErrBoundBytes, tdps->pwrErrBoundBytes_size, &signs, dataSeriesLength);
130 for (size_t i = 0; i < conf.num; i++) {
131 if (dec_data[i] < minLogValue) dec_data[i] = 0;
132 else dec_data[i] = exp2(dec_data[i]);
133 if (signs[i]) dec_data[i] = -dec_data[i];
134 }
135 } else {
136 for (size_t i = 0; i < conf.num; i++) {
137 if (dec_data[i] < minLogValue) dec_data[i] = 0;
138 else dec_data[i] = exp2(dec_data[i]);
139 }
140 }
141
142 SZ::verify<T>(data_.data(), dec_data, conf.num);
143
144 remove(compressed_file_name.c_str());
145 auto decompressed_file_name = compressed_file_name + ".out";
146 SZ::writefile(decompressed_file_name.c_str(), dec_data, conf.num);
147 std::cout << "Decompressed file = " << decompressed_file_name << std::endl;
148
149 delete[] dec_data;
150 return ratio;
151 }
152
153 template<typename T, uint N, class Frontend>
154 float SZ_compress_build_backend(std::unique_ptr<T[]> const &data,
155 SZ::Config &conf,
156 Frontend frontend) {
157 if (conf.lossless == 1) {
158 if (conf.encoder == 1) {
159 return SZ_compress<T, N>(data, conf, frontend, SZ::HuffmanEncoder<int>(), SZ::Lossless_zstd());
160 } else if (conf.encoder == 2) {
161 return SZ_compress<T, N>(data, conf, frontend, SZ::ArithmeticEncoder<int>(true), SZ::Lossless_zstd());
162 } else {
163 return SZ_compress<T, N>(data, conf, frontend, SZ::BypassEncoder<int>(), SZ::Lossless_zstd());
164 }
165 } else {
166 if (conf.encoder == 1) {
167 return SZ_compress<T, N>(data, conf, frontend, SZ::HuffmanEncoder<int>(), SZ::Lossless_bypass());
168 } else if (conf.encoder == 2) {
169 return SZ_compress<T, N>(data, conf, frontend, SZ::ArithmeticEncoder<int>(true), SZ::Lossless_bypass());
170 } else {
171 return SZ_compress<T, N>(data, conf, frontend, SZ::BypassEncoder<int>(), SZ::Lossless_bypass());
172 }
173 }
174 }
175
176 template<typename T, uint N>
177 float SZ_compress_build_frontend(std::unique_ptr<T[]> const &data, SZ::Config &conf) {
178 auto quantizer = SZ::LinearQuantizer<T>(conf.absErrorBound, conf.quantbinCnt / 2);
179 std::vector<std::shared_ptr<SZ::concepts::PredictorInterface<T, N>>> predictors;
180
181 int use_single_predictor =
182 (conf.lorenzo + conf.lorenzo2 + conf.regression) == 1;
183 if (conf.lorenzo) {
184 if (use_single_predictor) {
185 return SZ_compress_build_backend<T, N>(data, conf,
186 SZ::make_sz_general_frontend<T, N>(conf, SZ::LorenzoPredictor<T, N, 1>(conf.absErrorBound),
187 quantizer));
188 } else {
189 predictors.push_back(std::make_shared<SZ::LorenzoPredictor<T, N, 1>>(conf.absErrorBound));
190 }
191 }
192 if (conf.lorenzo2) {
193 if (use_single_predictor) {
194 return SZ_compress_build_backend<T, N>(data, conf,
195 SZ::make_sz_general_frontend<T, N>(conf, SZ::LorenzoPredictor<T, N, 2>(conf.absErrorBound),
196 quantizer));
197 } else {
198 predictors.push_back(std::make_shared<SZ::LorenzoPredictor<T, N, 2>>(conf.absErrorBound));
199 }
200 }
201 if (conf.regression) {
202 if (use_single_predictor) {
203 return SZ_compress_build_backend<T, N>(data, conf,
204 SZ::make_sz_general_frontend<T, N>(conf, SZ::RegressionPredictor<T, N>(conf.blockSize,
205 conf.absErrorBound),
206 quantizer));
207 } else {
208 predictors.push_back(std::make_shared<SZ::RegressionPredictor<T, N>>(conf.blockSize, conf.absErrorBound));
209 }
210 }
211
212 return SZ_compress_build_backend<T, N>(data, conf,
213 SZ::make_sz_general_frontend<T, N>(conf, SZ::ComposedPredictor<T, N>(predictors), quantizer));
214 }
215
216
217 template<class T, uint N>
218 float SZ_compress_parse_args(int argc, char **argv, int argp, std::unique_ptr<T[]> &data, float eb,
219 std::array<size_t, N> dims) {
220 SZ::Config conf;
221 conf.setDims(dims.begin(), dims.end());
222 conf.absErrorBound = eb;
223 if (argp < argc) {
224 int block_size = atoi(argv[argp++]);
225 conf.blockSize = block_size;
226 conf.stride = block_size;
227 }
228
229 int lorenzo_op = 1;
230 if (argp < argc) {
231 lorenzo_op = atoi(argv[argp++]);
232 conf.lorenzo = lorenzo_op == 1 || lorenzo_op == 3;
233 conf.lorenzo2 = lorenzo_op == 2 || lorenzo_op == 3;
234 }
235
236 int regression_op = 1;
237 if (argp < argc) {
238 regression_op = atoi(argv[argp++]);
239 conf.regression = regression_op == 1;
240 }
241
242 if (argp < argc) {
243 conf.encoder = atoi(argv[argp++]);
244 if (conf.encoder == 0) {
245 conf.quantbinCnt = 250;
246 } else if (conf.encoder == 2) {
247 conf.quantbinCnt = 1024;
248 }
249 }
250
251 if (argp < argc) {
252 conf.lossless = atoi(argv[argp++]);
253 }
254
255 if (argp < argc) {
256 conf.quantbinCnt = atoi(argv[argp++]);
257 }
258
259 auto ratio = SZ_compress_build_frontend<T, N>(data, conf);
260 return ratio;
261 }
262
263 int main(int argc, char **argv) {
264 if (argc < 2) {
265 std::cout << "SZ v" << SZ3_VER << std::endl;
266 std::cout << "usage: " << argv[0] <<
267 " data_file -num_dim dim0 .. dimn relative_eb [blocksize lorenzo_op regression_op encoder lossless quantbinCnt]"
268 << std::endl;
269 std::cout << "example: " << argv[0] <<
270 " qmcpack.dat -3 33120 69 69 1e-3 [6 1 1 1 1 32768]" << std::endl;
271 return 0;
272 }
273
274 size_t num = 0;
275 auto data = SZ::readfile<float>(argv[1], num);
276 src_file_name = argv[1];
277 std::cout << "Read " << num << " elements\n";
278
279 int dim = atoi(argv[2] + 1);
280 assert(1 <= dim && dim <= 4);
281 int argp = 3;
282 std::vector<size_t> dims(dim);
283 for (int i = 0; i < dim; i++) {
284 dims[i] = atoi(argv[argp++]);
285 }
286
287 // char *eb_op = argv[argp++] + 1;
288 float eb = 0;
289 // if (*eb_op == 'a') {
290 // eb = atof(argv[argp++]);
291 // } else {
292 eb = atof(argv[argp++]);
293 // float max = data[0];
294 // float min = data[0];
295 // for (int i = 1; i < num; i++) {
296 // if (max < data[i]) max = data[i];
297 // if (min > data[i]) min = data[i];
298 // }
299 // eb = relative_error_bound * (max - min);
300 // }
301
302 if (dim == 1) {
303 SZ_compress_parse_args<float, 1>(argc, argv, argp, data, eb, std::array<size_t, 1>{dims[0]});
304 } else if (dim == 2) {
305 SZ_compress_parse_args<float, 2>(argc, argv, argp, data, eb, std::array<size_t, 2>{dims[0], dims[1]});
306 } else if (dim == 3) {
307 SZ_compress_parse_args<float, 3>(argc, argv, argp, data, eb,
308 std::array<size_t, 3>{dims[0], dims[1], dims[2]});
309 } else if (dim == 4) {
310 SZ_compress_parse_args<float, 4>(argc, argv, argp, data, eb,
311 std::array<size_t, 4>{dims[0], dims[1], dims[2], dims[3]});
312 }
313
314
315 return 0;
316 }