Debian Codegraph @

src/SZ3/tools/mdz/include/ExaaltCompressor.hpp

raw
1 #ifndef _SZ_EXAALT_COMPRESSSOR_HPP
2 #define _SZ_EXAALT_COMPRESSSOR_HPP
3
4 #include <SZ3/predictor/Predictor.hpp>
5 #include <SZ3/predictor/LorenzoPredictor.hpp>
6 #include <SZ3/quantizer/Quantizer.hpp>
7 #include <SZ3/encoder/Encoder.hpp>
8 #include <SZ3/lossless/Lossless.hpp>
9 #include <SZ3/utils/Iterator.hpp>
10 #include <SZ3/utils/MemoryUtil.hpp>
11 #include <SZ3/utils/Config.hpp>
12 #include <SZ3/def.hpp>
13
14 namespace SZ {
15 template<class T, uint N, class Quantizer, class Encoder, class Lossless>
16 class SZ_Exaalt_Compressor {
17 public:
18
19
20 SZ_Exaalt_Compressor(const Config &conf,
21 Quantizer quantizer,
22 Encoder encoder,
23 Lossless lossless,
24 int timestep_op) :
25 quantizer(quantizer), encoder(encoder), lossless(lossless),
26 num_elements(conf.num), timestep_op(timestep_op) {
27 static_assert(std::is_base_of<concepts::QuantizerInterface<T>, Quantizer>::value,
28 "must implement the quatizer interface");
29 static_assert(std::is_base_of<concepts::EncoderInterface<int>, Encoder>::value,
30 "must implement the encoder interface");
31 static_assert(std::is_base_of<concepts::LosslessInterface, Lossless>::value,
32 "must implement the lossless interface");
33 assert(!(timestep_op > 0 && conf.dims.size() != 2) &&
34 "timestep prediction requires 2d dataset");
35 global_dimensions[0] = conf.dims[0];
36 global_dimensions[1] = conf.dims[1];
37 }
38
39 void set_level(float level_start_, float level_offset_, int level_num_) {
40 this->level_start = level_start_;
41 this->level_offset = level_offset_;
42 this->level_num = level_num_ + 200;
43 }
44
45 inline int quantize_to_level(T data) {
46 return round((data - level_start) / level_offset);
47 }
48
49 inline T level(int l) {
50 return level_start + l * level_offset;
51 }
52
53 // compress given the error bound
54 uchar *compress(T *data, size_t &compressed_size) {
55
56 std::vector<int> quant_inds(num_elements);
57 std::vector<int> pred_inds(num_elements);
58 quantizer.precompress_data();
59
60 // Timer timer(true);
61
62 auto l0 = quantize_to_level(data[0]);
63 pred_inds[0] = l0 + level_num;
64 quant_inds[0] = quantizer.quantize_and_overwrite(data[0], level(l0));
65
66 if (timestep_op == 0) {
67 for (size_t i = 1; i < num_elements; i++) {
68 auto l = quantize_to_level(data[i]);
69 pred_inds[i] = l - l0 + level_num;
70 quant_inds[i] = quantizer.quantize_and_overwrite(data[i], level(l));
71 l0 = l;
72 }
73 } else {
74
75 std::vector<int> levels(global_dimensions[1]);
76 levels[0] = l0;
77 for (size_t i = 1; i < global_dimensions[1]; i++) {
78 levels[i] = quantize_to_level(data[i]);
79 pred_inds[i] = levels[i] - levels[i - 1] + level_num;
80 quant_inds[i] = quantizer.quantize_and_overwrite(data[i], level(levels[i]));
81 }
82 auto pred_idx = global_dimensions[1];
83 if (timestep_op == 1) {
84 for (size_t i = 0; i < global_dimensions[1]; i++) {
85 for (size_t t = 1; t < global_dimensions[0]; t++) {
86 size_t idx = t * global_dimensions[1] + i;
87 quant_inds[pred_idx++] = quantizer.quantize_and_overwrite(
88 data[idx], data[(t - 1) * global_dimensions[1] + i]);
89 }
90 }
91 pred_inds.resize(global_dimensions[1]);
92 } else {
93 for (size_t i = 0; i < global_dimensions[1]; i++) {
94 l0 = levels[i];
95 for (size_t t = 1; t < global_dimensions[0]; t++) {
96 size_t idx = t * global_dimensions[1] + i;
97 auto l = quantize_to_level(data[idx]);
98 pred_inds[pred_idx] = l - l0 + level_num;
99 quant_inds[pred_idx++] = quantizer.quantize_and_overwrite(
100 data[idx], level(l));
101 l0 = l;
102 }
103 }
104 }
105 assert(pred_idx == num_elements);
106 }
107
108 // timer.stop("Predition & Quantization");
109
110 quantizer.postcompress_data();
111
112 uchar *compressed_data;
113 compressed_data = new uchar[4 * num_elements * sizeof(T)];
114 uchar *compressed_data_pos = compressed_data;
115 write(global_dimensions.data(), N, compressed_data_pos);
116 quantizer.save(compressed_data_pos);
117 // quantizer.print();
118
119 encoder.preprocess_encode(quant_inds, 4 * quantizer.get_radius());
120 encoder.save(compressed_data_pos);
121 encoder.encode(quant_inds, compressed_data_pos);
122 encoder.postprocess_encode();
123
124 // std::cout << *std::min_element(pred_inds.begin(), pred_inds.end()) << std::endl;
125 // std::cout << *std::max_element(pred_inds.begin(), pred_inds.end()) << std::endl;
126
127 encoder.preprocess_encode(pred_inds, level_num * 2 + 1);
128 encoder.save(compressed_data_pos);
129 encoder.encode(pred_inds, compressed_data_pos);
130 encoder.postprocess_encode();
131
132 uchar *lossless_data = lossless.compress(compressed_data, compressed_data_pos - compressed_data,
133 compressed_size);
134 lossless.postcompress_data(compressed_data);
135 return lossless_data;
136 }
137
138 T *decompress(uchar const *lossless_compressed_data, const size_t length) {
139 size_t remaining_length = length;
140 auto compressed_data = lossless.decompress(lossless_compressed_data, remaining_length);
141 uchar const *compressed_data_pos = compressed_data;
142 read(global_dimensions.data(), N, compressed_data_pos, remaining_length);
143 num_elements = 1;
144 for (const auto &d: global_dimensions) {
145 num_elements *= d;
146 std::cout << d << " ";
147 }
148 std::cout << std::endl;
149 quantizer.load(compressed_data_pos, remaining_length);
150 encoder.load(compressed_data_pos, remaining_length);
151 auto quant_inds = encoder.decode(compressed_data_pos, num_elements);
152 encoder.postprocess_decode();
153
154 encoder.load(compressed_data_pos, remaining_length);
155 auto pred_inds_num = (timestep_op == 1) ? global_dimensions[1] : num_elements;
156 auto pred_inds = encoder.decode(compressed_data_pos, pred_inds_num);
157 encoder.postprocess_decode();
158
159 lossless.postdecompress_data(compressed_data);
160
161 auto dec_data = new T[num_elements];
162
163 quantizer.predecompress_data();
164
165 auto l = pred_inds[0] - level_num;
166 dec_data[0] = quantizer.recover(level(l), quant_inds[0]);
167
168 if (timestep_op == 0) {
169 for (size_t i = 1; i < num_elements; i++) {
170 l += pred_inds[i] - level_num;
171 dec_data[i] = quantizer.recover(level(l), quant_inds[i]);
172 }
173 } else {
174 std::vector<int> levels(global_dimensions[1]);
175 levels[0] = l;
176 for (size_t i = 1; i < global_dimensions[1]; i++) {
177 l += pred_inds[i] - level_num;
178 dec_data[i] = quantizer.recover(level(l), quant_inds[i]);
179 levels[i] = l;
180 }
181 auto pred_idx = global_dimensions[1];
182 if (timestep_op == 1) {
183 for (size_t i = 0; i < global_dimensions[1]; i++) {
184 for (size_t t = 1; t < global_dimensions[0]; t++) {
185 dec_data[t * global_dimensions[1] + i] =
186 quantizer.recover(
187 dec_data[(t - 1) * global_dimensions[1] + i],
188 quant_inds[pred_idx++]);
189 }
190 }
191 } else {
192 for (size_t i = 0; i < global_dimensions[1]; i++) {
193 l = levels[i];
194 for (size_t t = 1; t < global_dimensions[0]; t++) {
195 size_t idx = t * global_dimensions[1] + i;
196 l += pred_inds[pred_idx] - level_num;
197 dec_data[idx] = quantizer.recover(level(l), quant_inds[pred_idx++]);
198 }
199 }
200 }
201 assert(pred_idx == num_elements);
202 }
203
204
205 quantizer.postdecompress_data();
206 encoder.postprocess_decode();
207 return dec_data;
208 }
209
210
211 private:
212 Quantizer quantizer;
213 Encoder encoder;
214 Lossless lossless;
215 size_t num_elements;
216 std::array<size_t, N> global_dimensions;
217 float level_start;
218 float level_offset;
219 int level_num;
220 int timestep_op;
221 };
222
223
224 }
225 #endif
226