| 1 |
#ifndef SZ_TIMEBASED_FRONTEND |
| 2 |
#define SZ_TIMEBASED_FRONTEND |
| 3 |
|
| 4 |
#include <SZ3/frontend/Frontend.hpp> |
| 5 |
#include <SZ3/def.hpp> |
| 6 |
#include <SZ3/predictor/Predictor.hpp> |
| 7 |
#include <SZ3/predictor/LorenzoPredictor.hpp> |
| 8 |
#include <SZ3/quantizer/Quantizer.hpp> |
| 9 |
#include <SZ3/utils/Iterator.hpp> |
| 10 |
#include <SZ3/utils/Config.hpp> |
| 11 |
#include <SZ3/utils/MemoryUtil.hpp> |
| 12 |
|
| 13 |
namespace SZ { |
| 14 |
|
| 15 |
|
| 16 |
template<class T, uint N, class Predictor, class Quantizer> |
| 17 |
class TimeBasedFrontend : public concepts::FrontendInterface<T, N> { |
| 18 |
public: |
| 19 |
|
| 20 |
TimeBasedFrontend(const Config &conf, Predictor predictor, Quantizer quantizer, T *data_ts0) : |
| 21 |
fallback_predictor(LorenzoPredictor<T, N - 1, 1>(conf.absErrorBound)), |
| 22 |
predictor(predictor), |
| 23 |
quantizer(quantizer), |
| 24 |
block_size(conf.blockSize), |
| 25 |
stride(conf.stride), |
| 26 |
num_elements(conf.num), |
| 27 |
data_ts0(data_ts0) { |
| 28 |
assert((conf.dims.size() == 2) && |
| 29 |
"timestep prediction requires 2d dataset"); |
| 30 |
global_dimensions[0] = conf.dims[0]; |
| 31 |
global_dimensions[1] = conf.dims[1]; |
| 32 |
} |
| 33 |
|
| 34 |
~TimeBasedFrontend() = default; |
| 35 |
|
| 36 |
std::vector<int> compress(T *data) { |
| 37 |
std::vector<int> quant_inds(num_elements); |
| 38 |
size_t quant_count = 0; |
| 39 |
if (data_ts0 != nullptr) { |
| 40 |
for (size_t j = 0; j < global_dimensions[1]; j++) { |
| 41 |
quant_inds[quant_count++] = quantizer.quantize_and_overwrite(data[j], data_ts0[j]); |
| 42 |
} |
| 43 |
} else { |
| 44 |
std::array<size_t, N - 1> global_dims; |
| 45 |
for (int i = 0; i < N - 1; i++) { |
| 46 |
global_dims[i] = global_dimensions[i + 1]; |
| 47 |
}; |
| 48 |
|
| 49 |
auto inter_block_range = std::make_shared<SZ::multi_dimensional_range<T, N - 1>>( |
| 50 |
data, std::begin(global_dims), std::end(global_dims), stride, 0); |
| 51 |
auto intra_block_range = std::make_shared<SZ::multi_dimensional_range<T, N - 1>>( |
| 52 |
data, std::begin(global_dims), std::end(global_dims), 1, 0); |
| 53 |
|
| 54 |
// std::array<size_t, N - 1> intra_block_dims; |
| 55 |
predictor.precompress_data(inter_block_range->begin()); |
| 56 |
quantizer.precompress_data(); |
| 57 |
auto inter_begin = inter_block_range->begin(); |
| 58 |
auto inter_end = inter_block_range->end(); |
| 59 |
for (auto block = inter_begin; block != inter_end; ++block) { |
| 60 |
|
| 61 |
intra_block_range->update_block_range(block, block_size); |
| 62 |
|
| 63 |
// // std::cout << *block << " " << lp.predict(block) << std::endl; |
| 64 |
// for (int i = 0; i < intra_block_dims.size(); i++) { |
| 65 |
// size_t cur_index = block.get_local_index(i); |
| 66 |
// size_t dims = inter_block_range->get_dimensions(i); |
| 67 |
// intra_block_dims[i] = (cur_index == dims - 1 && |
| 68 |
// global_dims[i] - cur_index * stride < block_size) ? |
| 69 |
// global_dims[i] - cur_index * stride : block_size; |
| 70 |
// } |
| 71 |
// |
| 72 |
// intra_block_range->set_dimensions(intra_block_dims.begin(), intra_block_dims.end()); |
| 73 |
// intra_block_range->set_offsets(block.get_offset()); |
| 74 |
// intra_block_range->set_starting_position(block.get_local_index()); |
| 75 |
concepts::PredictorInterface<T, N - 1> *predictor_withfallback = &predictor; |
| 76 |
if (!predictor.precompress_block(intra_block_range)) { |
| 77 |
predictor_withfallback = &fallback_predictor; |
| 78 |
} |
| 79 |
predictor_withfallback->precompress_block_commit(); |
| 80 |
// quantizer.precompress_block(); |
| 81 |
auto intra_begin = intra_block_range->begin(); |
| 82 |
auto intra_end = intra_block_range->end(); |
| 83 |
for (auto element = intra_begin; element != intra_end; ++element) { |
| 84 |
quant_inds[quant_count++] = quantizer.quantize_and_overwrite( |
| 85 |
*element, predictor_withfallback->predict(element)); |
| 86 |
} |
| 87 |
} |
| 88 |
predictor.postcompress_data(inter_block_range->begin()); |
| 89 |
} |
| 90 |
for (size_t j = 0; j < global_dimensions[1]; j++) { |
| 91 |
for (size_t i = 1; i < global_dimensions[0]; i++) { |
| 92 |
size_t idx = i * global_dimensions[1] + j; |
| 93 |
size_t idx_prev = (i - 1) * global_dimensions[1] + j; |
| 94 |
quant_inds[quant_count++] = quantizer.quantize_and_overwrite(data[idx], data[idx_prev]); |
| 95 |
} |
| 96 |
} |
| 97 |
assert(quant_count == num_elements); |
| 98 |
quantizer.postcompress_data(); |
| 99 |
return quant_inds; |
| 100 |
} |
| 101 |
|
| 102 |
T *decompress(std::vector<int> &quant_inds, T *dec_data) { |
| 103 |
|
| 104 |
int const *quant_inds_pos = (int const *) quant_inds.data(); |
| 105 |
std::array<size_t, N - 1> intra_block_dims; |
| 106 |
// auto dec_data = new T[num_elements]; |
| 107 |
|
| 108 |
if (data_ts0 != nullptr) { |
| 109 |
for (size_t j = 0; j < global_dimensions[1]; j++) { |
| 110 |
dec_data[j] = quantizer.recover(data_ts0[j], *(quant_inds_pos++)); |
| 111 |
} |
| 112 |
} else { |
| 113 |
std::array<size_t, N - 1> global_dims; |
| 114 |
for (int i = 0; i < N - 1; i++) { |
| 115 |
global_dims[i] = global_dimensions[i + 1]; |
| 116 |
}; |
| 117 |
auto inter_block_range = std::make_shared<SZ::multi_dimensional_range<T, N - 1>>( |
| 118 |
dec_data, std::begin(global_dims), std::end(global_dims), stride, 0); |
| 119 |
|
| 120 |
auto intra_block_range = std::make_shared<SZ::multi_dimensional_range<T, N - 1>>( |
| 121 |
dec_data, std::begin(global_dims), std::end(global_dims), 1, 0); |
| 122 |
|
| 123 |
predictor.predecompress_data(inter_block_range->begin()); |
| 124 |
quantizer.predecompress_data(); |
| 125 |
|
| 126 |
auto inter_begin = inter_block_range->begin(); |
| 127 |
auto inter_end = inter_block_range->end(); |
| 128 |
for (auto block = inter_begin; block != inter_end; block++) { |
| 129 |
intra_block_range->update_block_range(block, block_size); |
| 130 |
// for (int i = 0; i < intra_block_dims.size(); i++) { |
| 131 |
// size_t cur_index = block.get_local_index(i); |
| 132 |
// size_t dims = inter_block_range->get_dimensions(i); |
| 133 |
// intra_block_dims[i] = (cur_index == dims - 1) ? global_dims[i] - cur_index * block_size |
| 134 |
// : block_size; |
| 135 |
// } |
| 136 |
// intra_block_range->set_dimensions(intra_block_dims.begin(), intra_block_dims.end()); |
| 137 |
// intra_block_range->set_offsets(block.get_offset()); |
| 138 |
// intra_block_range->set_starting_position(block.get_local_index()); |
| 139 |
|
| 140 |
concepts::PredictorInterface<T, N - 1> *predictor_withfallback = &predictor; |
| 141 |
if (!predictor.predecompress_block(intra_block_range)) { |
| 142 |
predictor_withfallback = &fallback_predictor; |
| 143 |
} |
| 144 |
auto intra_begin = intra_block_range->begin(); |
| 145 |
auto intra_end = intra_block_range->end(); |
| 146 |
for (auto element = intra_begin; element != intra_end; ++element) { |
| 147 |
*element = quantizer.recover(predictor_withfallback->predict(element), *(quant_inds_pos++)); |
| 148 |
} |
| 149 |
} |
| 150 |
predictor.postdecompress_data(inter_block_range->begin()); |
| 151 |
} |
| 152 |
|
| 153 |
for (size_t j = 0; j < global_dimensions[1]; j++) { |
| 154 |
for (size_t i = 1; i < global_dimensions[0]; i++) { |
| 155 |
size_t idx = i * global_dimensions[1] + j; |
| 156 |
size_t idx_prev = (i - 1) * global_dimensions[1] + j; |
| 157 |
dec_data[idx] = quantizer.recover(dec_data[idx_prev], *(quant_inds_pos++)); |
| 158 |
} |
| 159 |
} |
| 160 |
|
| 161 |
quantizer.postdecompress_data(); |
| 162 |
return dec_data; |
| 163 |
} |
| 164 |
|
| 165 |
void save(uchar *&c) { |
| 166 |
write(global_dimensions.data(), N, c); |
| 167 |
write(block_size, c); |
| 168 |
|
| 169 |
predictor.save(c); |
| 170 |
quantizer.save(c); |
| 171 |
} |
| 172 |
|
| 173 |
void load(const uchar *&c, size_t &remaining_length) { |
| 174 |
read(global_dimensions.data(), N, c, remaining_length); |
| 175 |
num_elements = 1; |
| 176 |
for (const auto &d: global_dimensions) { |
| 177 |
num_elements *= d; |
| 178 |
std::cout << d << " "; |
| 179 |
} |
| 180 |
std::cout << std::endl; |
| 181 |
read(block_size, c, remaining_length); |
| 182 |
stride = block_size; |
| 183 |
predictor.load(c, remaining_length); |
| 184 |
quantizer.load(c, remaining_length); |
| 185 |
} |
| 186 |
|
| 187 |
void print() { |
| 188 |
// predictor.print(); |
| 189 |
// quantizer.print(); |
| 190 |
} |
| 191 |
|
| 192 |
void clear() { |
| 193 |
predictor.clear(); |
| 194 |
fallback_predictor.clear(); |
| 195 |
quantizer.clear(); |
| 196 |
} |
| 197 |
|
| 198 |
int get_radius() const { return quantizer.get_radius(); } |
| 199 |
|
| 200 |
size_t get_num_elements() const { return num_elements; }; |
| 201 |
|
| 202 |
size_t size_est() { return 0; }; |
| 203 |
private: |
| 204 |
Predictor predictor; |
| 205 |
LorenzoPredictor<T, N - 1, 1> fallback_predictor; |
| 206 |
Quantizer quantizer; |
| 207 |
uint block_size; |
| 208 |
uint stride; |
| 209 |
size_t num_elements; |
| 210 |
std::array<size_t, N> global_dimensions; |
| 211 |
T *data_ts0 = nullptr; |
| 212 |
}; |
| 213 |
|
| 214 |
template<class T, uint N, class Predictor, class Quantizer> |
| 215 |
TimeBasedFrontend<T, N, Predictor, Quantizer> |
| 216 |
make_sz3_timebased_frontend(const Config &conf, Predictor predictor, Quantizer quantizer, T *data_ts0) { |
| 217 |
return TimeBasedFrontend<T, N, Predictor, Quantizer>(conf, predictor, quantizer, data_ts0); |
| 218 |
} |
| 219 |
} |
| 220 |
|
| 221 |
#endif |