| 1 |
// Copyright (c) the JPEG XL Project Authors. All rights reserved. |
| 2 |
// |
| 3 |
// Use of this source code is governed by a BSD-style |
| 4 |
// license that can be found in the LICENSE file. |
| 5 |
|
| 6 |
#include "lib/jxl/render_pipeline/stage_write.h" |
| 7 |
|
| 8 |
#include <jxl/memory_manager.h> |
| 9 |
|
| 10 |
#include <cstdint> |
| 11 |
#include <type_traits> |
| 12 |
|
| 13 |
#include "lib/jxl/alpha.h" |
| 14 |
#include "lib/jxl/base/common.h" |
| 15 |
#include "lib/jxl/base/compiler_specific.h" |
| 16 |
#include "lib/jxl/base/sanitizers.h" |
| 17 |
#include "lib/jxl/base/status.h" |
| 18 |
#include "lib/jxl/dec_cache.h" |
| 19 |
#include "lib/jxl/dec_xyb.h" |
| 20 |
#include "lib/jxl/image.h" |
| 21 |
#include "lib/jxl/image_bundle.h" |
| 22 |
#include "lib/jxl/memory_manager_internal.h" |
| 23 |
|
| 24 |
#undef HWY_TARGET_INCLUDE |
| 25 |
#define HWY_TARGET_INCLUDE "lib/jxl/render_pipeline/stage_write.cc" |
| 26 |
#include <hwy/foreach_target.h> |
| 27 |
#include <hwy/highway.h> |
| 28 |
|
| 29 |
HWY_BEFORE_NAMESPACE(); |
| 30 |
namespace jxl { |
| 31 |
namespace HWY_NAMESPACE { |
| 32 |
|
| 33 |
// These templates are not found via ADL. |
| 34 |
using hwy::HWY_NAMESPACE::Add; |
| 35 |
using hwy::HWY_NAMESPACE::Clamp; |
| 36 |
using hwy::HWY_NAMESPACE::Div; |
| 37 |
using hwy::HWY_NAMESPACE::Max; |
| 38 |
using hwy::HWY_NAMESPACE::Mul; |
| 39 |
using hwy::HWY_NAMESPACE::NearestInt; |
| 40 |
using hwy::HWY_NAMESPACE::Or; |
| 41 |
using hwy::HWY_NAMESPACE::Rebind; |
| 42 |
using hwy::HWY_NAMESPACE::ShiftLeftSame; |
| 43 |
using hwy::HWY_NAMESPACE::ShiftRightSame; |
| 44 |
using hwy::HWY_NAMESPACE::VFromD; |
| 45 |
|
| 46 |
// 8x8 ordered dithering pattern from |
| 47 |
// https://en.wikipedia.org/wiki/Ordered_dithering |
| 48 |
// scaled to have an average of 0 and be fully contained in (-0.5, 0.5). |
| 49 |
// Matrix is duplicated in width to avoid inconsistencies or out-of-bound-reads |
| 50 |
// if doing unaligned operations. |
| 51 |
const float kDither[(2 * 8) * 8] = { |
| 52 |
-0.4921875, 0.0078125, -0.3671875, 0.1328125, // |
| 53 |
-0.4609375, 0.0390625, -0.3359375, 0.1640625, // |
| 54 |
-0.4921875, 0.0078125, -0.3671875, 0.1328125, // |
| 55 |
-0.4609375, 0.0390625, -0.3359375, 0.1640625, // |
| 56 |
// |
| 57 |
0.2578125, -0.2421875, 0.3828125, -0.1171875, // |
| 58 |
0.2890625, -0.2109375, 0.4140625, -0.0859375, // |
| 59 |
0.2578125, -0.2421875, 0.3828125, -0.1171875, // |
| 60 |
0.2890625, -0.2109375, 0.4140625, -0.0859375, // |
| 61 |
// |
| 62 |
-0.3046875, 0.1953125, -0.4296875, 0.0703125, // |
| 63 |
-0.2734375, 0.2265625, -0.3984375, 0.1015625, // |
| 64 |
-0.3046875, 0.1953125, -0.4296875, 0.0703125, // |
| 65 |
-0.2734375, 0.2265625, -0.3984375, 0.1015625, // |
| 66 |
// |
| 67 |
0.4453125, -0.0546875, 0.3203125, -0.1796875, // |
| 68 |
0.4765625, -0.0234375, 0.3515625, -0.1484375, // |
| 69 |
0.4453125, -0.0546875, 0.3203125, -0.1796875, // |
| 70 |
0.4765625, -0.0234375, 0.3515625, -0.1484375, // |
| 71 |
// |
| 72 |
-0.4453125, 0.0546875, -0.3203125, 0.1796875, // |
| 73 |
-0.4765625, 0.0234375, -0.3515625, 0.1484375, // |
| 74 |
-0.4453125, 0.0546875, -0.3203125, 0.1796875, // |
| 75 |
-0.4765625, 0.0234375, -0.3515625, 0.1484375, // |
| 76 |
// |
| 77 |
0.3046875, -0.1953125, 0.4296875, -0.0703125, // |
| 78 |
0.2734375, -0.2265625, 0.3984375, -0.1015625, // |
| 79 |
0.3046875, -0.1953125, 0.4296875, -0.0703125, // |
| 80 |
0.2734375, -0.2265625, 0.3984375, -0.1015625, // |
| 81 |
// |
| 82 |
-0.2578125, 0.2421875, -0.3828125, 0.1171875, // |
| 83 |
-0.2890625, 0.2109375, -0.4140625, 0.0859375, // |
| 84 |
-0.2578125, 0.2421875, -0.3828125, 0.1171875, // |
| 85 |
-0.2890625, 0.2109375, -0.4140625, 0.0859375, // |
| 86 |
// |
| 87 |
0.4921875, -0.0078125, 0.3671875, -0.1328125, // |
| 88 |
0.4609375, -0.0390625, 0.3359375, -0.1640625, // |
| 89 |
0.4921875, -0.0078125, 0.3671875, -0.1328125, // |
| 90 |
0.4609375, -0.0390625, 0.3359375, -0.1640625, // |
| 91 |
}; |
| 92 |
|
| 93 |
using DF = HWY_FULL(float); |
| 94 |
|
| 95 |
// Converts `v` to an appropriate value for the given unsigned type. |
| 96 |
// If the unsigned type is an 8-bit type, performs ordered dithering. |
| 97 |
template <typename T> |
| 98 |
VFromD<Rebind<T, DF>> MakeUnsigned(VFromD<DF> v, size_t x0, size_t y0, |
| 99 |
VFromD<DF> mul) { |
| 100 |
static_assert(std::is_unsigned<T>::value, "T must be an unsigned type"); |
| 101 |
using DU = Rebind<T, DF>; |
| 102 |
v = Mul(v, mul); |
| 103 |
// TODO(veluca): if constexpr with C++17 |
| 104 |
if (sizeof(T) == 1) { |
| 105 |
size_t pos = (y0 % 8) * (2 * 8) + (x0 % 8); |
| 106 |
#if HWY_TARGET != HWY_SCALAR |
| 107 |
auto dither = LoadDup128(DF(), kDither + pos); |
| 108 |
#else |
| 109 |
auto dither = LoadU(DF(), kDither + pos); |
| 110 |
#endif |
| 111 |
v = Add(v, dither); |
| 112 |
} |
| 113 |
v = Clamp(Zero(DF()), v, mul); |
| 114 |
return DemoteTo(DU(), NearestInt(v)); |
| 115 |
} |
| 116 |
|
| 117 |
class WriteToOutputStage : public RenderPipelineStage { |
| 118 |
public: |
| 119 |
WriteToOutputStage(const ImageOutput& main_output, size_t width, |
| 120 |
size_t height, bool has_alpha, bool unpremul_alpha, |
| 121 |
size_t alpha_c, Orientation undo_orientation, |
| 122 |
const std::vector<ImageOutput>& extra_output, |
| 123 |
JxlMemoryManager* memory_manager) |
| 124 |
: RenderPipelineStage(RenderPipelineStage::Settings()), |
| 125 |
width_(width), |
| 126 |
height_(height), |
| 127 |
main_(main_output), |
| 128 |
num_color_(main_.num_channels_ < 3 ? 1 : 3), |
| 129 |
want_alpha_(main_.num_channels_ == 2 || main_.num_channels_ == 4), |
| 130 |
has_alpha_(has_alpha), |
| 131 |
unpremul_alpha_(unpremul_alpha), |
| 132 |
alpha_c_(alpha_c), |
| 133 |
flip_x_(ShouldFlipX(undo_orientation)), |
| 134 |
flip_y_(ShouldFlipY(undo_orientation)), |
| 135 |
transpose_(ShouldTranspose(undo_orientation)), |
| 136 |
opaque_alpha_(kMaxPixelsPerCall, 1.0f), |
| 137 |
memory_manager_(memory_manager) { |
| 138 |
for (size_t ec = 0; ec < extra_output.size(); ++ec) { |
| 139 |
if (extra_output[ec].callback.IsPresent() || extra_output[ec].buffer) { |
| 140 |
Output extra(extra_output[ec]); |
| 141 |
extra.channel_index_ = 3 + ec; |
| 142 |
extra_channels_.push_back(extra); |
| 143 |
} |
| 144 |
} |
| 145 |
} |
| 146 |
|
| 147 |
WriteToOutputStage(const WriteToOutputStage&) = delete; |
| 148 |
WriteToOutputStage& operator=(const WriteToOutputStage&) = delete; |
| 149 |
WriteToOutputStage(WriteToOutputStage&&) = delete; |
| 150 |
WriteToOutputStage& operator=(WriteToOutputStage&&) = delete; |
| 151 |
|
| 152 |
~WriteToOutputStage() override { |
| 153 |
if (main_.run_opaque_) { |
| 154 |
main_.pixel_callback_.destroy(main_.run_opaque_); |
| 155 |
} |
| 156 |
for (auto& extra : extra_channels_) { |
| 157 |
if (extra.run_opaque_) { |
| 158 |
extra.pixel_callback_.destroy(extra.run_opaque_); |
| 159 |
} |
| 160 |
} |
| 161 |
} |
| 162 |
|
| 163 |
Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, |
| 164 |
size_t xextra, size_t xsize, size_t xpos, size_t ypos, |
| 165 |
size_t thread_id) const final { |
| 166 |
JXL_ENSURE(xextra == 0); |
| 167 |
JXL_ENSURE(main_.run_opaque_ || main_.buffer_); |
| 168 |
if (ypos >= height_) return true; |
| 169 |
if (xpos >= width_) return true; |
| 170 |
if (flip_y_) { |
| 171 |
ypos = height_ - 1u - ypos; |
| 172 |
} |
| 173 |
size_t limit = std::min(xsize, width_ - xpos); |
| 174 |
for (size_t x0 = 0; x0 < limit; x0 += kMaxPixelsPerCall) { |
| 175 |
size_t xstart = xpos + x0; |
| 176 |
size_t len = std::min<size_t>(kMaxPixelsPerCall, limit - x0); |
| 177 |
|
| 178 |
const float* line_buffers[4]; |
| 179 |
for (size_t c = 0; c < num_color_; c++) { |
| 180 |
line_buffers[c] = GetInputRow(input_rows, c, 0) + x0; |
| 181 |
} |
| 182 |
if (has_alpha_) { |
| 183 |
line_buffers[num_color_] = GetInputRow(input_rows, alpha_c_, 0) + x0; |
| 184 |
} else { |
| 185 |
// opaque_alpha_ is a way to set all values to 1.0f. |
| 186 |
line_buffers[num_color_] = opaque_alpha_.data(); |
| 187 |
} |
| 188 |
if (has_alpha_ && want_alpha_ && unpremul_alpha_) { |
| 189 |
UnpremulAlpha(thread_id, len, line_buffers); |
| 190 |
} |
| 191 |
OutputBuffers(main_, thread_id, ypos, xstart, len, line_buffers); |
| 192 |
for (const auto& extra : extra_channels_) { |
| 193 |
line_buffers[0] = GetInputRow(input_rows, extra.channel_index_, 0) + x0; |
| 194 |
OutputBuffers(extra, thread_id, ypos, xstart, len, line_buffers); |
| 195 |
} |
| 196 |
} |
| 197 |
return true; |
| 198 |
} |
| 199 |
|
| 200 |
RenderPipelineChannelMode GetChannelMode(size_t c) const final { |
| 201 |
if (c < num_color_ || (has_alpha_ && c == alpha_c_)) { |
| 202 |
return RenderPipelineChannelMode::kInput; |
| 203 |
} |
| 204 |
for (const auto& extra : extra_channels_) { |
| 205 |
if (c == extra.channel_index_) { |
| 206 |
return RenderPipelineChannelMode::kInput; |
| 207 |
} |
| 208 |
} |
| 209 |
return RenderPipelineChannelMode::kIgnored; |
| 210 |
} |
| 211 |
|
| 212 |
const char* GetName() const override { return "WritePixelCB"; } |
| 213 |
|
| 214 |
private: |
| 215 |
struct Output { |
| 216 |
explicit Output(const ImageOutput& image_out) |
| 217 |
: pixel_callback_(image_out.callback), |
| 218 |
buffer_(image_out.buffer), |
| 219 |
buffer_size_(image_out.buffer_size), |
| 220 |
stride_(image_out.stride), |
| 221 |
num_channels_(image_out.format.num_channels), |
| 222 |
swap_endianness_(SwapEndianness(image_out.format.endianness)), |
| 223 |
data_type_(image_out.format.data_type), |
| 224 |
bits_per_sample_(image_out.bits_per_sample) {} |
| 225 |
|
| 226 |
Status PrepareForThreads(size_t num_threads) { |
| 227 |
if (pixel_callback_.IsPresent()) { |
| 228 |
run_opaque_ = |
| 229 |
pixel_callback_.Init(num_threads, /*num_pixels=*/kMaxPixelsPerCall); |
| 230 |
JXL_RETURN_IF_ERROR(run_opaque_ != nullptr); |
| 231 |
} else { |
| 232 |
JXL_RETURN_IF_ERROR(buffer_ != nullptr); |
| 233 |
} |
| 234 |
return true; |
| 235 |
} |
| 236 |
|
| 237 |
PixelCallback pixel_callback_; |
| 238 |
void* run_opaque_ = nullptr; |
| 239 |
void* buffer_ = nullptr; |
| 240 |
size_t buffer_size_; |
| 241 |
size_t stride_; |
| 242 |
size_t num_channels_; |
| 243 |
bool swap_endianness_; |
| 244 |
JxlDataType data_type_; |
| 245 |
size_t bits_per_sample_; |
| 246 |
size_t channel_index_; // used for extra_channels |
| 247 |
}; |
| 248 |
|
| 249 |
Status PrepareForThreads(size_t num_threads) override { |
| 250 |
JXL_RETURN_IF_ERROR(main_.PrepareForThreads(num_threads)); |
| 251 |
for (auto& extra : extra_channels_) { |
| 252 |
JXL_RETURN_IF_ERROR(extra.PrepareForThreads(num_threads)); |
| 253 |
} |
| 254 |
temp_out_.resize(num_threads); |
| 255 |
for (AlignedMemory& temp : temp_out_) { |
| 256 |
size_t alloc_size = |
| 257 |
sizeof(float) * kMaxPixelsPerCall * main_.num_channels_; |
| 258 |
JXL_ASSIGN_OR_RETURN(temp, |
| 259 |
AlignedMemory::Create(memory_manager_, alloc_size)); |
| 260 |
} |
| 261 |
if ((has_alpha_ && want_alpha_ && unpremul_alpha_) || flip_x_) { |
| 262 |
temp_in_.resize(num_threads * main_.num_channels_); |
| 263 |
for (AlignedMemory& temp : temp_in_) { |
| 264 |
size_t alloc_size = sizeof(float) * kMaxPixelsPerCall; |
| 265 |
JXL_ASSIGN_OR_RETURN( |
| 266 |
temp, AlignedMemory::Create(memory_manager_, alloc_size)); |
| 267 |
} |
| 268 |
} |
| 269 |
return true; |
| 270 |
} |
| 271 |
static bool ShouldFlipX(Orientation undo_orientation) { |
| 272 |
return (undo_orientation == Orientation::kFlipHorizontal || |
| 273 |
undo_orientation == Orientation::kRotate180 || |
| 274 |
undo_orientation == Orientation::kRotate270 || |
| 275 |
undo_orientation == Orientation::kAntiTranspose); |
| 276 |
} |
| 277 |
static bool ShouldFlipY(Orientation undo_orientation) { |
| 278 |
return (undo_orientation == Orientation::kFlipVertical || |
| 279 |
undo_orientation == Orientation::kRotate180 || |
| 280 |
undo_orientation == Orientation::kRotate90 || |
| 281 |
undo_orientation == Orientation::kAntiTranspose); |
| 282 |
} |
| 283 |
static bool ShouldTranspose(Orientation undo_orientation) { |
| 284 |
return (undo_orientation == Orientation::kTranspose || |
| 285 |
undo_orientation == Orientation::kRotate90 || |
| 286 |
undo_orientation == Orientation::kRotate270 || |
| 287 |
undo_orientation == Orientation::kAntiTranspose); |
| 288 |
} |
| 289 |
|
| 290 |
void UnpremulAlpha(size_t thread_id, size_t len, |
| 291 |
const float** line_buffers) const { |
| 292 |
const HWY_FULL(float) d; |
| 293 |
auto one = Set(d, 1.0f); |
| 294 |
float* temp_in[4]; |
| 295 |
for (size_t c = 0; c < main_.num_channels_; ++c) { |
| 296 |
size_t tix = thread_id * main_.num_channels_ + c; |
| 297 |
temp_in[c] = temp_in_[tix].address<float>(); |
| 298 |
memcpy(temp_in[c], line_buffers[c], sizeof(float) * len); |
| 299 |
} |
| 300 |
auto small_alpha = Set(d, kSmallAlpha); |
| 301 |
for (size_t ix = 0; ix < len; ix += Lanes(d)) { |
| 302 |
auto alpha = LoadU(d, temp_in[num_color_] + ix); |
| 303 |
auto mul = Div(one, Max(small_alpha, alpha)); |
| 304 |
for (size_t c = 0; c < num_color_; ++c) { |
| 305 |
auto val = LoadU(d, temp_in[c] + ix); |
| 306 |
StoreU(Mul(val, mul), d, temp_in[c] + ix); |
| 307 |
} |
| 308 |
} |
| 309 |
for (size_t c = 0; c < main_.num_channels_; ++c) { |
| 310 |
line_buffers[c] = temp_in[c]; |
| 311 |
} |
| 312 |
} |
| 313 |
|
| 314 |
void OutputBuffers(const Output& out, size_t thread_id, size_t ypos, |
| 315 |
size_t xstart, size_t len, const float* input[4]) const { |
| 316 |
if (flip_x_) { |
| 317 |
FlipX(out, thread_id, len, &xstart, input); |
| 318 |
} |
| 319 |
if (out.data_type_ == JXL_TYPE_UINT8) { |
| 320 |
uint8_t* JXL_RESTRICT temp = temp_out_[thread_id].address<uint8_t>(); |
| 321 |
StoreUnsignedRow(out, input, len, temp, xstart, ypos); |
| 322 |
WriteToOutput(out, thread_id, ypos, xstart, len, temp); |
| 323 |
} else if (out.data_type_ == JXL_TYPE_UINT16 || |
| 324 |
out.data_type_ == JXL_TYPE_FLOAT16) { |
| 325 |
uint16_t* JXL_RESTRICT temp = temp_out_[thread_id].address<uint16_t>(); |
| 326 |
if (out.data_type_ == JXL_TYPE_UINT16) { |
| 327 |
StoreUnsignedRow(out, input, len, temp, xstart, ypos); |
| 328 |
} else { |
| 329 |
StoreFloat16Row(out, input, len, temp); |
| 330 |
} |
| 331 |
if (out.swap_endianness_) { |
| 332 |
const HWY_FULL(uint16_t) du; |
| 333 |
size_t output_len = len * out.num_channels_; |
| 334 |
for (size_t j = 0; j < output_len; j += Lanes(du)) { |
| 335 |
auto v = LoadU(du, temp + j); |
| 336 |
auto vswap = Or(ShiftRightSame(v, 8), ShiftLeftSame(v, 8)); |
| 337 |
StoreU(vswap, du, temp + j); |
| 338 |
} |
| 339 |
} |
| 340 |
WriteToOutput(out, thread_id, ypos, xstart, len, temp); |
| 341 |
} else if (out.data_type_ == JXL_TYPE_FLOAT) { |
| 342 |
float* JXL_RESTRICT temp = temp_out_[thread_id].address<float>(); |
| 343 |
StoreFloatRow(out, input, len, temp); |
| 344 |
if (out.swap_endianness_) { |
| 345 |
size_t output_len = len * out.num_channels_; |
| 346 |
for (size_t j = 0; j < output_len; ++j) { |
| 347 |
temp[j] = BSwapFloat(temp[j]); |
| 348 |
} |
| 349 |
} |
| 350 |
WriteToOutput(out, thread_id, ypos, xstart, len, temp); |
| 351 |
} |
| 352 |
} |
| 353 |
|
| 354 |
void FlipX(const Output& out, size_t thread_id, size_t len, size_t* xstart, |
| 355 |
const float** line_buffers) const { |
| 356 |
float* temp_in[4]; |
| 357 |
for (size_t c = 0; c < out.num_channels_; ++c) { |
| 358 |
size_t tix = thread_id * main_.num_channels_ + c; |
| 359 |
temp_in[c] = temp_in_[tix].address<float>(); |
| 360 |
if (temp_in[c] != line_buffers[c]) { |
| 361 |
memcpy(temp_in[c], line_buffers[c], sizeof(float) * len); |
| 362 |
} |
| 363 |
} |
| 364 |
size_t last = (len - 1u); |
| 365 |
size_t num = (len / 2); |
| 366 |
for (size_t i = 0; i < num; ++i) { |
| 367 |
for (size_t c = 0; c < out.num_channels_; ++c) { |
| 368 |
std::swap(temp_in[c][i], temp_in[c][last - i]); |
| 369 |
} |
| 370 |
} |
| 371 |
for (size_t c = 0; c < out.num_channels_; ++c) { |
| 372 |
line_buffers[c] = temp_in[c]; |
| 373 |
} |
| 374 |
*xstart = width_ - *xstart - len; |
| 375 |
} |
| 376 |
|
| 377 |
template <typename T> |
| 378 |
void StoreUnsignedRow(const Output& out, const float* input[4], size_t len, |
| 379 |
T* output, size_t xstart, size_t ypos) const { |
| 380 |
const HWY_FULL(float) d; |
| 381 |
auto mul = Set(d, (1u << (out.bits_per_sample_)) - 1); |
| 382 |
const Rebind<T, decltype(d)> du; |
| 383 |
const size_t padding = RoundUpTo(len, Lanes(d)) - len; |
| 384 |
for (size_t c = 0; c < out.num_channels_; ++c) { |
| 385 |
msan::UnpoisonMemory(input[c] + len, sizeof(input[c][0]) * padding); |
| 386 |
} |
| 387 |
if (out.num_channels_ == 1) { |
| 388 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 389 |
StoreU(MakeUnsigned<T>(LoadU(d, &input[0][i]), xstart + i, ypos, mul), |
| 390 |
du, &output[i]); |
| 391 |
} |
| 392 |
} else if (out.num_channels_ == 2) { |
| 393 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 394 |
StoreInterleaved2( |
| 395 |
MakeUnsigned<T>(LoadU(d, &input[0][i]), xstart + i, ypos, mul), |
| 396 |
MakeUnsigned<T>(LoadU(d, &input[1][i]), xstart + i, ypos, mul), du, |
| 397 |
&output[2 * i]); |
| 398 |
} |
| 399 |
} else if (out.num_channels_ == 3) { |
| 400 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 401 |
StoreInterleaved3( |
| 402 |
MakeUnsigned<T>(LoadU(d, &input[0][i]), xstart + i, ypos, mul), |
| 403 |
MakeUnsigned<T>(LoadU(d, &input[1][i]), xstart + i, ypos, mul), |
| 404 |
MakeUnsigned<T>(LoadU(d, &input[2][i]), xstart + i, ypos, mul), du, |
| 405 |
&output[3 * i]); |
| 406 |
} |
| 407 |
} else if (out.num_channels_ == 4) { |
| 408 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 409 |
StoreInterleaved4( |
| 410 |
MakeUnsigned<T>(LoadU(d, &input[0][i]), xstart + i, ypos, mul), |
| 411 |
MakeUnsigned<T>(LoadU(d, &input[1][i]), xstart + i, ypos, mul), |
| 412 |
MakeUnsigned<T>(LoadU(d, &input[2][i]), xstart + i, ypos, mul), |
| 413 |
MakeUnsigned<T>(LoadU(d, &input[3][i]), xstart + i, ypos, mul), du, |
| 414 |
&output[4 * i]); |
| 415 |
} |
| 416 |
} |
| 417 |
msan::PoisonMemory(output + out.num_channels_ * len, |
| 418 |
sizeof(output[0]) * out.num_channels_ * padding); |
| 419 |
} |
| 420 |
|
| 421 |
static void StoreFloat16Row(const Output& out, const float* input[4], |
| 422 |
size_t len, uint16_t* output) { |
| 423 |
const HWY_FULL(float) d; |
| 424 |
const Rebind<uint16_t, decltype(d)> du; |
| 425 |
const Rebind<hwy::float16_t, decltype(d)> df16; |
| 426 |
const size_t padding = RoundUpTo(len, Lanes(d)) - len; |
| 427 |
for (size_t c = 0; c < out.num_channels_; ++c) { |
| 428 |
msan::UnpoisonMemory(input[c] + len, sizeof(input[c][0]) * padding); |
| 429 |
} |
| 430 |
if (out.num_channels_ == 1) { |
| 431 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 432 |
auto v0 = LoadU(d, &input[0][i]); |
| 433 |
StoreU(BitCast(du, DemoteTo(df16, v0)), du, &output[i]); |
| 434 |
} |
| 435 |
} else if (out.num_channels_ == 2) { |
| 436 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 437 |
auto v0 = LoadU(d, &input[0][i]); |
| 438 |
auto v1 = LoadU(d, &input[1][i]); |
| 439 |
StoreInterleaved2(BitCast(du, DemoteTo(df16, v0)), |
| 440 |
BitCast(du, DemoteTo(df16, v1)), du, &output[2 * i]); |
| 441 |
} |
| 442 |
} else if (out.num_channels_ == 3) { |
| 443 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 444 |
auto v0 = LoadU(d, &input[0][i]); |
| 445 |
auto v1 = LoadU(d, &input[1][i]); |
| 446 |
auto v2 = LoadU(d, &input[2][i]); |
| 447 |
StoreInterleaved3(BitCast(du, DemoteTo(df16, v0)), |
| 448 |
BitCast(du, DemoteTo(df16, v1)), |
| 449 |
BitCast(du, DemoteTo(df16, v2)), du, &output[3 * i]); |
| 450 |
} |
| 451 |
} else if (out.num_channels_ == 4) { |
| 452 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 453 |
auto v0 = LoadU(d, &input[0][i]); |
| 454 |
auto v1 = LoadU(d, &input[1][i]); |
| 455 |
auto v2 = LoadU(d, &input[2][i]); |
| 456 |
auto v3 = LoadU(d, &input[3][i]); |
| 457 |
StoreInterleaved4(BitCast(du, DemoteTo(df16, v0)), |
| 458 |
BitCast(du, DemoteTo(df16, v1)), |
| 459 |
BitCast(du, DemoteTo(df16, v2)), |
| 460 |
BitCast(du, DemoteTo(df16, v3)), du, &output[4 * i]); |
| 461 |
} |
| 462 |
} |
| 463 |
msan::PoisonMemory(output + out.num_channels_ * len, |
| 464 |
sizeof(output[0]) * out.num_channels_ * padding); |
| 465 |
} |
| 466 |
|
| 467 |
static void StoreFloatRow(const Output& out, const float* input[4], |
| 468 |
size_t len, float* output) { |
| 469 |
const HWY_FULL(float) d; |
| 470 |
if (out.num_channels_ == 1) { |
| 471 |
memcpy(output, input[0], len * sizeof(output[0])); |
| 472 |
} else if (out.num_channels_ == 2) { |
| 473 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 474 |
StoreInterleaved2(LoadU(d, &input[0][i]), LoadU(d, &input[1][i]), d, |
| 475 |
&output[2 * i]); |
| 476 |
} |
| 477 |
} else if (out.num_channels_ == 3) { |
| 478 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 479 |
StoreInterleaved3(LoadU(d, &input[0][i]), LoadU(d, &input[1][i]), |
| 480 |
LoadU(d, &input[2][i]), d, &output[3 * i]); |
| 481 |
} |
| 482 |
} else { |
| 483 |
for (size_t i = 0; i < len; i += Lanes(d)) { |
| 484 |
StoreInterleaved4(LoadU(d, &input[0][i]), LoadU(d, &input[1][i]), |
| 485 |
LoadU(d, &input[2][i]), LoadU(d, &input[3][i]), d, |
| 486 |
&output[4 * i]); |
| 487 |
} |
| 488 |
} |
| 489 |
} |
| 490 |
|
| 491 |
template <typename T> |
| 492 |
void WriteToOutput(const Output& out, size_t thread_id, size_t ypos, |
| 493 |
size_t xstart, size_t len, T* output) const { |
| 494 |
if (transpose_) { |
| 495 |
// TODO(szabadka) Buffer 8x8 chunks and transpose with SIMD. |
| 496 |
if (out.run_opaque_) { |
| 497 |
for (size_t i = 0, j = 0; i < len; ++i, j += out.num_channels_) { |
| 498 |
out.pixel_callback_.run(out.run_opaque_, thread_id, ypos, xstart + i, |
| 499 |
1, output + j); |
| 500 |
} |
| 501 |
} else { |
| 502 |
const size_t pixel_stride = out.num_channels_ * sizeof(T); |
| 503 |
const size_t offset = xstart * out.stride_ + ypos * pixel_stride; |
| 504 |
for (size_t i = 0, j = 0; i < len; ++i, j += out.num_channels_) { |
| 505 |
const size_t ix = offset + i * out.stride_; |
| 506 |
JXL_DASSERT(ix + pixel_stride <= out.buffer_size_); |
| 507 |
memcpy(reinterpret_cast<uint8_t*>(out.buffer_) + ix, output + j, |
| 508 |
pixel_stride); |
| 509 |
} |
| 510 |
} |
| 511 |
} else { |
| 512 |
if (out.run_opaque_) { |
| 513 |
out.pixel_callback_.run(out.run_opaque_, thread_id, xstart, ypos, len, |
| 514 |
output); |
| 515 |
} else { |
| 516 |
const size_t pixel_stride = out.num_channels_ * sizeof(T); |
| 517 |
const size_t offset = ypos * out.stride_ + xstart * pixel_stride; |
| 518 |
JXL_DASSERT(offset + len * pixel_stride <= out.buffer_size_); |
| 519 |
memcpy(reinterpret_cast<uint8_t*>(out.buffer_) + offset, output, |
| 520 |
len * pixel_stride); |
| 521 |
} |
| 522 |
} |
| 523 |
} |
| 524 |
|
| 525 |
static constexpr size_t kMaxPixelsPerCall = 1024; |
| 526 |
size_t width_; |
| 527 |
size_t height_; |
| 528 |
Output main_; // color + alpha |
| 529 |
size_t num_color_; |
| 530 |
bool want_alpha_; |
| 531 |
bool has_alpha_; |
| 532 |
bool unpremul_alpha_; |
| 533 |
size_t alpha_c_; |
| 534 |
bool flip_x_; |
| 535 |
bool flip_y_; |
| 536 |
bool transpose_; |
| 537 |
std::vector<Output> extra_channels_; |
| 538 |
std::vector<float> opaque_alpha_; |
| 539 |
JxlMemoryManager* memory_manager_; |
| 540 |
std::vector<AlignedMemory> temp_in_; |
| 541 |
std::vector<AlignedMemory> temp_out_; |
| 542 |
}; |
| 543 |
|
| 544 |
#if JXL_CXX_LANG < JXL_CXX_17 |
| 545 |
constexpr size_t WriteToOutputStage::kMaxPixelsPerCall; |
| 546 |
#endif |
| 547 |
|
| 548 |
std::unique_ptr<RenderPipelineStage> GetWriteToOutputStage( |
| 549 |
const ImageOutput& main_output, size_t width, size_t height, bool has_alpha, |
| 550 |
bool unpremul_alpha, size_t alpha_c, Orientation undo_orientation, |
| 551 |
std::vector<ImageOutput>& extra_output, JxlMemoryManager* memory_manager) { |
| 552 |
return jxl::make_unique<WriteToOutputStage>( |
| 553 |
main_output, width, height, has_alpha, unpremul_alpha, alpha_c, |
| 554 |
undo_orientation, extra_output, memory_manager); |
| 555 |
} |
| 556 |
|
| 557 |
// NOLINTNEXTLINE(google-readability-namespace-comments) |
| 558 |
} // namespace HWY_NAMESPACE |
| 559 |
} // namespace jxl |
| 560 |
HWY_AFTER_NAMESPACE(); |
| 561 |
|
| 562 |
#if HWY_ONCE |
| 563 |
|
| 564 |
namespace jxl { |
| 565 |
|
| 566 |
HWY_EXPORT(GetWriteToOutputStage); |
| 567 |
|
| 568 |
namespace { |
| 569 |
class WriteToImageBundleStage : public RenderPipelineStage { |
| 570 |
public: |
| 571 |
explicit WriteToImageBundleStage( |
| 572 |
ImageBundle* image_bundle, const OutputEncodingInfo& output_encoding_info) |
| 573 |
: RenderPipelineStage(RenderPipelineStage::Settings()), |
| 574 |
image_bundle_(image_bundle), |
| 575 |
color_encoding_(output_encoding_info.color_encoding) {} |
| 576 |
|
| 577 |
Status SetInputSizes( |
| 578 |
const std::vector<std::pair<size_t, size_t>>& input_sizes) override { |
| 579 |
JxlMemoryManager* memory_manager = image_bundle_->memory_manager(); |
| 580 |
JXL_ENSURE(input_sizes.size() >= 3); |
| 581 |
for (size_t c = 1; c < input_sizes.size(); c++) { |
| 582 |
JXL_ENSURE(input_sizes[c].first == input_sizes[0].first); |
| 583 |
JXL_ENSURE(input_sizes[c].second == input_sizes[0].second); |
| 584 |
} |
| 585 |
// TODO(eustas): what should we do in the case of "want only ECs"? |
| 586 |
JXL_ASSIGN_OR_RETURN(Image3F tmp, |
| 587 |
Image3F::Create(memory_manager, input_sizes[0].first, |
| 588 |
input_sizes[0].second)); |
| 589 |
JXL_RETURN_IF_ERROR( |
| 590 |
image_bundle_->SetFromImage(std::move(tmp), color_encoding_)); |
| 591 |
// TODO(veluca): consider not reallocating ECs if not needed. |
| 592 |
image_bundle_->extra_channels().clear(); |
| 593 |
for (size_t c = 3; c < input_sizes.size(); c++) { |
| 594 |
JXL_ASSIGN_OR_RETURN(ImageF ch, |
| 595 |
ImageF::Create(memory_manager, input_sizes[c].first, |
| 596 |
input_sizes[c].second)); |
| 597 |
image_bundle_->extra_channels().emplace_back(std::move(ch)); |
| 598 |
} |
| 599 |
return true; |
| 600 |
} |
| 601 |
|
| 602 |
Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, |
| 603 |
size_t xextra, size_t xsize, size_t xpos, size_t ypos, |
| 604 |
size_t thread_id) const final { |
| 605 |
for (size_t c = 0; c < 3; c++) { |
| 606 |
memcpy(image_bundle_->color()->PlaneRow(c, ypos) + xpos - xextra, |
| 607 |
GetInputRow(input_rows, c, 0) - xextra, |
| 608 |
sizeof(float) * (xsize + 2 * xextra)); |
| 609 |
} |
| 610 |
for (size_t ec = 0; ec < image_bundle_->extra_channels().size(); ec++) { |
| 611 |
JXL_ENSURE(image_bundle_->extra_channels()[ec].xsize() >= |
| 612 |
xpos + xsize + xextra); |
| 613 |
memcpy(image_bundle_->extra_channels()[ec].Row(ypos) + xpos - xextra, |
| 614 |
GetInputRow(input_rows, 3 + ec, 0) - xextra, |
| 615 |
sizeof(float) * (xsize + 2 * xextra)); |
| 616 |
} |
| 617 |
return true; |
| 618 |
} |
| 619 |
|
| 620 |
RenderPipelineChannelMode GetChannelMode(size_t c) const final { |
| 621 |
return RenderPipelineChannelMode::kInput; |
| 622 |
} |
| 623 |
|
| 624 |
const char* GetName() const override { return "WriteIB"; } |
| 625 |
|
| 626 |
private: |
| 627 |
ImageBundle* image_bundle_; |
| 628 |
ColorEncoding color_encoding_; |
| 629 |
}; |
| 630 |
|
| 631 |
class WriteToImage3FStage : public RenderPipelineStage { |
| 632 |
public: |
| 633 |
WriteToImage3FStage(JxlMemoryManager* memory_manager, Image3F* image) |
| 634 |
: RenderPipelineStage(RenderPipelineStage::Settings()), |
| 635 |
memory_manager_(memory_manager), |
| 636 |
image_(image) {} |
| 637 |
|
| 638 |
Status SetInputSizes( |
| 639 |
const std::vector<std::pair<size_t, size_t>>& input_sizes) override { |
| 640 |
JXL_ENSURE(input_sizes.size() >= 3); |
| 641 |
for (size_t c = 1; c < 3; ++c) { |
| 642 |
JXL_ENSURE(input_sizes[c].first == input_sizes[0].first); |
| 643 |
JXL_ENSURE(input_sizes[c].second == input_sizes[0].second); |
| 644 |
} |
| 645 |
JXL_ASSIGN_OR_RETURN(*image_, |
| 646 |
Image3F::Create(memory_manager_, input_sizes[0].first, |
| 647 |
input_sizes[0].second)); |
| 648 |
return true; |
| 649 |
} |
| 650 |
|
| 651 |
Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, |
| 652 |
size_t xextra, size_t xsize, size_t xpos, size_t ypos, |
| 653 |
size_t thread_id) const final { |
| 654 |
for (size_t c = 0; c < 3; c++) { |
| 655 |
memcpy(image_->PlaneRow(c, ypos) + xpos - xextra, |
| 656 |
GetInputRow(input_rows, c, 0) - xextra, |
| 657 |
sizeof(float) * (xsize + 2 * xextra)); |
| 658 |
} |
| 659 |
return true; |
| 660 |
} |
| 661 |
|
| 662 |
RenderPipelineChannelMode GetChannelMode(size_t c) const final { |
| 663 |
return c < 3 ? RenderPipelineChannelMode::kInput |
| 664 |
: RenderPipelineChannelMode::kIgnored; |
| 665 |
} |
| 666 |
|
| 667 |
const char* GetName() const override { return "WriteI3F"; } |
| 668 |
|
| 669 |
private: |
| 670 |
JxlMemoryManager* memory_manager_; |
| 671 |
Image3F* image_; |
| 672 |
}; |
| 673 |
|
| 674 |
} // namespace |
| 675 |
|
| 676 |
std::unique_ptr<RenderPipelineStage> GetWriteToImageBundleStage( |
| 677 |
ImageBundle* image_bundle, const OutputEncodingInfo& output_encoding_info) { |
| 678 |
return jxl::make_unique<WriteToImageBundleStage>(image_bundle, |
| 679 |
output_encoding_info); |
| 680 |
} |
| 681 |
|
| 682 |
std::unique_ptr<RenderPipelineStage> GetWriteToImage3FStage( |
| 683 |
JxlMemoryManager* memory_manager, Image3F* image) { |
| 684 |
return jxl::make_unique<WriteToImage3FStage>(memory_manager, image); |
| 685 |
} |
| 686 |
|
| 687 |
std::unique_ptr<RenderPipelineStage> GetWriteToOutputStage( |
| 688 |
const ImageOutput& main_output, size_t width, size_t height, bool has_alpha, |
| 689 |
bool unpremul_alpha, size_t alpha_c, Orientation undo_orientation, |
| 690 |
std::vector<ImageOutput>& extra_output, JxlMemoryManager* memory_manager) { |
| 691 |
return HWY_DYNAMIC_DISPATCH(GetWriteToOutputStage)( |
| 692 |
main_output, width, height, has_alpha, unpremul_alpha, alpha_c, |
| 693 |
undo_orientation, extra_output, memory_manager); |
| 694 |
} |
| 695 |
|
| 696 |
} // namespace jxl |
| 697 |
|
| 698 |
#endif |