Debian Codegraph @

lib/jxl/render_pipeline/stage_write.cc

raw
1 // Copyright (c) the JPEG XL Project Authors. All rights reserved.
2 //
3 // Use of this source code is governed by a BSD-style
4 // license that can be found in the LICENSE file.
5
6 #include "lib/jxl/render_pipeline/stage_write.h"
7
8 #include <jxl/memory_manager.h>
9
10 #include <cstdint>
11 #include <type_traits>
12
13 #include "lib/jxl/alpha.h"
14 #include "lib/jxl/base/common.h"
15 #include "lib/jxl/base/compiler_specific.h"
16 #include "lib/jxl/base/sanitizers.h"
17 #include "lib/jxl/base/status.h"
18 #include "lib/jxl/dec_cache.h"
19 #include "lib/jxl/dec_xyb.h"
20 #include "lib/jxl/image.h"
21 #include "lib/jxl/image_bundle.h"
22 #include "lib/jxl/memory_manager_internal.h"
23
24 #undef HWY_TARGET_INCLUDE
25 #define HWY_TARGET_INCLUDE "lib/jxl/render_pipeline/stage_write.cc"
26 #include <hwy/foreach_target.h>
27 #include <hwy/highway.h>
28
29 HWY_BEFORE_NAMESPACE();
30 namespace jxl {
31 namespace HWY_NAMESPACE {
32
33 // These templates are not found via ADL.
34 using hwy::HWY_NAMESPACE::Add;
35 using hwy::HWY_NAMESPACE::Clamp;
36 using hwy::HWY_NAMESPACE::Div;
37 using hwy::HWY_NAMESPACE::Max;
38 using hwy::HWY_NAMESPACE::Mul;
39 using hwy::HWY_NAMESPACE::NearestInt;
40 using hwy::HWY_NAMESPACE::Or;
41 using hwy::HWY_NAMESPACE::Rebind;
42 using hwy::HWY_NAMESPACE::ShiftLeftSame;
43 using hwy::HWY_NAMESPACE::ShiftRightSame;
44 using hwy::HWY_NAMESPACE::VFromD;
45
46 // 8x8 ordered dithering pattern from
47 // https://en.wikipedia.org/wiki/Ordered_dithering
48 // scaled to have an average of 0 and be fully contained in (-0.5, 0.5).
49 // Matrix is duplicated in width to avoid inconsistencies or out-of-bound-reads
50 // if doing unaligned operations.
51 const float kDither[(2 * 8) * 8] = {
52 -0.4921875, 0.0078125, -0.3671875, 0.1328125, //
53 -0.4609375, 0.0390625, -0.3359375, 0.1640625, //
54 -0.4921875, 0.0078125, -0.3671875, 0.1328125, //
55 -0.4609375, 0.0390625, -0.3359375, 0.1640625, //
56 //
57 0.2578125, -0.2421875, 0.3828125, -0.1171875, //
58 0.2890625, -0.2109375, 0.4140625, -0.0859375, //
59 0.2578125, -0.2421875, 0.3828125, -0.1171875, //
60 0.2890625, -0.2109375, 0.4140625, -0.0859375, //
61 //
62 -0.3046875, 0.1953125, -0.4296875, 0.0703125, //
63 -0.2734375, 0.2265625, -0.3984375, 0.1015625, //
64 -0.3046875, 0.1953125, -0.4296875, 0.0703125, //
65 -0.2734375, 0.2265625, -0.3984375, 0.1015625, //
66 //
67 0.4453125, -0.0546875, 0.3203125, -0.1796875, //
68 0.4765625, -0.0234375, 0.3515625, -0.1484375, //
69 0.4453125, -0.0546875, 0.3203125, -0.1796875, //
70 0.4765625, -0.0234375, 0.3515625, -0.1484375, //
71 //
72 -0.4453125, 0.0546875, -0.3203125, 0.1796875, //
73 -0.4765625, 0.0234375, -0.3515625, 0.1484375, //
74 -0.4453125, 0.0546875, -0.3203125, 0.1796875, //
75 -0.4765625, 0.0234375, -0.3515625, 0.1484375, //
76 //
77 0.3046875, -0.1953125, 0.4296875, -0.0703125, //
78 0.2734375, -0.2265625, 0.3984375, -0.1015625, //
79 0.3046875, -0.1953125, 0.4296875, -0.0703125, //
80 0.2734375, -0.2265625, 0.3984375, -0.1015625, //
81 //
82 -0.2578125, 0.2421875, -0.3828125, 0.1171875, //
83 -0.2890625, 0.2109375, -0.4140625, 0.0859375, //
84 -0.2578125, 0.2421875, -0.3828125, 0.1171875, //
85 -0.2890625, 0.2109375, -0.4140625, 0.0859375, //
86 //
87 0.4921875, -0.0078125, 0.3671875, -0.1328125, //
88 0.4609375, -0.0390625, 0.3359375, -0.1640625, //
89 0.4921875, -0.0078125, 0.3671875, -0.1328125, //
90 0.4609375, -0.0390625, 0.3359375, -0.1640625, //
91 };
92
93 using DF = HWY_FULL(float);
94
95 // Converts `v` to an appropriate value for the given unsigned type.
96 // If the unsigned type is an 8-bit type, performs ordered dithering.
97 template <typename T>
98 VFromD<Rebind<T, DF>> MakeUnsigned(VFromD<DF> v, size_t x0, size_t y0,
99 VFromD<DF> mul) {
100 static_assert(std::is_unsigned<T>::value, "T must be an unsigned type");
101 using DU = Rebind<T, DF>;
102 v = Mul(v, mul);
103 // TODO(veluca): if constexpr with C++17
104 if (sizeof(T) == 1) {
105 size_t pos = (y0 % 8) * (2 * 8) + (x0 % 8);
106 #if HWY_TARGET != HWY_SCALAR
107 auto dither = LoadDup128(DF(), kDither + pos);
108 #else
109 auto dither = LoadU(DF(), kDither + pos);
110 #endif
111 v = Add(v, dither);
112 }
113 v = Clamp(Zero(DF()), v, mul);
114 return DemoteTo(DU(), NearestInt(v));
115 }
116
117 class WriteToOutputStage : public RenderPipelineStage {
118 public:
119 WriteToOutputStage(const ImageOutput& main_output, size_t width,
120 size_t height, bool has_alpha, bool unpremul_alpha,
121 size_t alpha_c, Orientation undo_orientation,
122 const std::vector<ImageOutput>& extra_output,
123 JxlMemoryManager* memory_manager)
124 : RenderPipelineStage(RenderPipelineStage::Settings()),
125 width_(width),
126 height_(height),
127 main_(main_output),
128 num_color_(main_.num_channels_ < 3 ? 1 : 3),
129 want_alpha_(main_.num_channels_ == 2 || main_.num_channels_ == 4),
130 has_alpha_(has_alpha),
131 unpremul_alpha_(unpremul_alpha),
132 alpha_c_(alpha_c),
133 flip_x_(ShouldFlipX(undo_orientation)),
134 flip_y_(ShouldFlipY(undo_orientation)),
135 transpose_(ShouldTranspose(undo_orientation)),
136 opaque_alpha_(kMaxPixelsPerCall, 1.0f),
137 memory_manager_(memory_manager) {
138 for (size_t ec = 0; ec < extra_output.size(); ++ec) {
139 if (extra_output[ec].callback.IsPresent() || extra_output[ec].buffer) {
140 Output extra(extra_output[ec]);
141 extra.channel_index_ = 3 + ec;
142 extra_channels_.push_back(extra);
143 }
144 }
145 }
146
147 WriteToOutputStage(const WriteToOutputStage&) = delete;
148 WriteToOutputStage& operator=(const WriteToOutputStage&) = delete;
149 WriteToOutputStage(WriteToOutputStage&&) = delete;
150 WriteToOutputStage& operator=(WriteToOutputStage&&) = delete;
151
152 ~WriteToOutputStage() override {
153 if (main_.run_opaque_) {
154 main_.pixel_callback_.destroy(main_.run_opaque_);
155 }
156 for (auto& extra : extra_channels_) {
157 if (extra.run_opaque_) {
158 extra.pixel_callback_.destroy(extra.run_opaque_);
159 }
160 }
161 }
162
163 Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows,
164 size_t xextra, size_t xsize, size_t xpos, size_t ypos,
165 size_t thread_id) const final {
166 JXL_ENSURE(xextra == 0);
167 JXL_ENSURE(main_.run_opaque_ || main_.buffer_);
168 if (ypos >= height_) return true;
169 if (xpos >= width_) return true;
170 if (flip_y_) {
171 ypos = height_ - 1u - ypos;
172 }
173 size_t limit = std::min(xsize, width_ - xpos);
174 for (size_t x0 = 0; x0 < limit; x0 += kMaxPixelsPerCall) {
175 size_t xstart = xpos + x0;
176 size_t len = std::min<size_t>(kMaxPixelsPerCall, limit - x0);
177
178 const float* line_buffers[4];
179 for (size_t c = 0; c < num_color_; c++) {
180 line_buffers[c] = GetInputRow(input_rows, c, 0) + x0;
181 }
182 if (has_alpha_) {
183 line_buffers[num_color_] = GetInputRow(input_rows, alpha_c_, 0) + x0;
184 } else {
185 // opaque_alpha_ is a way to set all values to 1.0f.
186 line_buffers[num_color_] = opaque_alpha_.data();
187 }
188 if (has_alpha_ && want_alpha_ && unpremul_alpha_) {
189 UnpremulAlpha(thread_id, len, line_buffers);
190 }
191 OutputBuffers(main_, thread_id, ypos, xstart, len, line_buffers);
192 for (const auto& extra : extra_channels_) {
193 line_buffers[0] = GetInputRow(input_rows, extra.channel_index_, 0) + x0;
194 OutputBuffers(extra, thread_id, ypos, xstart, len, line_buffers);
195 }
196 }
197 return true;
198 }
199
200 RenderPipelineChannelMode GetChannelMode(size_t c) const final {
201 if (c < num_color_ || (has_alpha_ && c == alpha_c_)) {
202 return RenderPipelineChannelMode::kInput;
203 }
204 for (const auto& extra : extra_channels_) {
205 if (c == extra.channel_index_) {
206 return RenderPipelineChannelMode::kInput;
207 }
208 }
209 return RenderPipelineChannelMode::kIgnored;
210 }
211
212 const char* GetName() const override { return "WritePixelCB"; }
213
214 private:
215 struct Output {
216 explicit Output(const ImageOutput& image_out)
217 : pixel_callback_(image_out.callback),
218 buffer_(image_out.buffer),
219 buffer_size_(image_out.buffer_size),
220 stride_(image_out.stride),
221 num_channels_(image_out.format.num_channels),
222 swap_endianness_(SwapEndianness(image_out.format.endianness)),
223 data_type_(image_out.format.data_type),
224 bits_per_sample_(image_out.bits_per_sample) {}
225
226 Status PrepareForThreads(size_t num_threads) {
227 if (pixel_callback_.IsPresent()) {
228 run_opaque_ =
229 pixel_callback_.Init(num_threads, /*num_pixels=*/kMaxPixelsPerCall);
230 JXL_RETURN_IF_ERROR(run_opaque_ != nullptr);
231 } else {
232 JXL_RETURN_IF_ERROR(buffer_ != nullptr);
233 }
234 return true;
235 }
236
237 PixelCallback pixel_callback_;
238 void* run_opaque_ = nullptr;
239 void* buffer_ = nullptr;
240 size_t buffer_size_;
241 size_t stride_;
242 size_t num_channels_;
243 bool swap_endianness_;
244 JxlDataType data_type_;
245 size_t bits_per_sample_;
246 size_t channel_index_; // used for extra_channels
247 };
248
249 Status PrepareForThreads(size_t num_threads) override {
250 JXL_RETURN_IF_ERROR(main_.PrepareForThreads(num_threads));
251 for (auto& extra : extra_channels_) {
252 JXL_RETURN_IF_ERROR(extra.PrepareForThreads(num_threads));
253 }
254 temp_out_.resize(num_threads);
255 for (AlignedMemory& temp : temp_out_) {
256 size_t alloc_size =
257 sizeof(float) * kMaxPixelsPerCall * main_.num_channels_;
258 JXL_ASSIGN_OR_RETURN(temp,
259 AlignedMemory::Create(memory_manager_, alloc_size));
260 }
261 if ((has_alpha_ && want_alpha_ && unpremul_alpha_) || flip_x_) {
262 temp_in_.resize(num_threads * main_.num_channels_);
263 for (AlignedMemory& temp : temp_in_) {
264 size_t alloc_size = sizeof(float) * kMaxPixelsPerCall;
265 JXL_ASSIGN_OR_RETURN(
266 temp, AlignedMemory::Create(memory_manager_, alloc_size));
267 }
268 }
269 return true;
270 }
271 static bool ShouldFlipX(Orientation undo_orientation) {
272 return (undo_orientation == Orientation::kFlipHorizontal ||
273 undo_orientation == Orientation::kRotate180 ||
274 undo_orientation == Orientation::kRotate270 ||
275 undo_orientation == Orientation::kAntiTranspose);
276 }
277 static bool ShouldFlipY(Orientation undo_orientation) {
278 return (undo_orientation == Orientation::kFlipVertical ||
279 undo_orientation == Orientation::kRotate180 ||
280 undo_orientation == Orientation::kRotate90 ||
281 undo_orientation == Orientation::kAntiTranspose);
282 }
283 static bool ShouldTranspose(Orientation undo_orientation) {
284 return (undo_orientation == Orientation::kTranspose ||
285 undo_orientation == Orientation::kRotate90 ||
286 undo_orientation == Orientation::kRotate270 ||
287 undo_orientation == Orientation::kAntiTranspose);
288 }
289
290 void UnpremulAlpha(size_t thread_id, size_t len,
291 const float** line_buffers) const {
292 const HWY_FULL(float) d;
293 auto one = Set(d, 1.0f);
294 float* temp_in[4];
295 for (size_t c = 0; c < main_.num_channels_; ++c) {
296 size_t tix = thread_id * main_.num_channels_ + c;
297 temp_in[c] = temp_in_[tix].address<float>();
298 memcpy(temp_in[c], line_buffers[c], sizeof(float) * len);
299 }
300 auto small_alpha = Set(d, kSmallAlpha);
301 for (size_t ix = 0; ix < len; ix += Lanes(d)) {
302 auto alpha = LoadU(d, temp_in[num_color_] + ix);
303 auto mul = Div(one, Max(small_alpha, alpha));
304 for (size_t c = 0; c < num_color_; ++c) {
305 auto val = LoadU(d, temp_in[c] + ix);
306 StoreU(Mul(val, mul), d, temp_in[c] + ix);
307 }
308 }
309 for (size_t c = 0; c < main_.num_channels_; ++c) {
310 line_buffers[c] = temp_in[c];
311 }
312 }
313
314 void OutputBuffers(const Output& out, size_t thread_id, size_t ypos,
315 size_t xstart, size_t len, const float* input[4]) const {
316 if (flip_x_) {
317 FlipX(out, thread_id, len, &xstart, input);
318 }
319 if (out.data_type_ == JXL_TYPE_UINT8) {
320 uint8_t* JXL_RESTRICT temp = temp_out_[thread_id].address<uint8_t>();
321 StoreUnsignedRow(out, input, len, temp, xstart, ypos);
322 WriteToOutput(out, thread_id, ypos, xstart, len, temp);
323 } else if (out.data_type_ == JXL_TYPE_UINT16 ||
324 out.data_type_ == JXL_TYPE_FLOAT16) {
325 uint16_t* JXL_RESTRICT temp = temp_out_[thread_id].address<uint16_t>();
326 if (out.data_type_ == JXL_TYPE_UINT16) {
327 StoreUnsignedRow(out, input, len, temp, xstart, ypos);
328 } else {
329 StoreFloat16Row(out, input, len, temp);
330 }
331 if (out.swap_endianness_) {
332 const HWY_FULL(uint16_t) du;
333 size_t output_len = len * out.num_channels_;
334 for (size_t j = 0; j < output_len; j += Lanes(du)) {
335 auto v = LoadU(du, temp + j);
336 auto vswap = Or(ShiftRightSame(v, 8), ShiftLeftSame(v, 8));
337 StoreU(vswap, du, temp + j);
338 }
339 }
340 WriteToOutput(out, thread_id, ypos, xstart, len, temp);
341 } else if (out.data_type_ == JXL_TYPE_FLOAT) {
342 float* JXL_RESTRICT temp = temp_out_[thread_id].address<float>();
343 StoreFloatRow(out, input, len, temp);
344 if (out.swap_endianness_) {
345 size_t output_len = len * out.num_channels_;
346 for (size_t j = 0; j < output_len; ++j) {
347 temp[j] = BSwapFloat(temp[j]);
348 }
349 }
350 WriteToOutput(out, thread_id, ypos, xstart, len, temp);
351 }
352 }
353
354 void FlipX(const Output& out, size_t thread_id, size_t len, size_t* xstart,
355 const float** line_buffers) const {
356 float* temp_in[4];
357 for (size_t c = 0; c < out.num_channels_; ++c) {
358 size_t tix = thread_id * main_.num_channels_ + c;
359 temp_in[c] = temp_in_[tix].address<float>();
360 if (temp_in[c] != line_buffers[c]) {
361 memcpy(temp_in[c], line_buffers[c], sizeof(float) * len);
362 }
363 }
364 size_t last = (len - 1u);
365 size_t num = (len / 2);
366 for (size_t i = 0; i < num; ++i) {
367 for (size_t c = 0; c < out.num_channels_; ++c) {
368 std::swap(temp_in[c][i], temp_in[c][last - i]);
369 }
370 }
371 for (size_t c = 0; c < out.num_channels_; ++c) {
372 line_buffers[c] = temp_in[c];
373 }
374 *xstart = width_ - *xstart - len;
375 }
376
377 template <typename T>
378 void StoreUnsignedRow(const Output& out, const float* input[4], size_t len,
379 T* output, size_t xstart, size_t ypos) const {
380 const HWY_FULL(float) d;
381 auto mul = Set(d, (1u << (out.bits_per_sample_)) - 1);
382 const Rebind<T, decltype(d)> du;
383 const size_t padding = RoundUpTo(len, Lanes(d)) - len;
384 for (size_t c = 0; c < out.num_channels_; ++c) {
385 msan::UnpoisonMemory(input[c] + len, sizeof(input[c][0]) * padding);
386 }
387 if (out.num_channels_ == 1) {
388 for (size_t i = 0; i < len; i += Lanes(d)) {
389 StoreU(MakeUnsigned<T>(LoadU(d, &input[0][i]), xstart + i, ypos, mul),
390 du, &output[i]);
391 }
392 } else if (out.num_channels_ == 2) {
393 for (size_t i = 0; i < len; i += Lanes(d)) {
394 StoreInterleaved2(
395 MakeUnsigned<T>(LoadU(d, &input[0][i]), xstart + i, ypos, mul),
396 MakeUnsigned<T>(LoadU(d, &input[1][i]), xstart + i, ypos, mul), du,
397 &output[2 * i]);
398 }
399 } else if (out.num_channels_ == 3) {
400 for (size_t i = 0; i < len; i += Lanes(d)) {
401 StoreInterleaved3(
402 MakeUnsigned<T>(LoadU(d, &input[0][i]), xstart + i, ypos, mul),
403 MakeUnsigned<T>(LoadU(d, &input[1][i]), xstart + i, ypos, mul),
404 MakeUnsigned<T>(LoadU(d, &input[2][i]), xstart + i, ypos, mul), du,
405 &output[3 * i]);
406 }
407 } else if (out.num_channels_ == 4) {
408 for (size_t i = 0; i < len; i += Lanes(d)) {
409 StoreInterleaved4(
410 MakeUnsigned<T>(LoadU(d, &input[0][i]), xstart + i, ypos, mul),
411 MakeUnsigned<T>(LoadU(d, &input[1][i]), xstart + i, ypos, mul),
412 MakeUnsigned<T>(LoadU(d, &input[2][i]), xstart + i, ypos, mul),
413 MakeUnsigned<T>(LoadU(d, &input[3][i]), xstart + i, ypos, mul), du,
414 &output[4 * i]);
415 }
416 }
417 msan::PoisonMemory(output + out.num_channels_ * len,
418 sizeof(output[0]) * out.num_channels_ * padding);
419 }
420
421 static void StoreFloat16Row(const Output& out, const float* input[4],
422 size_t len, uint16_t* output) {
423 const HWY_FULL(float) d;
424 const Rebind<uint16_t, decltype(d)> du;
425 const Rebind<hwy::float16_t, decltype(d)> df16;
426 const size_t padding = RoundUpTo(len, Lanes(d)) - len;
427 for (size_t c = 0; c < out.num_channels_; ++c) {
428 msan::UnpoisonMemory(input[c] + len, sizeof(input[c][0]) * padding);
429 }
430 if (out.num_channels_ == 1) {
431 for (size_t i = 0; i < len; i += Lanes(d)) {
432 auto v0 = LoadU(d, &input[0][i]);
433 StoreU(BitCast(du, DemoteTo(df16, v0)), du, &output[i]);
434 }
435 } else if (out.num_channels_ == 2) {
436 for (size_t i = 0; i < len; i += Lanes(d)) {
437 auto v0 = LoadU(d, &input[0][i]);
438 auto v1 = LoadU(d, &input[1][i]);
439 StoreInterleaved2(BitCast(du, DemoteTo(df16, v0)),
440 BitCast(du, DemoteTo(df16, v1)), du, &output[2 * i]);
441 }
442 } else if (out.num_channels_ == 3) {
443 for (size_t i = 0; i < len; i += Lanes(d)) {
444 auto v0 = LoadU(d, &input[0][i]);
445 auto v1 = LoadU(d, &input[1][i]);
446 auto v2 = LoadU(d, &input[2][i]);
447 StoreInterleaved3(BitCast(du, DemoteTo(df16, v0)),
448 BitCast(du, DemoteTo(df16, v1)),
449 BitCast(du, DemoteTo(df16, v2)), du, &output[3 * i]);
450 }
451 } else if (out.num_channels_ == 4) {
452 for (size_t i = 0; i < len; i += Lanes(d)) {
453 auto v0 = LoadU(d, &input[0][i]);
454 auto v1 = LoadU(d, &input[1][i]);
455 auto v2 = LoadU(d, &input[2][i]);
456 auto v3 = LoadU(d, &input[3][i]);
457 StoreInterleaved4(BitCast(du, DemoteTo(df16, v0)),
458 BitCast(du, DemoteTo(df16, v1)),
459 BitCast(du, DemoteTo(df16, v2)),
460 BitCast(du, DemoteTo(df16, v3)), du, &output[4 * i]);
461 }
462 }
463 msan::PoisonMemory(output + out.num_channels_ * len,
464 sizeof(output[0]) * out.num_channels_ * padding);
465 }
466
467 static void StoreFloatRow(const Output& out, const float* input[4],
468 size_t len, float* output) {
469 const HWY_FULL(float) d;
470 if (out.num_channels_ == 1) {
471 memcpy(output, input[0], len * sizeof(output[0]));
472 } else if (out.num_channels_ == 2) {
473 for (size_t i = 0; i < len; i += Lanes(d)) {
474 StoreInterleaved2(LoadU(d, &input[0][i]), LoadU(d, &input[1][i]), d,
475 &output[2 * i]);
476 }
477 } else if (out.num_channels_ == 3) {
478 for (size_t i = 0; i < len; i += Lanes(d)) {
479 StoreInterleaved3(LoadU(d, &input[0][i]), LoadU(d, &input[1][i]),
480 LoadU(d, &input[2][i]), d, &output[3 * i]);
481 }
482 } else {
483 for (size_t i = 0; i < len; i += Lanes(d)) {
484 StoreInterleaved4(LoadU(d, &input[0][i]), LoadU(d, &input[1][i]),
485 LoadU(d, &input[2][i]), LoadU(d, &input[3][i]), d,
486 &output[4 * i]);
487 }
488 }
489 }
490
491 template <typename T>
492 void WriteToOutput(const Output& out, size_t thread_id, size_t ypos,
493 size_t xstart, size_t len, T* output) const {
494 if (transpose_) {
495 // TODO(szabadka) Buffer 8x8 chunks and transpose with SIMD.
496 if (out.run_opaque_) {
497 for (size_t i = 0, j = 0; i < len; ++i, j += out.num_channels_) {
498 out.pixel_callback_.run(out.run_opaque_, thread_id, ypos, xstart + i,
499 1, output + j);
500 }
501 } else {
502 const size_t pixel_stride = out.num_channels_ * sizeof(T);
503 const size_t offset = xstart * out.stride_ + ypos * pixel_stride;
504 for (size_t i = 0, j = 0; i < len; ++i, j += out.num_channels_) {
505 const size_t ix = offset + i * out.stride_;
506 JXL_DASSERT(ix + pixel_stride <= out.buffer_size_);
507 memcpy(reinterpret_cast<uint8_t*>(out.buffer_) + ix, output + j,
508 pixel_stride);
509 }
510 }
511 } else {
512 if (out.run_opaque_) {
513 out.pixel_callback_.run(out.run_opaque_, thread_id, xstart, ypos, len,
514 output);
515 } else {
516 const size_t pixel_stride = out.num_channels_ * sizeof(T);
517 const size_t offset = ypos * out.stride_ + xstart * pixel_stride;
518 JXL_DASSERT(offset + len * pixel_stride <= out.buffer_size_);
519 memcpy(reinterpret_cast<uint8_t*>(out.buffer_) + offset, output,
520 len * pixel_stride);
521 }
522 }
523 }
524
525 static constexpr size_t kMaxPixelsPerCall = 1024;
526 size_t width_;
527 size_t height_;
528 Output main_; // color + alpha
529 size_t num_color_;
530 bool want_alpha_;
531 bool has_alpha_;
532 bool unpremul_alpha_;
533 size_t alpha_c_;
534 bool flip_x_;
535 bool flip_y_;
536 bool transpose_;
537 std::vector<Output> extra_channels_;
538 std::vector<float> opaque_alpha_;
539 JxlMemoryManager* memory_manager_;
540 std::vector<AlignedMemory> temp_in_;
541 std::vector<AlignedMemory> temp_out_;
542 };
543
544 #if JXL_CXX_LANG < JXL_CXX_17
545 constexpr size_t WriteToOutputStage::kMaxPixelsPerCall;
546 #endif
547
548 std::unique_ptr<RenderPipelineStage> GetWriteToOutputStage(
549 const ImageOutput& main_output, size_t width, size_t height, bool has_alpha,
550 bool unpremul_alpha, size_t alpha_c, Orientation undo_orientation,
551 std::vector<ImageOutput>& extra_output, JxlMemoryManager* memory_manager) {
552 return jxl::make_unique<WriteToOutputStage>(
553 main_output, width, height, has_alpha, unpremul_alpha, alpha_c,
554 undo_orientation, extra_output, memory_manager);
555 }
556
557 // NOLINTNEXTLINE(google-readability-namespace-comments)
558 } // namespace HWY_NAMESPACE
559 } // namespace jxl
560 HWY_AFTER_NAMESPACE();
561
562 #if HWY_ONCE
563
564 namespace jxl {
565
566 HWY_EXPORT(GetWriteToOutputStage);
567
568 namespace {
569 class WriteToImageBundleStage : public RenderPipelineStage {
570 public:
571 explicit WriteToImageBundleStage(
572 ImageBundle* image_bundle, const OutputEncodingInfo& output_encoding_info)
573 : RenderPipelineStage(RenderPipelineStage::Settings()),
574 image_bundle_(image_bundle),
575 color_encoding_(output_encoding_info.color_encoding) {}
576
577 Status SetInputSizes(
578 const std::vector<std::pair<size_t, size_t>>& input_sizes) override {
579 JxlMemoryManager* memory_manager = image_bundle_->memory_manager();
580 JXL_ENSURE(input_sizes.size() >= 3);
581 for (size_t c = 1; c < input_sizes.size(); c++) {
582 JXL_ENSURE(input_sizes[c].first == input_sizes[0].first);
583 JXL_ENSURE(input_sizes[c].second == input_sizes[0].second);
584 }
585 // TODO(eustas): what should we do in the case of "want only ECs"?
586 JXL_ASSIGN_OR_RETURN(Image3F tmp,
587 Image3F::Create(memory_manager, input_sizes[0].first,
588 input_sizes[0].second));
589 JXL_RETURN_IF_ERROR(
590 image_bundle_->SetFromImage(std::move(tmp), color_encoding_));
591 // TODO(veluca): consider not reallocating ECs if not needed.
592 image_bundle_->extra_channels().clear();
593 for (size_t c = 3; c < input_sizes.size(); c++) {
594 JXL_ASSIGN_OR_RETURN(ImageF ch,
595 ImageF::Create(memory_manager, input_sizes[c].first,
596 input_sizes[c].second));
597 image_bundle_->extra_channels().emplace_back(std::move(ch));
598 }
599 return true;
600 }
601
602 Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows,
603 size_t xextra, size_t xsize, size_t xpos, size_t ypos,
604 size_t thread_id) const final {
605 for (size_t c = 0; c < 3; c++) {
606 memcpy(image_bundle_->color()->PlaneRow(c, ypos) + xpos - xextra,
607 GetInputRow(input_rows, c, 0) - xextra,
608 sizeof(float) * (xsize + 2 * xextra));
609 }
610 for (size_t ec = 0; ec < image_bundle_->extra_channels().size(); ec++) {
611 JXL_ENSURE(image_bundle_->extra_channels()[ec].xsize() >=
612 xpos + xsize + xextra);
613 memcpy(image_bundle_->extra_channels()[ec].Row(ypos) + xpos - xextra,
614 GetInputRow(input_rows, 3 + ec, 0) - xextra,
615 sizeof(float) * (xsize + 2 * xextra));
616 }
617 return true;
618 }
619
620 RenderPipelineChannelMode GetChannelMode(size_t c) const final {
621 return RenderPipelineChannelMode::kInput;
622 }
623
624 const char* GetName() const override { return "WriteIB"; }
625
626 private:
627 ImageBundle* image_bundle_;
628 ColorEncoding color_encoding_;
629 };
630
631 class WriteToImage3FStage : public RenderPipelineStage {
632 public:
633 WriteToImage3FStage(JxlMemoryManager* memory_manager, Image3F* image)
634 : RenderPipelineStage(RenderPipelineStage::Settings()),
635 memory_manager_(memory_manager),
636 image_(image) {}
637
638 Status SetInputSizes(
639 const std::vector<std::pair<size_t, size_t>>& input_sizes) override {
640 JXL_ENSURE(input_sizes.size() >= 3);
641 for (size_t c = 1; c < 3; ++c) {
642 JXL_ENSURE(input_sizes[c].first == input_sizes[0].first);
643 JXL_ENSURE(input_sizes[c].second == input_sizes[0].second);
644 }
645 JXL_ASSIGN_OR_RETURN(*image_,
646 Image3F::Create(memory_manager_, input_sizes[0].first,
647 input_sizes[0].second));
648 return true;
649 }
650
651 Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows,
652 size_t xextra, size_t xsize, size_t xpos, size_t ypos,
653 size_t thread_id) const final {
654 for (size_t c = 0; c < 3; c++) {
655 memcpy(image_->PlaneRow(c, ypos) + xpos - xextra,
656 GetInputRow(input_rows, c, 0) - xextra,
657 sizeof(float) * (xsize + 2 * xextra));
658 }
659 return true;
660 }
661
662 RenderPipelineChannelMode GetChannelMode(size_t c) const final {
663 return c < 3 ? RenderPipelineChannelMode::kInput
664 : RenderPipelineChannelMode::kIgnored;
665 }
666
667 const char* GetName() const override { return "WriteI3F"; }
668
669 private:
670 JxlMemoryManager* memory_manager_;
671 Image3F* image_;
672 };
673
674 } // namespace
675
676 std::unique_ptr<RenderPipelineStage> GetWriteToImageBundleStage(
677 ImageBundle* image_bundle, const OutputEncodingInfo& output_encoding_info) {
678 return jxl::make_unique<WriteToImageBundleStage>(image_bundle,
679 output_encoding_info);
680 }
681
682 std::unique_ptr<RenderPipelineStage> GetWriteToImage3FStage(
683 JxlMemoryManager* memory_manager, Image3F* image) {
684 return jxl::make_unique<WriteToImage3FStage>(memory_manager, image);
685 }
686
687 std::unique_ptr<RenderPipelineStage> GetWriteToOutputStage(
688 const ImageOutput& main_output, size_t width, size_t height, bool has_alpha,
689 bool unpremul_alpha, size_t alpha_c, Orientation undo_orientation,
690 std::vector<ImageOutput>& extra_output, JxlMemoryManager* memory_manager) {
691 return HWY_DYNAMIC_DISPATCH(GetWriteToOutputStage)(
692 main_output, width, height, has_alpha, unpremul_alpha, alpha_c,
693 undo_orientation, extra_output, memory_manager);
694 }
695
696 } // namespace jxl
697
698 #endif