Debian Codegraph @

lib/jxl/render_pipeline/low_memory_render_pipeline.cc

raw
1 // Copyright (c) the JPEG XL Project Authors. All rights reserved.
2 //
3 // Use of this source code is governed by a BSD-style
4 // license that can be found in the LICENSE file.
5
6 #include "lib/jxl/render_pipeline/low_memory_render_pipeline.h"
7
8 #include <algorithm>
9 #include <cstddef>
10 #include <cstdint>
11 #include <utility>
12 #include <vector>
13
14 #include "lib/jxl/base/arch_macros.h"
15 #include "lib/jxl/base/common.h"
16 #include "lib/jxl/base/rect.h"
17 #include "lib/jxl/base/status.h"
18 #include "lib/jxl/image.h"
19 #include "lib/jxl/image_ops.h"
20
21 namespace jxl {
22 std::pair<size_t, size_t>
23 LowMemoryRenderPipeline::ColorDimensionsToChannelDimensions(
24 std::pair<size_t, size_t> in, size_t c, size_t stage) const {
25 std::pair<size_t, size_t> ret;
26 std::pair<size_t, size_t> shift = channel_shifts_[stage][c];
27 ret.first =
28 ((in.first << base_color_shift_) + (1 << shift.first) - 1) >> shift.first;
29 ret.second = ((in.second << base_color_shift_) + (1 << shift.second) - 1) >>
30 shift.second;
31 return ret;
32 }
33
34 std::pair<size_t, size_t> LowMemoryRenderPipeline::BorderToStore(
35 size_t c) const {
36 auto ret = ColorDimensionsToChannelDimensions(group_border_, c, 0);
37 ret.first += padding_[0][c].first;
38 ret.second += padding_[0][c].second;
39 return ret;
40 }
41
42 Status LowMemoryRenderPipeline::SaveBorders(size_t group_id, size_t c,
43 const ImageF& in) {
44 size_t gy = group_id / frame_dimensions_.xsize_groups;
45 size_t gx = group_id % frame_dimensions_.xsize_groups;
46 size_t hshift = channel_shifts_[0][c].first;
47 size_t vshift = channel_shifts_[0][c].second;
48 size_t x0 = gx * GroupInputXSize(c);
49 size_t x1 = std::min((gx + 1) * GroupInputXSize(c),
50 DivCeil(frame_dimensions_.xsize_upsampled, 1 << hshift));
51 size_t y0 = gy * GroupInputYSize(c);
52 size_t y1 = std::min((gy + 1) * GroupInputYSize(c),
53 DivCeil(frame_dimensions_.ysize_upsampled, 1 << vshift));
54
55 auto borders = BorderToStore(c);
56 size_t borderx_write = borders.first;
57 size_t bordery_write = borders.second;
58
59 if (gy > 0) {
60 Rect from(group_data_x_border_, group_data_y_border_, x1 - x0,
61 bordery_write);
62 Rect to(x0, (gy * 2 - 1) * bordery_write, x1 - x0, bordery_write);
63 JXL_RETURN_IF_ERROR(CopyImageTo(from, in, to, &borders_horizontal_[c]));
64 }
65 if (gy + 1 < frame_dimensions_.ysize_groups) {
66 Rect from(group_data_x_border_,
67 group_data_y_border_ + y1 - y0 - bordery_write, x1 - x0,
68 bordery_write);
69 Rect to(x0, (gy * 2) * bordery_write, x1 - x0, bordery_write);
70 JXL_RETURN_IF_ERROR(CopyImageTo(from, in, to, &borders_horizontal_[c]));
71 }
72 if (gx > 0) {
73 Rect from(group_data_x_border_, group_data_y_border_, borderx_write,
74 y1 - y0);
75 Rect to((gx * 2 - 1) * borderx_write, y0, borderx_write, y1 - y0);
76 JXL_RETURN_IF_ERROR(CopyImageTo(from, in, to, &borders_vertical_[c]));
77 }
78 if (gx + 1 < frame_dimensions_.xsize_groups) {
79 Rect from(group_data_x_border_ + x1 - x0 - borderx_write,
80 group_data_y_border_, borderx_write, y1 - y0);
81 Rect to((gx * 2) * borderx_write, y0, borderx_write, y1 - y0);
82 JXL_RETURN_IF_ERROR(CopyImageTo(from, in, to, &borders_vertical_[c]));
83 }
84 return true;
85 }
86
87 Status LowMemoryRenderPipeline::LoadBorders(size_t group_id, size_t c,
88 const Rect& r, ImageF* out) {
89 size_t gy = group_id / frame_dimensions_.xsize_groups;
90 size_t gx = group_id % frame_dimensions_.xsize_groups;
91 size_t hshift = channel_shifts_[0][c].first;
92 size_t vshift = channel_shifts_[0][c].second;
93 // Coordinates of the group in the image.
94 size_t x0 = gx * GroupInputXSize(c);
95 size_t x1 = std::min((gx + 1) * GroupInputXSize(c),
96 DivCeil(frame_dimensions_.xsize_upsampled, 1 << hshift));
97 size_t y0 = gy * GroupInputYSize(c);
98 size_t y1 = std::min((gy + 1) * GroupInputYSize(c),
99 DivCeil(frame_dimensions_.ysize_upsampled, 1 << vshift));
100
101 size_t paddingx = padding_[0][c].first;
102 size_t paddingy = padding_[0][c].second;
103
104 auto borders = BorderToStore(c);
105 size_t borderx_write = borders.first;
106 size_t bordery_write = borders.second;
107
108 // Limits of the area to copy from, in image coordinates.
109 JXL_ENSURE(r.x0() == 0 || (r.x0() << base_color_shift_) >= paddingx);
110 size_t x0src = DivCeil(r.x0() << base_color_shift_, 1 << hshift);
111 if (x0src != 0) {
112 x0src -= paddingx;
113 }
114 // r may be such that r.x1 (namely x0() + xsize()) is within paddingx of the
115 // right side of the image, so we use min() here.
116 size_t x1src =
117 DivCeil((r.x0() + r.xsize()) << base_color_shift_, 1 << hshift);
118 x1src = std::min(x1src + paddingx,
119 DivCeil(frame_dimensions_.xsize_upsampled, 1 << hshift));
120
121 // Similar computation for y.
122 JXL_ENSURE(r.y0() == 0 || (r.y0() << base_color_shift_) >= paddingy);
123 size_t y0src = DivCeil(r.y0() << base_color_shift_, 1 << vshift);
124 if (y0src != 0) {
125 y0src -= paddingy;
126 }
127 size_t y1src =
128 DivCeil((r.y0() + r.ysize()) << base_color_shift_, 1 << vshift);
129 y1src = std::min(y1src + paddingy,
130 DivCeil(frame_dimensions_.ysize_upsampled, 1 << vshift));
131
132 // Copy other groups' borders from the border storage.
133 if (y0src < y0) {
134 JXL_ENSURE(gy > 0);
135 JXL_RETURN_IF_ERROR(CopyImageTo(
136 Rect(x0src, (gy * 2 - 2) * bordery_write, x1src - x0src, bordery_write),
137 borders_horizontal_[c],
138 Rect(group_data_x_border_ + x0src - x0,
139 group_data_y_border_ - bordery_write, x1src - x0src,
140 bordery_write),
141 out));
142 }
143 if (y1src > y1) {
144 // When copying the bottom border we must not be on the bottom groups.
145 JXL_ENSURE(gy + 1 < frame_dimensions_.ysize_groups);
146 JXL_RETURN_IF_ERROR(CopyImageTo(
147 Rect(x0src, (gy * 2 + 1) * bordery_write, x1src - x0src, bordery_write),
148 borders_horizontal_[c],
149 Rect(group_data_x_border_ + x0src - x0, group_data_y_border_ + y1 - y0,
150 x1src - x0src, bordery_write),
151 out));
152 }
153 if (x0src < x0) {
154 JXL_ENSURE(gx > 0);
155 JXL_RETURN_IF_ERROR(CopyImageTo(
156 Rect((gx * 2 - 2) * borderx_write, y0src, borderx_write, y1src - y0src),
157 borders_vertical_[c],
158 Rect(group_data_x_border_ - borderx_write,
159 group_data_y_border_ + y0src - y0, borderx_write, y1src - y0src),
160 out));
161 }
162 if (x1src > x1) {
163 // When copying the right border we must not be on the rightmost groups.
164 JXL_ENSURE(gx + 1 < frame_dimensions_.xsize_groups);
165 JXL_RETURN_IF_ERROR(CopyImageTo(
166 Rect((gx * 2 + 1) * borderx_write, y0src, borderx_write, y1src - y0src),
167 borders_vertical_[c],
168 Rect(group_data_x_border_ + x1 - x0, group_data_y_border_ + y0src - y0,
169 borderx_write, y1src - y0src),
170 out));
171 }
172 return true;
173 }
174
175 size_t LowMemoryRenderPipeline::GroupInputXSize(size_t c) const {
176 return (frame_dimensions_.group_dim << base_color_shift_) >>
177 channel_shifts_[0][c].first;
178 }
179
180 size_t LowMemoryRenderPipeline::GroupInputYSize(size_t c) const {
181 return (frame_dimensions_.group_dim << base_color_shift_) >>
182 channel_shifts_[0][c].second;
183 }
184
185 Status LowMemoryRenderPipeline::EnsureBordersStorage() {
186 const auto& shifts = channel_shifts_[0];
187 if (borders_horizontal_.size() < shifts.size()) {
188 borders_horizontal_.resize(shifts.size());
189 borders_vertical_.resize(shifts.size());
190 }
191 for (size_t c = 0; c < shifts.size(); c++) {
192 auto borders = BorderToStore(c);
193 size_t borderx = borders.first;
194 size_t bordery = borders.second;
195 JXL_ENSURE(frame_dimensions_.xsize_groups > 0);
196 size_t num_xborders = (frame_dimensions_.xsize_groups - 1) * 2;
197 JXL_ENSURE(frame_dimensions_.ysize_groups > 0);
198 size_t num_yborders = (frame_dimensions_.ysize_groups - 1) * 2;
199 size_t downsampled_xsize =
200 DivCeil(frame_dimensions_.xsize_upsampled_padded, 1 << shifts[c].first);
201 size_t downsampled_ysize = DivCeil(frame_dimensions_.ysize_upsampled_padded,
202 1 << shifts[c].second);
203 Rect horizontal = Rect(0, 0, downsampled_xsize, bordery * num_yborders);
204 if (!SameSize(horizontal, borders_horizontal_[c])) {
205 JXL_ASSIGN_OR_RETURN(borders_horizontal_[c],
206 ImageF::Create(memory_manager_, horizontal.xsize(),
207 horizontal.ysize()));
208 }
209 Rect vertical = Rect(0, 0, borderx * num_xborders, downsampled_ysize);
210 if (!SameSize(vertical, borders_vertical_[c])) {
211 JXL_ASSIGN_OR_RETURN(
212 borders_vertical_[c],
213 ImageF::Create(memory_manager_, vertical.xsize(), vertical.ysize()));
214 }
215 }
216 return true;
217 }
218
219 Status LowMemoryRenderPipeline::Init() {
220 group_border_ = {0, 0};
221 base_color_shift_ = CeilLog2Nonzero(frame_dimensions_.xsize_upsampled_padded /
222 frame_dimensions_.xsize_padded);
223
224 const auto& shifts = channel_shifts_[0];
225
226 // Ensure that each channel has enough many border pixels.
227 for (size_t c = 0; c < shifts.size(); c++) {
228 group_border_.first =
229 std::max(group_border_.first,
230 DivCeil(padding_[0][c].first << channel_shifts_[0][c].first,
231 1 << base_color_shift_));
232 group_border_.second =
233 std::max(group_border_.second,
234 DivCeil(padding_[0][c].second << channel_shifts_[0][c].second,
235 1 << base_color_shift_));
236 }
237
238 // Ensure that all channels have an integer number of border pixels in the
239 // input.
240 for (size_t c = 0; c < shifts.size(); c++) {
241 if (channel_shifts_[0][c].first >= base_color_shift_) {
242 group_border_.first =
243 RoundUpTo(group_border_.first,
244 1 << (channel_shifts_[0][c].first - base_color_shift_));
245 }
246 if (channel_shifts_[0][c].second >= base_color_shift_) {
247 group_border_.second =
248 RoundUpTo(group_border_.second,
249 1 << (channel_shifts_[0][c].second - base_color_shift_));
250 }
251 }
252 // Ensure that the X border on color channels is a multiple of kBlockDim or
253 // the vector size (required for EPF stages). Vectors on ARM NEON are never
254 // wider than 4 floats, so rounding to multiples of 4 is enough.
255 #if JXL_ARCH_ARM
256 constexpr size_t kGroupXAlign = 4;
257 #else
258 constexpr size_t kGroupXAlign = 16;
259 #endif
260 group_border_.first = RoundUpTo(group_border_.first, kGroupXAlign);
261 // Allocate borders in group images that are just enough for storing the
262 // borders to be copied in, plus any rounding to ensure alignment.
263 std::pair<size_t, size_t> max_border = {0, 0};
264 for (size_t c = 0; c < shifts.size(); c++) {
265 max_border.first = std::max(BorderToStore(c).first, max_border.first);
266 max_border.second = std::max(BorderToStore(c).second, max_border.second);
267 }
268 group_data_x_border_ = RoundUpTo(max_border.first, kGroupXAlign);
269 group_data_y_border_ = max_border.second;
270
271 JXL_RETURN_IF_ERROR(EnsureBordersStorage());
272 group_border_assigner_.Init(frame_dimensions_);
273
274 for (first_trailing_stage_ = stages_.size(); first_trailing_stage_ > 0;
275 first_trailing_stage_--) {
276 bool has_inout_c = false;
277 for (size_t c = 0; c < shifts.size(); c++) {
278 if (stages_[first_trailing_stage_ - 1]->GetChannelMode(c) ==
279 RenderPipelineChannelMode::kInOut) {
280 has_inout_c = true;
281 }
282 }
283 if (has_inout_c) {
284 break;
285 }
286 }
287
288 first_image_dim_stage_ = stages_.size();
289 for (size_t i = 0; i < stages_.size(); i++) {
290 std::vector<std::pair<size_t, size_t>> input_sizes(shifts.size());
291 for (size_t c = 0; c < shifts.size(); c++) {
292 input_sizes[c] =
293 std::make_pair(DivCeil(frame_dimensions_.xsize_upsampled,
294 1 << channel_shifts_[i][c].first),
295 DivCeil(frame_dimensions_.ysize_upsampled,
296 1 << channel_shifts_[i][c].second));
297 }
298 JXL_RETURN_IF_ERROR(stages_[i]->SetInputSizes(input_sizes));
299 if (stages_[i]->SwitchToImageDimensions()) {
300 // We don't allow kInOut after switching to image dimensions.
301 JXL_ENSURE(i >= first_trailing_stage_);
302 first_image_dim_stage_ = i + 1;
303 stages_[i]->GetImageDimensions(&full_image_xsize_, &full_image_ysize_,
304 &frame_origin_);
305 break;
306 }
307 }
308 for (size_t i = first_image_dim_stage_; i < stages_.size(); i++) {
309 if (stages_[i]->SwitchToImageDimensions()) {
310 return JXL_UNREACHABLE(
311 "cannot switch to image dimensions multiple times");
312 }
313 std::vector<std::pair<size_t, size_t>> input_sizes(shifts.size());
314 for (size_t c = 0; c < shifts.size(); c++) {
315 input_sizes[c] = {full_image_xsize_, full_image_ysize_};
316 }
317 JXL_RETURN_IF_ERROR(stages_[i]->SetInputSizes(input_sizes));
318 }
319
320 anyc_.resize(stages_.size());
321 for (size_t i = 0; i < stages_.size(); i++) {
322 for (size_t c = 0; c < shifts.size(); c++) {
323 if (stages_[i]->GetChannelMode(c) !=
324 RenderPipelineChannelMode::kIgnored) {
325 anyc_[i] = c;
326 }
327 }
328 }
329
330 stage_input_for_channel_ = std::vector<std::vector<int32_t>>(
331 stages_.size(), std::vector<int32_t>(shifts.size()));
332 for (size_t c = 0; c < shifts.size(); c++) {
333 int input = -1;
334 for (size_t i = 0; i < stages_.size(); i++) {
335 stage_input_for_channel_[i][c] = input;
336 if (stages_[i]->GetChannelMode(c) == RenderPipelineChannelMode::kInOut) {
337 input = i;
338 }
339 }
340 }
341
342 image_rect_.resize(stages_.size());
343 for (size_t i = 0; i < stages_.size(); i++) {
344 size_t x1 = DivCeil(frame_dimensions_.xsize_upsampled,
345 1 << channel_shifts_[i][anyc_[i]].first);
346 size_t y1 = DivCeil(frame_dimensions_.ysize_upsampled,
347 1 << channel_shifts_[i][anyc_[i]].second);
348 image_rect_[i] = Rect(0, 0, x1, y1);
349 }
350
351 virtual_ypadding_for_output_.resize(stages_.size());
352 xpadding_for_output_.resize(stages_.size());
353 for (size_t c = 0; c < shifts.size(); c++) {
354 int ypad = 0;
355 int xpad = 0;
356 for (size_t i = stages_.size(); i-- > 0;) {
357 if (stages_[i]->GetChannelMode(c) !=
358 RenderPipelineChannelMode::kIgnored) {
359 virtual_ypadding_for_output_[i] =
360 std::max(ypad, virtual_ypadding_for_output_[i]);
361 xpadding_for_output_[i] = std::max(xpad, xpadding_for_output_[i]);
362 }
363 if (stages_[i]->GetChannelMode(c) == RenderPipelineChannelMode::kInOut) {
364 ypad = (DivCeil(ypad, 1 << channel_shifts_[i][c].second) +
365 stages_[i]->settings_.border_y)
366 << channel_shifts_[i][c].second;
367 xpad = DivCeil(xpad, 1 << stages_[i]->settings_.shift_x) +
368 stages_[i]->settings_.border_x;
369 }
370 }
371 }
372 return true;
373 }
374
375 Status LowMemoryRenderPipeline::PrepareForThreadsInternal(size_t num,
376 bool use_group_ids) {
377 const auto& shifts = channel_shifts_[0];
378 use_group_ids_ = use_group_ids;
379 size_t num_buffers = use_group_ids_ ? frame_dimensions_.num_groups : num;
380 for (size_t t = group_data_.size(); t < num_buffers; t++) {
381 group_data_.emplace_back();
382 group_data_[t].resize(shifts.size());
383 for (size_t c = 0; c < shifts.size(); c++) {
384 JXL_ASSIGN_OR_RETURN(
385 group_data_[t][c],
386 ImageF::Create(memory_manager_,
387 GroupInputXSize(c) + group_data_x_border_ * 2,
388 GroupInputYSize(c) + group_data_y_border_ * 2,
389 kRenderPipelineXOffset));
390 }
391 }
392 // TODO(veluca): avoid reallocating buffers if not needed.
393 stage_data_.resize(num);
394 size_t upsampling = 1u << base_color_shift_;
395 size_t group_dim = frame_dimensions_.group_dim * upsampling;
396 size_t padding =
397 2 * group_data_x_border_ * upsampling + // maximum size of a rect
398 2 * kRenderPipelineXOffset; // extra padding for processing
399 size_t stage_buffer_xsize = group_dim + padding;
400 for (size_t t = 0; t < num; t++) {
401 stage_data_[t].resize(shifts.size());
402 for (size_t c = 0; c < shifts.size(); c++) {
403 stage_data_[t][c].resize(stages_.size());
404 size_t next_y_border = 0;
405 for (size_t i = stages_.size(); i-- > 0;) {
406 if (stages_[i]->GetChannelMode(c) ==
407 RenderPipelineChannelMode::kInOut) {
408 size_t stage_buffer_ysize =
409 2 * next_y_border + (1 << stages_[i]->settings_.shift_y);
410 stage_buffer_ysize = 1 << CeilLog2Nonzero(stage_buffer_ysize);
411 next_y_border = stages_[i]->settings_.border_y;
412 JXL_ASSIGN_OR_RETURN(
413 stage_data_[t][c][i],
414 ImageF::Create(memory_manager_, stage_buffer_xsize,
415 stage_buffer_ysize));
416 }
417 }
418 }
419 }
420 if (first_image_dim_stage_ != stages_.size()) {
421 RectT<ssize_t> image_rect(0, 0, frame_dimensions_.xsize_upsampled,
422 frame_dimensions_.ysize_upsampled);
423 RectT<ssize_t> full_image_rect(0, 0, full_image_xsize_, full_image_ysize_);
424 image_rect = image_rect.Translate(frame_origin_.x0, frame_origin_.y0);
425 image_rect = image_rect.Intersection(full_image_rect);
426 if (image_rect.xsize() == 0 || image_rect.ysize() == 0) {
427 image_rect = RectT<ssize_t>(0, 0, 0, 0);
428 }
429 size_t left_padding = image_rect.x0();
430 size_t middle_padding = group_dim;
431 size_t right_padding = full_image_xsize_ - image_rect.x1();
432 size_t out_of_frame_xsize =
433 padding +
434 std::max(left_padding, std::max(middle_padding, right_padding));
435 out_of_frame_data_.resize(num);
436 for (size_t t = 0; t < num; t++) {
437 JXL_ASSIGN_OR_RETURN(
438 out_of_frame_data_[t],
439 ImageF::Create(memory_manager_, out_of_frame_xsize, shifts.size()));
440 }
441 }
442 return true;
443 }
444
445 std::vector<std::pair<ImageF*, Rect>> LowMemoryRenderPipeline::PrepareBuffers(
446 size_t group_id, size_t thread_id) {
447 std::vector<std::pair<ImageF*, Rect>> ret(channel_shifts_[0].size());
448 const size_t gx = group_id % frame_dimensions_.xsize_groups;
449 const size_t gy = group_id / frame_dimensions_.xsize_groups;
450 for (size_t c = 0; c < channel_shifts_[0].size(); c++) {
451 ret[c].first = &group_data_[use_group_ids_ ? group_id : thread_id][c];
452 ret[c].second = Rect(group_data_x_border_, group_data_y_border_,
453 GroupInputXSize(c), GroupInputYSize(c),
454 DivCeil(frame_dimensions_.xsize_upsampled,
455 1 << channel_shifts_[0][c].first) -
456 gx * GroupInputXSize(c) + group_data_x_border_,
457 DivCeil(frame_dimensions_.ysize_upsampled,
458 1 << channel_shifts_[0][c].second) -
459 gy * GroupInputYSize(c) + group_data_y_border_);
460 }
461 return ret;
462 }
463
464 namespace {
465
466 JXL_INLINE int GetMirroredY(int y, ssize_t group_y0, ssize_t image_ysize) {
467 if (group_y0 == 0 && (y < 0 || y + group_y0 >= image_ysize)) {
468 return Mirror(y, image_ysize);
469 }
470 if (y + group_y0 >= image_ysize) {
471 // Here we know that the one mirroring step is sufficient.
472 return 2 * image_ysize - (y + group_y0) - 1 - group_y0;
473 }
474 return y;
475 }
476
477 JXL_INLINE void ApplyXMirroring(float* row, ssize_t borderx, ssize_t group_x0,
478 ssize_t group_xsize, ssize_t image_xsize) {
479 if (image_xsize <= borderx) {
480 if (group_x0 == 0) {
481 for (ssize_t ix = 0; ix < borderx; ix++) {
482 row[kRenderPipelineXOffset - ix - 1] =
483 row[kRenderPipelineXOffset + Mirror(-ix - 1, image_xsize)];
484 }
485 }
486 if (group_xsize + borderx + group_x0 >= image_xsize) {
487 for (ssize_t ix = 0; ix < borderx; ix++) {
488 row[kRenderPipelineXOffset + image_xsize + ix - group_x0] =
489 row[kRenderPipelineXOffset + Mirror(image_xsize + ix, image_xsize) -
490 group_x0];
491 }
492 }
493 } else {
494 // Here we know that the one mirroring step is sufficient.
495 if (group_x0 == 0) {
496 for (ssize_t ix = 0; ix < borderx; ix++) {
497 row[kRenderPipelineXOffset - ix - 1] = row[kRenderPipelineXOffset + ix];
498 }
499 }
500 if (group_xsize + borderx + group_x0 >= image_xsize) {
501 for (ssize_t ix = 0; ix < borderx; ix++) {
502 row[kRenderPipelineXOffset + image_xsize - group_x0 + ix] =
503 row[kRenderPipelineXOffset + image_xsize - group_x0 - ix - 1];
504 }
505 }
506 }
507 }
508
509 // Information about where the *output* of each stage is stored.
510 class Rows {
511 public:
512 static StatusOr<Rows> Create(
513 const std::vector<std::unique_ptr<RenderPipelineStage>>& stages,
514 const Rect data_max_color_channel_rect, int group_data_x_border,
515 int group_data_y_border,
516 const std::vector<std::pair<size_t, size_t>>& group_data_shift,
517 size_t base_color_shift, std::vector<std::vector<ImageF>>& thread_data,
518 std::vector<ImageF>& input_data) {
519 size_t num_stages = stages.size();
520 size_t num_channels = input_data.size();
521
522 JXL_ENSURE(thread_data.size() == num_channels);
523 JXL_ENSURE(group_data_shift.size() == num_channels);
524
525 for (const auto& td : thread_data) {
526 JXL_ENSURE(td.size() == num_stages);
527 }
528
529 std::vector<std::vector<RowInfo>> rows;
530 rows.resize(num_stages + 1, std::vector<RowInfo>(num_channels));
531
532 for (size_t i = 0; i < num_stages; i++) {
533 for (size_t c = 0; c < input_data.size(); c++) {
534 if (stages[i]->GetChannelMode(c) == RenderPipelineChannelMode::kInOut) {
535 rows[i + 1][c].ymod_minus_1 = thread_data[c][i].ysize() - 1;
536 rows[i + 1][c].base_ptr = thread_data[c][i].Row(0);
537 rows[i + 1][c].stride = thread_data[c][i].PixelsPerRow();
538 }
539 }
540 }
541
542 for (size_t c = 0; c < input_data.size(); c++) {
543 auto tmp = data_max_color_channel_rect.As<ssize_t>()
544 .Translate(-group_data_x_border, -group_data_y_border)
545 .ShiftLeft(base_color_shift);
546 JXL_ASSIGN_OR_RETURN(tmp, tmp.CeilShiftRight(group_data_shift[c]));
547 auto channel_group_data_rect = tmp.Translate(
548 group_data_x_border - static_cast<ssize_t>(kRenderPipelineXOffset),
549 group_data_y_border);
550 rows[0][c].base_ptr = channel_group_data_rect.Row(&input_data[c], 0);
551 rows[0][c].stride = input_data[c].PixelsPerRow();
552 rows[0][c].ymod_minus_1 = -1;
553 }
554 return Rows(std::move(rows));
555 }
556
557 // Stage -1 refers to the input data; all other values must be nonnegative and
558 // refer to the data for the output of that stage.
559 JXL_INLINE float* GetBuffer(int stage, int y, size_t c) const {
560 JXL_DASSERT(stage >= -1);
561 const RowInfo& info = rows_[stage + 1][c];
562 return info.base_ptr +
563 static_cast<ssize_t>(info.stride) * (y & info.ymod_minus_1);
564 }
565
566 private:
567 struct RowInfo {
568 // Pointer to beginning of the first row.
569 float* base_ptr;
570 // Modulo value for the y axis minus 1 (ymod is guaranteed to be a power of
571 // 2, which allows efficient mod computation by masking).
572 int ymod_minus_1;
573 // Number of floats per row.
574 size_t stride;
575 };
576
577 explicit Rows(std::vector<std::vector<RowInfo>>&& rows)
578 : rows_(std::move(rows)) {}
579
580 std::vector<std::vector<RowInfo>> rows_;
581 };
582
583 } // namespace
584
585 Status LowMemoryRenderPipeline::RenderRect(size_t thread_id,
586 std::vector<ImageF>& input_data,
587 Rect data_max_color_channel_rect,
588 Rect image_max_color_channel_rect) {
589 // For each stage, the rect corresponding to the image area currently being
590 // processed, in the coordinates of that stage (i.e. with the scaling factor
591 // that that stage has).
592 std::vector<Rect> group_rect;
593 group_rect.resize(stages_.size());
594 Rect image_area_rect =
595 image_max_color_channel_rect.ShiftLeft(base_color_shift_)
596 .Crop(frame_dimensions_.xsize_upsampled,
597 frame_dimensions_.ysize_upsampled);
598 for (size_t i = 0; i < stages_.size(); i++) {
599 JXL_ASSIGN_OR_RETURN(group_rect[i], image_area_rect.CeilShiftRight(
600 channel_shifts_[i][anyc_[i]]));
601 }
602
603 ssize_t frame_x0 = frame_origin_.x0;
604 ssize_t frame_y0 = frame_origin_.y0;
605
606 // Compute actual x-axis bounds for the current image area in the context of
607 // the full image this frame is part of. As the left boundary may be negative,
608 // we also create the x_pixels_skip value, defined as follows:
609 // - both x_pixels_skip and full_image_x0 are >= 0, and at least one is 0;
610 // - full_image_x0 - x_pixels_skip is the position of the current frame area
611 // in the full image.
612 ssize_t full_image_x0 = frame_x0 + image_area_rect.x0();
613 ssize_t x_pixels_skip = 0;
614 if (full_image_x0 < 0) {
615 x_pixels_skip = -full_image_x0;
616 full_image_x0 = 0;
617 }
618 ssize_t full_image_x1 = frame_x0 + image_area_rect.x1();
619
620 std::vector<Rect> span(stages_.size());
621 for (size_t i = 0; i < stages_.size(); ++i) {
622 if (i < first_image_dim_stage_) {
623 span[i] = Rect(group_rect[i].x0(), 0, group_rect[i].xsize(),
624 image_rect_[i].ysize());
625 } else {
626 size_t x0 = full_image_x0;
627 size_t x1 = full_image_x1;
628 size_t x_max = full_image_xsize_;
629 size_t cropped_x1 = std::min<ssize_t>(x1, x_max);
630 span[i] =
631 Rect(x0, 0, std::max<ssize_t>(0, cropped_x1 - x0), full_image_ysize_);
632 }
633 }
634
635 // Data structures to hold information about input/output rows and their
636 // buffers.
637 JXL_ASSIGN_OR_RETURN(
638 Rows rows,
639 Rows::Create(stages_, data_max_color_channel_rect, group_data_x_border_,
640 group_data_y_border_, channel_shifts_[0], base_color_shift_,
641 stage_data_[thread_id], input_data));
642
643 std::vector<RenderPipelineStage::RowInfo> input_rows(first_trailing_stage_ +
644 1);
645 for (size_t i = 0; i < first_trailing_stage_; i++) {
646 input_rows[i].resize(input_data.size());
647 }
648 input_rows[first_trailing_stage_].resize(input_data.size(),
649 std::vector<float*>(1));
650
651 // Maximum possible shift is 3.
652 RenderPipelineStage::RowInfo output_rows(input_data.size(),
653 std::vector<float*>(8));
654
655 // Fills in input_rows and output_rows for a given y value (relative to the
656 // start of the group, measured in actual pixels at the appropriate vertical
657 // scaling factor) and a given stage, applying mirroring if necessary. This
658 // function is somewhat inefficient for trailing kInOut or kInput stages,
659 // where just filling the input row once ought to be sufficient.
660 auto prepare_io_rows = [&](int y, size_t i) {
661 ssize_t bordery = stages_[i]->settings_.border_y;
662 size_t shifty = stages_[i]->settings_.shift_y;
663 auto make_row = [&](size_t c, ssize_t iy) {
664 size_t mirrored_y = GetMirroredY(y + iy - bordery, group_rect[i].y0(),
665 image_rect_[i].ysize());
666 input_rows[i][c][iy] =
667 rows.GetBuffer(stage_input_for_channel_[i][c], mirrored_y, c);
668 ApplyXMirroring(input_rows[i][c][iy], stages_[i]->settings_.border_x,
669 group_rect[i].x0(), group_rect[i].xsize(),
670 image_rect_[i].xsize());
671 };
672 for (size_t c = 0; c < input_data.size(); c++) {
673 RenderPipelineChannelMode mode = stages_[i]->GetChannelMode(c);
674 if (mode == RenderPipelineChannelMode::kIgnored) {
675 continue;
676 }
677 // If we already have rows from a previous iteration, we can just shift
678 // the rows by 1 and insert the new one.
679 if (input_rows[i][c].size() == 2 * static_cast<size_t>(bordery) + 1) {
680 for (ssize_t iy = 0; iy < 2 * bordery; iy++) {
681 input_rows[i][c][iy] = input_rows[i][c][iy + 1];
682 }
683 make_row(c, bordery * 2);
684 } else {
685 input_rows[i][c].resize(2 * bordery + 1);
686 for (ssize_t iy = 0; iy < 2 * bordery + 1; iy++) {
687 make_row(c, iy);
688 }
689 }
690
691 // If necessary, get the output buffers.
692 if (mode == RenderPipelineChannelMode::kInOut) {
693 for (size_t iy = 0; iy < (1u << shifty); iy++) {
694 output_rows[c][iy] = rows.GetBuffer(i, y * (1 << shifty) + iy, c);
695 }
696 }
697 }
698 };
699
700 // We pretend that every stage has a vertical shift of 0, i.e. it is as tall
701 // as the final image.
702 // We call each such row a "virtual" row, because it may or may not correspond
703 // to an actual row of the current processing stage; actual processing happens
704 // when vy % (1<<vshift) == 0.
705
706 int num_extra_rows = *std::max_element(virtual_ypadding_for_output_.begin(),
707 virtual_ypadding_for_output_.end());
708
709 for (int vy = -num_extra_rows;
710 vy < static_cast<int>(image_area_rect.ysize()) + num_extra_rows; vy++) {
711 for (size_t i = 0; i < first_trailing_stage_; i++) {
712 int stage_vy = vy - num_extra_rows + virtual_ypadding_for_output_[i];
713
714 if (stage_vy % (1 << channel_shifts_[i][anyc_[i]].second) != 0) {
715 continue;
716 }
717
718 if (stage_vy < -virtual_ypadding_for_output_[i]) {
719 continue;
720 }
721
722 int y = stage_vy >> channel_shifts_[i][anyc_[i]].second;
723
724 ssize_t image_y = static_cast<ssize_t>(group_rect[i].y0()) + y;
725 // Do not produce rows in out-of-bounds areas.
726 if (image_y < 0) continue;
727 if (image_y >= static_cast<ssize_t>(span[i].y1())) continue;
728
729 // Get the input/output rows and potentially apply mirroring to the input.
730 prepare_io_rows(y, i);
731
732 // Produce output rows.
733 if (span[i].xsize() == 0) continue;
734 JXL_RETURN_IF_ERROR(stages_[i]->ProcessRow(
735 input_rows[i], output_rows, xpadding_for_output_[i], span[i].xsize(),
736 span[i].x0(), image_y, thread_id));
737 }
738
739 // Process trailing stages, i.e. the final set of non-kInOut stages; they
740 // all have the same input buffer and no need to use any mirroring.
741
742 int y = vy - num_extra_rows;
743
744 for (size_t c = 0; c < input_data.size(); c++) {
745 input_rows[first_trailing_stage_][c][0] = rows.GetBuffer(
746 stage_input_for_channel_[first_trailing_stage_][c], y, c);
747 }
748
749 // Check that we are not outside of the bounds for the current rendering
750 // rect. Not doing so might result in overwriting some rows that have been
751 // written (or will be written) by other threads.
752 if (y < 0 || y >= static_cast<ssize_t>(image_area_rect.ysize())) {
753 continue;
754 }
755
756 for (size_t i = first_trailing_stage_; i < first_image_dim_stage_; i++) {
757 if (span[i].xsize() == 0) continue;
758 size_t y0 = image_area_rect.y0() + y;
759 if (y0 >= span[i].y1()) continue;
760 JXL_RETURN_IF_ERROR(stages_[i]->ProcessRow(
761 input_rows[first_trailing_stage_], output_rows,
762 /*xextra=*/0, span[i].xsize(), span[i].x0(), y0, thread_id));
763 }
764
765 if (first_image_dim_stage_ == stages_.size()) continue;
766
767 // Skip pixels that are not part of the actual final image area.
768 for (size_t c = 0; c < input_data.size(); c++) {
769 input_rows[first_trailing_stage_][c][0] += x_pixels_skip;
770 }
771 // Avoid running pipeline stages on pixels that are outside the full image
772 // area. As trailing stages have no borders, this is a free optimization
773 // (and may be necessary for correctness, as some stages assume coordinates
774 // are within bounds).
775 ssize_t full_image_y = frame_y0 + image_area_rect.y0() + y;
776 if (full_image_y < 0) continue;
777
778 for (size_t i = first_image_dim_stage_; i < stages_.size(); i++) {
779 if (span[i].xsize() == 0) continue;
780 if (full_image_y >= static_cast<ssize_t>(span[i].y1())) continue;
781 JXL_RETURN_IF_ERROR(
782 stages_[i]->ProcessRow(input_rows[first_trailing_stage_], output_rows,
783 /*xextra=*/0, span[i].xsize(), span[i].x0(),
784 full_image_y, thread_id));
785 }
786 }
787 return true;
788 }
789
790 Status LowMemoryRenderPipeline::RenderPadding(size_t thread_id, Rect rect) {
791 if (rect.xsize() == 0) return true;
792 size_t numc = channel_shifts_[0].size();
793 RenderPipelineStage::RowInfo input_rows(numc, std::vector<float*>(1));
794 RenderPipelineStage::RowInfo output_rows;
795
796 for (size_t c = 0; c < numc; c++) {
797 input_rows[c][0] = out_of_frame_data_[thread_id].Row(c);
798 }
799
800 for (size_t y = 0; y < rect.ysize(); y++) {
801 stages_[first_image_dim_stage_ - 1]->ProcessPaddingRow(
802 input_rows, rect.xsize(), rect.x0(), rect.y0() + y);
803 for (size_t i = first_image_dim_stage_; i < stages_.size(); i++) {
804 JXL_RETURN_IF_ERROR(stages_[i]->ProcessRow(
805 input_rows, output_rows,
806 /*xextra=*/0, rect.xsize(), rect.x0(), rect.y0() + y, thread_id));
807 }
808 }
809 return true;
810 }
811
812 Status LowMemoryRenderPipeline::ProcessBuffers(size_t group_id,
813 size_t thread_id) {
814 std::vector<ImageF>& input_data =
815 group_data_[use_group_ids_ ? group_id : thread_id];
816
817 // Copy the group borders to the border storage.
818 for (size_t c = 0; c < input_data.size(); c++) {
819 JXL_RETURN_IF_ERROR(SaveBorders(group_id, c, input_data[c]));
820 }
821
822 size_t gy = group_id / frame_dimensions_.xsize_groups;
823 size_t gx = group_id % frame_dimensions_.xsize_groups;
824
825 if (first_image_dim_stage_ != stages_.size()) {
826 size_t group_dim = frame_dimensions_.group_dim << base_color_shift_;
827 RectT<ssize_t> group_rect(gx * group_dim, gy * group_dim, group_dim,
828 group_dim);
829 RectT<ssize_t> image_rect(0, 0, frame_dimensions_.xsize_upsampled,
830 frame_dimensions_.ysize_upsampled);
831 RectT<ssize_t> full_image_rect(0, 0, full_image_xsize_, full_image_ysize_);
832 group_rect = group_rect.Translate(frame_origin_.x0, frame_origin_.y0);
833 image_rect = image_rect.Translate(frame_origin_.x0, frame_origin_.y0);
834 image_rect = image_rect.Intersection(full_image_rect);
835 group_rect = group_rect.Intersection(image_rect);
836 size_t x0 = group_rect.x0();
837 size_t y0 = group_rect.y0();
838 size_t x1 = group_rect.x1();
839 size_t y1 = group_rect.y1();
840 JXL_DEBUG_V(6,
841 "Rendering padding for full image rect %s "
842 "outside group rect %s",
843 Description(full_image_rect).c_str(),
844 Description(group_rect).c_str());
845
846 if (group_id == 0 && (image_rect.xsize() == 0 || image_rect.ysize() == 0)) {
847 // If this frame does not intersect with the full image, we have to
848 // initialize the whole image area with RenderPadding.
849 JXL_RETURN_IF_ERROR(RenderPadding(
850 thread_id, Rect(0, 0, full_image_xsize_, full_image_ysize_)));
851 }
852
853 // Render padding for groups that intersect with the full image. The case
854 // where no groups intersect was handled above.
855 if (group_rect.xsize() > 0 && group_rect.ysize() > 0) {
856 if (gx == 0 && gy == 0) {
857 JXL_RETURN_IF_ERROR(RenderPadding(thread_id, Rect(0, 0, x0, y0)));
858 }
859 if (gy == 0) {
860 JXL_RETURN_IF_ERROR(RenderPadding(thread_id, Rect(x0, 0, x1 - x0, y0)));
861 }
862 if (gx == 0) {
863 JXL_RETURN_IF_ERROR(RenderPadding(thread_id, Rect(0, y0, x0, y1 - y0)));
864 }
865 if (gx == 0 && gy + 1 == frame_dimensions_.ysize_groups) {
866 JXL_RETURN_IF_ERROR(
867 RenderPadding(thread_id, Rect(0, y1, x0, full_image_ysize_ - y1)));
868 }
869 if (gy + 1 == frame_dimensions_.ysize_groups) {
870 JXL_RETURN_IF_ERROR(RenderPadding(
871 thread_id, Rect(x0, y1, x1 - x0, full_image_ysize_ - y1)));
872 }
873 if (gy == 0 && gx + 1 == frame_dimensions_.xsize_groups) {
874 JXL_RETURN_IF_ERROR(
875 RenderPadding(thread_id, Rect(x1, 0, full_image_xsize_ - x1, y0)));
876 }
877 if (gx + 1 == frame_dimensions_.xsize_groups) {
878 JXL_RETURN_IF_ERROR(RenderPadding(
879 thread_id, Rect(x1, y0, full_image_xsize_ - x1, y1 - y0)));
880 }
881 if (gy + 1 == frame_dimensions_.ysize_groups &&
882 gx + 1 == frame_dimensions_.xsize_groups) {
883 JXL_RETURN_IF_ERROR(RenderPadding(
884 thread_id,
885 Rect(x1, y1, full_image_xsize_ - x1, full_image_ysize_ - y1)));
886 }
887 }
888 }
889
890 Rect ready_rects[GroupBorderAssigner::kMaxToFinalize];
891 size_t num_ready_rects = 0;
892 group_border_assigner_.GroupDone(group_id, group_border_.first,
893 group_border_.second, ready_rects,
894 &num_ready_rects);
895 for (size_t i = 0; i < num_ready_rects; i++) {
896 const Rect& image_max_color_channel_rect = ready_rects[i];
897 for (size_t c = 0; c < input_data.size(); c++) {
898 JXL_RETURN_IF_ERROR(LoadBorders(group_id, c, image_max_color_channel_rect,
899 &input_data[c]));
900 }
901 Rect data_max_color_channel_rect(
902 group_data_x_border_ + image_max_color_channel_rect.x0() -
903 gx * frame_dimensions_.group_dim,
904 group_data_y_border_ + image_max_color_channel_rect.y0() -
905 gy * frame_dimensions_.group_dim,
906 image_max_color_channel_rect.xsize(),
907 image_max_color_channel_rect.ysize());
908 JXL_RETURN_IF_ERROR(RenderRect(thread_id, input_data,
909 data_max_color_channel_rect,
910 image_max_color_channel_rect));
911 }
912 return true;
913 }
914 } // namespace jxl