Debian Codegraph @

lib/jxl/render_pipeline/stage_epf.cc

raw
1 // Copyright (c) the JPEG XL Project Authors. All rights reserved.
2 //
3 // Use of this source code is governed by a BSD-style
4 // license that can be found in the LICENSE file.
5
6 #include "lib/jxl/render_pipeline/stage_epf.h"
7
8 #include "lib/jxl/base/common.h"
9 #include "lib/jxl/base/compiler_specific.h"
10 #include "lib/jxl/base/status.h"
11 #include "lib/jxl/common.h" // JXL_HIGH_PRECISION
12 #include "lib/jxl/epf.h"
13
14 #undef HWY_TARGET_INCLUDE
15 #define HWY_TARGET_INCLUDE "lib/jxl/render_pipeline/stage_epf.cc"
16 #include <hwy/foreach_target.h>
17 #include <hwy/highway.h>
18
19 HWY_BEFORE_NAMESPACE();
20 namespace jxl {
21 namespace HWY_NAMESPACE {
22 // TODO(veluca): In principle, vectors could be not capped, if we want to deal
23 // with having two different sigma values in a single vector.
24 using DF = HWY_CAPPED(float, 8);
25
26 // These templates are not found via ADL.
27 using hwy::HWY_NAMESPACE::AbsDiff;
28 using hwy::HWY_NAMESPACE::Add;
29 using hwy::HWY_NAMESPACE::Div;
30 using hwy::HWY_NAMESPACE::Mul;
31 using hwy::HWY_NAMESPACE::MulAdd;
32 using hwy::HWY_NAMESPACE::Vec;
33 using hwy::HWY_NAMESPACE::VFromD;
34 using hwy::HWY_NAMESPACE::ZeroIfNegative;
35
36 JXL_INLINE Vec<DF> Weight(Vec<DF> sad, Vec<DF> inv_sigma, Vec<DF> thres) {
37 auto v = MulAdd(sad, inv_sigma, Set(DF(), 1.0f));
38 return ZeroIfNegative(v);
39 }
40
41 // 5x5 plus-shaped kernel with 5 SADs per pixel (3x3 plus-shaped). So this makes
42 // this filter a 7x7 filter.
43 class EPF0Stage : public RenderPipelineStage {
44 public:
45 EPF0Stage(LoopFilter lf, const ImageF& sigma)
46 : RenderPipelineStage(RenderPipelineStage::Settings::Symmetric(
47 /*shift=*/0, /*border=*/3)),
48 lf_(std::move(lf)),
49 sigma_(&sigma) {}
50
51 template <bool aligned>
52 JXL_INLINE void AddPixel(int row, float* JXL_RESTRICT rows[3][7], ssize_t x,
53 Vec<DF> sad, Vec<DF> inv_sigma,
54 Vec<DF>* JXL_RESTRICT X, Vec<DF>* JXL_RESTRICT Y,
55 Vec<DF>* JXL_RESTRICT B,
56 Vec<DF>* JXL_RESTRICT w) const {
57 auto cx = aligned ? Load(DF(), rows[0][3 + row] + x)
58 : LoadU(DF(), rows[0][3 + row] + x);
59 auto cy = aligned ? Load(DF(), rows[1][3 + row] + x)
60 : LoadU(DF(), rows[1][3 + row] + x);
61 auto cb = aligned ? Load(DF(), rows[2][3 + row] + x)
62 : LoadU(DF(), rows[2][3 + row] + x);
63
64 auto weight = Weight(sad, inv_sigma, Set(DF(), lf_.epf_pass1_zeroflush));
65 *w = Add(*w, weight);
66 *X = MulAdd(weight, cx, *X);
67 *Y = MulAdd(weight, cy, *Y);
68 *B = MulAdd(weight, cb, *B);
69 }
70
71 Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows,
72 size_t xextra, size_t xsize, size_t xpos, size_t ypos,
73 size_t thread_id) const final {
74 DF df;
75
76 using V = decltype(Zero(df));
77 V t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, tA, tB; // NOLINT
78 V* sads[12] = {&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7, &t8, &t9, &tA, &tB};
79
80 xextra = RoundUpTo(xextra, Lanes(df));
81 const float* JXL_RESTRICT row_sigma =
82 sigma_->Row(ypos / kBlockDim + kSigmaPadding);
83
84 float sm = lf_.epf_pass0_sigma_scale * 1.65;
85 float bsm = sm * lf_.epf_border_sad_mul;
86
87 HWY_ALIGN float sad_mul_center[kBlockDim] = {bsm, sm, sm, sm,
88 sm, sm, sm, bsm};
89 HWY_ALIGN float sad_mul_border[kBlockDim] = {bsm, bsm, bsm, bsm,
90 bsm, bsm, bsm, bsm};
91 float* JXL_RESTRICT rows[3][7];
92 for (size_t c = 0; c < 3; c++) {
93 for (int i = 0; i < 7; i++) {
94 rows[c][i] = GetInputRow(input_rows, c, i - 3);
95 }
96 }
97
98 const float* sad_mul =
99 (ypos % kBlockDim == 0 || ypos % kBlockDim == kBlockDim - 1)
100 ? sad_mul_border
101 : sad_mul_center;
102
103 for (ssize_t x = -xextra; x < static_cast<ssize_t>(xsize + xextra);
104 x += Lanes(df)) {
105 size_t bx = (x + xpos + kSigmaPadding * kBlockDim) / kBlockDim;
106 size_t ix = (x + xpos) % kBlockDim;
107
108 if (row_sigma[bx] < kMinSigma) {
109 for (size_t c = 0; c < 3; c++) {
110 auto px = Load(df, rows[c][3 + 0] + x);
111 StoreU(px, df, GetOutputRow(output_rows, c, 0) + x);
112 }
113 continue;
114 }
115
116 const auto sm = Load(df, sad_mul + ix);
117 const auto inv_sigma = Mul(Set(df, row_sigma[bx]), sm);
118
119 for (auto& sad : sads) *sad = Zero(df);
120 constexpr std::array<int, 2> sads_off[12] = {
121 {{-2, 0}}, {{-1, -1}}, {{-1, 0}}, {{-1, 1}}, {{0, -2}}, {{0, -1}},
122 {{0, 1}}, {{0, 2}}, {{1, -1}}, {{1, 0}}, {{1, 1}}, {{2, 0}},
123 };
124
125 // compute sads
126 // TODO(veluca): consider unrolling and optimizing this.
127 for (size_t c = 0; c < 3; c++) {
128 auto scale = Set(df, lf_.epf_channel_scale[c]);
129 for (size_t i = 0; i < 12; i++) {
130 auto sad = Zero(df);
131 constexpr std::array<int, 2> plus_off[] = {
132 {{0, 0}}, {{-1, 0}}, {{0, -1}}, {{1, 0}}, {{0, 1}}};
133 for (const auto& off : plus_off) {
134 const auto r11 = LoadU(df, rows[c][3 + off[0]] + x + off[1]);
135 const auto c11 = LoadU(df, rows[c][3 + sads_off[i][0] + off[0]] +
136 x + sads_off[i][1] + off[1]);
137 sad = Add(sad, AbsDiff(r11, c11));
138 }
139 *sads[i] = MulAdd(sad, scale, *sads[i]);
140 }
141 }
142 const auto x_cc = Load(df, rows[0][3 + 0] + x);
143 const auto y_cc = Load(df, rows[1][3 + 0] + x);
144 const auto b_cc = Load(df, rows[2][3 + 0] + x);
145
146 auto w = Set(df, 1);
147 auto X = x_cc;
148 auto Y = y_cc;
149 auto B = b_cc;
150
151 for (size_t i = 0; i < 12; i++) {
152 AddPixel</*aligned=*/false>(/*row=*/sads_off[i][0], rows,
153 x + sads_off[i][1], *sads[i], inv_sigma, &X,
154 &Y, &B, &w);
155 }
156 #if JXL_HIGH_PRECISION
157 auto inv_w = Div(Set(df, 1.0f), w);
158 #else
159 auto inv_w = ApproximateReciprocal(w);
160 #endif
161 StoreU(Mul(X, inv_w), df, GetOutputRow(output_rows, 0, 0) + x);
162 StoreU(Mul(Y, inv_w), df, GetOutputRow(output_rows, 1, 0) + x);
163 StoreU(Mul(B, inv_w), df, GetOutputRow(output_rows, 2, 0) + x);
164 }
165 return true;
166 }
167
168 RenderPipelineChannelMode GetChannelMode(size_t c) const final {
169 return c < 3 ? RenderPipelineChannelMode::kInOut
170 : RenderPipelineChannelMode::kIgnored;
171 }
172
173 const char* GetName() const override { return "EPF0"; }
174
175 private:
176 LoopFilter lf_;
177 const ImageF* sigma_;
178 };
179
180 // 3x3 plus-shaped kernel with 5 SADs per pixel (also 3x3 plus-shaped). So this
181 // makes this filter a 5x5 filter.
182 class EPF1Stage : public RenderPipelineStage {
183 public:
184 EPF1Stage(LoopFilter lf, const ImageF& sigma)
185 : RenderPipelineStage(RenderPipelineStage::Settings::Symmetric(
186 /*shift=*/0, /*border=*/2)),
187 lf_(std::move(lf)),
188 sigma_(&sigma) {}
189
190 template <bool aligned>
191 JXL_INLINE void AddPixel(int row, float* JXL_RESTRICT rows[3][5], ssize_t x,
192 Vec<DF> sad, Vec<DF> inv_sigma,
193 Vec<DF>* JXL_RESTRICT X, Vec<DF>* JXL_RESTRICT Y,
194 Vec<DF>* JXL_RESTRICT B,
195 Vec<DF>* JXL_RESTRICT w) const {
196 auto cx = aligned ? Load(DF(), rows[0][2 + row] + x)
197 : LoadU(DF(), rows[0][2 + row] + x);
198 auto cy = aligned ? Load(DF(), rows[1][2 + row] + x)
199 : LoadU(DF(), rows[1][2 + row] + x);
200 auto cb = aligned ? Load(DF(), rows[2][2 + row] + x)
201 : LoadU(DF(), rows[2][2 + row] + x);
202
203 auto weight = Weight(sad, inv_sigma, Set(DF(), lf_.epf_pass1_zeroflush));
204 *w = Add(*w, weight);
205 *X = MulAdd(weight, cx, *X);
206 *Y = MulAdd(weight, cy, *Y);
207 *B = MulAdd(weight, cb, *B);
208 }
209
210 Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows,
211 size_t xextra, size_t xsize, size_t xpos, size_t ypos,
212 size_t thread_id) const final {
213 DF df;
214 xextra = RoundUpTo(xextra, Lanes(df));
215 const float* JXL_RESTRICT row_sigma =
216 sigma_->Row(ypos / kBlockDim + kSigmaPadding);
217
218 float sm = 1.65f;
219 float bsm = sm * lf_.epf_border_sad_mul;
220
221 HWY_ALIGN float sad_mul_center[kBlockDim] = {bsm, sm, sm, sm,
222 sm, sm, sm, bsm};
223 HWY_ALIGN float sad_mul_border[kBlockDim] = {bsm, bsm, bsm, bsm,
224 bsm, bsm, bsm, bsm};
225
226 float* JXL_RESTRICT rows[3][5];
227 for (size_t c = 0; c < 3; c++) {
228 for (int i = 0; i < 5; i++) {
229 rows[c][i] = GetInputRow(input_rows, c, i - 2);
230 }
231 }
232
233 const float* sad_mul =
234 (ypos % kBlockDim == 0 || ypos % kBlockDim == kBlockDim - 1)
235 ? sad_mul_border
236 : sad_mul_center;
237
238 for (ssize_t x = -xextra; x < static_cast<ssize_t>(xsize + xextra);
239 x += Lanes(df)) {
240 size_t bx = (x + xpos + kSigmaPadding * kBlockDim) / kBlockDim;
241 size_t ix = (x + xpos) % kBlockDim;
242
243 if (row_sigma[bx] < kMinSigma) {
244 for (size_t c = 0; c < 3; c++) {
245 auto px = Load(df, rows[c][2 + 0] + x);
246 Store(px, df, GetOutputRow(output_rows, c, 0) + x);
247 }
248 continue;
249 }
250
251 const auto sm = Load(df, sad_mul + ix);
252 const auto inv_sigma = Mul(Set(df, row_sigma[bx]), sm);
253 auto sad0 = Zero(df);
254 auto sad1 = Zero(df);
255 auto sad2 = Zero(df);
256 auto sad3 = Zero(df);
257
258 // compute sads
259 for (size_t c = 0; c < 3; c++) {
260 // center px = 22, px above = 21
261 auto t = Undefined(df);
262
263 const auto p20 = Load(df, rows[c][2 + -2] + x);
264 const auto p21 = Load(df, rows[c][2 + -1] + x);
265 auto sad0c = AbsDiff(p20, p21); // SAD 2, 1
266
267 const auto p11 = LoadU(df, rows[c][2 + -1] + x - 1);
268 auto sad1c = AbsDiff(p11, p21); // SAD 1, 2
269
270 const auto p31 = LoadU(df, rows[c][2 + -1] + x + 1);
271 auto sad2c = AbsDiff(p31, p21); // SAD 3, 2
272
273 const auto p02 = LoadU(df, rows[c][2 + 0] + x - 2);
274 const auto p12 = LoadU(df, rows[c][2 + 0] + x - 1);
275 sad1c = Add(sad1c, AbsDiff(p02, p12)); // SAD 1, 2
276 sad0c = Add(sad0c, AbsDiff(p11, p12)); // SAD 2, 1
277
278 const auto p22 = LoadU(df, rows[c][2 + 0] + x);
279 t = AbsDiff(p12, p22);
280 sad1c = Add(sad1c, t); // SAD 1, 2
281 sad2c = Add(sad2c, t); // SAD 3, 2
282 t = AbsDiff(p22, p21);
283 auto sad3c = t; // SAD 2, 3
284 sad0c = Add(sad0c, t); // SAD 2, 1
285
286 const auto p32 = LoadU(df, rows[c][2 + 0] + x + 1);
287 sad0c = Add(sad0c, AbsDiff(p31, p32)); // SAD 2, 1
288 t = AbsDiff(p22, p32);
289 sad1c = Add(sad1c, t); // SAD 1, 2
290 sad2c = Add(sad2c, t); // SAD 3, 2
291
292 const auto p42 = LoadU(df, rows[c][2 + 0] + x + 2);
293 sad2c = Add(sad2c, AbsDiff(p42, p32)); // SAD 3, 2
294
295 const auto p13 = LoadU(df, rows[c][2 + 1] + x - 1);
296 sad3c = Add(sad3c, AbsDiff(p13, p12)); // SAD 2, 3
297
298 const auto p23 = Load(df, rows[c][2 + 1] + x);
299 t = AbsDiff(p22, p23);
300 sad0c = Add(sad0c, t); // SAD 2, 1
301 sad3c = Add(sad3c, t); // SAD 2, 3
302 sad1c = Add(sad1c, AbsDiff(p13, p23)); // SAD 1, 2
303
304 const auto p33 = LoadU(df, rows[c][2 + 1] + x + 1);
305 sad2c = Add(sad2c, AbsDiff(p33, p23)); // SAD 3, 2
306 sad3c = Add(sad3c, AbsDiff(p33, p32)); // SAD 2, 3
307
308 const auto p24 = Load(df, rows[c][2 + 2] + x);
309 sad3c = Add(sad3c, AbsDiff(p24, p23)); // SAD 2, 3
310
311 auto scale = Set(df, lf_.epf_channel_scale[c]);
312 sad0 = MulAdd(sad0c, scale, sad0);
313 sad1 = MulAdd(sad1c, scale, sad1);
314 sad2 = MulAdd(sad2c, scale, sad2);
315 sad3 = MulAdd(sad3c, scale, sad3);
316 }
317 const auto x_cc = Load(df, rows[0][2 + 0] + x);
318 const auto y_cc = Load(df, rows[1][2 + 0] + x);
319 const auto b_cc = Load(df, rows[2][2 + 0] + x);
320
321 auto w = Set(df, 1);
322 auto X = x_cc;
323 auto Y = y_cc;
324 auto B = b_cc;
325
326 // Top row
327 AddPixel</*aligned=*/true>(/*row=*/-1, rows, x, sad0, inv_sigma, &X, &Y,
328 &B, &w);
329 // Center
330 AddPixel</*aligned=*/false>(/*row=*/0, rows, x - 1, sad1, inv_sigma, &X,
331 &Y, &B, &w);
332 AddPixel</*aligned=*/false>(/*row=*/0, rows, x + 1, sad2, inv_sigma, &X,
333 &Y, &B, &w);
334 // Bottom
335 AddPixel</*aligned=*/true>(/*row=*/1, rows, x, sad3, inv_sigma, &X, &Y,
336 &B, &w);
337 #if JXL_HIGH_PRECISION
338 auto inv_w = Div(Set(df, 1.0f), w);
339 #else
340 auto inv_w = ApproximateReciprocal(w);
341 #endif
342 Store(Mul(X, inv_w), df, GetOutputRow(output_rows, 0, 0) + x);
343 Store(Mul(Y, inv_w), df, GetOutputRow(output_rows, 1, 0) + x);
344 Store(Mul(B, inv_w), df, GetOutputRow(output_rows, 2, 0) + x);
345 }
346 return true;
347 }
348
349 RenderPipelineChannelMode GetChannelMode(size_t c) const final {
350 return c < 3 ? RenderPipelineChannelMode::kInOut
351 : RenderPipelineChannelMode::kIgnored;
352 }
353
354 const char* GetName() const override { return "EPF1"; }
355
356 private:
357 LoopFilter lf_;
358 const ImageF* sigma_;
359 };
360
361 // 3x3 plus-shaped kernel with 1 SAD per pixel. So this makes this filter a 3x3
362 // filter.
363 class EPF2Stage : public RenderPipelineStage {
364 public:
365 EPF2Stage(LoopFilter lf, const ImageF& sigma)
366 : RenderPipelineStage(RenderPipelineStage::Settings::Symmetric(
367 /*shift=*/0, /*border=*/1)),
368 lf_(std::move(lf)),
369 sigma_(&sigma) {}
370
371 template <bool aligned>
372 JXL_INLINE void AddPixel(int row, float* JXL_RESTRICT rows[3][3], ssize_t x,
373 Vec<DF> rx, Vec<DF> ry, Vec<DF> rb,
374 Vec<DF> inv_sigma, Vec<DF>* JXL_RESTRICT X,
375 Vec<DF>* JXL_RESTRICT Y, Vec<DF>* JXL_RESTRICT B,
376 Vec<DF>* JXL_RESTRICT w) const {
377 auto cx = aligned ? Load(DF(), rows[0][1 + row] + x)
378 : LoadU(DF(), rows[0][1 + row] + x);
379 auto cy = aligned ? Load(DF(), rows[1][1 + row] + x)
380 : LoadU(DF(), rows[1][1 + row] + x);
381 auto cb = aligned ? Load(DF(), rows[2][1 + row] + x)
382 : LoadU(DF(), rows[2][1 + row] + x);
383
384 auto sad = Mul(AbsDiff(cx, rx), Set(DF(), lf_.epf_channel_scale[0]));
385 sad = MulAdd(AbsDiff(cy, ry), Set(DF(), lf_.epf_channel_scale[1]), sad);
386 sad = MulAdd(AbsDiff(cb, rb), Set(DF(), lf_.epf_channel_scale[2]), sad);
387
388 auto weight = Weight(sad, inv_sigma, Set(DF(), lf_.epf_pass2_zeroflush));
389
390 *w = Add(*w, weight);
391 *X = MulAdd(weight, cx, *X);
392 *Y = MulAdd(weight, cy, *Y);
393 *B = MulAdd(weight, cb, *B);
394 }
395
396 Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows,
397 size_t xextra, size_t xsize, size_t xpos, size_t ypos,
398 size_t thread_id) const final {
399 DF df;
400 xextra = RoundUpTo(xextra, Lanes(df));
401 const float* JXL_RESTRICT row_sigma =
402 sigma_->Row(ypos / kBlockDim + kSigmaPadding);
403
404 float sm = lf_.epf_pass2_sigma_scale * 1.65;
405 float bsm = sm * lf_.epf_border_sad_mul;
406
407 HWY_ALIGN float sad_mul_center[kBlockDim] = {bsm, sm, sm, sm,
408 sm, sm, sm, bsm};
409 HWY_ALIGN float sad_mul_border[kBlockDim] = {bsm, bsm, bsm, bsm,
410 bsm, bsm, bsm, bsm};
411
412 float* JXL_RESTRICT rows[3][3];
413 for (size_t c = 0; c < 3; c++) {
414 for (int i = 0; i < 3; i++) {
415 rows[c][i] = GetInputRow(input_rows, c, i - 1);
416 }
417 }
418
419 const float* sad_mul =
420 (ypos % kBlockDim == 0 || ypos % kBlockDim == kBlockDim - 1)
421 ? sad_mul_border
422 : sad_mul_center;
423
424 for (ssize_t x = -xextra; x < static_cast<ssize_t>(xsize + xextra);
425 x += Lanes(df)) {
426 size_t bx = (x + xpos + kSigmaPadding * kBlockDim) / kBlockDim;
427 size_t ix = (x + xpos) % kBlockDim;
428
429 if (row_sigma[bx] < kMinSigma) {
430 for (size_t c = 0; c < 3; c++) {
431 auto px = Load(df, rows[c][1 + 0] + x);
432 Store(px, df, GetOutputRow(output_rows, c, 0) + x);
433 }
434 continue;
435 }
436
437 const auto sm = Load(df, sad_mul + ix);
438 const auto inv_sigma = Mul(Set(df, row_sigma[bx]), sm);
439
440 const auto x_cc = Load(df, rows[0][1 + 0] + x);
441 const auto y_cc = Load(df, rows[1][1 + 0] + x);
442 const auto b_cc = Load(df, rows[2][1 + 0] + x);
443
444 auto w = Set(df, 1);
445 auto X = x_cc;
446 auto Y = y_cc;
447 auto B = b_cc;
448
449 // Top row
450 AddPixel</*aligned=*/true>(/*row=*/-1, rows, x, x_cc, y_cc, b_cc,
451 inv_sigma, &X, &Y, &B, &w);
452 // Center
453 AddPixel</*aligned=*/false>(/*row=*/0, rows, x - 1, x_cc, y_cc, b_cc,
454 inv_sigma, &X, &Y, &B, &w);
455 AddPixel</*aligned=*/false>(/*row=*/0, rows, x + 1, x_cc, y_cc, b_cc,
456 inv_sigma, &X, &Y, &B, &w);
457 // Bottom
458 AddPixel</*aligned=*/true>(/*row=*/1, rows, x, x_cc, y_cc, b_cc,
459 inv_sigma, &X, &Y, &B, &w);
460 #if JXL_HIGH_PRECISION
461 auto inv_w = Div(Set(df, 1.0f), w);
462 #else
463 auto inv_w = ApproximateReciprocal(w);
464 #endif
465 Store(Mul(X, inv_w), df, GetOutputRow(output_rows, 0, 0) + x);
466 Store(Mul(Y, inv_w), df, GetOutputRow(output_rows, 1, 0) + x);
467 Store(Mul(B, inv_w), df, GetOutputRow(output_rows, 2, 0) + x);
468 }
469 return true;
470 }
471
472 RenderPipelineChannelMode GetChannelMode(size_t c) const final {
473 return c < 3 ? RenderPipelineChannelMode::kInOut
474 : RenderPipelineChannelMode::kIgnored;
475 }
476
477 const char* GetName() const override { return "EPF2"; }
478
479 private:
480 LoopFilter lf_;
481 const ImageF* sigma_;
482 };
483
484 std::unique_ptr<RenderPipelineStage> GetEPFStage0(const LoopFilter& lf,
485 const ImageF& sigma) {
486 return jxl::make_unique<EPF0Stage>(lf, sigma);
487 }
488
489 std::unique_ptr<RenderPipelineStage> GetEPFStage1(const LoopFilter& lf,
490 const ImageF& sigma) {
491 return jxl::make_unique<EPF1Stage>(lf, sigma);
492 }
493
494 std::unique_ptr<RenderPipelineStage> GetEPFStage2(const LoopFilter& lf,
495 const ImageF& sigma) {
496 return jxl::make_unique<EPF2Stage>(lf, sigma);
497 }
498
499 // NOLINTNEXTLINE(google-readability-namespace-comments)
500 } // namespace HWY_NAMESPACE
501 } // namespace jxl
502 HWY_AFTER_NAMESPACE();
503
504 #if HWY_ONCE
505 namespace jxl {
506
507 HWY_EXPORT(GetEPFStage0);
508 HWY_EXPORT(GetEPFStage1);
509 HWY_EXPORT(GetEPFStage2);
510
511 std::unique_ptr<RenderPipelineStage> GetEPFStage(const LoopFilter& lf,
512 const ImageF& sigma,
513 EpfStage epf_stage) {
514 if (lf.epf_iters == 0) return nullptr;
515 switch (epf_stage) {
516 case EpfStage::Zero:
517 return HWY_DYNAMIC_DISPATCH(GetEPFStage0)(lf, sigma);
518 case EpfStage::One:
519 return HWY_DYNAMIC_DISPATCH(GetEPFStage1)(lf, sigma);
520 case EpfStage::Two:
521 return HWY_DYNAMIC_DISPATCH(GetEPFStage2)(lf, sigma);
522 }
523 JXL_DEBUG_ABORT("internal: unexpected EpfStage: %d",
524 static_cast<int>(epf_stage));
525 return nullptr;
526 }
527
528 } // namespace jxl
529 #endif