Debian Codegraph @

lib/jxl/render_pipeline/stage_upsampling.cc

raw
1 // Copyright (c) the JPEG XL Project Authors. All rights reserved.
2 //
3 // Use of this source code is governed by a BSD-style
4 // license that can be found in the LICENSE file.
5
6 #include "lib/jxl/render_pipeline/stage_upsampling.h"
7
8 #include "lib/jxl/base/sanitizers.h"
9 #include "lib/jxl/base/status.h"
10
11 #undef HWY_TARGET_INCLUDE
12 #define HWY_TARGET_INCLUDE "lib/jxl/render_pipeline/stage_upsampling.cc"
13 #include <hwy/foreach_target.h>
14 #include <hwy/highway.h>
15
16 #include "lib/jxl/simd_util-inl.h"
17
18 HWY_BEFORE_NAMESPACE();
19 namespace jxl {
20 namespace HWY_NAMESPACE {
21
22 // These templates are not found via ADL.
23 using hwy::HWY_NAMESPACE::Clamp;
24 using hwy::HWY_NAMESPACE::Max;
25 using hwy::HWY_NAMESPACE::Min;
26 using hwy::HWY_NAMESPACE::MulAdd;
27
28 class UpsamplingStage : public RenderPipelineStage {
29 public:
30 explicit UpsamplingStage(const CustomTransformData& ups_factors, size_t c,
31 size_t shift)
32 : RenderPipelineStage(RenderPipelineStage::Settings::Symmetric(
33 /*shift=*/shift, /*border=*/2)),
34 c_(c) {
35 const float* weights = shift == 1 ? ups_factors.upsampling2_weights
36 : shift == 2 ? ups_factors.upsampling4_weights
37 : ups_factors.upsampling8_weights;
38 size_t N = 1 << (shift - 1);
39 for (size_t i = 0; i < 5 * N; i++) {
40 for (size_t j = 0; j < 5 * N; j++) {
41 size_t y = std::min(i, j);
42 size_t x = std::max(i, j);
43 kernel_[j / 5][i / 5][j % 5][i % 5] =
44 weights[5 * N * y - y * (y - 1) / 2 + x - y];
45 }
46 }
47 }
48
49 Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows,
50 size_t xextra, size_t xsize, size_t xpos, size_t ypos,
51 size_t thread_id) const final {
52 static HWY_FULL(float) df;
53 size_t shift = settings_.shift_x;
54 size_t N = 1 << shift;
55 const size_t xsize_v = RoundUpTo(xsize, Lanes(df));
56 for (ssize_t iy = -2; iy <= 2; iy++) {
57 msan::UnpoisonMemory(GetInputRow(input_rows, c_, iy) + xsize + 2,
58 sizeof(float) * (xsize_v - xsize));
59 }
60 JXL_ENSURE(xextra == 0);
61 ssize_t x0 = 0;
62 ssize_t x1 = xsize;
63 if (N == 2) {
64 ProcessRowImpl<2>(input_rows, output_rows, x0, x1);
65 }
66 if (N == 4) {
67 ProcessRowImpl<4>(input_rows, output_rows, x0, x1);
68 }
69 if (N == 8) {
70 ProcessRowImpl<8>(input_rows, output_rows, x0, x1);
71 }
72 for (size_t oy = 0; oy < N; oy++) {
73 float* dst_row = GetOutputRow(output_rows, c_, oy);
74 msan::PoisonMemory(dst_row + xsize * N,
75 sizeof(float) * (xsize_v - xsize) * N);
76 }
77 return true;
78 }
79
80 RenderPipelineChannelMode GetChannelMode(size_t c) const final {
81 return c == c_ ? RenderPipelineChannelMode::kInOut
82 : RenderPipelineChannelMode::kIgnored;
83 }
84
85 const char* GetName() const override { return "Upsample"; }
86
87 private:
88 template <size_t N>
89 JXL_INLINE float Kernel(size_t x, size_t y, ssize_t ix, ssize_t iy) const {
90 static_assert(N == 2 || N == 4 || N == 8, "N must be 2, 4, or 8");
91 ix += 2;
92 iy += 2;
93 if (N == 2) {
94 return kernel_[0][0][y % 2 ? 4 - iy : iy][x % 2 ? 4 - ix : ix];
95 }
96 if (N == 4) {
97 return kernel_[y % 4 < 2 ? y % 2 : 1 - y % 2]
98 [x % 4 < 2 ? x % 2 : 1 - x % 2][y % 4 < 2 ? iy : 4 - iy]
99 [x % 4 < 2 ? ix : 4 - ix];
100 }
101 if (N == 8) {
102 return kernel_[y % 8 < 4 ? y % 4 : 3 - y % 4]
103 [x % 8 < 4 ? x % 4 : 3 - x % 4][y % 8 < 4 ? iy : 4 - iy]
104 [x % 8 < 4 ? ix : 4 - ix];
105 }
106 }
107
108 template <ssize_t N>
109 void ProcessRowImpl(const RowInfo& input_rows, const RowInfo& output_rows,
110 ssize_t x0, ssize_t x1) const {
111 static HWY_FULL(float) df;
112 using V = hwy::HWY_NAMESPACE::Vec<HWY_FULL(float)>;
113 V ups0, ups1, ups2, ups3, ups4, ups5, ups6, ups7; // NOLINT
114 (void)ups2, (void)ups3, (void)ups4, (void)ups5, (void)ups6, (void)ups7;
115 // Once we have C++17 available, change this back to `V* ups[N]` and
116 // initialize using `if constexpr` below.
117 V* ups[8] = {};
118 static_assert(N == 2 || N == 4 || N == 8, "N must be 2, 4, or 8");
119 if (N >= 2) {
120 ups[0] = &ups0;
121 ups[1] = &ups1;
122 }
123 if (N >= 4) {
124 ups[2] = &ups2;
125 ups[3] = &ups3;
126 }
127 if (N == 8) {
128 ups[4] = &ups4;
129 ups[5] = &ups5;
130 ups[6] = &ups6;
131 ups[7] = &ups7;
132 }
133
134 for (size_t oy = 0; oy < N; oy++) {
135 float* dst_row = GetOutputRow(output_rows, c_, oy);
136 for (ssize_t x = x0; x < x1; x += Lanes(df)) {
137 for (size_t ox = 0; ox < N; ox++) {
138 auto result = Zero(df);
139 auto min = LoadU(df, GetInputRow(input_rows, c_, 0) + x);
140 auto max = min;
141 for (ssize_t iy = -2; iy <= 2; iy++) {
142 for (ssize_t ix = -2; ix <= 2; ix++) {
143 auto v = LoadU(df, GetInputRow(input_rows, c_, iy) + x + ix);
144 result = MulAdd(Set(df, Kernel<N>(ox, oy, ix, iy)), v, result);
145 min = Min(v, min);
146 max = Max(v, max);
147 }
148 }
149 // Avoid overshooting.
150 *ups[ox] = Clamp(result, min, max);
151 }
152 if (N == 2) {
153 StoreInterleaved(df, ups0, ups1, dst_row + x * N);
154 }
155 if (N == 4) {
156 StoreInterleaved(df, ups0, ups1, ups2, ups3, dst_row + x * N);
157 }
158 if (N == 8) {
159 StoreInterleaved(df, ups0, ups1, ups2, ups3, ups4, ups5, ups6, ups7,
160 dst_row + x * N);
161 }
162 }
163 }
164 }
165
166 size_t c_;
167 float kernel_[4][4][5][5];
168 };
169
170 std::unique_ptr<RenderPipelineStage> GetUpsamplingStage(
171 const CustomTransformData& ups_factors, size_t c, size_t shift) {
172 return jxl::make_unique<UpsamplingStage>(ups_factors, c, shift);
173 }
174
175 // NOLINTNEXTLINE(google-readability-namespace-comments)
176 } // namespace HWY_NAMESPACE
177 } // namespace jxl
178 HWY_AFTER_NAMESPACE();
179
180 #if HWY_ONCE
181 namespace jxl {
182
183 HWY_EXPORT(GetUpsamplingStage);
184
185 std::unique_ptr<RenderPipelineStage> GetUpsamplingStage(
186 const CustomTransformData& ups_factors, size_t c, size_t shift) {
187 if ((shift < 1) || (shift > 3)) {
188 JXL_DEBUG_ABORT("internal: (shift != 0) && (shift <= 3)");
189 return nullptr;
190 }
191 return HWY_DYNAMIC_DISPATCH(GetUpsamplingStage)(ups_factors, c, shift);
192 }
193
194 } // namespace jxl
195 #endif