| 1 |
// Copyright (c) the JPEG XL Project Authors. All rights reserved. |
| 2 |
// |
| 3 |
// Use of this source code is governed by a BSD-style |
| 4 |
// license that can be found in the LICENSE file. |
| 5 |
|
| 6 |
#include "lib/jxl/render_pipeline/stage_upsampling.h" |
| 7 |
|
| 8 |
#include "lib/jxl/base/sanitizers.h" |
| 9 |
#include "lib/jxl/base/status.h" |
| 10 |
|
| 11 |
#undef HWY_TARGET_INCLUDE |
| 12 |
#define HWY_TARGET_INCLUDE "lib/jxl/render_pipeline/stage_upsampling.cc" |
| 13 |
#include <hwy/foreach_target.h> |
| 14 |
#include <hwy/highway.h> |
| 15 |
|
| 16 |
#include "lib/jxl/simd_util-inl.h" |
| 17 |
|
| 18 |
HWY_BEFORE_NAMESPACE(); |
| 19 |
namespace jxl { |
| 20 |
namespace HWY_NAMESPACE { |
| 21 |
|
| 22 |
// These templates are not found via ADL. |
| 23 |
using hwy::HWY_NAMESPACE::Clamp; |
| 24 |
using hwy::HWY_NAMESPACE::Max; |
| 25 |
using hwy::HWY_NAMESPACE::Min; |
| 26 |
using hwy::HWY_NAMESPACE::MulAdd; |
| 27 |
|
| 28 |
class UpsamplingStage : public RenderPipelineStage { |
| 29 |
public: |
| 30 |
explicit UpsamplingStage(const CustomTransformData& ups_factors, size_t c, |
| 31 |
size_t shift) |
| 32 |
: RenderPipelineStage(RenderPipelineStage::Settings::Symmetric( |
| 33 |
/*shift=*/shift, /*border=*/2)), |
| 34 |
c_(c) { |
| 35 |
const float* weights = shift == 1 ? ups_factors.upsampling2_weights |
| 36 |
: shift == 2 ? ups_factors.upsampling4_weights |
| 37 |
: ups_factors.upsampling8_weights; |
| 38 |
size_t N = 1 << (shift - 1); |
| 39 |
for (size_t i = 0; i < 5 * N; i++) { |
| 40 |
for (size_t j = 0; j < 5 * N; j++) { |
| 41 |
size_t y = std::min(i, j); |
| 42 |
size_t x = std::max(i, j); |
| 43 |
kernel_[j / 5][i / 5][j % 5][i % 5] = |
| 44 |
weights[5 * N * y - y * (y - 1) / 2 + x - y]; |
| 45 |
} |
| 46 |
} |
| 47 |
} |
| 48 |
|
| 49 |
Status ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, |
| 50 |
size_t xextra, size_t xsize, size_t xpos, size_t ypos, |
| 51 |
size_t thread_id) const final { |
| 52 |
static HWY_FULL(float) df; |
| 53 |
size_t shift = settings_.shift_x; |
| 54 |
size_t N = 1 << shift; |
| 55 |
const size_t xsize_v = RoundUpTo(xsize, Lanes(df)); |
| 56 |
for (ssize_t iy = -2; iy <= 2; iy++) { |
| 57 |
msan::UnpoisonMemory(GetInputRow(input_rows, c_, iy) + xsize + 2, |
| 58 |
sizeof(float) * (xsize_v - xsize)); |
| 59 |
} |
| 60 |
JXL_ENSURE(xextra == 0); |
| 61 |
ssize_t x0 = 0; |
| 62 |
ssize_t x1 = xsize; |
| 63 |
if (N == 2) { |
| 64 |
ProcessRowImpl<2>(input_rows, output_rows, x0, x1); |
| 65 |
} |
| 66 |
if (N == 4) { |
| 67 |
ProcessRowImpl<4>(input_rows, output_rows, x0, x1); |
| 68 |
} |
| 69 |
if (N == 8) { |
| 70 |
ProcessRowImpl<8>(input_rows, output_rows, x0, x1); |
| 71 |
} |
| 72 |
for (size_t oy = 0; oy < N; oy++) { |
| 73 |
float* dst_row = GetOutputRow(output_rows, c_, oy); |
| 74 |
msan::PoisonMemory(dst_row + xsize * N, |
| 75 |
sizeof(float) * (xsize_v - xsize) * N); |
| 76 |
} |
| 77 |
return true; |
| 78 |
} |
| 79 |
|
| 80 |
RenderPipelineChannelMode GetChannelMode(size_t c) const final { |
| 81 |
return c == c_ ? RenderPipelineChannelMode::kInOut |
| 82 |
: RenderPipelineChannelMode::kIgnored; |
| 83 |
} |
| 84 |
|
| 85 |
const char* GetName() const override { return "Upsample"; } |
| 86 |
|
| 87 |
private: |
| 88 |
template <size_t N> |
| 89 |
JXL_INLINE float Kernel(size_t x, size_t y, ssize_t ix, ssize_t iy) const { |
| 90 |
static_assert(N == 2 || N == 4 || N == 8, "N must be 2, 4, or 8"); |
| 91 |
ix += 2; |
| 92 |
iy += 2; |
| 93 |
if (N == 2) { |
| 94 |
return kernel_[0][0][y % 2 ? 4 - iy : iy][x % 2 ? 4 - ix : ix]; |
| 95 |
} |
| 96 |
if (N == 4) { |
| 97 |
return kernel_[y % 4 < 2 ? y % 2 : 1 - y % 2] |
| 98 |
[x % 4 < 2 ? x % 2 : 1 - x % 2][y % 4 < 2 ? iy : 4 - iy] |
| 99 |
[x % 4 < 2 ? ix : 4 - ix]; |
| 100 |
} |
| 101 |
if (N == 8) { |
| 102 |
return kernel_[y % 8 < 4 ? y % 4 : 3 - y % 4] |
| 103 |
[x % 8 < 4 ? x % 4 : 3 - x % 4][y % 8 < 4 ? iy : 4 - iy] |
| 104 |
[x % 8 < 4 ? ix : 4 - ix]; |
| 105 |
} |
| 106 |
} |
| 107 |
|
| 108 |
template <ssize_t N> |
| 109 |
void ProcessRowImpl(const RowInfo& input_rows, const RowInfo& output_rows, |
| 110 |
ssize_t x0, ssize_t x1) const { |
| 111 |
static HWY_FULL(float) df; |
| 112 |
using V = hwy::HWY_NAMESPACE::Vec<HWY_FULL(float)>; |
| 113 |
V ups0, ups1, ups2, ups3, ups4, ups5, ups6, ups7; // NOLINT |
| 114 |
(void)ups2, (void)ups3, (void)ups4, (void)ups5, (void)ups6, (void)ups7; |
| 115 |
// Once we have C++17 available, change this back to `V* ups[N]` and |
| 116 |
// initialize using `if constexpr` below. |
| 117 |
V* ups[8] = {}; |
| 118 |
static_assert(N == 2 || N == 4 || N == 8, "N must be 2, 4, or 8"); |
| 119 |
if (N >= 2) { |
| 120 |
ups[0] = &ups0; |
| 121 |
ups[1] = &ups1; |
| 122 |
} |
| 123 |
if (N >= 4) { |
| 124 |
ups[2] = &ups2; |
| 125 |
ups[3] = &ups3; |
| 126 |
} |
| 127 |
if (N == 8) { |
| 128 |
ups[4] = &ups4; |
| 129 |
ups[5] = &ups5; |
| 130 |
ups[6] = &ups6; |
| 131 |
ups[7] = &ups7; |
| 132 |
} |
| 133 |
|
| 134 |
for (size_t oy = 0; oy < N; oy++) { |
| 135 |
float* dst_row = GetOutputRow(output_rows, c_, oy); |
| 136 |
for (ssize_t x = x0; x < x1; x += Lanes(df)) { |
| 137 |
for (size_t ox = 0; ox < N; ox++) { |
| 138 |
auto result = Zero(df); |
| 139 |
auto min = LoadU(df, GetInputRow(input_rows, c_, 0) + x); |
| 140 |
auto max = min; |
| 141 |
for (ssize_t iy = -2; iy <= 2; iy++) { |
| 142 |
for (ssize_t ix = -2; ix <= 2; ix++) { |
| 143 |
auto v = LoadU(df, GetInputRow(input_rows, c_, iy) + x + ix); |
| 144 |
result = MulAdd(Set(df, Kernel<N>(ox, oy, ix, iy)), v, result); |
| 145 |
min = Min(v, min); |
| 146 |
max = Max(v, max); |
| 147 |
} |
| 148 |
} |
| 149 |
// Avoid overshooting. |
| 150 |
*ups[ox] = Clamp(result, min, max); |
| 151 |
} |
| 152 |
if (N == 2) { |
| 153 |
StoreInterleaved(df, ups0, ups1, dst_row + x * N); |
| 154 |
} |
| 155 |
if (N == 4) { |
| 156 |
StoreInterleaved(df, ups0, ups1, ups2, ups3, dst_row + x * N); |
| 157 |
} |
| 158 |
if (N == 8) { |
| 159 |
StoreInterleaved(df, ups0, ups1, ups2, ups3, ups4, ups5, ups6, ups7, |
| 160 |
dst_row + x * N); |
| 161 |
} |
| 162 |
} |
| 163 |
} |
| 164 |
} |
| 165 |
|
| 166 |
size_t c_; |
| 167 |
float kernel_[4][4][5][5]; |
| 168 |
}; |
| 169 |
|
| 170 |
std::unique_ptr<RenderPipelineStage> GetUpsamplingStage( |
| 171 |
const CustomTransformData& ups_factors, size_t c, size_t shift) { |
| 172 |
return jxl::make_unique<UpsamplingStage>(ups_factors, c, shift); |
| 173 |
} |
| 174 |
|
| 175 |
// NOLINTNEXTLINE(google-readability-namespace-comments) |
| 176 |
} // namespace HWY_NAMESPACE |
| 177 |
} // namespace jxl |
| 178 |
HWY_AFTER_NAMESPACE(); |
| 179 |
|
| 180 |
#if HWY_ONCE |
| 181 |
namespace jxl { |
| 182 |
|
| 183 |
HWY_EXPORT(GetUpsamplingStage); |
| 184 |
|
| 185 |
std::unique_ptr<RenderPipelineStage> GetUpsamplingStage( |
| 186 |
const CustomTransformData& ups_factors, size_t c, size_t shift) { |
| 187 |
if ((shift < 1) || (shift > 3)) { |
| 188 |
JXL_DEBUG_ABORT("internal: (shift != 0) && (shift <= 3)"); |
| 189 |
return nullptr; |
| 190 |
} |
| 191 |
return HWY_DYNAMIC_DISPATCH(GetUpsamplingStage)(ups_factors, c, shift); |
| 192 |
} |
| 193 |
|
| 194 |
} // namespace jxl |
| 195 |
#endif |