Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/neon/t_lut3_to_3.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::conversions::LutBarycentricReduction;
30 use crate::conversions::interpolator::BarycentricWeight;
31 use crate::conversions::lut_transforms::Lut3x3Factory;
32 use crate::conversions::neon::interpolator::*;
33 use crate::conversions::neon::interpolator_q0_15::NeonAlignedI16x4;
34 use crate::conversions::neon::rgb_xyz::NeonAlignedF32;
35 use crate::conversions::neon::t_lut3_to_3_q0_15::TransformLut3x3NeonQ0_15;
36 use crate::transform::PointeeSizeExpressible;
37 use crate::{
38 BarycentricWeightScale, CmsError, DataColorSpace, InterpolationMethod, Layout,
39 TransformExecutor, TransformOptions,
40 };
41 use num_traits::AsPrimitive;
42 use std::arch::aarch64::*;
43 use std::marker::PhantomData;
44
45 struct TransformLut3x3Neon<
46 T,
47 U,
48 const SRC_LAYOUT: u8,
49 const DST_LAYOUT: u8,
50 const GRID_SIZE: usize,
51 const BIT_DEPTH: usize,
52 const BINS: usize,
53 const BARYCENTRIC_BINS: usize,
54 > {
55 lut: Vec<NeonAlignedF32>,
56 _phantom: PhantomData<T>,
57 _phantom1: PhantomData<U>,
58 interpolation_method: InterpolationMethod,
59 weights: Box<[BarycentricWeight<f32>; BINS]>,
60 color_space: DataColorSpace,
61 is_linear: bool,
62 }
63
64 impl<
65 T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible,
66 U: AsPrimitive<usize>,
67 const SRC_LAYOUT: u8,
68 const DST_LAYOUT: u8,
69 const GRID_SIZE: usize,
70 const BIT_DEPTH: usize,
71 const BINS: usize,
72 const BARYCENTRIC_BINS: usize,
73 > TransformLut3x3Neon<T, U, SRC_LAYOUT, DST_LAYOUT, GRID_SIZE, BIT_DEPTH, BINS, BARYCENTRIC_BINS>
74 where
75 f32: AsPrimitive<T>,
76 u32: AsPrimitive<T>,
77 (): LutBarycentricReduction<T, U>,
78 {
79 #[inline(never)]
80 fn transform_chunk(
81 &self,
82 src: &[T],
83 dst: &mut [T],
84 interpolator: Box<dyn NeonMdInterpolation + Send + Sync>,
85 ) {
86 unsafe {
87 let src_cn = Layout::from(SRC_LAYOUT);
88 let src_channels = src_cn.channels();
89
90 let dst_cn = Layout::from(DST_LAYOUT);
91 let dst_channels = dst_cn.channels();
92
93 let value_scale = vdupq_n_f32(((1 << BIT_DEPTH) - 1) as f32);
94 let max_value = ((1u32 << BIT_DEPTH) - 1).as_();
95
96 for (src, dst) in src
97 .chunks_exact(src_channels)
98 .zip(dst.chunks_exact_mut(dst_channels))
99 {
100 let x = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
101 src[src_cn.r_i()],
102 );
103 let y = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
104 src[src_cn.g_i()],
105 );
106 let z = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
107 src[src_cn.b_i()],
108 );
109
110 let a = if src_channels == 4 {
111 src[src_cn.a_i()]
112 } else {
113 max_value
114 };
115
116 let v = interpolator.inter3_neon(
117 &self.lut,
118 x.as_(),
119 y.as_(),
120 z.as_(),
121 self.weights.as_slice(),
122 );
123 if T::FINITE {
124 let mut r = vfmaq_f32(vdupq_n_f32(0.5f32), v.v, value_scale);
125 r = vminq_f32(r, value_scale);
126 let jvx = vcvtaq_u32_f32(r);
127
128 dst[dst_cn.r_i()] = vgetq_lane_u32::<0>(jvx).as_();
129 dst[dst_cn.g_i()] = vgetq_lane_u32::<1>(jvx).as_();
130 dst[dst_cn.b_i()] = vgetq_lane_u32::<2>(jvx).as_();
131 } else {
132 dst[dst_cn.r_i()] = vgetq_lane_f32::<0>(v.v).as_();
133 dst[dst_cn.g_i()] = vgetq_lane_f32::<1>(v.v).as_();
134 dst[dst_cn.b_i()] = vgetq_lane_f32::<2>(v.v).as_();
135 }
136 if dst_channels == 4 {
137 dst[dst_cn.a_i()] = a;
138 }
139 }
140 }
141 }
142 }
143
144 impl<
145 T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible,
146 U: AsPrimitive<usize>,
147 const SRC_LAYOUT: u8,
148 const DST_LAYOUT: u8,
149 const GRID_SIZE: usize,
150 const BIT_DEPTH: usize,
151 const BINS: usize,
152 const BARYCENTRIC_BINS: usize,
153 > TransformExecutor<T>
154 for TransformLut3x3Neon<
155 T,
156 U,
157 SRC_LAYOUT,
158 DST_LAYOUT,
159 GRID_SIZE,
160 BIT_DEPTH,
161 BINS,
162 BARYCENTRIC_BINS,
163 >
164 where
165 f32: AsPrimitive<T>,
166 u32: AsPrimitive<T>,
167 (): LutBarycentricReduction<T, U>,
168 {
169 fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
170 let src_cn = Layout::from(SRC_LAYOUT);
171 let src_channels = src_cn.channels();
172
173 let dst_cn = Layout::from(DST_LAYOUT);
174 let dst_channels = dst_cn.channels();
175 if src.len() % src_channels != 0 {
176 return Err(CmsError::LaneMultipleOfChannels);
177 }
178 if dst.len() % dst_channels != 0 {
179 return Err(CmsError::LaneMultipleOfChannels);
180 }
181 let src_chunks = src.len() / src_channels;
182 let dst_chunks = dst.len() / dst_channels;
183 if src_chunks != dst_chunks {
184 return Err(CmsError::LaneSizeMismatch);
185 }
186
187 if self.color_space == DataColorSpace::Lab
188 || (self.is_linear && self.color_space == DataColorSpace::Rgb)
189 || self.color_space == DataColorSpace::Xyz
190 {
191 self.transform_chunk(src, dst, Box::new(TrilinearNeon::<GRID_SIZE> {}));
192 } else {
193 match self.interpolation_method {
194 #[cfg(feature = "options")]
195 InterpolationMethod::Tetrahedral => {
196 self.transform_chunk(src, dst, Box::new(TetrahedralNeon::<GRID_SIZE> {}));
197 }
198 #[cfg(feature = "options")]
199 InterpolationMethod::Pyramid => {
200 self.transform_chunk(src, dst, Box::new(PyramidalNeon::<GRID_SIZE> {}));
201 }
202 #[cfg(feature = "options")]
203 InterpolationMethod::Prism => {
204 self.transform_chunk(src, dst, Box::new(PrismaticNeon::<GRID_SIZE> {}));
205 }
206 InterpolationMethod::Linear => {
207 self.transform_chunk(src, dst, Box::new(TrilinearNeon::<GRID_SIZE> {}));
208 }
209 }
210 }
211
212 Ok(())
213 }
214 }
215
216 pub(crate) struct NeonLut3x3Factory {}
217
218 impl Lut3x3Factory for NeonLut3x3Factory {
219 fn make_transform_3x3<
220 T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible + 'static + Send + Sync,
221 const SRC_LAYOUT: u8,
222 const DST_LAYOUT: u8,
223 const GRID_SIZE: usize,
224 const BIT_DEPTH: usize,
225 >(
226 lut: Vec<f32>,
227 options: TransformOptions,
228 color_space: DataColorSpace,
229 is_linear: bool,
230 ) -> Box<dyn TransformExecutor<T> + Send + Sync>
231 where
232 f32: AsPrimitive<T>,
233 u32: AsPrimitive<T>,
234 (): LutBarycentricReduction<T, u8>,
235 (): LutBarycentricReduction<T, u16>,
236 {
237 if options.prefer_fixed_point
238 && BIT_DEPTH < 16
239 && std::arch::is_aarch64_feature_detected!("rdm")
240 {
241 let q: f32 = if T::FINITE {
242 ((1i32 << BIT_DEPTH as i32) - 1) as f32
243 } else {
244 ((1i32 << 14i32) - 1) as f32
245 };
246 let lut = lut
247 .chunks_exact(3)
248 .map(|x| {
249 NeonAlignedI16x4([
250 (x[0] * q).round() as i16,
251 (x[1] * q).round() as i16,
252 (x[2] * q).round() as i16,
253 0,
254 ])
255 })
256 .collect::<Vec<_>>();
257 return match options.barycentric_weight_scale {
258 BarycentricWeightScale::Low => Box::new(TransformLut3x3NeonQ0_15::<
259 T,
260 u8,
261 SRC_LAYOUT,
262 DST_LAYOUT,
263 GRID_SIZE,
264 BIT_DEPTH,
265 256,
266 256,
267 > {
268 lut,
269 _phantom: PhantomData,
270 _phantom1: PhantomData,
271 interpolation_method: options.interpolation_method,
272 weights: BarycentricWeight::<i16>::create_ranged_256::<GRID_SIZE>(),
273 color_space,
274 is_linear,
275 }),
276 #[cfg(feature = "options")]
277 BarycentricWeightScale::High => Box::new(TransformLut3x3NeonQ0_15::<
278 T,
279 u16,
280 SRC_LAYOUT,
281 DST_LAYOUT,
282 GRID_SIZE,
283 BIT_DEPTH,
284 65536,
285 65536,
286 > {
287 lut,
288 _phantom: PhantomData,
289 _phantom1: PhantomData,
290 interpolation_method: options.interpolation_method,
291 weights: BarycentricWeight::<i16>::create_binned::<GRID_SIZE, 65536>(),
292 color_space,
293 is_linear,
294 }),
295 };
296 }
297 let lut = lut
298 .chunks_exact(3)
299 .map(|x| NeonAlignedF32([x[0], x[1], x[2], 0f32]))
300 .collect::<Vec<_>>();
301 match options.barycentric_weight_scale {
302 BarycentricWeightScale::Low => Box::new(TransformLut3x3Neon::<
303 T,
304 u8,
305 SRC_LAYOUT,
306 DST_LAYOUT,
307 GRID_SIZE,
308 BIT_DEPTH,
309 256,
310 256,
311 > {
312 lut,
313 _phantom: PhantomData,
314 _phantom1: PhantomData,
315 interpolation_method: options.interpolation_method,
316 weights: BarycentricWeight::<f32>::create_ranged_256::<GRID_SIZE>(),
317 color_space,
318 is_linear,
319 }),
320 #[cfg(feature = "options")]
321 BarycentricWeightScale::High => Box::new(TransformLut3x3Neon::<
322 T,
323 u16,
324 SRC_LAYOUT,
325 DST_LAYOUT,
326 GRID_SIZE,
327 BIT_DEPTH,
328 65536,
329 65536,
330 > {
331 lut,
332 _phantom: PhantomData,
333 _phantom1: PhantomData,
334 interpolation_method: options.interpolation_method,
335 weights: BarycentricWeight::<f32>::create_binned::<GRID_SIZE, 65536>(),
336 color_space,
337 is_linear,
338 }),
339 }
340 }
341 }