Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/neon/lut4_to_3.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::conversions::LutBarycentricReduction;
30 use crate::conversions::interpolator::BarycentricWeight;
31 use crate::conversions::lut_transforms::Lut4x3Factory;
32 use crate::conversions::neon::interpolator::*;
33 use crate::conversions::neon::interpolator_q0_15::NeonAlignedI16x4;
34 use crate::conversions::neon::lut4_to_3_q0_15::TransformLut4To3NeonQ0_15;
35 use crate::conversions::neon::rgb_xyz::NeonAlignedF32;
36 use crate::transform::PointeeSizeExpressible;
37 use crate::{
38 BarycentricWeightScale, CmsError, DataColorSpace, InterpolationMethod, Layout,
39 TransformExecutor, TransformOptions,
40 };
41 use num_traits::AsPrimitive;
42 use std::arch::aarch64::*;
43 use std::marker::PhantomData;
44
45 struct TransformLut4To3Neon<
46 T,
47 U,
48 const LAYOUT: u8,
49 const GRID_SIZE: usize,
50 const BIT_DEPTH: usize,
51 const BINS: usize,
52 const BARYCENTRIC_BINS: usize,
53 > {
54 lut: Vec<NeonAlignedF32>,
55 _phantom: PhantomData<T>,
56 _phantom1: PhantomData<U>,
57 interpolation_method: InterpolationMethod,
58 weights: Box<[BarycentricWeight<f32>; BINS]>,
59 color_space: DataColorSpace,
60 is_linear: bool,
61 }
62
63 impl<
64 T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible,
65 U: AsPrimitive<usize>,
66 const LAYOUT: u8,
67 const GRID_SIZE: usize,
68 const BIT_DEPTH: usize,
69 const BINS: usize,
70 const BARYCENTRIC_BINS: usize,
71 > TransformLut4To3Neon<T, U, LAYOUT, GRID_SIZE, BIT_DEPTH, BINS, BARYCENTRIC_BINS>
72 where
73 f32: AsPrimitive<T>,
74 u32: AsPrimitive<T>,
75 (): LutBarycentricReduction<T, U>,
76 {
77 #[allow(unused_unsafe)]
78 #[inline(never)]
79 fn transform_chunk(
80 &self,
81 src: &[T],
82 dst: &mut [T],
83 interpolator: Box<dyn NeonMdInterpolationDouble + Send + Sync>,
84 ) {
85 let cn = Layout::from(LAYOUT);
86 let channels = cn.channels();
87 let grid_size = GRID_SIZE as i32;
88 let grid_size3 = grid_size * grid_size * grid_size;
89
90 let value_scale = unsafe { vdupq_n_f32(((1 << BIT_DEPTH) - 1) as f32) };
91 let max_value = ((1 << BIT_DEPTH) - 1u32).as_();
92
93 for (src, dst) in src.chunks_exact(4).zip(dst.chunks_exact_mut(channels)) {
94 let c = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
95 src[0],
96 );
97 let m = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
98 src[1],
99 );
100 let y = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
101 src[2],
102 );
103 let k = <() as LutBarycentricReduction<T, U>>::reduce::<BIT_DEPTH, BARYCENTRIC_BINS>(
104 src[3],
105 );
106
107 let k_weights = self.weights[k.as_()];
108
109 let w: i32 = k_weights.x;
110 let w_n: i32 = k_weights.x_n;
111 let t: f32 = k_weights.w;
112
113 let table1 = &self.lut[(w * grid_size3) as usize..];
114 let table2 = &self.lut[(w_n * grid_size3) as usize..];
115
116 let (a0, b0) = interpolator.inter3_neon(
117 table1,
118 table2,
119 c.as_(),
120 m.as_(),
121 y.as_(),
122 self.weights.as_slice(),
123 );
124 let (a0, b0) = (a0.v, b0.v);
125
126 if T::FINITE {
127 unsafe {
128 let t0 = vdupq_n_f32(t);
129 let hp = vfmsq_f32(a0, a0, t0);
130 let mut v = vfmaq_f32(hp, b0, t0);
131 v = vmulq_f32(v, value_scale);
132 v = vminq_f32(v, value_scale);
133
134 let jvx = vcvtaq_u32_f32(v);
135
136 dst[cn.r_i()] = vgetq_lane_u32::<0>(jvx).as_();
137 dst[cn.g_i()] = vgetq_lane_u32::<1>(jvx).as_();
138 dst[cn.b_i()] = vgetq_lane_u32::<2>(jvx).as_();
139 }
140 } else {
141 unsafe {
142 let t0 = vdupq_n_f32(t);
143 let hp = vfmsq_f32(a0, a0, t0);
144 let v = vfmaq_f32(hp, b0, t0);
145
146 dst[cn.r_i()] = vgetq_lane_f32::<0>(v).as_();
147 dst[cn.g_i()] = vgetq_lane_f32::<1>(v).as_();
148 dst[cn.b_i()] = vgetq_lane_f32::<2>(v).as_();
149 }
150 }
151 if channels == 4 {
152 dst[cn.a_i()] = max_value;
153 }
154 }
155 }
156 }
157
158 impl<
159 T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible,
160 U: AsPrimitive<usize>,
161 const LAYOUT: u8,
162 const GRID_SIZE: usize,
163 const BIT_DEPTH: usize,
164 const BINS: usize,
165 const BARYCENTRIC_BINS: usize,
166 > TransformExecutor<T>
167 for TransformLut4To3Neon<T, U, LAYOUT, GRID_SIZE, BIT_DEPTH, BINS, BARYCENTRIC_BINS>
168 where
169 f32: AsPrimitive<T>,
170 u32: AsPrimitive<T>,
171 (): LutBarycentricReduction<T, U>,
172 {
173 fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
174 let cn = Layout::from(LAYOUT);
175 let channels = cn.channels();
176 if src.len() % 4 != 0 {
177 return Err(CmsError::LaneMultipleOfChannels);
178 }
179 if dst.len() % channels != 0 {
180 return Err(CmsError::LaneMultipleOfChannels);
181 }
182 let src_chunks = src.len() / 4;
183 let dst_chunks = dst.len() / channels;
184 if src_chunks != dst_chunks {
185 return Err(CmsError::LaneSizeMismatch);
186 }
187
188 if self.color_space == DataColorSpace::Lab
189 || (self.is_linear && self.color_space == DataColorSpace::Rgb)
190 || self.color_space == DataColorSpace::Xyz
191 {
192 self.transform_chunk(src, dst, Box::new(TrilinearNeonDouble::<GRID_SIZE> {}));
193 } else {
194 match self.interpolation_method {
195 #[cfg(feature = "options")]
196 InterpolationMethod::Tetrahedral => {
197 self.transform_chunk(src, dst, Box::new(TetrahedralNeonDouble::<GRID_SIZE> {}));
198 }
199 #[cfg(feature = "options")]
200 InterpolationMethod::Pyramid => {
201 self.transform_chunk(src, dst, Box::new(PyramidalNeonDouble::<GRID_SIZE> {}));
202 }
203 #[cfg(feature = "options")]
204 InterpolationMethod::Prism => {
205 self.transform_chunk(src, dst, Box::new(PrismaticNeonDouble::<GRID_SIZE> {}));
206 }
207 InterpolationMethod::Linear => {
208 self.transform_chunk(src, dst, Box::new(TrilinearNeonDouble::<GRID_SIZE> {}));
209 }
210 }
211 }
212
213 Ok(())
214 }
215 }
216
217 pub(crate) struct NeonLut4x3Factory {}
218
219 impl Lut4x3Factory for NeonLut4x3Factory {
220 fn make_transform_4x3<
221 T: Copy + AsPrimitive<f32> + Default + PointeeSizeExpressible + 'static + Send + Sync,
222 const LAYOUT: u8,
223 const GRID_SIZE: usize,
224 const BIT_DEPTH: usize,
225 >(
226 lut: Vec<f32>,
227 options: TransformOptions,
228 color_space: DataColorSpace,
229 is_linear: bool,
230 ) -> Box<dyn TransformExecutor<T> + Sync + Send>
231 where
232 f32: AsPrimitive<T>,
233 u32: AsPrimitive<T>,
234 (): LutBarycentricReduction<T, u8>,
235 (): LutBarycentricReduction<T, u16>,
236 {
237 if options.prefer_fixed_point
238 && BIT_DEPTH < 16
239 && std::arch::is_aarch64_feature_detected!("rdm")
240 {
241 let q: f32 = if T::FINITE {
242 ((1i32 << BIT_DEPTH as i32) - 1) as f32
243 } else {
244 ((1i32 << 14i32) - 1) as f32
245 };
246 let lut = lut
247 .chunks_exact(3)
248 .map(|x| {
249 NeonAlignedI16x4([
250 (x[0] * q).round() as i16,
251 (x[1] * q).round() as i16,
252 (x[2] * q).round() as i16,
253 0,
254 ])
255 })
256 .collect::<Vec<_>>();
257 return match options.barycentric_weight_scale {
258 BarycentricWeightScale::Low => Box::new(TransformLut4To3NeonQ0_15::<
259 T,
260 u8,
261 LAYOUT,
262 GRID_SIZE,
263 BIT_DEPTH,
264 256,
265 256,
266 > {
267 lut,
268 _phantom: PhantomData,
269 _phantom1: PhantomData,
270 interpolation_method: options.interpolation_method,
271 weights: BarycentricWeight::<i16>::create_ranged_256::<GRID_SIZE>(),
272 color_space,
273 is_linear,
274 }),
275 #[cfg(feature = "options")]
276 BarycentricWeightScale::High => Box::new(TransformLut4To3NeonQ0_15::<
277 T,
278 u16,
279 LAYOUT,
280 GRID_SIZE,
281 BIT_DEPTH,
282 65536,
283 65536,
284 > {
285 lut,
286 _phantom: PhantomData,
287 _phantom1: PhantomData,
288 interpolation_method: options.interpolation_method,
289 weights: BarycentricWeight::<i16>::create_binned::<GRID_SIZE, 65536>(),
290 color_space,
291 is_linear,
292 }),
293 };
294 }
295 let lut = lut
296 .chunks_exact(3)
297 .map(|x| NeonAlignedF32([x[0], x[1], x[2], 0f32]))
298 .collect::<Vec<_>>();
299 match options.barycentric_weight_scale {
300 BarycentricWeightScale::Low => {
301 Box::new(
302 TransformLut4To3Neon::<T, u8, LAYOUT, GRID_SIZE, BIT_DEPTH, 256, 256> {
303 lut,
304 _phantom: PhantomData,
305 _phantom1: PhantomData,
306 interpolation_method: options.interpolation_method,
307 weights: BarycentricWeight::<f32>::create_ranged_256::<GRID_SIZE>(),
308 color_space,
309 is_linear,
310 },
311 )
312 }
313 #[cfg(feature = "options")]
314 BarycentricWeightScale::High => {
315 Box::new(
316 TransformLut4To3Neon::<T, u16, LAYOUT, GRID_SIZE, BIT_DEPTH, 65536, 65536> {
317 lut,
318 _phantom: PhantomData,
319 _phantom1: PhantomData,
320 interpolation_method: options.interpolation_method,
321 weights: BarycentricWeight::<f32>::create_binned::<GRID_SIZE, 65536>(),
322 color_space,
323 is_linear,
324 },
325 )
326 }
327 }
328 }
329 }