Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/neon/rgb_xyz_q2_13_opt.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::conversions::neon::rgb_xyz_q2_13::{split_by_twos, split_by_twos_mut};
30 use crate::conversions::rgbxyz_fixed::TransformMatrixShaperFpOptVec;
31 use crate::transform::PointeeSizeExpressible;
32 use crate::{CmsError, Layout, TransformExecutor};
33 use num_traits::AsPrimitive;
34 use std::arch::aarch64::*;
35
36 pub(crate) struct TransformShaperQ2_13NeonOpt<
37 T: Copy,
38 const SRC_LAYOUT: u8,
39 const DST_LAYOUT: u8,
40 const PRECISION: i32,
41 > {
42 pub(crate) profile: TransformMatrixShaperFpOptVec<i16, i16, T>,
43 pub(crate) bit_depth: usize,
44 pub(crate) gamma_lut: usize,
45 }
46
47 impl<
48 T: Copy + PointeeSizeExpressible + 'static + Default,
49 const SRC_LAYOUT: u8,
50 const DST_LAYOUT: u8,
51 const PRECISION: i32,
52 > TransformExecutor<T> for TransformShaperQ2_13NeonOpt<T, SRC_LAYOUT, DST_LAYOUT, PRECISION>
53 where
54 u32: AsPrimitive<T>,
55 {
56 fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
57 let src_cn = Layout::from(SRC_LAYOUT);
58 let dst_cn = Layout::from(DST_LAYOUT);
59 let src_channels = src_cn.channels();
60 let dst_channels = dst_cn.channels();
61
62 if src.len() / src_channels != dst.len() / dst_channels {
63 return Err(CmsError::LaneSizeMismatch);
64 }
65 if src.len() % src_channels != 0 {
66 return Err(CmsError::LaneMultipleOfChannels);
67 }
68 if dst.len() % dst_channels != 0 {
69 return Err(CmsError::LaneMultipleOfChannels);
70 }
71
72 let t = self.profile.adaptation_matrix.transpose();
73 let max_colors: T = ((1 << self.bit_depth) - 1).as_();
74
75 // safety precondition for linearization table
76 if T::FINITE {
77 let cap = (1 << self.bit_depth) - 1;
78 assert!(self.profile.linear.len() >= cap);
79 } else {
80 assert!(self.profile.linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
81 }
82
83 let lut_lin = &self.profile.linear;
84
85 let (src_chunks, src_remainder) = split_by_twos(src, src_channels);
86 let (dst_chunks, dst_remainder) = split_by_twos_mut(dst, dst_channels);
87
88 unsafe {
89 let m0 = vld1_s16([t.v[0][0], t.v[0][1], t.v[0][2], 0].as_ptr());
90 let m1 = vld1_s16([t.v[1][0], t.v[1][1], t.v[1][2], 0].as_ptr());
91 let m2 = vld1_s16([t.v[2][0], t.v[2][1], t.v[2][2], 0].as_ptr());
92
93 let v_max_value = vdup_n_u16((self.gamma_lut - 1) as u16);
94
95 let rnd = vdupq_n_s32(1 << (PRECISION - 1));
96
97 if !src_chunks.is_empty() {
98 let (src0, src1) = src_chunks.split_at(src_chunks.len() / 2);
99 let (dst0, dst1) = dst_chunks.split_at_mut(dst_chunks.len() / 2);
100 let mut src_iter0 = src0.chunks_exact(src_channels * 2);
101 let mut src_iter1 = src1.chunks_exact(src_channels * 2);
102
103 let (mut r0, mut g0, mut b0, mut a0);
104 let (mut r1, mut g1, mut b1, mut a1);
105 let (mut r2, mut g2, mut b2, mut a2);
106 let (mut r3, mut g3, mut b3, mut a3);
107
108 if let (Some(src0), Some(src1)) = (src_iter0.next(), src_iter1.next()) {
109 let r0p = lut_lin.get_unchecked(src0[src_cn.r_i()]._as_usize());
110 let g0p = lut_lin.get_unchecked(src0[src_cn.g_i()]._as_usize());
111 let b0p = lut_lin.get_unchecked(src0[src_cn.b_i()]._as_usize());
112
113 let r1p = lut_lin.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize());
114 let g1p = lut_lin.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize());
115 let b1p = lut_lin.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize());
116
117 let r2p = lut_lin.get_unchecked(src1[src_cn.r_i()]._as_usize());
118 let g2p = lut_lin.get_unchecked(src1[src_cn.g_i()]._as_usize());
119 let b2p = lut_lin.get_unchecked(src1[src_cn.b_i()]._as_usize());
120
121 let r3p = lut_lin.get_unchecked(src1[src_cn.r_i() + src_channels]._as_usize());
122 let g3p = lut_lin.get_unchecked(src1[src_cn.g_i() + src_channels]._as_usize());
123 let b3p = lut_lin.get_unchecked(src1[src_cn.b_i() + src_channels]._as_usize());
124
125 r0 = vld1_dup_s16(r0p);
126 g0 = vld1_dup_s16(g0p);
127 b0 = vld1_dup_s16(b0p);
128
129 r1 = vld1_dup_s16(r1p);
130 g1 = vld1_dup_s16(g1p);
131 b1 = vld1_dup_s16(b1p);
132
133 r2 = vld1_dup_s16(r2p);
134 g2 = vld1_dup_s16(g2p);
135 b2 = vld1_dup_s16(b2p);
136
137 r3 = vld1_dup_s16(r3p);
138 g3 = vld1_dup_s16(g3p);
139 b3 = vld1_dup_s16(b3p);
140
141 a0 = if src_channels == 4 {
142 src0[src_cn.a_i()]
143 } else {
144 max_colors
145 };
146
147 a1 = if src_channels == 4 {
148 src0[src_cn.a_i() + src_channels]
149 } else {
150 max_colors
151 };
152
153 a2 = if src_channels == 4 {
154 src1[src_cn.a_i()]
155 } else {
156 max_colors
157 };
158
159 a3 = if src_channels == 4 {
160 src1[src_cn.a_i() + src_channels]
161 } else {
162 max_colors
163 };
164 } else {
165 r0 = vdup_n_s16(0);
166 g0 = vdup_n_s16(0);
167 b0 = vdup_n_s16(0);
168 r1 = vdup_n_s16(0);
169 g1 = vdup_n_s16(0);
170 b1 = vdup_n_s16(0);
171 r2 = vdup_n_s16(0);
172 g2 = vdup_n_s16(0);
173 b2 = vdup_n_s16(0);
174 r3 = vdup_n_s16(0);
175 g3 = vdup_n_s16(0);
176 b3 = vdup_n_s16(0);
177 a0 = max_colors;
178 a1 = max_colors;
179 a2 = max_colors;
180 a3 = max_colors;
181 }
182
183 for (((src0, src1), dst0), dst1) in src_iter0
184 .zip(src_iter1)
185 .zip(dst0.chunks_exact_mut(dst_channels * 2))
186 .zip(dst1.chunks_exact_mut(dst_channels * 2))
187 {
188 let v0_0 = vmlal_s16(rnd, r0, m0);
189 let v0_1 = vmlal_s16(rnd, r1, m0);
190 let v0_2 = vmlal_s16(rnd, r2, m0);
191 let v0_3 = vmlal_s16(rnd, r3, m0);
192
193 let v1_0 = vmlal_s16(v0_0, g0, m1);
194 let v1_1 = vmlal_s16(v0_1, g1, m1);
195 let v1_2 = vmlal_s16(v0_2, g2, m1);
196 let v1_3 = vmlal_s16(v0_3, g3, m1);
197
198 let vr0 = vmlal_s16(v1_0, b0, m2);
199 let vr1 = vmlal_s16(v1_1, b1, m2);
200 let vr2 = vmlal_s16(v1_2, b2, m2);
201 let vr3 = vmlal_s16(v1_3, b3, m2);
202
203 let mut vr0 = vqshrun_n_s32::<PRECISION>(vr0);
204 let mut vr1 = vqshrun_n_s32::<PRECISION>(vr1);
205 let mut vr2 = vqshrun_n_s32::<PRECISION>(vr2);
206 let mut vr3 = vqshrun_n_s32::<PRECISION>(vr3);
207
208 vr0 = vmin_u16(vr0, v_max_value);
209 vr1 = vmin_u16(vr1, v_max_value);
210 vr2 = vmin_u16(vr2, v_max_value);
211 vr3 = vmin_u16(vr3, v_max_value);
212
213 let r0p = lut_lin.get_unchecked(src0[src_cn.r_i()]._as_usize());
214 let g0p = lut_lin.get_unchecked(src0[src_cn.g_i()]._as_usize());
215 let b0p = lut_lin.get_unchecked(src0[src_cn.b_i()]._as_usize());
216
217 let r1p = lut_lin.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize());
218 let g1p = lut_lin.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize());
219 let b1p = lut_lin.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize());
220
221 let r2p = lut_lin.get_unchecked(src1[src_cn.r_i()]._as_usize());
222 let g2p = lut_lin.get_unchecked(src1[src_cn.g_i()]._as_usize());
223 let b2p = lut_lin.get_unchecked(src1[src_cn.b_i()]._as_usize());
224
225 let r3p = lut_lin.get_unchecked(src1[src_cn.r_i() + src_channels]._as_usize());
226 let g3p = lut_lin.get_unchecked(src1[src_cn.g_i() + src_channels]._as_usize());
227 let b3p = lut_lin.get_unchecked(src1[src_cn.b_i() + src_channels]._as_usize());
228
229 r0 = vld1_dup_s16(r0p);
230 g0 = vld1_dup_s16(g0p);
231 b0 = vld1_dup_s16(b0p);
232
233 r1 = vld1_dup_s16(r1p);
234 g1 = vld1_dup_s16(g1p);
235 b1 = vld1_dup_s16(b1p);
236
237 r2 = vld1_dup_s16(r2p);
238 g2 = vld1_dup_s16(g2p);
239 b2 = vld1_dup_s16(b2p);
240
241 r3 = vld1_dup_s16(r3p);
242 g3 = vld1_dup_s16(g3p);
243 b3 = vld1_dup_s16(b3p);
244
245 dst0[dst_cn.r_i()] = self.profile.gamma[vget_lane_u16::<0>(vr0) as usize];
246 dst0[dst_cn.g_i()] = self.profile.gamma[vget_lane_u16::<1>(vr0) as usize];
247 dst0[dst_cn.b_i()] = self.profile.gamma[vget_lane_u16::<2>(vr0) as usize];
248 if dst_channels == 4 {
249 dst0[dst_cn.a_i()] = a0;
250 }
251
252 dst0[dst_cn.r_i() + dst_channels] =
253 self.profile.gamma[vget_lane_u16::<0>(vr1) as usize];
254 dst0[dst_cn.g_i() + dst_channels] =
255 self.profile.gamma[vget_lane_u16::<1>(vr1) as usize];
256 dst0[dst_cn.b_i() + dst_channels] =
257 self.profile.gamma[vget_lane_u16::<2>(vr0) as usize];
258 if dst_channels == 4 {
259 dst0[dst_cn.a_i() + dst_channels] = a1;
260 }
261
262 dst1[dst_cn.r_i()] = self.profile.gamma[vget_lane_u16::<0>(vr2) as usize];
263 dst1[dst_cn.g_i()] = self.profile.gamma[vget_lane_u16::<1>(vr2) as usize];
264 dst1[dst_cn.b_i()] = self.profile.gamma[vget_lane_u16::<2>(vr2) as usize];
265 if dst_channels == 4 {
266 dst1[dst_cn.a_i()] = a2;
267 }
268
269 dst1[dst_cn.r_i() + dst_channels] =
270 self.profile.gamma[vget_lane_u16::<0>(vr3) as usize];
271 dst1[dst_cn.g_i() + dst_channels] =
272 self.profile.gamma[vget_lane_u16::<1>(vr3) as usize];
273 dst1[dst_cn.b_i() + dst_channels] =
274 self.profile.gamma[vget_lane_u16::<2>(vr3) as usize];
275 if dst_channels == 4 {
276 dst1[dst_cn.a_i() + dst_channels] = a3;
277 }
278
279 a0 = if src_channels == 4 {
280 src0[src_cn.a_i()]
281 } else {
282 max_colors
283 };
284
285 a1 = if src_channels == 4 {
286 src0[src_cn.a_i() + src_channels]
287 } else {
288 max_colors
289 };
290
291 a2 = if src_channels == 4 {
292 src1[src_cn.a_i()]
293 } else {
294 max_colors
295 };
296
297 a3 = if src_channels == 4 {
298 src1[src_cn.a_i() + src_channels]
299 } else {
300 max_colors
301 };
302 }
303
304 if let (Some(dst0), Some(dst1)) = (
305 dst0.chunks_exact_mut(dst_channels * 2).last(),
306 dst1.chunks_exact_mut(dst_channels * 2).last(),
307 ) {
308 let v0_0 = vmlal_s16(rnd, r0, m0);
309 let v0_1 = vmlal_s16(rnd, r1, m0);
310 let v0_2 = vmlal_s16(rnd, r2, m0);
311 let v0_3 = vmlal_s16(rnd, r3, m0);
312
313 let v1_0 = vmlal_s16(v0_0, g0, m1);
314 let v1_1 = vmlal_s16(v0_1, g1, m1);
315 let v1_2 = vmlal_s16(v0_2, g2, m1);
316 let v1_3 = vmlal_s16(v0_3, g3, m1);
317
318 let vr0 = vmlal_s16(v1_0, b0, m2);
319 let vr1 = vmlal_s16(v1_1, b1, m2);
320 let vr2 = vmlal_s16(v1_2, b2, m2);
321 let vr3 = vmlal_s16(v1_3, b3, m2);
322
323 let mut vr0 = vqshrun_n_s32::<PRECISION>(vr0);
324 let mut vr1 = vqshrun_n_s32::<PRECISION>(vr1);
325 let mut vr2 = vqshrun_n_s32::<PRECISION>(vr2);
326 let mut vr3 = vqshrun_n_s32::<PRECISION>(vr3);
327
328 vr0 = vmin_u16(vr0, v_max_value);
329 vr1 = vmin_u16(vr1, v_max_value);
330 vr2 = vmin_u16(vr2, v_max_value);
331 vr3 = vmin_u16(vr3, v_max_value);
332
333 dst0[dst_cn.r_i()] = self.profile.gamma[vget_lane_u16::<0>(vr0) as usize];
334 dst0[dst_cn.g_i()] = self.profile.gamma[vget_lane_u16::<1>(vr0) as usize];
335 dst0[dst_cn.b_i()] = self.profile.gamma[vget_lane_u16::<2>(vr0) as usize];
336 if dst_channels == 4 {
337 dst0[dst_cn.a_i()] = a0;
338 }
339
340 dst0[dst_cn.r_i() + dst_channels] =
341 self.profile.gamma[vget_lane_u16::<0>(vr1) as usize];
342 dst0[dst_cn.g_i() + dst_channels] =
343 self.profile.gamma[vget_lane_u16::<1>(vr1) as usize];
344 dst0[dst_cn.b_i() + dst_channels] =
345 self.profile.gamma[vget_lane_u16::<2>(vr0) as usize];
346 if dst_channels == 4 {
347 dst0[dst_cn.a_i() + dst_channels] = a1;
348 }
349
350 dst1[dst_cn.r_i()] = self.profile.gamma[vget_lane_u16::<0>(vr2) as usize];
351 dst1[dst_cn.g_i()] = self.profile.gamma[vget_lane_u16::<1>(vr2) as usize];
352 dst1[dst_cn.b_i()] = self.profile.gamma[vget_lane_u16::<2>(vr2) as usize];
353 if dst_channels == 4 {
354 dst1[dst_cn.a_i()] = a2;
355 }
356
357 dst1[dst_cn.r_i() + dst_channels] =
358 self.profile.gamma[vget_lane_u16::<0>(vr3) as usize];
359 dst1[dst_cn.g_i() + dst_channels] =
360 self.profile.gamma[vget_lane_u16::<1>(vr3) as usize];
361 dst1[dst_cn.b_i() + dst_channels] =
362 self.profile.gamma[vget_lane_u16::<2>(vr3) as usize];
363 if dst_channels == 4 {
364 dst1[dst_cn.a_i() + dst_channels] = a3;
365 }
366 }
367 }
368
369 for (src, dst) in src_remainder
370 .chunks_exact(src_channels)
371 .zip(dst_remainder.chunks_exact_mut(dst_channels))
372 {
373 let rp = lut_lin.get_unchecked(src[src_cn.r_i()]._as_usize());
374 let gp = lut_lin.get_unchecked(src[src_cn.g_i()]._as_usize());
375 let bp = lut_lin.get_unchecked(src[src_cn.b_i()]._as_usize());
376 let r = vld1_dup_s16(rp);
377 let g = vld1_dup_s16(gp);
378 let b = vld1_dup_s16(bp);
379 let a = if src_channels == 4 {
380 src[src_cn.a_i()]
381 } else {
382 max_colors
383 };
384
385 let v0 = vmlal_s16(rnd, r, m0);
386 let v1 = vmlal_s16(v0, g, m1);
387 let v = vmlal_s16(v1, b, m2);
388
389 let mut vr0 = vqshrun_n_s32::<PRECISION>(v);
390 vr0 = vmin_u16(vr0, v_max_value);
391
392 dst[dst_cn.r_i()] = self.profile.gamma[vget_lane_u16::<0>(vr0) as usize];
393 dst[dst_cn.g_i()] = self.profile.gamma[vget_lane_u16::<1>(vr0) as usize];
394 dst[dst_cn.b_i()] = self.profile.gamma[vget_lane_u16::<2>(vr0) as usize];
395 if dst_channels == 4 {
396 dst[dst_cn.a_i()] = a;
397 }
398 }
399 }
400
401 Ok(())
402 }
403 }