Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/neon/rgb_xyz_q2_13.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::conversions::rgbxyz_fixed::TransformMatrixShaperFp;
30 use crate::transform::PointeeSizeExpressible;
31 use crate::{CmsError, Layout, TransformExecutor};
32 use num_traits::AsPrimitive;
33 use std::arch::aarch64::*;
34
35 #[allow(dead_code)]
36 #[inline]
37 pub(crate) fn split_by_twos<T: Copy>(data: &[T], channels: usize) -> (&[T], &[T]) {
38 let len = data.len() / (channels * 4);
39 let split_point = len * 4;
40 data.split_at(split_point * channels)
41 }
42
43 #[allow(dead_code)]
44 #[inline]
45 pub(crate) fn split_by_twos_mut<T: Copy>(data: &mut [T], channels: usize) -> (&mut [T], &mut [T]) {
46 let len = data.len() / (channels * 4);
47 let split_point = len * 4;
48 data.split_at_mut(split_point * channels)
49 }
50
51 pub(crate) struct TransformShaperQ2_13Neon<
52 T: Copy,
53 const SRC_LAYOUT: u8,
54 const DST_LAYOUT: u8,
55 const PRECISION: i32,
56 > {
57 pub(crate) profile: TransformMatrixShaperFp<i16, T>,
58 pub(crate) bit_depth: usize,
59 pub(crate) gamma_lut: usize,
60 }
61
62 impl<
63 T: Copy + PointeeSizeExpressible + 'static + Default,
64 const SRC_LAYOUT: u8,
65 const DST_LAYOUT: u8,
66 const PRECISION: i32,
67 > TransformExecutor<T> for TransformShaperQ2_13Neon<T, SRC_LAYOUT, DST_LAYOUT, PRECISION>
68 where
69 u32: AsPrimitive<T>,
70 {
71 fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
72 let src_cn = Layout::from(SRC_LAYOUT);
73 let dst_cn = Layout::from(DST_LAYOUT);
74 let src_channels = src_cn.channels();
75 let dst_channels = dst_cn.channels();
76
77 if src.len() / src_channels != dst.len() / dst_channels {
78 return Err(CmsError::LaneSizeMismatch);
79 }
80 if src.len() % src_channels != 0 {
81 return Err(CmsError::LaneMultipleOfChannels);
82 }
83 if dst.len() % dst_channels != 0 {
84 return Err(CmsError::LaneMultipleOfChannels);
85 }
86
87 let t = self.profile.adaptation_matrix.transpose();
88 let max_colors: T = ((1 << self.bit_depth) - 1).as_();
89
90 // safety precondition for linearization table
91 if T::FINITE {
92 let cap = (1 << self.bit_depth) - 1;
93 assert!(self.profile.r_linear.len() >= cap);
94 assert!(self.profile.g_linear.len() >= cap);
95 assert!(self.profile.b_linear.len() >= cap);
96 } else {
97 assert!(self.profile.r_linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
98 assert!(self.profile.g_linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
99 assert!(self.profile.b_linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
100 }
101
102 let r_lin = &self.profile.r_linear;
103 let g_lin = &self.profile.g_linear;
104 let b_lin = &self.profile.b_linear;
105
106 let (src_chunks, src_remainder) = split_by_twos(src, src_channels);
107 let (dst_chunks, dst_remainder) = split_by_twos_mut(dst, dst_channels);
108
109 unsafe {
110 let m0 = vld1_s16([t.v[0][0], t.v[0][1], t.v[0][2], 0].as_ptr());
111 let m1 = vld1_s16([t.v[1][0], t.v[1][1], t.v[1][2], 0].as_ptr());
112 let m2 = vld1_s16([t.v[2][0], t.v[2][1], t.v[2][2], 0].as_ptr());
113
114 let v_max_value = vdup_n_u16((self.gamma_lut - 1) as u16);
115
116 let rnd = vdupq_n_s32(1 << (PRECISION - 1));
117
118 if !src_chunks.is_empty() {
119 let (src0, src1) = src_chunks.split_at(src_chunks.len() / 2);
120 let (dst0, dst1) = dst_chunks.split_at_mut(dst_chunks.len() / 2);
121 let mut src_iter0 = src0.chunks_exact(src_channels * 2);
122 let mut src_iter1 = src1.chunks_exact(src_channels * 2);
123
124 let (mut r0, mut g0, mut b0, mut a0);
125 let (mut r1, mut g1, mut b1, mut a1);
126 let (mut r2, mut g2, mut b2, mut a2);
127 let (mut r3, mut g3, mut b3, mut a3);
128
129 if let (Some(src0), Some(src1)) = (src_iter0.next(), src_iter1.next()) {
130 let r0p = r_lin.get_unchecked(src0[src_cn.r_i()]._as_usize());
131 let g0p = g_lin.get_unchecked(src0[src_cn.g_i()]._as_usize());
132 let b0p = b_lin.get_unchecked(src0[src_cn.b_i()]._as_usize());
133
134 let r1p = r_lin.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize());
135 let g1p = g_lin.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize());
136 let b1p = b_lin.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize());
137
138 let r2p = r_lin.get_unchecked(src1[src_cn.r_i()]._as_usize());
139 let g2p = g_lin.get_unchecked(src1[src_cn.g_i()]._as_usize());
140 let b2p = b_lin.get_unchecked(src1[src_cn.b_i()]._as_usize());
141
142 let r3p = r_lin.get_unchecked(src1[src_cn.r_i() + src_channels]._as_usize());
143 let g3p = g_lin.get_unchecked(src1[src_cn.g_i() + src_channels]._as_usize());
144 let b3p = b_lin.get_unchecked(src1[src_cn.b_i() + src_channels]._as_usize());
145
146 r0 = vld1_dup_s16(r0p);
147 g0 = vld1_dup_s16(g0p);
148 b0 = vld1_dup_s16(b0p);
149
150 r1 = vld1_dup_s16(r1p);
151 g1 = vld1_dup_s16(g1p);
152 b1 = vld1_dup_s16(b1p);
153
154 r2 = vld1_dup_s16(r2p);
155 g2 = vld1_dup_s16(g2p);
156 b2 = vld1_dup_s16(b2p);
157
158 r3 = vld1_dup_s16(r3p);
159 g3 = vld1_dup_s16(g3p);
160 b3 = vld1_dup_s16(b3p);
161
162 a0 = if src_channels == 4 {
163 src0[src_cn.a_i()]
164 } else {
165 max_colors
166 };
167
168 a1 = if src_channels == 4 {
169 src0[src_cn.a_i() + src_channels]
170 } else {
171 max_colors
172 };
173
174 a2 = if src_channels == 4 {
175 src1[src_cn.a_i()]
176 } else {
177 max_colors
178 };
179
180 a3 = if src_channels == 4 {
181 src1[src_cn.a_i() + src_channels]
182 } else {
183 max_colors
184 };
185 } else {
186 r0 = vdup_n_s16(0);
187 g0 = vdup_n_s16(0);
188 b0 = vdup_n_s16(0);
189 r1 = vdup_n_s16(0);
190 g1 = vdup_n_s16(0);
191 b1 = vdup_n_s16(0);
192 r2 = vdup_n_s16(0);
193 g2 = vdup_n_s16(0);
194 b2 = vdup_n_s16(0);
195 r3 = vdup_n_s16(0);
196 g3 = vdup_n_s16(0);
197 b3 = vdup_n_s16(0);
198 a0 = max_colors;
199 a1 = max_colors;
200 a2 = max_colors;
201 a3 = max_colors;
202 }
203
204 for (((src0, src1), dst0), dst1) in src_iter0
205 .zip(src_iter1)
206 .zip(dst0.chunks_exact_mut(dst_channels * 2))
207 .zip(dst1.chunks_exact_mut(dst_channels * 2))
208 {
209 let v0_0 = vmlal_s16(rnd, r0, m0);
210 let v0_1 = vmlal_s16(rnd, r1, m0);
211 let v0_2 = vmlal_s16(rnd, r2, m0);
212 let v0_3 = vmlal_s16(rnd, r3, m0);
213
214 let v1_0 = vmlal_s16(v0_0, g0, m1);
215 let v1_1 = vmlal_s16(v0_1, g1, m1);
216 let v1_2 = vmlal_s16(v0_2, g2, m1);
217 let v1_3 = vmlal_s16(v0_3, g3, m1);
218
219 let vr0 = vmlal_s16(v1_0, b0, m2);
220 let vr1 = vmlal_s16(v1_1, b1, m2);
221 let vr2 = vmlal_s16(v1_2, b2, m2);
222 let vr3 = vmlal_s16(v1_3, b3, m2);
223
224 let mut vr0 = vqshrun_n_s32::<PRECISION>(vr0);
225 let mut vr1 = vqshrun_n_s32::<PRECISION>(vr1);
226 let mut vr2 = vqshrun_n_s32::<PRECISION>(vr2);
227 let mut vr3 = vqshrun_n_s32::<PRECISION>(vr3);
228
229 vr0 = vmin_u16(vr0, v_max_value);
230 vr1 = vmin_u16(vr1, v_max_value);
231 vr2 = vmin_u16(vr2, v_max_value);
232 vr3 = vmin_u16(vr3, v_max_value);
233
234 let r0p = r_lin.get_unchecked(src0[src_cn.r_i()]._as_usize());
235 let g0p = g_lin.get_unchecked(src0[src_cn.g_i()]._as_usize());
236 let b0p = b_lin.get_unchecked(src0[src_cn.b_i()]._as_usize());
237
238 let r1p = r_lin.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize());
239 let g1p = g_lin.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize());
240 let b1p = b_lin.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize());
241
242 let r2p = r_lin.get_unchecked(src1[src_cn.r_i()]._as_usize());
243 let g2p = g_lin.get_unchecked(src1[src_cn.g_i()]._as_usize());
244 let b2p = b_lin.get_unchecked(src1[src_cn.b_i()]._as_usize());
245
246 let r3p = r_lin.get_unchecked(src1[src_cn.r_i() + src_channels]._as_usize());
247 let g3p = g_lin.get_unchecked(src1[src_cn.g_i() + src_channels]._as_usize());
248 let b3p = b_lin.get_unchecked(src1[src_cn.b_i() + src_channels]._as_usize());
249
250 r0 = vld1_dup_s16(r0p);
251 g0 = vld1_dup_s16(g0p);
252 b0 = vld1_dup_s16(b0p);
253
254 r1 = vld1_dup_s16(r1p);
255 g1 = vld1_dup_s16(g1p);
256 b1 = vld1_dup_s16(b1p);
257
258 r2 = vld1_dup_s16(r2p);
259 g2 = vld1_dup_s16(g2p);
260 b2 = vld1_dup_s16(b2p);
261
262 r3 = vld1_dup_s16(r3p);
263 g3 = vld1_dup_s16(g3p);
264 b3 = vld1_dup_s16(b3p);
265
266 dst0[dst_cn.r_i()] = self.profile.r_gamma[vget_lane_u16::<0>(vr0) as usize];
267 dst0[dst_cn.g_i()] = self.profile.g_gamma[vget_lane_u16::<1>(vr0) as usize];
268 dst0[dst_cn.b_i()] = self.profile.b_gamma[vget_lane_u16::<2>(vr0) as usize];
269 if dst_channels == 4 {
270 dst0[dst_cn.a_i()] = a0;
271 }
272
273 dst0[dst_cn.r_i() + dst_channels] =
274 self.profile.r_gamma[vget_lane_u16::<0>(vr1) as usize];
275 dst0[dst_cn.g_i() + dst_channels] =
276 self.profile.g_gamma[vget_lane_u16::<1>(vr1) as usize];
277 dst0[dst_cn.b_i() + dst_channels] =
278 self.profile.b_gamma[vget_lane_u16::<2>(vr0) as usize];
279 if dst_channels == 4 {
280 dst0[dst_cn.a_i() + dst_channels] = a1;
281 }
282
283 dst1[dst_cn.r_i()] = self.profile.r_gamma[vget_lane_u16::<0>(vr2) as usize];
284 dst1[dst_cn.g_i()] = self.profile.g_gamma[vget_lane_u16::<1>(vr2) as usize];
285 dst1[dst_cn.b_i()] = self.profile.b_gamma[vget_lane_u16::<2>(vr2) as usize];
286 if dst_channels == 4 {
287 dst1[dst_cn.a_i()] = a2;
288 }
289
290 dst1[dst_cn.r_i() + dst_channels] =
291 self.profile.r_gamma[vget_lane_u16::<0>(vr3) as usize];
292 dst1[dst_cn.g_i() + dst_channels] =
293 self.profile.g_gamma[vget_lane_u16::<1>(vr3) as usize];
294 dst1[dst_cn.b_i() + dst_channels] =
295 self.profile.b_gamma[vget_lane_u16::<2>(vr3) as usize];
296 if dst_channels == 4 {
297 dst1[dst_cn.a_i() + dst_channels] = a3;
298 }
299
300 a0 = if src_channels == 4 {
301 src0[src_cn.a_i()]
302 } else {
303 max_colors
304 };
305
306 a1 = if src_channels == 4 {
307 src0[src_cn.a_i() + src_channels]
308 } else {
309 max_colors
310 };
311
312 a2 = if src_channels == 4 {
313 src1[src_cn.a_i()]
314 } else {
315 max_colors
316 };
317
318 a3 = if src_channels == 4 {
319 src1[src_cn.a_i() + src_channels]
320 } else {
321 max_colors
322 };
323 }
324
325 if let (Some(dst0), Some(dst1)) = (
326 dst0.chunks_exact_mut(dst_channels * 2).last(),
327 dst1.chunks_exact_mut(dst_channels * 2).last(),
328 ) {
329 let v0_0 = vmlal_s16(rnd, r0, m0);
330 let v0_1 = vmlal_s16(rnd, r1, m0);
331 let v0_2 = vmlal_s16(rnd, r2, m0);
332 let v0_3 = vmlal_s16(rnd, r3, m0);
333
334 let v1_0 = vmlal_s16(v0_0, g0, m1);
335 let v1_1 = vmlal_s16(v0_1, g1, m1);
336 let v1_2 = vmlal_s16(v0_2, g2, m1);
337 let v1_3 = vmlal_s16(v0_3, g3, m1);
338
339 let vr0 = vmlal_s16(v1_0, b0, m2);
340 let vr1 = vmlal_s16(v1_1, b1, m2);
341 let vr2 = vmlal_s16(v1_2, b2, m2);
342 let vr3 = vmlal_s16(v1_3, b3, m2);
343
344 let mut vr0 = vqshrun_n_s32::<PRECISION>(vr0);
345 let mut vr1 = vqshrun_n_s32::<PRECISION>(vr1);
346 let mut vr2 = vqshrun_n_s32::<PRECISION>(vr2);
347 let mut vr3 = vqshrun_n_s32::<PRECISION>(vr3);
348
349 vr0 = vmin_u16(vr0, v_max_value);
350 vr1 = vmin_u16(vr1, v_max_value);
351 vr2 = vmin_u16(vr2, v_max_value);
352 vr3 = vmin_u16(vr3, v_max_value);
353
354 dst0[dst_cn.r_i()] = self.profile.r_gamma[vget_lane_u16::<0>(vr0) as usize];
355 dst0[dst_cn.g_i()] = self.profile.g_gamma[vget_lane_u16::<1>(vr0) as usize];
356 dst0[dst_cn.b_i()] = self.profile.b_gamma[vget_lane_u16::<2>(vr0) as usize];
357 if dst_channels == 4 {
358 dst0[dst_cn.a_i()] = a0;
359 }
360
361 dst0[dst_cn.r_i() + dst_channels] =
362 self.profile.r_gamma[vget_lane_u16::<0>(vr1) as usize];
363 dst0[dst_cn.g_i() + dst_channels] =
364 self.profile.g_gamma[vget_lane_u16::<1>(vr1) as usize];
365 dst0[dst_cn.b_i() + dst_channels] =
366 self.profile.b_gamma[vget_lane_u16::<2>(vr0) as usize];
367 if dst_channels == 4 {
368 dst0[dst_cn.a_i() + dst_channels] = a1;
369 }
370
371 dst1[dst_cn.r_i()] = self.profile.r_gamma[vget_lane_u16::<0>(vr2) as usize];
372 dst1[dst_cn.g_i()] = self.profile.g_gamma[vget_lane_u16::<1>(vr2) as usize];
373 dst1[dst_cn.b_i()] = self.profile.b_gamma[vget_lane_u16::<2>(vr2) as usize];
374 if dst_channels == 4 {
375 dst1[dst_cn.a_i()] = a2;
376 }
377
378 dst1[dst_cn.r_i() + dst_channels] =
379 self.profile.r_gamma[vget_lane_u16::<0>(vr3) as usize];
380 dst1[dst_cn.g_i() + dst_channels] =
381 self.profile.g_gamma[vget_lane_u16::<1>(vr3) as usize];
382 dst1[dst_cn.b_i() + dst_channels] =
383 self.profile.b_gamma[vget_lane_u16::<2>(vr3) as usize];
384 if dst_channels == 4 {
385 dst1[dst_cn.a_i() + dst_channels] = a3;
386 }
387 }
388 }
389
390 for (src, dst) in src_remainder
391 .chunks_exact(src_channels)
392 .zip(dst_remainder.chunks_exact_mut(dst_channels))
393 {
394 let rp = r_lin.get_unchecked(src[src_cn.r_i()]._as_usize());
395 let gp = g_lin.get_unchecked(src[src_cn.g_i()]._as_usize());
396 let bp = b_lin.get_unchecked(src[src_cn.b_i()]._as_usize());
397 let r = vld1_dup_s16(rp);
398 let g = vld1_dup_s16(gp);
399 let b = vld1_dup_s16(bp);
400 let a = if src_channels == 4 {
401 src[src_cn.a_i()]
402 } else {
403 max_colors
404 };
405
406 let v0 = vmlal_s16(rnd, r, m0);
407 let v1 = vmlal_s16(v0, g, m1);
408 let v = vmlal_s16(v1, b, m2);
409
410 let mut vr0 = vqshrun_n_s32::<PRECISION>(v);
411 vr0 = vmin_u16(vr0, v_max_value);
412
413 dst[dst_cn.r_i()] = self.profile.r_gamma[vget_lane_u16::<0>(vr0) as usize];
414 dst[dst_cn.g_i()] = self.profile.g_gamma[vget_lane_u16::<1>(vr0) as usize];
415 dst[dst_cn.b_i()] = self.profile.b_gamma[vget_lane_u16::<2>(vr0) as usize];
416 if dst_channels == 4 {
417 dst[dst_cn.a_i()] = a;
418 }
419 }
420 }
421
422 Ok(())
423 }
424 }