Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/neon/rgb_xyz_q1_30_opt.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::conversions::neon::rgb_xyz_q2_13::{split_by_twos, split_by_twos_mut};
30 use crate::conversions::rgbxyz_fixed::TransformMatrixShaperFpOptVec;
31 use crate::transform::PointeeSizeExpressible;
32 use crate::{CmsError, Layout, TransformExecutor};
33 use num_traits::AsPrimitive;
34 use std::arch::aarch64::*;
35
36 pub(crate) struct TransformShaperQ1_30NeonOpt<T: Copy, const SRC_LAYOUT: u8, const DST_LAYOUT: u8> {
37 pub(crate) profile: TransformMatrixShaperFpOptVec<i32, i32, T>,
38 pub(crate) gamma_lut: usize,
39 pub(crate) bit_depth: usize,
40 }
41
42 impl<
43 T: Copy + PointeeSizeExpressible + 'static + Default,
44 const SRC_LAYOUT: u8,
45 const DST_LAYOUT: u8,
46 > TransformShaperQ1_30NeonOpt<T, SRC_LAYOUT, DST_LAYOUT>
47 where
48 u32: AsPrimitive<T>,
49 {
50 #[target_feature(enable = "rdm")]
51 unsafe fn transform_impl(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
52 let src_cn = Layout::from(SRC_LAYOUT);
53 let dst_cn = Layout::from(DST_LAYOUT);
54 let src_channels = src_cn.channels();
55 let dst_channels = dst_cn.channels();
56
57 if src.len() / src_channels != dst.len() / dst_channels {
58 return Err(CmsError::LaneSizeMismatch);
59 }
60 if src.len() % src_channels != 0 {
61 return Err(CmsError::LaneMultipleOfChannels);
62 }
63 if dst.len() % dst_channels != 0 {
64 return Err(CmsError::LaneMultipleOfChannels);
65 }
66
67 let t = self.profile.adaptation_matrix.transpose();
68 let max_colors: T = ((1 << self.bit_depth) - 1).as_();
69
70 let (src_chunks, src_remainder) = split_by_twos(src, src_channels);
71 let (dst_chunks, dst_remainder) = split_by_twos_mut(dst, dst_channels);
72
73 // safety precondition for linearization table
74 if T::FINITE {
75 assert!(self.profile.linear.len() >= (1 << self.bit_depth) - 1);
76 } else {
77 assert!(self.profile.linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
78 }
79
80 unsafe {
81 let m0 = vld1q_s32([t.v[0][0], t.v[0][1], t.v[0][2], 0].as_ptr());
82 let m1 = vld1q_s32([t.v[1][0], t.v[1][1], t.v[1][2], 0].as_ptr());
83 let m2 = vld1q_s32([t.v[2][0], t.v[2][1], t.v[2][2], 0].as_ptr());
84
85 let lin_lut = &self.profile.linear;
86
87 let v_max_value = vdup_n_u16((self.gamma_lut - 1) as u16);
88
89 if !src_chunks.is_empty() {
90 let (src0, src1) = src_chunks.split_at(src_chunks.len() / 2);
91 let (dst0, dst1) = dst_chunks.split_at_mut(dst_chunks.len() / 2);
92 let mut src_iter0 = src0.chunks_exact(src_channels * 2);
93 let mut src_iter1 = src1.chunks_exact(src_channels * 2);
94
95 let (mut r0, mut g0, mut b0, mut a0);
96 let (mut r1, mut g1, mut b1, mut a1);
97 let (mut r2, mut g2, mut b2, mut a2);
98 let (mut r3, mut g3, mut b3, mut a3);
99
100 if let (Some(src0), Some(src1)) = (src_iter0.next(), src_iter1.next()) {
101 let r0p = lin_lut.get_unchecked(src0[src_cn.r_i()]._as_usize());
102 let g0p = lin_lut.get_unchecked(src0[src_cn.g_i()]._as_usize());
103 let b0p = lin_lut.get_unchecked(src0[src_cn.b_i()]._as_usize());
104
105 let r1p = lin_lut.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize());
106 let g1p = lin_lut.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize());
107 let b1p = lin_lut.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize());
108
109 let r2p = lin_lut.get_unchecked(src1[src_cn.r_i()]._as_usize());
110 let g2p = lin_lut.get_unchecked(src1[src_cn.g_i()]._as_usize());
111 let b2p = lin_lut.get_unchecked(src1[src_cn.b_i()]._as_usize());
112
113 let r3p = lin_lut.get_unchecked(src1[src_cn.r_i() + src_channels]._as_usize());
114 let g3p = lin_lut.get_unchecked(src1[src_cn.g_i() + src_channels]._as_usize());
115 let b3p = lin_lut.get_unchecked(src1[src_cn.b_i() + src_channels]._as_usize());
116
117 r0 = vld1q_dup_s32(r0p);
118 g0 = vld1q_dup_s32(g0p);
119 b0 = vld1q_dup_s32(b0p);
120
121 r1 = vld1q_dup_s32(r1p);
122 g1 = vld1q_dup_s32(g1p);
123 b1 = vld1q_dup_s32(b1p);
124
125 r2 = vld1q_dup_s32(r2p);
126 g2 = vld1q_dup_s32(g2p);
127 b2 = vld1q_dup_s32(b2p);
128
129 r3 = vld1q_dup_s32(r3p);
130 g3 = vld1q_dup_s32(g3p);
131 b3 = vld1q_dup_s32(b3p);
132
133 a0 = if src_channels == 4 {
134 src0[src_cn.a_i()]
135 } else {
136 max_colors
137 };
138
139 a1 = if src_channels == 4 {
140 src0[src_cn.a_i() + src_channels]
141 } else {
142 max_colors
143 };
144
145 a2 = if src_channels == 4 {
146 src1[src_cn.a_i()]
147 } else {
148 max_colors
149 };
150
151 a3 = if src_channels == 4 {
152 src1[src_cn.a_i() + src_channels]
153 } else {
154 max_colors
155 };
156 } else {
157 r0 = vdupq_n_s32(0);
158 g0 = vdupq_n_s32(0);
159 b0 = vdupq_n_s32(0);
160 r1 = vdupq_n_s32(0);
161 g1 = vdupq_n_s32(0);
162 b1 = vdupq_n_s32(0);
163 r2 = vdupq_n_s32(0);
164 g2 = vdupq_n_s32(0);
165 b2 = vdupq_n_s32(0);
166 r3 = vdupq_n_s32(0);
167 g3 = vdupq_n_s32(0);
168 b3 = vdupq_n_s32(0);
169 a0 = max_colors;
170 a1 = max_colors;
171 a2 = max_colors;
172 a3 = max_colors;
173 }
174
175 for (((src0, src1), dst0), dst1) in src_iter0
176 .zip(src_iter1)
177 .zip(dst0.chunks_exact_mut(dst_channels * 2))
178 .zip(dst1.chunks_exact_mut(dst_channels * 2))
179 {
180 let v0_0 = vqrdmulhq_s32(r0, m0);
181 let v0_1 = vqrdmulhq_s32(r1, m0);
182 let v0_2 = vqrdmulhq_s32(r2, m0);
183 let v0_3 = vqrdmulhq_s32(r3, m0);
184
185 let v1_0 = vqrdmlahq_s32(v0_0, g0, m1);
186 let v1_1 = vqrdmlahq_s32(v0_1, g1, m1);
187 let v1_2 = vqrdmlahq_s32(v0_2, g2, m1);
188 let v1_3 = vqrdmlahq_s32(v0_3, g3, m1);
189
190 let vr0 = vqrdmlahq_s32(v1_0, b0, m2);
191 let vr1 = vqrdmlahq_s32(v1_1, b1, m2);
192 let vr2 = vqrdmlahq_s32(v1_2, b2, m2);
193 let vr3 = vqrdmlahq_s32(v1_3, b3, m2);
194
195 let mut vr0 = vqmovun_s32(vr0);
196 let mut vr1 = vqmovun_s32(vr1);
197 let mut vr2 = vqmovun_s32(vr2);
198 let mut vr3 = vqmovun_s32(vr3);
199
200 vr0 = vmin_u16(vr0, v_max_value);
201 vr1 = vmin_u16(vr1, v_max_value);
202 vr2 = vmin_u16(vr2, v_max_value);
203 vr3 = vmin_u16(vr3, v_max_value);
204
205 let r0p = lin_lut.get_unchecked(src0[src_cn.r_i()]._as_usize());
206 let g0p = lin_lut.get_unchecked(src0[src_cn.g_i()]._as_usize());
207 let b0p = lin_lut.get_unchecked(src0[src_cn.b_i()]._as_usize());
208
209 let r1p = lin_lut.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize());
210 let g1p = lin_lut.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize());
211 let b1p = lin_lut.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize());
212
213 let r2p = lin_lut.get_unchecked(src1[src_cn.r_i()]._as_usize());
214 let g2p = lin_lut.get_unchecked(src1[src_cn.g_i()]._as_usize());
215 let b2p = lin_lut.get_unchecked(src1[src_cn.b_i()]._as_usize());
216
217 let r3p = lin_lut.get_unchecked(src1[src_cn.r_i() + src_channels]._as_usize());
218 let g3p = lin_lut.get_unchecked(src1[src_cn.g_i() + src_channels]._as_usize());
219 let b3p = lin_lut.get_unchecked(src1[src_cn.b_i() + src_channels]._as_usize());
220
221 r0 = vld1q_dup_s32(r0p);
222 g0 = vld1q_dup_s32(g0p);
223 b0 = vld1q_dup_s32(b0p);
224
225 r1 = vld1q_dup_s32(r1p);
226 g1 = vld1q_dup_s32(g1p);
227 b1 = vld1q_dup_s32(b1p);
228
229 r2 = vld1q_dup_s32(r2p);
230 g2 = vld1q_dup_s32(g2p);
231 b2 = vld1q_dup_s32(b2p);
232
233 r3 = vld1q_dup_s32(r3p);
234 g3 = vld1q_dup_s32(g3p);
235 b3 = vld1q_dup_s32(b3p);
236
237 dst0[dst_cn.r_i()] = self.profile.gamma[vget_lane_u16::<0>(vr0) as usize];
238 dst0[dst_cn.g_i()] = self.profile.gamma[vget_lane_u16::<1>(vr0) as usize];
239 dst0[dst_cn.b_i()] = self.profile.gamma[vget_lane_u16::<2>(vr0) as usize];
240 if dst_channels == 4 {
241 dst0[dst_cn.a_i()] = a0;
242 }
243
244 dst0[dst_cn.r_i() + dst_channels] =
245 self.profile.gamma[vget_lane_u16::<0>(vr1) as usize];
246 dst0[dst_cn.g_i() + dst_channels] =
247 self.profile.gamma[vget_lane_u16::<1>(vr1) as usize];
248 dst0[dst_cn.b_i() + dst_channels] =
249 self.profile.gamma[vget_lane_u16::<2>(vr0) as usize];
250 if dst_channels == 4 {
251 dst0[dst_cn.a_i() + dst_channels] = a1;
252 }
253
254 dst1[dst_cn.r_i()] = self.profile.gamma[vget_lane_u16::<0>(vr2) as usize];
255 dst1[dst_cn.g_i()] = self.profile.gamma[vget_lane_u16::<1>(vr2) as usize];
256 dst1[dst_cn.b_i()] = self.profile.gamma[vget_lane_u16::<2>(vr2) as usize];
257 if dst_channels == 4 {
258 dst1[dst_cn.a_i()] = a2;
259 }
260
261 dst1[dst_cn.r_i() + dst_channels] =
262 self.profile.gamma[vget_lane_u16::<0>(vr3) as usize];
263 dst1[dst_cn.g_i() + dst_channels] =
264 self.profile.gamma[vget_lane_u16::<1>(vr3) as usize];
265 dst1[dst_cn.b_i() + dst_channels] =
266 self.profile.gamma[vget_lane_u16::<2>(vr3) as usize];
267 if dst_channels == 4 {
268 dst1[dst_cn.a_i() + dst_channels] = a3;
269 }
270
271 a0 = if src_channels == 4 {
272 src0[src_cn.a_i()]
273 } else {
274 max_colors
275 };
276
277 a1 = if src_channels == 4 {
278 src0[src_cn.a_i() + src_channels]
279 } else {
280 max_colors
281 };
282
283 a2 = if src_channels == 4 {
284 src1[src_cn.a_i()]
285 } else {
286 max_colors
287 };
288
289 a3 = if src_channels == 4 {
290 src1[src_cn.a_i() + src_channels]
291 } else {
292 max_colors
293 };
294 }
295
296 if let (Some(dst0), Some(dst1)) = (
297 dst0.chunks_exact_mut(dst_channels * 2).last(),
298 dst1.chunks_exact_mut(dst_channels * 2).last(),
299 ) {
300 let v0_0 = vqrdmulhq_s32(r0, m0);
301 let v0_1 = vqrdmulhq_s32(r1, m0);
302 let v0_2 = vqrdmulhq_s32(r2, m0);
303 let v0_3 = vqrdmulhq_s32(r3, m0);
304
305 let v1_0 = vqrdmlahq_s32(v0_0, g0, m1);
306 let v1_1 = vqrdmlahq_s32(v0_1, g1, m1);
307 let v1_2 = vqrdmlahq_s32(v0_2, g2, m1);
308 let v1_3 = vqrdmlahq_s32(v0_3, g3, m1);
309
310 let vr0 = vqrdmlahq_s32(v1_0, b0, m2);
311 let vr1 = vqrdmlahq_s32(v1_1, b1, m2);
312 let vr2 = vqrdmlahq_s32(v1_2, b2, m2);
313 let vr3 = vqrdmlahq_s32(v1_3, b3, m2);
314
315 let mut vr0 = vqmovun_s32(vr0);
316 let mut vr1 = vqmovun_s32(vr1);
317 let mut vr2 = vqmovun_s32(vr2);
318 let mut vr3 = vqmovun_s32(vr3);
319
320 vr0 = vmin_u16(vr0, v_max_value);
321 vr1 = vmin_u16(vr1, v_max_value);
322 vr2 = vmin_u16(vr2, v_max_value);
323 vr3 = vmin_u16(vr3, v_max_value);
324
325 dst0[dst_cn.r_i()] = self.profile.gamma[vget_lane_u16::<0>(vr0) as usize];
326 dst0[dst_cn.g_i()] = self.profile.gamma[vget_lane_u16::<1>(vr0) as usize];
327 dst0[dst_cn.b_i()] = self.profile.gamma[vget_lane_u16::<2>(vr0) as usize];
328 if dst_channels == 4 {
329 dst0[dst_cn.a_i()] = a0;
330 }
331
332 dst0[dst_cn.r_i() + dst_channels] =
333 self.profile.gamma[vget_lane_u16::<0>(vr1) as usize];
334 dst0[dst_cn.g_i() + dst_channels] =
335 self.profile.gamma[vget_lane_u16::<1>(vr1) as usize];
336 dst0[dst_cn.b_i() + dst_channels] =
337 self.profile.gamma[vget_lane_u16::<2>(vr0) as usize];
338 if dst_channels == 4 {
339 dst0[dst_cn.a_i() + dst_channels] = a1;
340 }
341
342 dst1[dst_cn.r_i()] = self.profile.gamma[vget_lane_u16::<0>(vr2) as usize];
343 dst1[dst_cn.g_i()] = self.profile.gamma[vget_lane_u16::<1>(vr2) as usize];
344 dst1[dst_cn.b_i()] = self.profile.gamma[vget_lane_u16::<2>(vr2) as usize];
345 if dst_channels == 4 {
346 dst1[dst_cn.a_i()] = a2;
347 }
348
349 dst1[dst_cn.r_i() + dst_channels] =
350 self.profile.gamma[vget_lane_u16::<0>(vr3) as usize];
351 dst1[dst_cn.g_i() + dst_channels] =
352 self.profile.gamma[vget_lane_u16::<1>(vr3) as usize];
353 dst1[dst_cn.b_i() + dst_channels] =
354 self.profile.gamma[vget_lane_u16::<2>(vr3) as usize];
355 if dst_channels == 4 {
356 dst1[dst_cn.a_i() + dst_channels] = a3;
357 }
358 }
359 }
360
361 for (src, dst) in src_remainder
362 .chunks_exact(src_channels)
363 .zip(dst_remainder.chunks_exact_mut(dst_channels))
364 {
365 let rp = lin_lut.get_unchecked(src[src_cn.r_i()]._as_usize());
366 let gp = lin_lut.get_unchecked(src[src_cn.g_i()]._as_usize());
367 let bp = lin_lut.get_unchecked(src[src_cn.b_i()]._as_usize());
368 let r = vld1q_dup_s32(rp);
369 let g = vld1q_dup_s32(gp);
370 let b = vld1q_dup_s32(bp);
371 let a = if src_channels == 4 {
372 src[src_cn.a_i()]
373 } else {
374 max_colors
375 };
376
377 let v0 = vqrdmulhq_s32(r, m0);
378 let v1 = vqrdmlahq_s32(v0, g, m1);
379 let v = vqrdmlahq_s32(v1, b, m2);
380
381 let mut vr0 = vqmovun_s32(v);
382 vr0 = vmin_u16(vr0, v_max_value);
383
384 dst[dst_cn.r_i()] = self.profile.gamma[vget_lane_u16::<0>(vr0) as usize];
385 dst[dst_cn.g_i()] = self.profile.gamma[vget_lane_u16::<1>(vr0) as usize];
386 dst[dst_cn.b_i()] = self.profile.gamma[vget_lane_u16::<2>(vr0) as usize];
387 if dst_channels == 4 {
388 dst[dst_cn.a_i()] = a;
389 }
390 }
391 }
392
393 Ok(())
394 }
395 }
396
397 impl<
398 T: Copy + PointeeSizeExpressible + 'static + Default,
399 const SRC_LAYOUT: u8,
400 const DST_LAYOUT: u8,
401 > TransformExecutor<T> for TransformShaperQ1_30NeonOpt<T, SRC_LAYOUT, DST_LAYOUT>
402 where
403 u32: AsPrimitive<T>,
404 {
405 fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
406 unsafe { self.transform_impl(src, dst) }
407 }
408 }