Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/neon/rgb_xyz.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::conversions::neon::rgb_xyz_q2_13::{split_by_twos, split_by_twos_mut};
30 use crate::conversions::rgbxyz::TransformMatrixShaperV;
31 use crate::transform::PointeeSizeExpressible;
32 use crate::{CmsError, Layout, TransformExecutor};
33 use num_traits::AsPrimitive;
34 use std::arch::aarch64::*;
35
36 #[repr(align(16), C)]
37 pub(crate) struct NeonAlignedU16(pub(crate) [u16; 8]);
38
39 #[repr(align(16), C)]
40 pub(crate) struct NeonAlignedF32(pub(crate) [f32; 4]);
41
42 pub(crate) struct TransformShaperRgbNeon<
43 T: Clone + PointeeSizeExpressible + Copy + Default + 'static,
44 const SRC_LAYOUT: u8,
45 const DST_LAYOUT: u8,
46 > {
47 pub(crate) profile: TransformMatrixShaperV<T>,
48 pub(crate) bit_depth: usize,
49 pub(crate) gamma_lut: usize,
50 }
51
52 impl<
53 T: Clone + PointeeSizeExpressible + Copy + Default + 'static,
54 const SRC_LAYOUT: u8,
55 const DST_LAYOUT: u8,
56 > TransformExecutor<T> for TransformShaperRgbNeon<T, SRC_LAYOUT, DST_LAYOUT>
57 where
58 u32: AsPrimitive<T>,
59 {
60 fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
61 let src_cn = Layout::from(SRC_LAYOUT);
62 let dst_cn = Layout::from(DST_LAYOUT);
63 let src_channels = src_cn.channels();
64 let dst_channels = dst_cn.channels();
65
66 let mut temporary0 = NeonAlignedU16([0; 8]);
67 let mut temporary1 = NeonAlignedU16([0; 8]);
68 let mut temporary2 = NeonAlignedU16([0; 8]);
69 let mut temporary3 = NeonAlignedU16([0; 8]);
70
71 if src.len() / src_channels != dst.len() / dst_channels {
72 return Err(CmsError::LaneSizeMismatch);
73 }
74 if src.len() % src_channels != 0 {
75 return Err(CmsError::LaneMultipleOfChannels);
76 }
77 if dst.len() % dst_channels != 0 {
78 return Err(CmsError::LaneMultipleOfChannels);
79 }
80
81 let t = self.profile.adaptation_matrix.transpose();
82 let scale = (self.gamma_lut - 1) as f32;
83 let max_colors: T = ((1 << self.bit_depth) - 1).as_();
84
85 // safety precondition for linearization table
86 if T::FINITE {
87 let cap = (1 << self.bit_depth) - 1;
88 assert!(self.profile.r_linear.len() >= cap);
89 assert!(self.profile.g_linear.len() >= cap);
90 assert!(self.profile.b_linear.len() >= cap);
91 } else {
92 assert!(self.profile.r_linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
93 assert!(self.profile.g_linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
94 assert!(self.profile.b_linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
95 }
96
97 let r_lin = &self.profile.r_linear;
98 let g_lin = &self.profile.g_linear;
99 let b_lin = &self.profile.b_linear;
100
101 let (src_chunks, src_remainder) = split_by_twos(src, src_channels);
102 let (dst_chunks, dst_remainder) = split_by_twos_mut(dst, dst_channels);
103
104 unsafe {
105 let m0 = vld1q_f32([t.v[0][0], t.v[0][1], t.v[0][2], 0.].as_ptr());
106 let m1 = vld1q_f32([t.v[1][0], t.v[1][1], t.v[1][2], 0.].as_ptr());
107 let m2 = vld1q_f32([t.v[2][0], t.v[2][1], t.v[2][2], 0.].as_ptr());
108
109 let v_scale = vdupq_n_f32(scale);
110
111 let rnd = vdupq_n_f32(0.5);
112
113 if !src_chunks.is_empty() {
114 let (src0, src1) = src_chunks.split_at(src_chunks.len() / 2);
115 let (dst0, dst1) = dst_chunks.split_at_mut(dst_chunks.len() / 2);
116 let mut src_iter0 = src0.chunks_exact(src_channels * 2);
117 let mut src_iter1 = src1.chunks_exact(src_channels * 2);
118
119 let (mut r0, mut g0, mut b0, mut a0);
120 let (mut r1, mut g1, mut b1, mut a1);
121 let (mut r2, mut g2, mut b2, mut a2);
122 let (mut r3, mut g3, mut b3, mut a3);
123
124 if let (Some(src0), Some(src1)) = (src_iter0.next(), src_iter1.next()) {
125 let r0p = r_lin.get_unchecked(src0[src_cn.r_i()]._as_usize());
126 let g0p = g_lin.get_unchecked(src0[src_cn.g_i()]._as_usize());
127 let b0p = b_lin.get_unchecked(src0[src_cn.b_i()]._as_usize());
128
129 let r1p = r_lin.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize());
130 let g1p = g_lin.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize());
131 let b1p = b_lin.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize());
132
133 let r2p = r_lin.get_unchecked(src1[src_cn.r_i()]._as_usize());
134 let g2p = g_lin.get_unchecked(src1[src_cn.g_i()]._as_usize());
135 let b2p = b_lin.get_unchecked(src1[src_cn.b_i()]._as_usize());
136
137 let r3p = r_lin.get_unchecked(src1[src_cn.r_i() + src_channels]._as_usize());
138 let g3p = g_lin.get_unchecked(src1[src_cn.g_i() + src_channels]._as_usize());
139 let b3p = b_lin.get_unchecked(src1[src_cn.b_i() + src_channels]._as_usize());
140
141 r0 = vld1q_dup_f32(r0p);
142 g0 = vld1q_dup_f32(g0p);
143 b0 = vld1q_dup_f32(b0p);
144
145 r1 = vld1q_dup_f32(r1p);
146 g1 = vld1q_dup_f32(g1p);
147 b1 = vld1q_dup_f32(b1p);
148
149 r2 = vld1q_dup_f32(r2p);
150 g2 = vld1q_dup_f32(g2p);
151 b2 = vld1q_dup_f32(b2p);
152
153 r3 = vld1q_dup_f32(r3p);
154 g3 = vld1q_dup_f32(g3p);
155 b3 = vld1q_dup_f32(b3p);
156
157 a0 = if src_channels == 4 {
158 src0[src_cn.a_i()]
159 } else {
160 max_colors
161 };
162
163 a1 = if src_channels == 4 {
164 src0[src_cn.a_i() + src_channels]
165 } else {
166 max_colors
167 };
168
169 a2 = if src_channels == 4 {
170 src1[src_cn.a_i()]
171 } else {
172 max_colors
173 };
174
175 a3 = if src_channels == 4 {
176 src1[src_cn.a_i() + src_channels]
177 } else {
178 max_colors
179 };
180 } else {
181 r0 = vdupq_n_f32(0.);
182 g0 = vdupq_n_f32(0.);
183 b0 = vdupq_n_f32(0.);
184 r1 = vdupq_n_f32(0.);
185 g1 = vdupq_n_f32(0.);
186 b1 = vdupq_n_f32(0.);
187 r2 = vdupq_n_f32(0.);
188 g2 = vdupq_n_f32(0.);
189 b2 = vdupq_n_f32(0.);
190 r3 = vdupq_n_f32(0.);
191 g3 = vdupq_n_f32(0.);
192 b3 = vdupq_n_f32(0.);
193 a0 = max_colors;
194 a1 = max_colors;
195 a2 = max_colors;
196 a3 = max_colors;
197 }
198
199 for (((src0, src1), dst0), dst1) in src_iter0
200 .zip(src_iter1)
201 .zip(dst0.chunks_exact_mut(dst_channels * 2))
202 .zip(dst1.chunks_exact_mut(dst_channels * 2))
203 {
204 let v0_0 = vmulq_f32(r0, m0);
205 let v0_1 = vmulq_f32(r1, m0);
206 let v0_2 = vmulq_f32(r2, m0);
207 let v0_3 = vmulq_f32(r3, m0);
208
209 let v1_0 = vfmaq_f32(v0_0, g0, m1);
210 let v1_1 = vfmaq_f32(v0_1, g1, m1);
211 let v1_2 = vfmaq_f32(v0_2, g2, m1);
212 let v1_3 = vfmaq_f32(v0_3, g3, m1);
213
214 let mut vr0 = vfmaq_f32(v1_0, b0, m2);
215 let mut vr1 = vfmaq_f32(v1_1, b1, m2);
216 let mut vr2 = vfmaq_f32(v1_2, b2, m2);
217 let mut vr3 = vfmaq_f32(v1_3, b3, m2);
218
219 vr0 = vfmaq_f32(rnd, vr0, v_scale);
220 vr1 = vfmaq_f32(rnd, vr1, v_scale);
221 vr2 = vfmaq_f32(rnd, vr2, v_scale);
222 vr3 = vfmaq_f32(rnd, vr3, v_scale);
223
224 vr0 = vminq_f32(vr0, v_scale);
225 vr1 = vminq_f32(vr1, v_scale);
226 vr2 = vminq_f32(vr2, v_scale);
227 vr3 = vminq_f32(vr3, v_scale);
228
229 let zx0 = vcvtq_u32_f32(vr0);
230 let zx1 = vcvtq_u32_f32(vr1);
231 let zx2 = vcvtq_u32_f32(vr2);
232 let zx3 = vcvtq_u32_f32(vr3);
233 vst1q_u32(temporary0.0.as_mut_ptr() as *mut _, zx0);
234 vst1q_u32(temporary1.0.as_mut_ptr() as *mut _, zx1);
235 vst1q_u32(temporary2.0.as_mut_ptr() as *mut _, zx2);
236 vst1q_u32(temporary3.0.as_mut_ptr() as *mut _, zx3);
237
238 let r0p = r_lin.get_unchecked(src0[src_cn.r_i()]._as_usize());
239 let g0p = g_lin.get_unchecked(src0[src_cn.g_i()]._as_usize());
240 let b0p = b_lin.get_unchecked(src0[src_cn.b_i()]._as_usize());
241
242 let r1p = r_lin.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize());
243 let g1p = g_lin.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize());
244 let b1p = b_lin.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize());
245
246 let r2p = r_lin.get_unchecked(src1[src_cn.r_i()]._as_usize());
247 let g2p = g_lin.get_unchecked(src1[src_cn.g_i()]._as_usize());
248 let b2p = b_lin.get_unchecked(src1[src_cn.b_i()]._as_usize());
249
250 let r3p = r_lin.get_unchecked(src1[src_cn.r_i() + src_channels]._as_usize());
251 let g3p = g_lin.get_unchecked(src1[src_cn.g_i() + src_channels]._as_usize());
252 let b3p = b_lin.get_unchecked(src1[src_cn.b_i() + src_channels]._as_usize());
253
254 r0 = vld1q_dup_f32(r0p);
255 g0 = vld1q_dup_f32(g0p);
256 b0 = vld1q_dup_f32(b0p);
257
258 r1 = vld1q_dup_f32(r1p);
259 g1 = vld1q_dup_f32(g1p);
260 b1 = vld1q_dup_f32(b1p);
261
262 r2 = vld1q_dup_f32(r2p);
263 g2 = vld1q_dup_f32(g2p);
264 b2 = vld1q_dup_f32(b2p);
265
266 r3 = vld1q_dup_f32(r3p);
267 g3 = vld1q_dup_f32(g3p);
268 b3 = vld1q_dup_f32(b3p);
269
270 dst0[dst_cn.r_i()] = self.profile.r_gamma[temporary0.0[0] as usize];
271 dst0[dst_cn.g_i()] = self.profile.g_gamma[temporary0.0[2] as usize];
272 dst0[dst_cn.b_i()] = self.profile.b_gamma[temporary0.0[4] as usize];
273 if dst_channels == 4 {
274 dst0[dst_cn.a_i()] = a0;
275 }
276
277 dst0[dst_cn.r_i() + dst_channels] =
278 self.profile.r_gamma[temporary1.0[0] as usize];
279 dst0[dst_cn.g_i() + dst_channels] =
280 self.profile.g_gamma[temporary1.0[2] as usize];
281 dst0[dst_cn.b_i() + dst_channels] =
282 self.profile.b_gamma[temporary1.0[4] as usize];
283 if dst_channels == 4 {
284 dst0[dst_cn.a_i() + dst_channels] = a1;
285 }
286
287 dst1[dst_cn.r_i()] = self.profile.r_gamma[temporary2.0[0] as usize];
288 dst1[dst_cn.g_i()] = self.profile.g_gamma[temporary2.0[2] as usize];
289 dst1[dst_cn.b_i()] = self.profile.b_gamma[temporary2.0[4] as usize];
290 if dst_channels == 4 {
291 dst1[dst_cn.a_i()] = a2;
292 }
293
294 dst1[dst_cn.r_i() + dst_channels] =
295 self.profile.r_gamma[temporary3.0[0] as usize];
296 dst1[dst_cn.g_i() + dst_channels] =
297 self.profile.g_gamma[temporary3.0[2] as usize];
298 dst1[dst_cn.b_i() + dst_channels] =
299 self.profile.b_gamma[temporary3.0[4] as usize];
300 if dst_channels == 4 {
301 dst1[dst_cn.a_i() + dst_channels] = a3;
302 }
303
304 a0 = if src_channels == 4 {
305 src0[src_cn.a_i()]
306 } else {
307 max_colors
308 };
309
310 a1 = if src_channels == 4 {
311 src0[src_cn.a_i() + src_channels]
312 } else {
313 max_colors
314 };
315
316 a2 = if src_channels == 4 {
317 src1[src_cn.a_i()]
318 } else {
319 max_colors
320 };
321
322 a3 = if src_channels == 4 {
323 src1[src_cn.a_i() + src_channels]
324 } else {
325 max_colors
326 };
327 }
328
329 if let (Some(dst0), Some(dst1)) = (
330 dst0.chunks_exact_mut(dst_channels * 2).last(),
331 dst1.chunks_exact_mut(dst_channels * 2).last(),
332 ) {
333 let v0_0 = vmulq_f32(r0, m0);
334 let v0_1 = vmulq_f32(r1, m0);
335 let v0_2 = vmulq_f32(r2, m0);
336 let v0_3 = vmulq_f32(r3, m0);
337
338 let v1_0 = vfmaq_f32(v0_0, g0, m1);
339 let v1_1 = vfmaq_f32(v0_1, g1, m1);
340 let v1_2 = vfmaq_f32(v0_2, g2, m1);
341 let v1_3 = vfmaq_f32(v0_3, g3, m1);
342
343 let mut vr0 = vfmaq_f32(v1_0, b0, m2);
344 let mut vr1 = vfmaq_f32(v1_1, b1, m2);
345 let mut vr2 = vfmaq_f32(v1_2, b2, m2);
346 let mut vr3 = vfmaq_f32(v1_3, b3, m2);
347
348 vr0 = vfmaq_f32(rnd, vr0, v_scale);
349 vr1 = vfmaq_f32(rnd, vr1, v_scale);
350 vr2 = vfmaq_f32(rnd, vr2, v_scale);
351 vr3 = vfmaq_f32(rnd, vr3, v_scale);
352
353 vr0 = vminq_f32(vr0, v_scale);
354 vr1 = vminq_f32(vr1, v_scale);
355 vr2 = vminq_f32(vr2, v_scale);
356 vr3 = vminq_f32(vr3, v_scale);
357
358 let zx0 = vcvtq_u32_f32(vr0);
359 let zx1 = vcvtq_u32_f32(vr1);
360 let zx2 = vcvtq_u32_f32(vr2);
361 let zx3 = vcvtq_u32_f32(vr3);
362 vst1q_u32(temporary0.0.as_mut_ptr() as *mut _, zx0);
363 vst1q_u32(temporary1.0.as_mut_ptr() as *mut _, zx1);
364 vst1q_u32(temporary2.0.as_mut_ptr() as *mut _, zx2);
365 vst1q_u32(temporary3.0.as_mut_ptr() as *mut _, zx3);
366
367 dst0[dst_cn.r_i()] = self.profile.r_gamma[temporary0.0[0] as usize];
368 dst0[dst_cn.g_i()] = self.profile.g_gamma[temporary0.0[2] as usize];
369 dst0[dst_cn.b_i()] = self.profile.b_gamma[temporary0.0[4] as usize];
370 if dst_channels == 4 {
371 dst0[dst_cn.a_i()] = a0;
372 }
373
374 dst0[dst_cn.r_i() + dst_channels] =
375 self.profile.r_gamma[temporary1.0[0] as usize];
376 dst0[dst_cn.g_i() + dst_channels] =
377 self.profile.g_gamma[temporary1.0[2] as usize];
378 dst0[dst_cn.b_i() + dst_channels] =
379 self.profile.b_gamma[temporary1.0[4] as usize];
380 if dst_channels == 4 {
381 dst0[dst_cn.a_i() + dst_channels] = a1;
382 }
383
384 dst1[dst_cn.r_i()] = self.profile.r_gamma[temporary2.0[0] as usize];
385 dst1[dst_cn.g_i()] = self.profile.g_gamma[temporary2.0[2] as usize];
386 dst1[dst_cn.b_i()] = self.profile.b_gamma[temporary2.0[4] as usize];
387 if dst_channels == 4 {
388 dst1[dst_cn.a_i()] = a2;
389 }
390
391 dst1[dst_cn.r_i() + dst_channels] =
392 self.profile.r_gamma[temporary3.0[0] as usize];
393 dst1[dst_cn.g_i() + dst_channels] =
394 self.profile.g_gamma[temporary3.0[2] as usize];
395 dst1[dst_cn.b_i() + dst_channels] =
396 self.profile.b_gamma[temporary3.0[4] as usize];
397 if dst_channels == 4 {
398 dst1[dst_cn.a_i() + dst_channels] = a3;
399 }
400 }
401 }
402
403 for (src, dst) in src_remainder
404 .chunks_exact(src_channels)
405 .zip(dst_remainder.chunks_exact_mut(dst_channels))
406 {
407 let rp = r_lin.get_unchecked(src[src_cn.r_i()]._as_usize());
408 let gp = g_lin.get_unchecked(src[src_cn.g_i()]._as_usize());
409 let bp = b_lin.get_unchecked(src[src_cn.b_i()]._as_usize());
410 let r = vld1q_dup_f32(rp);
411 let g = vld1q_dup_f32(gp);
412 let b = vld1q_dup_f32(bp);
413 let a = if src_channels == 4 {
414 src[src_cn.a_i()]
415 } else {
416 max_colors
417 };
418
419 let v0 = vmulq_f32(r, m0);
420 let v1 = vfmaq_f32(v0, g, m1);
421 let mut v = vfmaq_f32(v1, b, m2);
422
423 v = vfmaq_f32(rnd, v, v_scale);
424 v = vminq_f32(v, v_scale);
425
426 let zx = vcvtq_u32_f32(v);
427 vst1q_u32(temporary0.0.as_mut_ptr() as *mut _, zx);
428
429 dst[dst_cn.r_i()] = self.profile.r_gamma[temporary0.0[0] as usize];
430 dst[dst_cn.g_i()] = self.profile.g_gamma[temporary0.0[2] as usize];
431 dst[dst_cn.b_i()] = self.profile.b_gamma[temporary0.0[4] as usize];
432 if dst_channels == 4 {
433 dst[dst_cn.a_i()] = a;
434 }
435 }
436 }
437
438 Ok(())
439 }
440 }