Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/neon/rgb_xyz_opt.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::conversions::neon::rgb_xyz::NeonAlignedU16;
30 use crate::conversions::neon::rgb_xyz_q2_13::{split_by_twos, split_by_twos_mut};
31 use crate::conversions::rgbxyz::TransformMatrixShaperOptimizedV;
32 use crate::transform::PointeeSizeExpressible;
33 use crate::{CmsError, Layout, TransformExecutor};
34 use num_traits::AsPrimitive;
35 use std::arch::aarch64::*;
36
37 pub(crate) struct TransformShaperRgbOptNeon<
38 T: Clone + PointeeSizeExpressible + Copy + Default + 'static,
39 const SRC_LAYOUT: u8,
40 const DST_LAYOUT: u8,
41 > {
42 pub(crate) profile: TransformMatrixShaperOptimizedV<T>,
43 pub(crate) bit_depth: usize,
44 pub(crate) gamma_lut: usize,
45 }
46
47 impl<
48 T: Clone + PointeeSizeExpressible + Copy + Default + 'static,
49 const SRC_LAYOUT: u8,
50 const DST_LAYOUT: u8,
51 > TransformExecutor<T> for TransformShaperRgbOptNeon<T, SRC_LAYOUT, DST_LAYOUT>
52 where
53 u32: AsPrimitive<T>,
54 {
55 fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
56 let src_cn = Layout::from(SRC_LAYOUT);
57 let dst_cn = Layout::from(DST_LAYOUT);
58 let src_channels = src_cn.channels();
59 let dst_channels = dst_cn.channels();
60
61 let mut temporary0 = NeonAlignedU16([0; 8]);
62 let mut temporary1 = NeonAlignedU16([0; 8]);
63 let mut temporary2 = NeonAlignedU16([0; 8]);
64 let mut temporary3 = NeonAlignedU16([0; 8]);
65
66 if src.len() / src_channels != dst.len() / dst_channels {
67 return Err(CmsError::LaneSizeMismatch);
68 }
69 if src.len() % src_channels != 0 {
70 return Err(CmsError::LaneMultipleOfChannels);
71 }
72 if dst.len() % dst_channels != 0 {
73 return Err(CmsError::LaneMultipleOfChannels);
74 }
75
76 let t = self.profile.adaptation_matrix.transpose();
77 let scale = (self.gamma_lut - 1) as f32;
78 let max_colors: T = ((1 << self.bit_depth) - 1).as_();
79
80 // safety precondition for linearization table
81 if T::FINITE {
82 let cap = (1 << self.bit_depth) - 1;
83 assert!(self.profile.linear.len() >= cap);
84 } else {
85 assert!(self.profile.linear.len() >= T::NOT_FINITE_LINEAR_TABLE_SIZE);
86 }
87
88 let lut_lin = &self.profile.linear;
89
90 let (src_chunks, src_remainder) = split_by_twos(src, src_channels);
91 let (dst_chunks, dst_remainder) = split_by_twos_mut(dst, dst_channels);
92
93 unsafe {
94 let m0 = vld1q_f32([t.v[0][0], t.v[0][1], t.v[0][2], 0.].as_ptr());
95 let m1 = vld1q_f32([t.v[1][0], t.v[1][1], t.v[1][2], 0.].as_ptr());
96 let m2 = vld1q_f32([t.v[2][0], t.v[2][1], t.v[2][2], 0.].as_ptr());
97
98 let v_scale = vdupq_n_f32(scale);
99
100 let rnd = vdupq_n_f32(0.5);
101
102 if !src_chunks.is_empty() {
103 let (src0, src1) = src_chunks.split_at(src_chunks.len() / 2);
104 let (dst0, dst1) = dst_chunks.split_at_mut(dst_chunks.len() / 2);
105 let mut src_iter0 = src0.chunks_exact(src_channels * 2);
106 let mut src_iter1 = src1.chunks_exact(src_channels * 2);
107
108 let (mut r0, mut g0, mut b0, mut a0);
109 let (mut r1, mut g1, mut b1, mut a1);
110 let (mut r2, mut g2, mut b2, mut a2);
111 let (mut r3, mut g3, mut b3, mut a3);
112
113 if let (Some(src0), Some(src1)) = (src_iter0.next(), src_iter1.next()) {
114 let r0p = lut_lin.get_unchecked(src0[src_cn.r_i()]._as_usize());
115 let g0p = lut_lin.get_unchecked(src0[src_cn.g_i()]._as_usize());
116 let b0p = lut_lin.get_unchecked(src0[src_cn.b_i()]._as_usize());
117
118 let r1p = lut_lin.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize());
119 let g1p = lut_lin.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize());
120 let b1p = lut_lin.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize());
121
122 let r2p = lut_lin.get_unchecked(src1[src_cn.r_i()]._as_usize());
123 let g2p = lut_lin.get_unchecked(src1[src_cn.g_i()]._as_usize());
124 let b2p = lut_lin.get_unchecked(src1[src_cn.b_i()]._as_usize());
125
126 let r3p = lut_lin.get_unchecked(src1[src_cn.r_i() + src_channels]._as_usize());
127 let g3p = lut_lin.get_unchecked(src1[src_cn.g_i() + src_channels]._as_usize());
128 let b3p = lut_lin.get_unchecked(src1[src_cn.b_i() + src_channels]._as_usize());
129
130 r0 = vld1q_dup_f32(r0p);
131 g0 = vld1q_dup_f32(g0p);
132 b0 = vld1q_dup_f32(b0p);
133
134 r1 = vld1q_dup_f32(r1p);
135 g1 = vld1q_dup_f32(g1p);
136 b1 = vld1q_dup_f32(b1p);
137
138 r2 = vld1q_dup_f32(r2p);
139 g2 = vld1q_dup_f32(g2p);
140 b2 = vld1q_dup_f32(b2p);
141
142 r3 = vld1q_dup_f32(r3p);
143 g3 = vld1q_dup_f32(g3p);
144 b3 = vld1q_dup_f32(b3p);
145
146 a0 = if src_channels == 4 {
147 src0[src_cn.a_i()]
148 } else {
149 max_colors
150 };
151
152 a1 = if src_channels == 4 {
153 src0[src_cn.a_i() + src_channels]
154 } else {
155 max_colors
156 };
157
158 a2 = if src_channels == 4 {
159 src1[src_cn.a_i()]
160 } else {
161 max_colors
162 };
163
164 a3 = if src_channels == 4 {
165 src1[src_cn.a_i() + src_channels]
166 } else {
167 max_colors
168 };
169 } else {
170 r0 = vdupq_n_f32(0.);
171 g0 = vdupq_n_f32(0.);
172 b0 = vdupq_n_f32(0.);
173 r1 = vdupq_n_f32(0.);
174 g1 = vdupq_n_f32(0.);
175 b1 = vdupq_n_f32(0.);
176 r2 = vdupq_n_f32(0.);
177 g2 = vdupq_n_f32(0.);
178 b2 = vdupq_n_f32(0.);
179 r3 = vdupq_n_f32(0.);
180 g3 = vdupq_n_f32(0.);
181 b3 = vdupq_n_f32(0.);
182 a0 = max_colors;
183 a1 = max_colors;
184 a2 = max_colors;
185 a3 = max_colors;
186 }
187
188 for (((src0, src1), dst0), dst1) in src_iter0
189 .zip(src_iter1)
190 .zip(dst0.chunks_exact_mut(dst_channels * 2))
191 .zip(dst1.chunks_exact_mut(dst_channels * 2))
192 {
193 let v0_0 = vmulq_f32(r0, m0);
194 let v0_1 = vmulq_f32(r1, m0);
195 let v0_2 = vmulq_f32(r2, m0);
196 let v0_3 = vmulq_f32(r3, m0);
197
198 let v1_0 = vfmaq_f32(v0_0, g0, m1);
199 let v1_1 = vfmaq_f32(v0_1, g1, m1);
200 let v1_2 = vfmaq_f32(v0_2, g2, m1);
201 let v1_3 = vfmaq_f32(v0_3, g3, m1);
202
203 let mut vr0 = vfmaq_f32(v1_0, b0, m2);
204 let mut vr1 = vfmaq_f32(v1_1, b1, m2);
205 let mut vr2 = vfmaq_f32(v1_2, b2, m2);
206 let mut vr3 = vfmaq_f32(v1_3, b3, m2);
207
208 vr0 = vfmaq_f32(rnd, vr0, v_scale);
209 vr1 = vfmaq_f32(rnd, vr1, v_scale);
210 vr2 = vfmaq_f32(rnd, vr2, v_scale);
211 vr3 = vfmaq_f32(rnd, vr3, v_scale);
212
213 vr0 = vminq_f32(vr0, v_scale);
214 vr1 = vminq_f32(vr1, v_scale);
215 vr2 = vminq_f32(vr2, v_scale);
216 vr3 = vminq_f32(vr3, v_scale);
217
218 let zx0 = vcvtq_u32_f32(vr0);
219 let zx1 = vcvtq_u32_f32(vr1);
220 let zx2 = vcvtq_u32_f32(vr2);
221 let zx3 = vcvtq_u32_f32(vr3);
222 vst1q_u32(temporary0.0.as_mut_ptr() as *mut _, zx0);
223 vst1q_u32(temporary1.0.as_mut_ptr() as *mut _, zx1);
224 vst1q_u32(temporary2.0.as_mut_ptr() as *mut _, zx2);
225 vst1q_u32(temporary3.0.as_mut_ptr() as *mut _, zx3);
226
227 let r0p = lut_lin.get_unchecked(src0[src_cn.r_i()]._as_usize());
228 let g0p = lut_lin.get_unchecked(src0[src_cn.g_i()]._as_usize());
229 let b0p = lut_lin.get_unchecked(src0[src_cn.b_i()]._as_usize());
230
231 let r1p = lut_lin.get_unchecked(src0[src_cn.r_i() + src_channels]._as_usize());
232 let g1p = lut_lin.get_unchecked(src0[src_cn.g_i() + src_channels]._as_usize());
233 let b1p = lut_lin.get_unchecked(src0[src_cn.b_i() + src_channels]._as_usize());
234
235 let r2p = lut_lin.get_unchecked(src1[src_cn.r_i()]._as_usize());
236 let g2p = lut_lin.get_unchecked(src1[src_cn.g_i()]._as_usize());
237 let b2p = lut_lin.get_unchecked(src1[src_cn.b_i()]._as_usize());
238
239 let r3p = lut_lin.get_unchecked(src1[src_cn.r_i() + src_channels]._as_usize());
240 let g3p = lut_lin.get_unchecked(src1[src_cn.g_i() + src_channels]._as_usize());
241 let b3p = lut_lin.get_unchecked(src1[src_cn.b_i() + src_channels]._as_usize());
242
243 r0 = vld1q_dup_f32(r0p);
244 g0 = vld1q_dup_f32(g0p);
245 b0 = vld1q_dup_f32(b0p);
246
247 r1 = vld1q_dup_f32(r1p);
248 g1 = vld1q_dup_f32(g1p);
249 b1 = vld1q_dup_f32(b1p);
250
251 r2 = vld1q_dup_f32(r2p);
252 g2 = vld1q_dup_f32(g2p);
253 b2 = vld1q_dup_f32(b2p);
254
255 r3 = vld1q_dup_f32(r3p);
256 g3 = vld1q_dup_f32(g3p);
257 b3 = vld1q_dup_f32(b3p);
258
259 dst0[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
260 dst0[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
261 dst0[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
262 if dst_channels == 4 {
263 dst0[dst_cn.a_i()] = a0;
264 }
265
266 dst0[dst_cn.r_i() + dst_channels] =
267 self.profile.gamma[temporary1.0[0] as usize];
268 dst0[dst_cn.g_i() + dst_channels] =
269 self.profile.gamma[temporary1.0[2] as usize];
270 dst0[dst_cn.b_i() + dst_channels] =
271 self.profile.gamma[temporary1.0[4] as usize];
272 if dst_channels == 4 {
273 dst0[dst_cn.a_i() + dst_channels] = a1;
274 }
275
276 dst1[dst_cn.r_i()] = self.profile.gamma[temporary2.0[0] as usize];
277 dst1[dst_cn.g_i()] = self.profile.gamma[temporary2.0[2] as usize];
278 dst1[dst_cn.b_i()] = self.profile.gamma[temporary2.0[4] as usize];
279 if dst_channels == 4 {
280 dst1[dst_cn.a_i()] = a2;
281 }
282
283 dst1[dst_cn.r_i() + dst_channels] =
284 self.profile.gamma[temporary3.0[0] as usize];
285 dst1[dst_cn.g_i() + dst_channels] =
286 self.profile.gamma[temporary3.0[2] as usize];
287 dst1[dst_cn.b_i() + dst_channels] =
288 self.profile.gamma[temporary3.0[4] as usize];
289 if dst_channels == 4 {
290 dst1[dst_cn.a_i() + dst_channels] = a3;
291 }
292
293 a0 = if src_channels == 4 {
294 src0[src_cn.a_i()]
295 } else {
296 max_colors
297 };
298
299 a1 = if src_channels == 4 {
300 src0[src_cn.a_i() + src_channels]
301 } else {
302 max_colors
303 };
304
305 a2 = if src_channels == 4 {
306 src1[src_cn.a_i()]
307 } else {
308 max_colors
309 };
310
311 a3 = if src_channels == 4 {
312 src1[src_cn.a_i() + src_channels]
313 } else {
314 max_colors
315 };
316 }
317
318 if let (Some(dst0), Some(dst1)) = (
319 dst0.chunks_exact_mut(dst_channels * 2).last(),
320 dst1.chunks_exact_mut(dst_channels * 2).last(),
321 ) {
322 let v0_0 = vmulq_f32(r0, m0);
323 let v0_1 = vmulq_f32(r1, m0);
324 let v0_2 = vmulq_f32(r2, m0);
325 let v0_3 = vmulq_f32(r3, m0);
326
327 let v1_0 = vfmaq_f32(v0_0, g0, m1);
328 let v1_1 = vfmaq_f32(v0_1, g1, m1);
329 let v1_2 = vfmaq_f32(v0_2, g2, m1);
330 let v1_3 = vfmaq_f32(v0_3, g3, m1);
331
332 let mut vr0 = vfmaq_f32(v1_0, b0, m2);
333 let mut vr1 = vfmaq_f32(v1_1, b1, m2);
334 let mut vr2 = vfmaq_f32(v1_2, b2, m2);
335 let mut vr3 = vfmaq_f32(v1_3, b3, m2);
336
337 vr0 = vfmaq_f32(rnd, vr0, v_scale);
338 vr1 = vfmaq_f32(rnd, vr1, v_scale);
339 vr2 = vfmaq_f32(rnd, vr2, v_scale);
340 vr3 = vfmaq_f32(rnd, vr3, v_scale);
341
342 vr0 = vminq_f32(vr0, v_scale);
343 vr1 = vminq_f32(vr1, v_scale);
344 vr2 = vminq_f32(vr2, v_scale);
345 vr3 = vminq_f32(vr3, v_scale);
346
347 let zx0 = vcvtq_u32_f32(vr0);
348 let zx1 = vcvtq_u32_f32(vr1);
349 let zx2 = vcvtq_u32_f32(vr2);
350 let zx3 = vcvtq_u32_f32(vr3);
351 vst1q_u32(temporary0.0.as_mut_ptr() as *mut _, zx0);
352 vst1q_u32(temporary1.0.as_mut_ptr() as *mut _, zx1);
353 vst1q_u32(temporary2.0.as_mut_ptr() as *mut _, zx2);
354 vst1q_u32(temporary3.0.as_mut_ptr() as *mut _, zx3);
355
356 dst0[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
357 dst0[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
358 dst0[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
359 if dst_channels == 4 {
360 dst0[dst_cn.a_i()] = a0;
361 }
362
363 dst0[dst_cn.r_i() + dst_channels] =
364 self.profile.gamma[temporary1.0[0] as usize];
365 dst0[dst_cn.g_i() + dst_channels] =
366 self.profile.gamma[temporary1.0[2] as usize];
367 dst0[dst_cn.b_i() + dst_channels] =
368 self.profile.gamma[temporary1.0[4] as usize];
369 if dst_channels == 4 {
370 dst0[dst_cn.a_i() + dst_channels] = a1;
371 }
372
373 dst1[dst_cn.r_i()] = self.profile.gamma[temporary2.0[0] as usize];
374 dst1[dst_cn.g_i()] = self.profile.gamma[temporary2.0[2] as usize];
375 dst1[dst_cn.b_i()] = self.profile.gamma[temporary2.0[4] as usize];
376 if dst_channels == 4 {
377 dst1[dst_cn.a_i()] = a2;
378 }
379
380 dst1[dst_cn.r_i() + dst_channels] =
381 self.profile.gamma[temporary3.0[0] as usize];
382 dst1[dst_cn.g_i() + dst_channels] =
383 self.profile.gamma[temporary3.0[2] as usize];
384 dst1[dst_cn.b_i() + dst_channels] =
385 self.profile.gamma[temporary3.0[4] as usize];
386 if dst_channels == 4 {
387 dst1[dst_cn.a_i() + dst_channels] = a3;
388 }
389 }
390 }
391
392 for (src, dst) in src_remainder
393 .chunks_exact(src_channels)
394 .zip(dst_remainder.chunks_exact_mut(dst_channels))
395 {
396 let rp = lut_lin.get_unchecked(src[src_cn.r_i()]._as_usize());
397 let gp = lut_lin.get_unchecked(src[src_cn.g_i()]._as_usize());
398 let bp = lut_lin.get_unchecked(src[src_cn.b_i()]._as_usize());
399 let r = vld1q_dup_f32(rp);
400 let g = vld1q_dup_f32(gp);
401 let b = vld1q_dup_f32(bp);
402 let a = if src_channels == 4 {
403 src[src_cn.a_i()]
404 } else {
405 max_colors
406 };
407
408 let v0 = vmulq_f32(r, m0);
409 let v1 = vfmaq_f32(v0, g, m1);
410 let mut v = vfmaq_f32(v1, b, m2);
411
412 v = vfmaq_f32(rnd, v, v_scale);
413 v = vminq_f32(v, v_scale);
414
415 let zx = vcvtq_u32_f32(v);
416 vst1q_u32(temporary0.0.as_mut_ptr() as *mut _, zx);
417
418 dst[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
419 dst[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
420 dst[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
421 if dst_channels == 4 {
422 dst[dst_cn.a_i()] = a;
423 }
424 }
425 }
426
427 Ok(())
428 }
429 }