Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/avx512/rgb_xyz_q2_13_opt.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::conversions::rgbxyz_fixed::TransformMatrixShaperFixedPointOpt;
30 use crate::transform::PointeeSizeExpressible;
31 use crate::{CmsError, Layout, TransformExecutor};
32 use num_traits::AsPrimitive;
33 use std::arch::x86_64::*;
34
35 pub(crate) struct TransformShaperRgbQ2_13OptAvx512<
36 T: Copy,
37 const SRC_LAYOUT: u8,
38 const DST_LAYOUT: u8,
39 const LINEAR_CAP: usize,
40 const PRECISION: i32,
41 > {
42 pub(crate) profile: TransformMatrixShaperFixedPointOpt<i32, i16, T, LINEAR_CAP>,
43 pub(crate) bit_depth: usize,
44 pub(crate) gamma_lut: usize,
45 }
46
47 #[inline(always)]
48 pub(crate) unsafe fn _xmm_broadcast_epi32(f: &i32) -> __m128i {
49 let float_ref: &f32 = unsafe { &*(f as *const i32 as *const f32) };
50 unsafe { _mm_castps_si128(_mm_broadcast_ss(float_ref)) }
51 }
52
53 #[repr(align(32), C)]
54 #[derive(Debug)]
55 pub(crate) struct AvxAlignedU16(pub(crate) [u16; 16]);
56
57 #[inline]
58 pub(crate) fn split_by_twos<T: Copy>(data: &[T], channels: usize) -> (&[T], &[T]) {
59 let len = data.len() / (channels * 4);
60 let split_point = len * 4;
61 data.split_at(split_point * channels)
62 }
63 #[inline]
64 pub(crate) fn split_by_twos_mut<T: Copy>(data: &mut [T], channels: usize) -> (&mut [T], &mut [T]) {
65 let len = data.len() / (channels * 4);
66 let split_point = len * 4;
67 data.split_at_mut(split_point * channels)
68 }
69
70 impl<
71 T: Copy + PointeeSizeExpressible + 'static,
72 const SRC_LAYOUT: u8,
73 const DST_LAYOUT: u8,
74 const LINEAR_CAP: usize,
75 const PRECISION: i32,
76 > TransformShaperRgbQ2_13OptAvx512<T, SRC_LAYOUT, DST_LAYOUT, LINEAR_CAP, PRECISION>
77 where
78 u32: AsPrimitive<T>,
79 {
80 #[target_feature(enable = "avx512bw", enable = "avx512vl")]
81 unsafe fn transform_avx512(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
82 let src_cn = Layout::from(SRC_LAYOUT);
83 let dst_cn = Layout::from(DST_LAYOUT);
84 let src_channels = src_cn.channels();
85 let dst_channels = dst_cn.channels();
86
87 if src.len() / src_channels != dst.len() / dst_channels {
88 return Err(CmsError::LaneSizeMismatch);
89 }
90 if src.len() % src_channels != 0 {
91 return Err(CmsError::LaneMultipleOfChannels);
92 }
93 if dst.len() % dst_channels != 0 {
94 return Err(CmsError::LaneMultipleOfChannels);
95 }
96
97 let t = self.profile.adaptation_matrix.transpose();
98
99 let max_colors = ((1 << self.bit_depth) - 1).as_();
100
101 // If precision changed in another place it should be also changed here
102 assert_eq!(PRECISION, 13);
103
104 let (src_chunks, src_remainder) = split_by_twos(src, src_channels);
105 let (dst_chunks, dst_remainder) = split_by_twos_mut(dst, dst_channels);
106
107 let mut temporary0 = AvxAlignedU16([0; 16]);
108 let mut temporary1 = AvxAlignedU16([0; 16]);
109
110 unsafe {
111 let m0 = _mm256_set_epi16(
112 0, 0, t.v[1][2], t.v[0][2], t.v[1][1], t.v[0][1], t.v[1][0], t.v[0][0], 0, 0,
113 t.v[1][2], t.v[0][2], t.v[1][1], t.v[0][1], t.v[1][0], t.v[0][0],
114 );
115 let m2 = _mm256_set_epi16(
116 0, 0, 1, t.v[2][2], 1, t.v[2][1], 1, t.v[2][0], 0, 0, 1, t.v[2][2], 1, t.v[2][1],
117 1, t.v[2][0],
118 );
119
120 let rnd_val = ((1i32 << (PRECISION - 1)) as i16).to_ne_bytes();
121 let rnd = _mm256_set1_epi32(i32::from_ne_bytes([0, 0, rnd_val[0], rnd_val[1]]));
122
123 let zeros = _mm256_setzero_si256();
124
125 let v_max_value = _mm256_set1_epi32(self.gamma_lut as i32 - 1);
126
127 let (mut r0, mut g0, mut b0, mut a0);
128 let (mut r1, mut g1, mut b1, mut a1);
129 let (mut r2, mut g2, mut b2, mut a2);
130 let (mut r3, mut g3, mut b3, mut a3);
131
132 if !src_chunks.is_empty() {
133 let (src0, src1) = src_chunks.split_at(src_chunks.len() / 2);
134 let (dst0, dst1) = dst_chunks.split_at_mut(dst_chunks.len() / 2);
135 let mut src_iter0 = src0.chunks_exact(src_channels * 2);
136 let mut src_iter1 = src1.chunks_exact(src_channels * 2);
137
138 if let (Some(src0), Some(src1)) = (src_iter0.next(), src_iter1.next()) {
139 r0 = _xmm_broadcast_epi32(&self.profile.linear[src0[src_cn.r_i()]._as_usize()]);
140 g0 = _xmm_broadcast_epi32(&self.profile.linear[src0[src_cn.g_i()]._as_usize()]);
141 b0 = _xmm_broadcast_epi32(&self.profile.linear[src0[src_cn.b_i()]._as_usize()]);
142
143 r1 = _xmm_broadcast_epi32(
144 &self.profile.linear[src0[src_cn.r_i() + src_channels]._as_usize()],
145 );
146 g1 = _xmm_broadcast_epi32(
147 &self.profile.linear[src0[src_cn.g_i() + src_channels]._as_usize()],
148 );
149 b1 = _xmm_broadcast_epi32(
150 &self.profile.linear[src0[src_cn.b_i() + src_channels]._as_usize()],
151 );
152
153 r2 = _xmm_broadcast_epi32(&self.profile.linear[src1[src_cn.r_i()]._as_usize()]);
154 g2 = _xmm_broadcast_epi32(&self.profile.linear[src1[src_cn.g_i()]._as_usize()]);
155 b2 = _xmm_broadcast_epi32(&self.profile.linear[src1[src_cn.b_i()]._as_usize()]);
156
157 r3 = _xmm_broadcast_epi32(
158 &self.profile.linear[src1[src_cn.r_i() + src_channels]._as_usize()],
159 );
160 g3 = _xmm_broadcast_epi32(
161 &self.profile.linear[src1[src_cn.g_i() + src_channels]._as_usize()],
162 );
163 b3 = _xmm_broadcast_epi32(
164 &self.profile.linear[src1[src_cn.b_i() + src_channels]._as_usize()],
165 );
166
167 a0 = if src_channels == 4 {
168 src0[src_cn.a_i()]
169 } else {
170 max_colors
171 };
172 a1 = if src_channels == 4 {
173 src0[src_cn.a_i() + src_channels]
174 } else {
175 max_colors
176 };
177
178 a2 = if src_channels == 4 {
179 src1[src_cn.a_i()]
180 } else {
181 max_colors
182 };
183 a3 = if src_channels == 4 {
184 src1[src_cn.a_i() + src_channels]
185 } else {
186 max_colors
187 };
188 } else {
189 r0 = _mm_setzero_si128();
190 g0 = _mm_setzero_si128();
191 b0 = _mm_setzero_si128();
192 a0 = max_colors;
193 r1 = _mm_setzero_si128();
194 g1 = _mm_setzero_si128();
195 b1 = _mm_setzero_si128();
196 a1 = max_colors;
197 r2 = _mm_setzero_si128();
198 g2 = _mm_setzero_si128();
199 b2 = _mm_setzero_si128();
200 a2 = max_colors;
201 r3 = _mm_setzero_si128();
202 g3 = _mm_setzero_si128();
203 b3 = _mm_setzero_si128();
204 a3 = max_colors;
205 }
206
207 for (((src0, src1), dst0), dst1) in src_iter0
208 .zip(src_iter1)
209 .zip(dst0.chunks_exact_mut(dst_channels * 2))
210 .zip(dst1.chunks_exact_mut(dst_channels * 2))
211 {
212 let zr0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(r0), r1);
213 let mut zg0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(g0), g1);
214 let zb0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(b0), b1);
215 zg0 = _mm256_slli_epi32::<16>(zg0);
216
217 let zr1 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(r2), r3);
218 let mut zg1 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(g2), g3);
219 let zb1 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(b2), b3);
220 zg1 = _mm256_slli_epi32::<16>(zg1);
221
222 let zrg0 = _mm256_or_si256(zr0, zg0);
223 let zbz0 = _mm256_or_si256(zb0, rnd);
224
225 let zrg1 = _mm256_or_si256(zr1, zg1);
226 let zbz1 = _mm256_or_si256(zb1, rnd);
227
228 let va0 = _mm256_madd_epi16(zrg0, m0);
229 let va1 = _mm256_madd_epi16(zbz0, m2);
230
231 let va2 = _mm256_madd_epi16(zrg1, m0);
232 let va3 = _mm256_madd_epi16(zbz1, m2);
233
234 let mut v0 = _mm256_add_epi32(va0, va1);
235 let mut v1 = _mm256_add_epi32(va2, va3);
236
237 v0 = _mm256_srai_epi32::<PRECISION>(v0);
238 v0 = _mm256_max_epi32(v0, zeros);
239 v0 = _mm256_min_epi32(v0, v_max_value);
240
241 v1 = _mm256_srai_epi32::<PRECISION>(v1);
242 v1 = _mm256_max_epi32(v1, zeros);
243 v1 = _mm256_min_epi32(v1, v_max_value);
244
245 _mm256_store_si256(temporary0.0.as_mut_ptr() as *mut _, v0);
246 _mm256_store_si256(temporary1.0.as_mut_ptr() as *mut _, v1);
247
248 r0 = _xmm_broadcast_epi32(&self.profile.linear[src0[src_cn.r_i()]._as_usize()]);
249 g0 = _xmm_broadcast_epi32(&self.profile.linear[src0[src_cn.g_i()]._as_usize()]);
250 b0 = _xmm_broadcast_epi32(&self.profile.linear[src0[src_cn.b_i()]._as_usize()]);
251
252 r1 = _xmm_broadcast_epi32(
253 &self.profile.linear[src0[src_cn.r_i() + src_channels]._as_usize()],
254 );
255 g1 = _xmm_broadcast_epi32(
256 &self.profile.linear[src0[src_cn.g_i() + src_channels]._as_usize()],
257 );
258 b1 = _xmm_broadcast_epi32(
259 &self.profile.linear[src0[src_cn.b_i() + src_channels]._as_usize()],
260 );
261
262 r2 = _xmm_broadcast_epi32(&self.profile.linear[src1[src_cn.r_i()]._as_usize()]);
263 g2 = _xmm_broadcast_epi32(&self.profile.linear[src1[src_cn.g_i()]._as_usize()]);
264 b2 = _xmm_broadcast_epi32(&self.profile.linear[src1[src_cn.b_i()]._as_usize()]);
265
266 r3 = _xmm_broadcast_epi32(
267 &self.profile.linear[src1[src_cn.r_i() + src_channels]._as_usize()],
268 );
269 g3 = _xmm_broadcast_epi32(
270 &self.profile.linear[src1[src_cn.g_i() + src_channels]._as_usize()],
271 );
272 b3 = _xmm_broadcast_epi32(
273 &self.profile.linear[src1[src_cn.b_i() + src_channels]._as_usize()],
274 );
275
276 dst0[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
277 dst0[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
278 dst0[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
279 if dst_channels == 4 {
280 dst0[dst_cn.a_i()] = a0;
281 }
282
283 dst0[dst_cn.r_i() + dst_channels] =
284 self.profile.gamma[temporary0.0[8] as usize];
285 dst0[dst_cn.g_i() + dst_channels] =
286 self.profile.gamma[temporary0.0[10] as usize];
287 dst0[dst_cn.b_i() + dst_channels] =
288 self.profile.gamma[temporary0.0[12] as usize];
289 if dst_channels == 4 {
290 dst0[dst_cn.a_i() + dst_channels] = a1;
291 }
292
293 dst1[dst_cn.r_i()] = self.profile.gamma[temporary1.0[0] as usize];
294 dst1[dst_cn.g_i()] = self.profile.gamma[temporary1.0[2] as usize];
295 dst1[dst_cn.b_i()] = self.profile.gamma[temporary1.0[4] as usize];
296 if dst_channels == 4 {
297 dst1[dst_cn.a_i()] = a2;
298 }
299
300 dst1[dst_cn.r_i() + dst_channels] =
301 self.profile.gamma[temporary1.0[8] as usize];
302 dst1[dst_cn.g_i() + dst_channels] =
303 self.profile.gamma[temporary1.0[10] as usize];
304 dst1[dst_cn.b_i() + dst_channels] =
305 self.profile.gamma[temporary1.0[12] as usize];
306 if dst_channels == 4 {
307 dst1[dst_cn.a_i() + dst_channels] = a3;
308 }
309
310 a0 = if src_channels == 4 {
311 src0[src_cn.a_i()]
312 } else {
313 max_colors
314 };
315 a1 = if src_channels == 4 {
316 src0[src_cn.a_i() + src_channels]
317 } else {
318 max_colors
319 };
320
321 a2 = if src_channels == 4 {
322 src1[src_cn.a_i()]
323 } else {
324 max_colors
325 };
326 a3 = if src_channels == 4 {
327 src1[src_cn.a_i() + src_channels]
328 } else {
329 max_colors
330 };
331 }
332
333 if let (Some(dst0), Some(dst1)) = (
334 dst0.chunks_exact_mut(dst_channels * 2).last(),
335 dst1.chunks_exact_mut(dst_channels * 2).last(),
336 ) {
337 let zr0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(r0), r1);
338 let mut zg0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(g0), g1);
339 let zb0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(b0), b1);
340 zg0 = _mm256_slli_epi32::<16>(zg0);
341
342 let zr1 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(r2), r3);
343 let mut zg1 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(g2), g3);
344 let zb1 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(b2), b3);
345 zg1 = _mm256_slli_epi32::<16>(zg1);
346
347 let zrg0 = _mm256_or_si256(zr0, zg0);
348 let zbz0 = _mm256_or_si256(zb0, rnd);
349
350 let zrg1 = _mm256_or_si256(zr1, zg1);
351 let zbz1 = _mm256_or_si256(zb1, rnd);
352
353 let va0 = _mm256_madd_epi16(zrg0, m0);
354 let va1 = _mm256_madd_epi16(zbz0, m2);
355
356 let va2 = _mm256_madd_epi16(zrg1, m0);
357 let va3 = _mm256_madd_epi16(zbz1, m2);
358
359 let mut v0 = _mm256_add_epi32(va0, va1);
360 let mut v1 = _mm256_add_epi32(va2, va3);
361
362 v0 = _mm256_srai_epi32::<PRECISION>(v0);
363 v0 = _mm256_max_epi32(v0, zeros);
364 v0 = _mm256_min_epi32(v0, v_max_value);
365
366 v1 = _mm256_srai_epi32::<PRECISION>(v1);
367 v1 = _mm256_max_epi32(v1, zeros);
368 v1 = _mm256_min_epi32(v1, v_max_value);
369
370 _mm256_store_si256(temporary0.0.as_mut_ptr() as *mut _, v0);
371 _mm256_store_si256(temporary1.0.as_mut_ptr() as *mut _, v1);
372
373 dst0[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
374 dst0[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
375 dst0[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
376 if dst_channels == 4 {
377 dst0[dst_cn.a_i()] = a0;
378 }
379
380 dst0[dst_cn.r_i() + dst_channels] =
381 self.profile.gamma[temporary0.0[8] as usize];
382 dst0[dst_cn.g_i() + dst_channels] =
383 self.profile.gamma[temporary0.0[10] as usize];
384 dst0[dst_cn.b_i() + dst_channels] =
385 self.profile.gamma[temporary0.0[12] as usize];
386 if dst_channels == 4 {
387 dst0[dst_cn.a_i() + dst_channels] = a1;
388 }
389
390 dst1[dst_cn.r_i()] = self.profile.gamma[temporary1.0[0] as usize];
391 dst1[dst_cn.g_i()] = self.profile.gamma[temporary1.0[2] as usize];
392 dst1[dst_cn.b_i()] = self.profile.gamma[temporary1.0[4] as usize];
393 if dst_channels == 4 {
394 dst1[dst_cn.a_i()] = a2;
395 }
396
397 dst1[dst_cn.r_i() + dst_channels] =
398 self.profile.gamma[temporary1.0[8] as usize];
399 dst1[dst_cn.g_i() + dst_channels] =
400 self.profile.gamma[temporary1.0[10] as usize];
401 dst1[dst_cn.b_i() + dst_channels] =
402 self.profile.gamma[temporary1.0[12] as usize];
403 if dst_channels == 4 {
404 dst1[dst_cn.a_i() + dst_channels] = a3;
405 }
406 }
407 }
408
409 for (src, dst) in src_remainder
410 .chunks_exact(src_channels)
411 .zip(dst_remainder.chunks_exact_mut(dst_channels))
412 {
413 let r = _xmm_broadcast_epi32(&self.profile.linear[src[src_cn.r_i()]._as_usize()]);
414 let mut g =
415 _xmm_broadcast_epi32(&self.profile.linear[src[src_cn.g_i()]._as_usize()]);
416 let b = _xmm_broadcast_epi32(&self.profile.linear[src[src_cn.b_i()]._as_usize()]);
417
418 g = _mm_slli_epi32::<16>(g);
419
420 let a = if src_channels == 4 {
421 src[src_cn.a_i()]
422 } else {
423 max_colors
424 };
425
426 let zrg0 = _mm_or_si128(r, g);
427 let zbz0 = _mm_or_si128(b, _mm256_castsi256_si128(rnd));
428
429 let v0 = _mm_madd_epi16(zrg0, _mm256_castsi256_si128(m0));
430 let v1 = _mm_madd_epi16(zbz0, _mm256_castsi256_si128(m2));
431
432 let mut v = _mm_add_epi32(v0, v1);
433
434 v = _mm_srai_epi32::<PRECISION>(v);
435 v = _mm_max_epi32(v, _mm_setzero_si128());
436 v = _mm_min_epi32(v, _mm256_castsi256_si128(v_max_value));
437
438 _mm_store_si128(temporary0.0.as_mut_ptr() as *mut _, v);
439
440 dst[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
441 dst[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
442 dst[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
443 if dst_channels == 4 {
444 dst[dst_cn.a_i()] = a;
445 }
446 }
447 }
448
449 Ok(())
450 }
451 }
452
453 impl<
454 T: Copy + PointeeSizeExpressible + 'static + Default,
455 const SRC_LAYOUT: u8,
456 const DST_LAYOUT: u8,
457 const LINEAR_CAP: usize,
458 const PRECISION: i32,
459 > TransformExecutor<T>
460 for TransformShaperRgbQ2_13OptAvx512<T, SRC_LAYOUT, DST_LAYOUT, LINEAR_CAP, PRECISION>
461 where
462 u32: AsPrimitive<T>,
463 {
464 fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
465 unsafe { self.transform_avx512(src, dst) }
466 }
467 }