Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/avx512/rgb_xyz_opt.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 5/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::conversions::TransformMatrixShaperOptimized;
30 use crate::conversions::avx512::rgb_xyz_q2_13_opt::{
31 AvxAlignedU16, split_by_twos, split_by_twos_mut,
32 };
33 use crate::transform::PointeeSizeExpressible;
34 use crate::{CmsError, Layout, TransformExecutor};
35 use num_traits::AsPrimitive;
36 use std::arch::x86_64::*;
37
38 pub(crate) struct TransformShaperRgbOptAvx512<
39 T: Clone + Copy + 'static + PointeeSizeExpressible + Default,
40 const SRC_LAYOUT: u8,
41 const DST_LAYOUT: u8,
42 const LINEAR_CAP: usize,
43 > {
44 pub(crate) profile: TransformMatrixShaperOptimized<T, LINEAR_CAP>,
45 pub(crate) bit_depth: usize,
46 pub(crate) gamma_lut: usize,
47 }
48
49 impl<
50 T: Clone + Copy + 'static + PointeeSizeExpressible + Default,
51 const SRC_LAYOUT: u8,
52 const DST_LAYOUT: u8,
53 const LINEAR_CAP: usize,
54 > TransformShaperRgbOptAvx512<T, SRC_LAYOUT, DST_LAYOUT, LINEAR_CAP>
55 where
56 u32: AsPrimitive<T>,
57 {
58 #[target_feature(enable = "avx512bw", enable = "avx512vl", enable = "fma")]
59 unsafe fn transform_impl(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
60 let src_cn = Layout::from(SRC_LAYOUT);
61 let dst_cn = Layout::from(DST_LAYOUT);
62 let src_channels = src_cn.channels();
63 let dst_channels = dst_cn.channels();
64
65 if src.len() / src_channels != dst.len() / dst_channels {
66 return Err(CmsError::LaneSizeMismatch);
67 }
68 if src.len() % src_channels != 0 {
69 return Err(CmsError::LaneMultipleOfChannels);
70 }
71 if dst.len() % dst_channels != 0 {
72 return Err(CmsError::LaneMultipleOfChannels);
73 }
74
75 let t = self.profile.adaptation_matrix.transpose();
76
77 let scale = (self.gamma_lut - 1) as f32;
78 let max_colors: T = ((1 << self.bit_depth) - 1).as_();
79
80 let (src_chunks, src_remainder) = split_by_twos(src, src_channels);
81 let (dst_chunks, dst_remainder) = split_by_twos_mut(dst, dst_channels);
82
83 let mut temporary0 = AvxAlignedU16([0; 16]);
84 let mut temporary1 = AvxAlignedU16([0; 16]);
85
86 unsafe {
87 let m0 = _mm256_setr_ps(
88 t.v[0][0], t.v[0][1], t.v[0][2], 0f32, t.v[0][0], t.v[0][1], t.v[0][2], 0f32,
89 );
90 let m1 = _mm256_setr_ps(
91 t.v[1][0], t.v[1][1], t.v[1][2], 0f32, t.v[1][0], t.v[1][1], t.v[1][2], 0f32,
92 );
93 let m2 = _mm256_setr_ps(
94 t.v[2][0], t.v[2][1], t.v[2][2], 0f32, t.v[2][0], t.v[2][1], t.v[2][2], 0f32,
95 );
96
97 let zeros = _mm_setzero_ps();
98
99 let v_scale = _mm256_set1_ps(scale);
100
101 if !src_chunks.is_empty() {
102 let (src0, src1) = src_chunks.split_at(src_chunks.len() / 2);
103 let (dst0, dst1) = dst_chunks.split_at_mut(dst_chunks.len() / 2);
104 let mut src_iter0 = src0.chunks_exact(src_channels * 2);
105 let mut src_iter1 = src1.chunks_exact(src_channels * 2);
106
107 let (mut r0, mut g0, mut b0, mut a0);
108 let (mut r1, mut g1, mut b1, mut a1);
109 let (mut r2, mut g2, mut b2, mut a2);
110 let (mut r3, mut g3, mut b3, mut a3);
111
112 if let (Some(src0), Some(src1)) = (src_iter0.next(), src_iter1.next()) {
113 r0 = _mm_broadcast_ss(&self.profile.linear[src0[src_cn.r_i()]._as_usize()]);
114 g0 = _mm_broadcast_ss(&self.profile.linear[src0[src_cn.g_i()]._as_usize()]);
115 b0 = _mm_broadcast_ss(&self.profile.linear[src0[src_cn.b_i()]._as_usize()]);
116
117 r1 = _mm_broadcast_ss(
118 &self.profile.linear[src0[src_cn.r_i() + src_channels]._as_usize()],
119 );
120 g1 = _mm_broadcast_ss(
121 &self.profile.linear[src0[src_cn.g_i() + src_channels]._as_usize()],
122 );
123 b1 = _mm_broadcast_ss(
124 &self.profile.linear[src0[src_cn.b_i() + src_channels]._as_usize()],
125 );
126
127 r2 = _mm_broadcast_ss(&self.profile.linear[src1[src_cn.r_i()]._as_usize()]);
128 g2 = _mm_broadcast_ss(&self.profile.linear[src1[src_cn.g_i()]._as_usize()]);
129 b2 = _mm_broadcast_ss(&self.profile.linear[src1[src_cn.b_i()]._as_usize()]);
130
131 r3 = _mm_broadcast_ss(
132 &self.profile.linear[src1[src_cn.r_i() + src_channels]._as_usize()],
133 );
134 g3 = _mm_broadcast_ss(
135 &self.profile.linear[src1[src_cn.g_i() + src_channels]._as_usize()],
136 );
137 b3 = _mm_broadcast_ss(
138 &self.profile.linear[src1[src_cn.b_i() + src_channels]._as_usize()],
139 );
140
141 a0 = if src_channels == 4 {
142 src0[src_cn.a_i()]
143 } else {
144 max_colors
145 };
146 a1 = if src_channels == 4 {
147 src0[src_cn.a_i() + src_channels]
148 } else {
149 max_colors
150 };
151
152 a2 = if src_channels == 4 {
153 src1[src_cn.a_i()]
154 } else {
155 max_colors
156 };
157 a3 = if src_channels == 4 {
158 src1[src_cn.a_i() + src_channels]
159 } else {
160 max_colors
161 };
162 } else {
163 r0 = _mm_setzero_ps();
164 g0 = _mm_setzero_ps();
165 b0 = _mm_setzero_ps();
166 a0 = max_colors;
167 r1 = _mm_setzero_ps();
168 g1 = _mm_setzero_ps();
169 b1 = _mm_setzero_ps();
170 a1 = max_colors;
171 r2 = _mm_setzero_ps();
172 g2 = _mm_setzero_ps();
173 b2 = _mm_setzero_ps();
174 a2 = max_colors;
175 r3 = _mm_setzero_ps();
176 g3 = _mm_setzero_ps();
177 b3 = _mm_setzero_ps();
178 a3 = max_colors;
179 }
180
181 for (((src0, src1), dst0), dst1) in src_iter0
182 .zip(src_iter1)
183 .zip(dst0.chunks_exact_mut(dst_channels * 2))
184 .zip(dst1.chunks_exact_mut(dst_channels * 2))
185 {
186 let rz0 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(r0), r1);
187 let gz0 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(g0), g1);
188 let bz0 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(b0), b1);
189
190 let rz1 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(r2), r3);
191 let gz1 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(g2), g3);
192 let bz1 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(b2), b3);
193
194 let v0 = _mm256_mul_ps(rz0, m0);
195 let v1 = _mm256_fmadd_ps(gz0, m1, v0);
196 let mut vz0 = _mm256_fmadd_ps(bz0, m2, v1);
197
198 let v2 = _mm256_mul_ps(rz1, m0);
199 let v3 = _mm256_fmadd_ps(gz1, m1, v2);
200 let mut vz1 = _mm256_fmadd_ps(bz1, m2, v3);
201
202 vz0 = _mm256_max_ps(vz0, _mm256_setzero_ps());
203 vz0 = _mm256_mul_ps(vz0, v_scale);
204 vz0 = _mm256_min_ps(vz0, v_scale);
205
206 vz1 = _mm256_max_ps(vz1, _mm256_setzero_ps());
207 vz1 = _mm256_mul_ps(vz1, v_scale);
208 vz1 = _mm256_min_ps(vz1, v_scale);
209
210 let zx0 = _mm256_cvtps_epi32(vz0);
211 let zx1 = _mm256_cvtps_epi32(vz1);
212 _mm256_store_si256(temporary0.0.as_mut_ptr() as *mut _, zx0);
213 _mm256_store_si256(temporary1.0.as_mut_ptr() as *mut _, zx1);
214
215 r0 = _mm_broadcast_ss(&self.profile.linear[src0[src_cn.r_i()]._as_usize()]);
216 g0 = _mm_broadcast_ss(&self.profile.linear[src0[src_cn.g_i()]._as_usize()]);
217 b0 = _mm_broadcast_ss(&self.profile.linear[src0[src_cn.b_i()]._as_usize()]);
218 r1 = _mm_broadcast_ss(
219 &self.profile.linear[src0[src_cn.r_i() + src_channels]._as_usize()],
220 );
221 g1 = _mm_broadcast_ss(
222 &self.profile.linear[src0[src_cn.g_i() + src_channels]._as_usize()],
223 );
224 b1 = _mm_broadcast_ss(
225 &self.profile.linear[src0[src_cn.b_i() + src_channels]._as_usize()],
226 );
227
228 r2 = _mm_broadcast_ss(&self.profile.linear[src1[src_cn.r_i()]._as_usize()]);
229 g2 = _mm_broadcast_ss(&self.profile.linear[src1[src_cn.g_i()]._as_usize()]);
230 b2 = _mm_broadcast_ss(&self.profile.linear[src1[src_cn.b_i()]._as_usize()]);
231
232 r3 = _mm_broadcast_ss(
233 &self.profile.linear[src1[src_cn.r_i() + src_channels]._as_usize()],
234 );
235 g3 = _mm_broadcast_ss(
236 &self.profile.linear[src1[src_cn.g_i() + src_channels]._as_usize()],
237 );
238 b3 = _mm_broadcast_ss(
239 &self.profile.linear[src1[src_cn.b_i() + src_channels]._as_usize()],
240 );
241
242 dst0[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
243 dst0[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
244 dst0[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
245 if dst_channels == 4 {
246 dst0[dst_cn.a_i()] = a0;
247 }
248
249 dst0[dst_cn.r_i() + dst_channels] =
250 self.profile.gamma[temporary0.0[8] as usize];
251 dst0[dst_cn.g_i() + dst_channels] =
252 self.profile.gamma[temporary0.0[10] as usize];
253 dst0[dst_cn.b_i() + dst_channels] =
254 self.profile.gamma[temporary0.0[12] as usize];
255 if dst_channels == 4 {
256 dst0[dst_cn.a_i() + dst_channels] = a1;
257 }
258
259 dst1[dst_cn.r_i()] = self.profile.gamma[temporary1.0[0] as usize];
260 dst1[dst_cn.g_i()] = self.profile.gamma[temporary1.0[2] as usize];
261 dst1[dst_cn.b_i()] = self.profile.gamma[temporary1.0[4] as usize];
262 if dst_channels == 4 {
263 dst1[dst_cn.a_i()] = a2;
264 }
265
266 dst1[dst_cn.r_i() + dst_channels] =
267 self.profile.gamma[temporary1.0[8] as usize];
268 dst1[dst_cn.g_i() + dst_channels] =
269 self.profile.gamma[temporary1.0[10] as usize];
270 dst1[dst_cn.b_i() + dst_channels] =
271 self.profile.gamma[temporary1.0[12] as usize];
272 if dst_channels == 4 {
273 dst1[dst_cn.a_i() + dst_channels] = a3;
274 }
275
276 a0 = if src_channels == 4 {
277 src[src_cn.a_i()]
278 } else {
279 max_colors
280 };
281 a1 = if src_channels == 4 {
282 src[src_cn.a_i() + src_channels]
283 } else {
284 max_colors
285 };
286
287 a2 = if src_channels == 4 {
288 src1[src_cn.a_i()]
289 } else {
290 max_colors
291 };
292 a3 = if src_channels == 4 {
293 src1[src_cn.a_i() + src_channels]
294 } else {
295 max_colors
296 };
297 }
298
299 if let (Some(dst0), Some(dst1)) = (
300 dst0.chunks_exact_mut(dst_channels * 2).last(),
301 dst1.chunks_exact_mut(dst_channels * 2).last(),
302 ) {
303 let rz0 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(r0), r1);
304 let gz0 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(g0), g1);
305 let bz0 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(b0), b1);
306
307 let rz1 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(r2), r3);
308 let gz1 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(g2), g3);
309 let bz1 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(b2), b3);
310
311 let v0 = _mm256_mul_ps(rz0, m0);
312 let v1 = _mm256_fmadd_ps(gz0, m1, v0);
313 let mut vz0 = _mm256_fmadd_ps(bz0, m2, v1);
314
315 let v2 = _mm256_mul_ps(rz1, m0);
316 let v3 = _mm256_fmadd_ps(gz1, m1, v2);
317 let mut vz1 = _mm256_fmadd_ps(bz1, m2, v3);
318
319 vz0 = _mm256_max_ps(vz0, _mm256_setzero_ps());
320 vz0 = _mm256_mul_ps(vz0, v_scale);
321 vz0 = _mm256_min_ps(vz0, v_scale);
322
323 vz1 = _mm256_max_ps(vz1, _mm256_setzero_ps());
324 vz1 = _mm256_mul_ps(vz1, v_scale);
325 vz1 = _mm256_min_ps(vz1, v_scale);
326
327 let zx0 = _mm256_cvtps_epi32(vz0);
328 let zx1 = _mm256_cvtps_epi32(vz1);
329 _mm256_store_si256(temporary0.0.as_mut_ptr() as *mut _, zx0);
330 _mm256_store_si256(temporary1.0.as_mut_ptr() as *mut _, zx1);
331
332 dst0[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
333 dst0[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
334 dst0[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
335 if dst_channels == 4 {
336 dst0[dst_cn.a_i()] = a0;
337 }
338
339 dst0[dst_cn.r_i() + dst_channels] =
340 self.profile.gamma[temporary0.0[8] as usize];
341 dst0[dst_cn.g_i() + dst_channels] =
342 self.profile.gamma[temporary0.0[10] as usize];
343 dst0[dst_cn.b_i() + dst_channels] =
344 self.profile.gamma[temporary0.0[12] as usize];
345 if dst_channels == 4 {
346 dst0[dst_cn.a_i() + dst_channels] = a1;
347 }
348
349 dst1[dst_cn.r_i()] = self.profile.gamma[temporary1.0[0] as usize];
350 dst1[dst_cn.g_i()] = self.profile.gamma[temporary1.0[2] as usize];
351 dst1[dst_cn.b_i()] = self.profile.gamma[temporary1.0[4] as usize];
352 if dst_channels == 4 {
353 dst1[dst_cn.a_i()] = a2;
354 }
355
356 dst1[dst_cn.r_i() + dst_channels] =
357 self.profile.gamma[temporary1.0[8] as usize];
358 dst1[dst_cn.g_i() + dst_channels] =
359 self.profile.gamma[temporary1.0[10] as usize];
360 dst1[dst_cn.b_i() + dst_channels] =
361 self.profile.gamma[temporary1.0[12] as usize];
362 if dst_channels == 4 {
363 dst1[dst_cn.a_i() + dst_channels] = a3;
364 }
365 }
366 }
367
368 for (src, dst) in src_remainder
369 .chunks_exact(src_channels)
370 .zip(dst_remainder.chunks_exact_mut(dst_channels))
371 {
372 let r = _mm_broadcast_ss(&self.profile.linear[src[src_cn.r_i()]._as_usize()]);
373 let g = _mm_broadcast_ss(&self.profile.linear[src[src_cn.g_i()]._as_usize()]);
374 let b = _mm_broadcast_ss(&self.profile.linear[src[src_cn.b_i()]._as_usize()]);
375 let a = if src_channels == 4 {
376 src[src_cn.a_i()]
377 } else {
378 max_colors
379 };
380
381 let v0 = _mm_mul_ps(r, _mm256_castps256_ps128(m0));
382 let v1 = _mm_fmadd_ps(g, _mm256_castps256_ps128(m1), v0);
383 let mut v = _mm_fmadd_ps(b, _mm256_castps256_ps128(m2), v1);
384
385 v = _mm_max_ps(v, zeros);
386 v = _mm_mul_ps(v, _mm256_castps256_ps128(v_scale));
387 v = _mm_min_ps(v, _mm256_castps256_ps128(v_scale));
388
389 let zx = _mm_cvtps_epi32(v);
390 _mm_store_si128(temporary0.0.as_mut_ptr() as *mut _, zx);
391
392 dst[dst_cn.r_i()] = self.profile.gamma[temporary0.0[0] as usize];
393 dst[dst_cn.g_i()] = self.profile.gamma[temporary0.0[2] as usize];
394 dst[dst_cn.b_i()] = self.profile.gamma[temporary0.0[4] as usize];
395 if dst_channels == 4 {
396 dst[dst_cn.a_i()] = a;
397 }
398 }
399 }
400
401 Ok(())
402 }
403 }
404
405 impl<
406 T: Clone + Copy + 'static + PointeeSizeExpressible + Default,
407 const SRC_LAYOUT: u8,
408 const DST_LAYOUT: u8,
409 const LINEAR_CAP: usize,
410 > TransformExecutor<T> for TransformShaperRgbOptAvx512<T, SRC_LAYOUT, DST_LAYOUT, LINEAR_CAP>
411 where
412 u32: AsPrimitive<T>,
413 {
414 fn transform(&self, src: &[T], dst: &mut [T]) -> Result<(), CmsError> {
415 unsafe { self.transform_impl(src, dst) }
416 }
417 }