Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/neon/interpolator_q0_15.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 use crate::conversions::interpolator::BarycentricWeight;
30 use crate::math::FusedMultiplyAdd;
31 use std::arch::aarch64::*;
32 use std::ops::{Add, Mul, Sub};
33
34 #[repr(align(8), C)]
35 pub(crate) struct NeonAlignedI16x4(pub(crate) [i16; 4]);
36
37 #[cfg(feature = "options")]
38 pub(crate) struct TetrahedralNeonQ0_15<const GRID_SIZE: usize> {}
39
40 #[cfg(feature = "options")]
41 pub(crate) struct PyramidalNeonQ0_15<const GRID_SIZE: usize> {}
42
43 pub(crate) struct TrilinearNeonQ0_15<const GRID_SIZE: usize> {}
44
45 #[cfg(feature = "options")]
46 pub(crate) struct PrismaticNeonQ0_15<const GRID_SIZE: usize> {}
47
48 #[cfg(feature = "options")]
49 pub(crate) struct PyramidalNeonQ0_15Double<const GRID_SIZE: usize> {}
50
51 #[cfg(feature = "options")]
52 pub(crate) struct PrismaticNeonQ0_15Double<const GRID_SIZE: usize> {}
53
54 pub(crate) struct TrilinearNeonQ0_15Double<const GRID_SIZE: usize> {}
55
56 #[cfg(feature = "options")]
57 pub(crate) struct TetrahedralNeonQ0_15Double<const GRID_SIZE: usize> {}
58
59 trait Fetcher<T> {
60 fn fetch(&self, x: i32, y: i32, z: i32) -> T;
61 }
62
63 struct TetrahedralNeonQ0_15FetchVector<'a, const GRID_SIZE: usize> {
64 cube: &'a [NeonAlignedI16x4],
65 }
66
67 struct TetrahedralNeonQ0_15FetchVectorDouble<'a, const GRID_SIZE: usize> {
68 cube0: &'a [NeonAlignedI16x4],
69 cube1: &'a [NeonAlignedI16x4],
70 }
71
72 #[derive(Copy, Clone)]
73 pub(crate) struct NeonVectorQ0_15 {
74 pub(crate) v: int16x4_t,
75 }
76
77 #[derive(Copy, Clone)]
78 pub(crate) struct NeonVectorQ0_15Double {
79 pub(crate) v: int16x8_t,
80 }
81
82 impl From<i16> for NeonVectorQ0_15 {
83 #[inline(always)]
84 fn from(v: i16) -> Self {
85 NeonVectorQ0_15 {
86 v: unsafe { vdup_n_s16(v) },
87 }
88 }
89 }
90
91 impl From<i16> for NeonVectorQ0_15Double {
92 #[inline(always)]
93 fn from(v: i16) -> Self {
94 NeonVectorQ0_15Double {
95 v: unsafe { vdupq_n_s16(v) },
96 }
97 }
98 }
99
100 impl Sub<NeonVectorQ0_15> for NeonVectorQ0_15 {
101 type Output = Self;
102 #[inline(always)]
103 fn sub(self, rhs: NeonVectorQ0_15) -> Self::Output {
104 NeonVectorQ0_15 {
105 v: unsafe { vsub_s16(self.v, rhs.v) },
106 }
107 }
108 }
109
110 impl Mul<NeonVectorQ0_15> for NeonVectorQ0_15 {
111 type Output = Self;
112 #[inline(always)]
113 fn mul(self, rhs: NeonVectorQ0_15) -> Self::Output {
114 NeonVectorQ0_15 {
115 v: unsafe { vqrdmulh_s16(self.v, rhs.v) },
116 }
117 }
118 }
119
120 impl Sub<NeonVectorQ0_15Double> for NeonVectorQ0_15Double {
121 type Output = Self;
122 #[inline(always)]
123 fn sub(self, rhs: NeonVectorQ0_15Double) -> Self::Output {
124 NeonVectorQ0_15Double {
125 v: unsafe { vsubq_s16(self.v, rhs.v) },
126 }
127 }
128 }
129
130 impl Mul<NeonVectorQ0_15Double> for NeonVectorQ0_15Double {
131 type Output = Self;
132 #[inline(always)]
133 fn mul(self, rhs: NeonVectorQ0_15Double) -> Self::Output {
134 NeonVectorQ0_15Double {
135 v: unsafe { vqrdmulhq_s16(self.v, rhs.v) },
136 }
137 }
138 }
139
140 impl Add<NeonVectorQ0_15> for NeonVectorQ0_15 {
141 type Output = Self;
142 #[inline(always)]
143 fn add(self, rhs: NeonVectorQ0_15) -> Self::Output {
144 NeonVectorQ0_15 {
145 v: unsafe { vadd_s16(self.v, rhs.v) },
146 }
147 }
148 }
149
150 impl Add<NeonVectorQ0_15Double> for NeonVectorQ0_15Double {
151 type Output = Self;
152 #[inline(always)]
153 fn add(self, rhs: NeonVectorQ0_15Double) -> Self::Output {
154 NeonVectorQ0_15Double {
155 v: unsafe { vaddq_s16(self.v, rhs.v) },
156 }
157 }
158 }
159
160 impl FusedMultiplyAdd<NeonVectorQ0_15> for NeonVectorQ0_15 {
161 #[inline(always)]
162 fn mla(&self, b: NeonVectorQ0_15, c: NeonVectorQ0_15) -> NeonVectorQ0_15 {
163 NeonVectorQ0_15 {
164 v: unsafe { vqrdmlah_s16(self.v, b.v, c.v) },
165 }
166 }
167 }
168
169 impl NeonVectorQ0_15 {
170 #[inline(always)]
171 fn neg_mla(&self, b: NeonVectorQ0_15, c: NeonVectorQ0_15) -> NeonVectorQ0_15 {
172 NeonVectorQ0_15 {
173 v: unsafe { vqrdmlsh_s16(self.v, b.v, c.v) },
174 }
175 }
176 }
177
178 impl NeonVectorQ0_15Double {
179 #[inline(always)]
180 fn neg_mla(&self, b: NeonVectorQ0_15Double, c: NeonVectorQ0_15Double) -> NeonVectorQ0_15Double {
181 NeonVectorQ0_15Double {
182 v: unsafe { vqrdmlshq_s16(self.v, b.v, c.v) },
183 }
184 }
185 }
186
187 impl NeonVectorQ0_15Double {
188 #[inline(always)]
189 fn mla(&self, b: NeonVectorQ0_15Double, c: NeonVectorQ0_15) -> NeonVectorQ0_15Double {
190 NeonVectorQ0_15Double {
191 v: unsafe { vqrdmlahq_s16(self.v, b.v, vcombine_s16(c.v, c.v)) },
192 }
193 }
194
195 #[inline(always)]
196 pub(crate) fn split(self) -> (NeonVectorQ0_15, NeonVectorQ0_15) {
197 unsafe {
198 (
199 NeonVectorQ0_15 {
200 v: vget_low_s16(self.v),
201 },
202 NeonVectorQ0_15 {
203 v: vget_high_s16(self.v),
204 },
205 )
206 }
207 }
208 }
209
210 impl<const GRID_SIZE: usize> Fetcher<NeonVectorQ0_15>
211 for TetrahedralNeonQ0_15FetchVector<'_, GRID_SIZE>
212 {
213 fn fetch(&self, x: i32, y: i32, z: i32) -> NeonVectorQ0_15 {
214 let offset = (x as u32 * (GRID_SIZE as u32 * GRID_SIZE as u32)
215 + y as u32 * GRID_SIZE as u32
216 + z as u32) as usize;
217 let jx = unsafe { self.cube.get_unchecked(offset..) };
218 NeonVectorQ0_15 {
219 v: unsafe { vld1_s16(jx.as_ptr() as *const i16) },
220 }
221 }
222 }
223
224 impl<const GRID_SIZE: usize> Fetcher<NeonVectorQ0_15Double>
225 for TetrahedralNeonQ0_15FetchVectorDouble<'_, GRID_SIZE>
226 {
227 fn fetch(&self, x: i32, y: i32, z: i32) -> NeonVectorQ0_15Double {
228 let offset = (x as u32 * (GRID_SIZE as u32 * GRID_SIZE as u32)
229 + y as u32 * GRID_SIZE as u32
230 + z as u32) as usize;
231 let jx0 = unsafe { self.cube0.get_unchecked(offset..) };
232 let jx1 = unsafe { self.cube1.get_unchecked(offset..) };
233 NeonVectorQ0_15Double {
234 v: unsafe {
235 vcombine_s16(
236 vld1_s16(jx0.as_ptr() as *const i16),
237 vld1_s16(jx1.as_ptr() as *const i16),
238 )
239 },
240 }
241 }
242 }
243
244 pub(crate) trait NeonMdInterpolationQ0_15 {
245 fn inter3_neon(
246 &self,
247 cube: &[NeonAlignedI16x4],
248 in_r: usize,
249 in_g: usize,
250 in_b: usize,
251 lut: &[BarycentricWeight<i16>],
252 ) -> NeonVectorQ0_15;
253 }
254
255 pub(crate) trait NeonMdInterpolationQ0_15Double {
256 fn inter3_neon(
257 &self,
258 table0: &[NeonAlignedI16x4],
259 table1: &[NeonAlignedI16x4],
260 in_r: usize,
261 in_g: usize,
262 in_b: usize,
263 lut: &[BarycentricWeight<i16>],
264 ) -> (NeonVectorQ0_15, NeonVectorQ0_15);
265 }
266
267 #[cfg(feature = "options")]
268 impl<const GRID_SIZE: usize> TetrahedralNeonQ0_15<GRID_SIZE> {
269 #[target_feature(enable = "rdm")]
270 unsafe fn interpolate(
271 &self,
272 in_r: usize,
273 in_g: usize,
274 in_b: usize,
275 lut: &[BarycentricWeight<i16>],
276 r: impl Fetcher<NeonVectorQ0_15>,
277 ) -> NeonVectorQ0_15 {
278 let lut_r = unsafe { *lut.get_unchecked(in_r) };
279 let lut_g = unsafe { *lut.get_unchecked(in_g) };
280 let lut_b = unsafe { *lut.get_unchecked(in_b) };
281
282 let x: i32 = lut_r.x;
283 let y: i32 = lut_g.x;
284 let z: i32 = lut_b.x;
285
286 let x_n: i32 = lut_r.x_n;
287 let y_n: i32 = lut_g.x_n;
288 let z_n: i32 = lut_b.x_n;
289
290 let rx = lut_r.w;
291 let ry = lut_g.w;
292 let rz = lut_b.w;
293
294 let c0 = r.fetch(x, y, z);
295
296 let c2;
297 let c1;
298 let c3;
299 if rx >= ry {
300 if ry >= rz {
301 //rx >= ry && ry >= rz
302 c1 = r.fetch(x_n, y, z) - c0;
303 c2 = r.fetch(x_n, y_n, z) - r.fetch(x_n, y, z);
304 c3 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y_n, z);
305 } else if rx >= rz {
306 //rx >= rz && rz >= ry
307 c1 = r.fetch(x_n, y, z) - c0;
308 c2 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y, z_n);
309 c3 = r.fetch(x_n, y, z_n) - r.fetch(x_n, y, z);
310 } else {
311 //rz > rx && rx >= ry
312 c1 = r.fetch(x_n, y, z_n) - r.fetch(x, y, z_n);
313 c2 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y, z_n);
314 c3 = r.fetch(x, y, z_n) - c0;
315 }
316 } else if rx >= rz {
317 //ry > rx && rx >= rz
318 c1 = r.fetch(x_n, y_n, z) - r.fetch(x, y_n, z);
319 c2 = r.fetch(x, y_n, z) - c0;
320 c3 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y_n, z);
321 } else if ry >= rz {
322 //ry >= rz && rz > rx
323 c1 = r.fetch(x_n, y_n, z_n) - r.fetch(x, y_n, z_n);
324 c2 = r.fetch(x, y_n, z) - c0;
325 c3 = r.fetch(x, y_n, z_n) - r.fetch(x, y_n, z);
326 } else {
327 //rz > ry && ry > rx
328 c1 = r.fetch(x_n, y_n, z_n) - r.fetch(x, y_n, z_n);
329 c2 = r.fetch(x, y_n, z_n) - r.fetch(x, y, z_n);
330 c3 = r.fetch(x, y, z_n) - c0;
331 }
332 let s0 = c0.mla(c1, NeonVectorQ0_15::from(rx));
333 let s1 = s0.mla(c2, NeonVectorQ0_15::from(ry));
334 s1.mla(c3, NeonVectorQ0_15::from(rz))
335 }
336 }
337
338 #[cfg(feature = "options")]
339 impl<const GRID_SIZE: usize> TetrahedralNeonQ0_15Double<GRID_SIZE> {
340 #[target_feature(enable = "rdm")]
341 unsafe fn interpolate(
342 &self,
343 in_r: usize,
344 in_g: usize,
345 in_b: usize,
346 lut: &[BarycentricWeight<i16>],
347 r: impl Fetcher<NeonVectorQ0_15Double>,
348 ) -> (NeonVectorQ0_15, NeonVectorQ0_15) {
349 let lut_r = unsafe { *lut.get_unchecked(in_r) };
350 let lut_g = unsafe { *lut.get_unchecked(in_g) };
351 let lut_b = unsafe { *lut.get_unchecked(in_b) };
352
353 let x: i32 = lut_r.x;
354 let y: i32 = lut_g.x;
355 let z: i32 = lut_b.x;
356
357 let x_n: i32 = lut_r.x_n;
358 let y_n: i32 = lut_g.x_n;
359 let z_n: i32 = lut_b.x_n;
360
361 let rx = lut_r.w;
362 let ry = lut_g.w;
363 let rz = lut_b.w;
364
365 let c0 = r.fetch(x, y, z);
366
367 let c2;
368 let c1;
369 let c3;
370 if rx >= ry {
371 if ry >= rz {
372 //rx >= ry && ry >= rz
373 c1 = r.fetch(x_n, y, z) - c0;
374 c2 = r.fetch(x_n, y_n, z) - r.fetch(x_n, y, z);
375 c3 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y_n, z);
376 } else if rx >= rz {
377 //rx >= rz && rz >= ry
378 c1 = r.fetch(x_n, y, z) - c0;
379 c2 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y, z_n);
380 c3 = r.fetch(x_n, y, z_n) - r.fetch(x_n, y, z);
381 } else {
382 //rz > rx && rx >= ry
383 c1 = r.fetch(x_n, y, z_n) - r.fetch(x, y, z_n);
384 c2 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y, z_n);
385 c3 = r.fetch(x, y, z_n) - c0;
386 }
387 } else if rx >= rz {
388 //ry > rx && rx >= rz
389 c1 = r.fetch(x_n, y_n, z) - r.fetch(x, y_n, z);
390 c2 = r.fetch(x, y_n, z) - c0;
391 c3 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y_n, z);
392 } else if ry >= rz {
393 //ry >= rz && rz > rx
394 c1 = r.fetch(x_n, y_n, z_n) - r.fetch(x, y_n, z_n);
395 c2 = r.fetch(x, y_n, z) - c0;
396 c3 = r.fetch(x, y_n, z_n) - r.fetch(x, y_n, z);
397 } else {
398 //rz > ry && ry > rx
399 c1 = r.fetch(x_n, y_n, z_n) - r.fetch(x, y_n, z_n);
400 c2 = r.fetch(x, y_n, z_n) - r.fetch(x, y, z_n);
401 c3 = r.fetch(x, y, z_n) - c0;
402 }
403 let s0 = c0.mla(c1, NeonVectorQ0_15::from(rx));
404 let s1 = s0.mla(c2, NeonVectorQ0_15::from(ry));
405 s1.mla(c3, NeonVectorQ0_15::from(rz)).split()
406 }
407 }
408
409 macro_rules! define_md_inter_neon {
410 ($interpolator: ident) => {
411 impl<const GRID_SIZE: usize> NeonMdInterpolationQ0_15 for $interpolator<GRID_SIZE> {
412 fn inter3_neon(
413 &self,
414 cube: &[NeonAlignedI16x4],
415 in_r: usize,
416 in_g: usize,
417 in_b: usize,
418 lut: &[BarycentricWeight<i16>],
419 ) -> NeonVectorQ0_15 {
420 unsafe {
421 self.interpolate(
422 in_r,
423 in_g,
424 in_b,
425 lut,
426 TetrahedralNeonQ0_15FetchVector::<GRID_SIZE> { cube },
427 )
428 }
429 }
430 }
431 };
432 }
433
434 macro_rules! define_md_inter_neon_d {
435 ($interpolator: ident) => {
436 impl<const GRID_SIZE: usize> NeonMdInterpolationQ0_15Double for $interpolator<GRID_SIZE> {
437 fn inter3_neon(
438 &self,
439 table0: &[NeonAlignedI16x4],
440 table1: &[NeonAlignedI16x4],
441 in_r: usize,
442 in_g: usize,
443 in_b: usize,
444 lut: &[BarycentricWeight<i16>],
445 ) -> (NeonVectorQ0_15, NeonVectorQ0_15) {
446 unsafe {
447 self.interpolate(
448 in_r,
449 in_g,
450 in_b,
451 lut,
452 TetrahedralNeonQ0_15FetchVectorDouble::<GRID_SIZE> {
453 cube0: table0,
454 cube1: table1,
455 },
456 )
457 }
458 }
459 }
460 };
461 }
462
463 #[cfg(feature = "options")]
464 define_md_inter_neon!(TetrahedralNeonQ0_15);
465 #[cfg(feature = "options")]
466 define_md_inter_neon!(PyramidalNeonQ0_15);
467 #[cfg(feature = "options")]
468 define_md_inter_neon!(PrismaticNeonQ0_15);
469 define_md_inter_neon!(TrilinearNeonQ0_15);
470 #[cfg(feature = "options")]
471 define_md_inter_neon_d!(PrismaticNeonQ0_15Double);
472 #[cfg(feature = "options")]
473 define_md_inter_neon_d!(PyramidalNeonQ0_15Double);
474 #[cfg(feature = "options")]
475 define_md_inter_neon_d!(TetrahedralNeonQ0_15Double);
476 define_md_inter_neon_d!(TrilinearNeonQ0_15Double);
477
478 #[cfg(feature = "options")]
479 impl<const GRID_SIZE: usize> PyramidalNeonQ0_15<GRID_SIZE> {
480 #[target_feature(enable = "rdm")]
481 unsafe fn interpolate(
482 &self,
483 in_r: usize,
484 in_g: usize,
485 in_b: usize,
486 lut: &[BarycentricWeight<i16>],
487 r: impl Fetcher<NeonVectorQ0_15>,
488 ) -> NeonVectorQ0_15 {
489 let lut_r = unsafe { *lut.get_unchecked(in_r) };
490 let lut_g = unsafe { *lut.get_unchecked(in_g) };
491 let lut_b = unsafe { *lut.get_unchecked(in_b) };
492
493 let x: i32 = lut_r.x;
494 let y: i32 = lut_g.x;
495 let z: i32 = lut_b.x;
496
497 let x_n: i32 = lut_r.x_n;
498 let y_n: i32 = lut_g.x_n;
499 let z_n: i32 = lut_b.x_n;
500
501 let dr = lut_r.w;
502 let dg = lut_g.w;
503 let db = lut_b.w;
504
505 let c0 = r.fetch(x, y, z);
506
507 let w0 = NeonVectorQ0_15::from(db);
508 let w1 = NeonVectorQ0_15::from(dr);
509 let w2 = NeonVectorQ0_15::from(dg);
510
511 if dr > db && dg > db {
512 let x0 = r.fetch(x_n, y_n, z_n);
513 let x1 = r.fetch(x_n, y_n, z);
514 let x2 = r.fetch(x_n, y, z);
515 let x3 = r.fetch(x, y_n, z);
516
517 let w3 = w1 * w2;
518
519 let c1 = x0 - x1;
520 let c2 = x2 - c0;
521 let c3 = x3 - c0;
522 let c4 = c0 - x3 - x2 + x1;
523
524 let s0 = c0.mla(c1, w0);
525 let s1 = s0.mla(c2, w1);
526 let s2 = s1.mla(c3, w2);
527 s2.mla(c4, w3)
528 } else if db > dr && dg > dr {
529 let x0 = r.fetch(x, y, z_n);
530 let x1 = r.fetch(x_n, y_n, z_n);
531 let x2 = r.fetch(x, y_n, z_n);
532 let x3 = r.fetch(x, y_n, z);
533
534 let w3 = w2 * w0;
535
536 let c1 = x0 - c0;
537 let c2 = x1 - x2;
538 let c3 = x3 - c0;
539 let c4 = c0 - x3 - x0 + x2;
540
541 let s0 = c0.mla(c1, w0);
542 let s1 = s0.mla(c2, w1);
543 let s2 = s1.mla(c3, w2);
544 s2.mla(c4, w3)
545 } else {
546 let x0 = r.fetch(x, y, z_n);
547 let x1 = r.fetch(x_n, y, z);
548 let x2 = r.fetch(x_n, y, z_n);
549 let x3 = r.fetch(x_n, y_n, z_n);
550
551 let w3 = w0 * w1;
552
553 let c1 = x0 - c0;
554 let c2 = x1 - c0;
555 let c3 = x3 - x2;
556 let c4 = c0 - x1 - x0 + x2;
557
558 let s0 = c0.mla(c1, w0);
559 let s1 = s0.mla(c2, w1);
560 let s2 = s1.mla(c3, w2);
561 s2.mla(c4, w3)
562 }
563 }
564 }
565
566 #[cfg(feature = "options")]
567 impl<const GRID_SIZE: usize> PyramidalNeonQ0_15Double<GRID_SIZE> {
568 #[target_feature(enable = "rdm")]
569 unsafe fn interpolate(
570 &self,
571 in_r: usize,
572 in_g: usize,
573 in_b: usize,
574 lut: &[BarycentricWeight<i16>],
575 r: impl Fetcher<NeonVectorQ0_15Double>,
576 ) -> (NeonVectorQ0_15, NeonVectorQ0_15) {
577 let lut_r = unsafe { *lut.get_unchecked(in_r) };
578 let lut_g = unsafe { *lut.get_unchecked(in_g) };
579 let lut_b = unsafe { *lut.get_unchecked(in_b) };
580
581 let x: i32 = lut_r.x;
582 let y: i32 = lut_g.x;
583 let z: i32 = lut_b.x;
584
585 let x_n: i32 = lut_r.x_n;
586 let y_n: i32 = lut_g.x_n;
587 let z_n: i32 = lut_b.x_n;
588
589 let dr = lut_r.w;
590 let dg = lut_g.w;
591 let db = lut_b.w;
592
593 let c0 = r.fetch(x, y, z);
594
595 let w0 = NeonVectorQ0_15::from(db);
596 let w1 = NeonVectorQ0_15::from(dr);
597 let w2 = NeonVectorQ0_15::from(dg);
598
599 if dr > db && dg > db {
600 let w3 = NeonVectorQ0_15::from(dr) * NeonVectorQ0_15::from(dg);
601 let x0 = r.fetch(x_n, y_n, z_n);
602 let x1 = r.fetch(x_n, y_n, z);
603 let x2 = r.fetch(x_n, y, z);
604 let x3 = r.fetch(x, y_n, z);
605
606 let c1 = x0 - x1;
607 let c2 = x2 - c0;
608 let c3 = x3 - c0;
609 let c4 = c0 - x3 - x2 + x1;
610
611 let s0 = c0.mla(c1, w0);
612 let s1 = s0.mla(c2, w1);
613 let s2 = s1.mla(c3, w2);
614 s2.mla(c4, w3).split()
615 } else if db > dr && dg > dr {
616 let w3 = NeonVectorQ0_15::from(dg) * NeonVectorQ0_15::from(db);
617 let x0 = r.fetch(x, y, z_n);
618 let x1 = r.fetch(x_n, y_n, z_n);
619 let x2 = r.fetch(x, y_n, z_n);
620 let x3 = r.fetch(x, y_n, z);
621
622 let c1 = x0 - c0;
623 let c2 = x1 - x2;
624 let c3 = x3 - c0;
625 let c4 = c0 - x3 - x0 + x2;
626
627 let s0 = c0.mla(c1, w0);
628 let s1 = s0.mla(c2, w1);
629 let s2 = s1.mla(c3, w2);
630 s2.mla(c4, w3).split()
631 } else {
632 let w3 = NeonVectorQ0_15::from(db) * NeonVectorQ0_15::from(dr);
633 let x0 = r.fetch(x, y, z_n);
634 let x1 = r.fetch(x_n, y, z);
635 let x2 = r.fetch(x_n, y, z_n);
636 let x3 = r.fetch(x_n, y_n, z_n);
637
638 let c1 = x0 - c0;
639 let c2 = x1 - c0;
640 let c3 = x3 - x2;
641 let c4 = c0 - x1 - x0 + x2;
642
643 let s0 = c0.mla(c1, w0);
644 let s1 = s0.mla(c2, w1);
645 let s2 = s1.mla(c3, w2);
646 s2.mla(c4, w3).split()
647 }
648 }
649 }
650
651 #[cfg(feature = "options")]
652 impl<const GRID_SIZE: usize> PrismaticNeonQ0_15<GRID_SIZE> {
653 #[target_feature(enable = "rdm")]
654 unsafe fn interpolate(
655 &self,
656 in_r: usize,
657 in_g: usize,
658 in_b: usize,
659 lut: &[BarycentricWeight<i16>],
660 r: impl Fetcher<NeonVectorQ0_15>,
661 ) -> NeonVectorQ0_15 {
662 let lut_r = unsafe { *lut.get_unchecked(in_r) };
663 let lut_g = unsafe { *lut.get_unchecked(in_g) };
664 let lut_b = unsafe { *lut.get_unchecked(in_b) };
665
666 let x: i32 = lut_r.x;
667 let y: i32 = lut_g.x;
668 let z: i32 = lut_b.x;
669
670 let x_n: i32 = lut_r.x_n;
671 let y_n: i32 = lut_g.x_n;
672 let z_n: i32 = lut_b.x_n;
673
674 let dr = lut_r.w;
675 let dg = lut_g.w;
676 let db = lut_b.w;
677
678 let c0 = r.fetch(x, y, z);
679
680 let w0 = NeonVectorQ0_15::from(db);
681 let w1 = NeonVectorQ0_15::from(dr);
682 let w2 = NeonVectorQ0_15::from(dg);
683
684 if db > dr {
685 let w3 = w2 * w0;
686 let w4 = w1 * w2;
687 let x0 = r.fetch(x, y, z_n);
688 let x1 = r.fetch(x_n, y, z_n);
689 let x2 = r.fetch(x, y_n, z);
690 let x3 = r.fetch(x, y_n, z_n);
691 let x4 = r.fetch(x_n, y_n, z_n);
692
693 let c1 = x0 - c0;
694 let c2 = x1 - x0;
695 let c3 = x2 - c0;
696 let c4 = c0 - x2 - x0 + x3;
697 let c5 = x0 - x3 - x1 + x4;
698
699 let s0 = c0.mla(c1, w0);
700 let s1 = s0.mla(c2, w1);
701 let s2 = s1.mla(c3, w2);
702 let s3 = s2.mla(c4, w3);
703 s3.mla(c5, w4)
704 } else {
705 let w3 = w2 * w0;
706 let w4 = w1 * w2;
707 let x0 = r.fetch(x_n, y, z);
708 let x1 = r.fetch(x_n, y, z_n);
709 let x2 = r.fetch(x, y_n, z);
710 let x3 = r.fetch(x_n, y_n, z);
711 let x4 = r.fetch(x_n, y_n, z_n);
712
713 let c1 = x1 - x0;
714 let c2 = x0 - c0;
715 let c3 = x2 - c0;
716 let c4 = x0 - x3 - x1 + x4;
717 let c5 = c0 - x2 - x0 + x3;
718
719 let s0 = c0.mla(c1, w0);
720 let s1 = s0.mla(c2, w1);
721 let s2 = s1.mla(c3, w2);
722 let s3 = s2.mla(c4, w3);
723 s3.mla(c5, w4)
724 }
725 }
726 }
727
728 #[cfg(feature = "options")]
729 impl<const GRID_SIZE: usize> PrismaticNeonQ0_15Double<GRID_SIZE> {
730 #[target_feature(enable = "rdm")]
731 unsafe fn interpolate(
732 &self,
733 in_r: usize,
734 in_g: usize,
735 in_b: usize,
736 lut: &[BarycentricWeight<i16>],
737 rv: impl Fetcher<NeonVectorQ0_15Double>,
738 ) -> (NeonVectorQ0_15, NeonVectorQ0_15) {
739 let lut_r = unsafe { *lut.get_unchecked(in_r) };
740 let lut_g = unsafe { *lut.get_unchecked(in_g) };
741 let lut_b = unsafe { *lut.get_unchecked(in_b) };
742
743 let x: i32 = lut_r.x;
744 let y: i32 = lut_g.x;
745 let z: i32 = lut_b.x;
746
747 let x_n: i32 = lut_r.x_n;
748 let y_n: i32 = lut_g.x_n;
749 let z_n: i32 = lut_b.x_n;
750
751 let dr = lut_r.w;
752 let dg = lut_g.w;
753 let db = lut_b.w;
754
755 let c0 = rv.fetch(x, y, z);
756
757 let w0 = NeonVectorQ0_15::from(db);
758 let w1 = NeonVectorQ0_15::from(dr);
759 let w2 = NeonVectorQ0_15::from(dg);
760 let w3 = NeonVectorQ0_15::from(dg) * NeonVectorQ0_15::from(db);
761 let w4 = NeonVectorQ0_15::from(dr) * NeonVectorQ0_15::from(dg);
762
763 if db > dr {
764 let x0 = rv.fetch(x, y, z_n);
765 let x1 = rv.fetch(x_n, y, z_n);
766 let x2 = rv.fetch(x, y_n, z);
767 let x3 = rv.fetch(x, y_n, z_n);
768 let x4 = rv.fetch(x_n, y_n, z_n);
769
770 let c1 = x0 - c0;
771 let c2 = x1 - x0;
772 let c3 = x2 - c0;
773 let c4 = c0 - x2 - x0 + x3;
774 let c5 = x0 - x3 - x1 + x4;
775
776 let s0 = c0.mla(c1, w0);
777 let s1 = s0.mla(c2, w1);
778 let s2 = s1.mla(c3, w2);
779 let s3 = s2.mla(c4, w3);
780 s3.mla(c5, w4).split()
781 } else {
782 let x0 = rv.fetch(x_n, y, z);
783 let x1 = rv.fetch(x_n, y, z_n);
784 let x2 = rv.fetch(x, y_n, z);
785 let x3 = rv.fetch(x_n, y_n, z);
786 let x4 = rv.fetch(x_n, y_n, z_n);
787
788 let c1 = x1 - x0;
789 let c2 = x0 - c0;
790 let c3 = x2 - c0;
791 let c4 = x0 - x3 - x1 + x4;
792 let c5 = c0 - x2 - x0 + x3;
793
794 let s0 = c0.mla(c1, w0);
795 let s1 = s0.mla(c2, w1);
796 let s2 = s1.mla(c3, w2);
797 let s3 = s2.mla(c4, w3);
798 s3.mla(c5, w4).split()
799 }
800 }
801 }
802
803 impl<const GRID_SIZE: usize> TrilinearNeonQ0_15Double<GRID_SIZE> {
804 #[target_feature(enable = "rdm")]
805 unsafe fn interpolate(
806 &self,
807 in_r: usize,
808 in_g: usize,
809 in_b: usize,
810 lut: &[BarycentricWeight<i16>],
811 r: impl Fetcher<NeonVectorQ0_15Double>,
812 ) -> (NeonVectorQ0_15, NeonVectorQ0_15) {
813 let lut_r = unsafe { *lut.get_unchecked(in_r) };
814 let lut_g = unsafe { *lut.get_unchecked(in_g) };
815 let lut_b = unsafe { *lut.get_unchecked(in_b) };
816
817 let x: i32 = lut_r.x;
818 let y: i32 = lut_g.x;
819 let z: i32 = lut_b.x;
820
821 let x_n: i32 = lut_r.x_n;
822 let y_n: i32 = lut_g.x_n;
823 let z_n: i32 = lut_b.x_n;
824
825 let dr = lut_r.w;
826 let dg = lut_g.w;
827 let db = lut_b.w;
828
829 let w0 = NeonVectorQ0_15::from(dr);
830 let w1 = NeonVectorQ0_15::from(dg);
831 let w2 = NeonVectorQ0_15::from(db);
832
833 let c000 = r.fetch(x, y, z);
834 let c100 = r.fetch(x_n, y, z);
835 let c010 = r.fetch(x, y_n, z);
836 let c110 = r.fetch(x_n, y_n, z);
837 let c001 = r.fetch(x, y, z_n);
838 let c101 = r.fetch(x_n, y, z_n);
839 let c011 = r.fetch(x, y_n, z_n);
840 let c111 = r.fetch(x_n, y_n, z_n);
841
842 let dx = NeonVectorQ0_15Double::from(dr);
843
844 let c00 = c000.neg_mla(c000, dx).mla(c100, w0);
845 let c10 = c010.neg_mla(c010, dx).mla(c110, w0);
846 let c01 = c001.neg_mla(c001, dx).mla(c101, w0);
847 let c11 = c011.neg_mla(c011, dx).mla(c111, w0);
848
849 let dy = NeonVectorQ0_15Double::from(dg);
850
851 let c0 = c00.neg_mla(c00, dy).mla(c10, w1);
852 let c1 = c01.neg_mla(c01, dy).mla(c11, w1);
853
854 let dz = NeonVectorQ0_15Double::from(db);
855
856 c0.neg_mla(c0, dz).mla(c1, w2).split()
857 }
858 }
859
860 impl<const GRID_SIZE: usize> TrilinearNeonQ0_15<GRID_SIZE> {
861 #[target_feature(enable = "rdm")]
862 unsafe fn interpolate(
863 &self,
864 in_r: usize,
865 in_g: usize,
866 in_b: usize,
867 lut: &[BarycentricWeight<i16>],
868 r: impl Fetcher<NeonVectorQ0_15>,
869 ) -> NeonVectorQ0_15 {
870 let lut_r = unsafe { *lut.get_unchecked(in_r) };
871 let lut_g = unsafe { *lut.get_unchecked(in_g) };
872 let lut_b = unsafe { *lut.get_unchecked(in_b) };
873
874 let x: i32 = lut_r.x;
875 let y: i32 = lut_g.x;
876 let z: i32 = lut_b.x;
877
878 let x_n: i32 = lut_r.x_n;
879 let y_n: i32 = lut_g.x_n;
880 let z_n: i32 = lut_b.x_n;
881
882 let dr = lut_r.w;
883 let dg = lut_g.w;
884 let db = lut_b.w;
885
886 let w0 = NeonVectorQ0_15::from(dr);
887 let w1 = NeonVectorQ0_15::from(dg);
888 let w2 = NeonVectorQ0_15::from(db);
889
890 let c000 = r.fetch(x, y, z);
891 let c100 = r.fetch(x_n, y, z);
892 let c010 = r.fetch(x, y_n, z);
893 let c110 = r.fetch(x_n, y_n, z);
894 let c001 = r.fetch(x, y, z_n);
895 let c101 = r.fetch(x_n, y, z_n);
896 let c011 = r.fetch(x, y_n, z_n);
897 let c111 = r.fetch(x_n, y_n, z_n);
898
899 let dx = NeonVectorQ0_15::from(dr);
900
901 let c00 = c000.neg_mla(c000, dx).mla(c100, w0);
902 let c10 = c010.neg_mla(c010, dx).mla(c110, w0);
903 let c01 = c001.neg_mla(c001, dx).mla(c101, w0);
904 let c11 = c011.neg_mla(c011, dx).mla(c111, w0);
905
906 let dy = NeonVectorQ0_15::from(dg);
907
908 let c0 = c00.neg_mla(c00, dy).mla(c10, w1);
909 let c1 = c01.neg_mla(c01, dy).mla(c11, w1);
910
911 let dz = NeonVectorQ0_15::from(db);
912
913 c0.neg_mla(c0, dz).mla(c1, w2)
914 }
915 }