Debian Codegraph @

.cargo/vendor/moxcms/src/conversions/neon/interpolator.rs

raw
1 /*
2 * // Copyright (c) Radzivon Bartoshyk 3/2025. All rights reserved.
3 * //
4 * // Redistribution and use in source and binary forms, with or without modification,
5 * // are permitted provided that the following conditions are met:
6 * //
7 * // 1. Redistributions of source code must retain the above copyright notice, this
8 * // list of conditions and the following disclaimer.
9 * //
10 * // 2. Redistributions in binary form must reproduce the above copyright notice,
11 * // this list of conditions and the following disclaimer in the documentation
12 * // and/or other materials provided with the distribution.
13 * //
14 * // 3. Neither the name of the copyright holder nor the names of its
15 * // contributors may be used to endorse or promote products derived from
16 * // this software without specific prior written permission.
17 * //
18 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29 #![allow(dead_code)]
30 use crate::conversions::interpolator::BarycentricWeight;
31 use crate::conversions::neon::rgb_xyz::NeonAlignedF32;
32 use crate::math::{FusedMultiplyAdd, FusedMultiplyNegAdd};
33 use std::arch::aarch64::*;
34 use std::ops::{Add, Mul, Sub};
35
36 pub(crate) struct TetrahedralNeon<const GRID_SIZE: usize> {}
37
38 pub(crate) struct PyramidalNeon<const GRID_SIZE: usize> {}
39
40 pub(crate) struct TrilinearNeon<const GRID_SIZE: usize> {}
41
42 pub(crate) struct PyramidalNeonDouble<const GRID_SIZE: usize> {}
43
44 pub(crate) struct PrismaticNeonDouble<const GRID_SIZE: usize> {}
45
46 pub(crate) struct TrilinearNeonDouble<const GRID_SIZE: usize> {}
47
48 pub(crate) struct TetrahedralNeonDouble<const GRID_SIZE: usize> {}
49
50 pub(crate) struct PrismaticNeon<const GRID_SIZE: usize> {}
51
52 trait Fetcher<T> {
53 fn fetch(&self, x: i32, y: i32, z: i32) -> T;
54 }
55
56 struct TetrahedralNeonFetchVector<'a, const GRID_SIZE: usize> {
57 cube: &'a [NeonAlignedF32],
58 }
59
60 struct TetrahedralNeonFetchVectorDouble<'a, const GRID_SIZE: usize> {
61 cube0: &'a [NeonAlignedF32],
62 cube1: &'a [NeonAlignedF32],
63 }
64
65 #[derive(Copy, Clone)]
66 pub(crate) struct NeonVector {
67 pub(crate) v: float32x4_t,
68 }
69
70 #[derive(Copy, Clone)]
71 pub(crate) struct NeonVectorDouble {
72 pub(crate) v0: float32x4_t,
73 pub(crate) v1: float32x4_t,
74 }
75
76 impl From<f32> for NeonVector {
77 #[inline(always)]
78 fn from(v: f32) -> Self {
79 NeonVector {
80 v: unsafe { vdupq_n_f32(v) },
81 }
82 }
83 }
84
85 impl From<f32> for NeonVectorDouble {
86 #[inline(always)]
87 fn from(v: f32) -> Self {
88 NeonVectorDouble {
89 v0: unsafe { vdupq_n_f32(v) },
90 v1: unsafe { vdupq_n_f32(v) },
91 }
92 }
93 }
94
95 impl Sub<NeonVector> for NeonVector {
96 type Output = Self;
97 #[inline(always)]
98 fn sub(self, rhs: NeonVector) -> Self::Output {
99 NeonVector {
100 v: unsafe { vsubq_f32(self.v, rhs.v) },
101 }
102 }
103 }
104
105 impl Mul<NeonVector> for NeonVector {
106 type Output = Self;
107 #[inline(always)]
108 fn mul(self, rhs: NeonVector) -> Self::Output {
109 NeonVector {
110 v: unsafe { vmulq_f32(self.v, rhs.v) },
111 }
112 }
113 }
114
115 impl Sub<NeonVectorDouble> for NeonVectorDouble {
116 type Output = Self;
117 #[inline(always)]
118 fn sub(self, rhs: NeonVectorDouble) -> Self::Output {
119 NeonVectorDouble {
120 v0: unsafe { vsubq_f32(self.v0, rhs.v0) },
121 v1: unsafe { vsubq_f32(self.v1, rhs.v1) },
122 }
123 }
124 }
125
126 impl Mul<NeonVectorDouble> for NeonVectorDouble {
127 type Output = Self;
128 #[inline(always)]
129 fn mul(self, rhs: NeonVectorDouble) -> Self::Output {
130 NeonVectorDouble {
131 v0: unsafe { vmulq_f32(self.v0, rhs.v0) },
132 v1: unsafe { vmulq_f32(self.v1, rhs.v1) },
133 }
134 }
135 }
136
137 impl Add<NeonVector> for NeonVector {
138 type Output = Self;
139 #[inline(always)]
140 fn add(self, rhs: NeonVector) -> Self::Output {
141 NeonVector {
142 v: unsafe { vaddq_f32(self.v, rhs.v) },
143 }
144 }
145 }
146
147 impl Add<NeonVectorDouble> for NeonVectorDouble {
148 type Output = Self;
149 #[inline(always)]
150 fn add(self, rhs: NeonVectorDouble) -> Self::Output {
151 NeonVectorDouble {
152 v0: unsafe { vaddq_f32(self.v0, rhs.v0) },
153 v1: unsafe { vaddq_f32(self.v1, rhs.v1) },
154 }
155 }
156 }
157
158 impl FusedMultiplyAdd<NeonVector> for NeonVector {
159 #[inline(always)]
160 fn mla(&self, b: NeonVector, c: NeonVector) -> NeonVector {
161 NeonVector {
162 v: unsafe { vfmaq_f32(self.v, b.v, c.v) },
163 }
164 }
165 }
166
167 impl FusedMultiplyNegAdd<NeonVector> for NeonVector {
168 #[inline(always)]
169 fn neg_mla(&self, b: NeonVector, c: NeonVector) -> NeonVector {
170 NeonVector {
171 v: unsafe { vfmsq_f32(self.v, b.v, c.v) },
172 }
173 }
174 }
175
176 impl NeonVectorDouble {
177 #[inline(always)]
178 fn neg_mla(&self, b: NeonVectorDouble, c: NeonVectorDouble) -> NeonVectorDouble {
179 NeonVectorDouble {
180 v0: unsafe { vfmsq_f32(self.v0, b.v0, c.v0) },
181 v1: unsafe { vfmsq_f32(self.v1, b.v1, c.v1) },
182 }
183 }
184 }
185
186 impl NeonVectorDouble {
187 #[inline(always)]
188 fn mla(&self, b: NeonVectorDouble, c: NeonVector) -> NeonVectorDouble {
189 NeonVectorDouble {
190 v0: unsafe { vfmaq_f32(self.v0, b.v0, c.v) },
191 v1: unsafe { vfmaq_f32(self.v1, b.v1, c.v) },
192 }
193 }
194
195 #[inline(always)]
196 pub(crate) fn split(self) -> (NeonVector, NeonVector) {
197 (NeonVector { v: self.v0 }, NeonVector { v: self.v1 })
198 }
199 }
200
201 impl<const GRID_SIZE: usize> Fetcher<NeonVector> for TetrahedralNeonFetchVector<'_, GRID_SIZE> {
202 fn fetch(&self, x: i32, y: i32, z: i32) -> NeonVector {
203 let offset = (x as u32 * (GRID_SIZE as u32 * GRID_SIZE as u32)
204 + y as u32 * GRID_SIZE as u32
205 + z as u32) as usize;
206 let jx = unsafe { self.cube.get_unchecked(offset..) };
207 NeonVector {
208 v: unsafe { vld1q_f32(jx.as_ptr() as *const f32) },
209 }
210 }
211 }
212
213 impl<const GRID_SIZE: usize> Fetcher<NeonVectorDouble>
214 for TetrahedralNeonFetchVectorDouble<'_, GRID_SIZE>
215 {
216 fn fetch(&self, x: i32, y: i32, z: i32) -> NeonVectorDouble {
217 let offset = (x as u32 * (GRID_SIZE as u32 * GRID_SIZE as u32)
218 + y as u32 * GRID_SIZE as u32
219 + z as u32) as usize;
220 let jx0 = unsafe { self.cube0.get_unchecked(offset..) };
221 let jx1 = unsafe { self.cube1.get_unchecked(offset..) };
222 NeonVectorDouble {
223 v0: unsafe { vld1q_f32(jx0.as_ptr() as *const f32) },
224 v1: unsafe { vld1q_f32(jx1.as_ptr() as *const f32) },
225 }
226 }
227 }
228
229 pub(crate) trait NeonMdInterpolation {
230 fn inter3_neon(
231 &self,
232 cube: &[NeonAlignedF32],
233 in_r: usize,
234 in_g: usize,
235 in_b: usize,
236 lut: &[BarycentricWeight<f32>],
237 ) -> NeonVector;
238 }
239
240 pub(crate) trait NeonMdInterpolationDouble {
241 fn inter3_neon(
242 &self,
243 table0: &[NeonAlignedF32],
244 table1: &[NeonAlignedF32],
245 in_r: usize,
246 in_g: usize,
247 in_b: usize,
248 lut: &[BarycentricWeight<f32>],
249 ) -> (NeonVector, NeonVector);
250 }
251
252 impl<const GRID_SIZE: usize> TetrahedralNeon<GRID_SIZE> {
253 #[inline]
254 fn interpolate(
255 &self,
256 in_r: usize,
257 in_g: usize,
258 in_b: usize,
259 lut: &[BarycentricWeight<f32>],
260 r: impl Fetcher<NeonVector>,
261 ) -> NeonVector {
262 let lut_r = unsafe { *lut.get_unchecked(in_r) };
263 let lut_g = unsafe { *lut.get_unchecked(in_g) };
264 let lut_b = unsafe { *lut.get_unchecked(in_b) };
265
266 let x: i32 = lut_r.x;
267 let y: i32 = lut_g.x;
268 let z: i32 = lut_b.x;
269
270 let x_n: i32 = lut_r.x_n;
271 let y_n: i32 = lut_g.x_n;
272 let z_n: i32 = lut_b.x_n;
273
274 let rx = lut_r.w;
275 let ry = lut_g.w;
276 let rz = lut_b.w;
277
278 let c0 = r.fetch(x, y, z);
279
280 let c2;
281 let c1;
282 let c3;
283 if rx >= ry {
284 if ry >= rz {
285 //rx >= ry && ry >= rz
286 c1 = r.fetch(x_n, y, z) - c0;
287 c2 = r.fetch(x_n, y_n, z) - r.fetch(x_n, y, z);
288 c3 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y_n, z);
289 } else if rx >= rz {
290 //rx >= rz && rz >= ry
291 c1 = r.fetch(x_n, y, z) - c0;
292 c2 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y, z_n);
293 c3 = r.fetch(x_n, y, z_n) - r.fetch(x_n, y, z);
294 } else {
295 //rz > rx && rx >= ry
296 c1 = r.fetch(x_n, y, z_n) - r.fetch(x, y, z_n);
297 c2 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y, z_n);
298 c3 = r.fetch(x, y, z_n) - c0;
299 }
300 } else if rx >= rz {
301 //ry > rx && rx >= rz
302 c1 = r.fetch(x_n, y_n, z) - r.fetch(x, y_n, z);
303 c2 = r.fetch(x, y_n, z) - c0;
304 c3 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y_n, z);
305 } else if ry >= rz {
306 //ry >= rz && rz > rx
307 c1 = r.fetch(x_n, y_n, z_n) - r.fetch(x, y_n, z_n);
308 c2 = r.fetch(x, y_n, z) - c0;
309 c3 = r.fetch(x, y_n, z_n) - r.fetch(x, y_n, z);
310 } else {
311 //rz > ry && ry > rx
312 c1 = r.fetch(x_n, y_n, z_n) - r.fetch(x, y_n, z_n);
313 c2 = r.fetch(x, y_n, z_n) - r.fetch(x, y, z_n);
314 c3 = r.fetch(x, y, z_n) - c0;
315 }
316 let s0 = c0.mla(c1, NeonVector::from(rx));
317 let s1 = s0.mla(c2, NeonVector::from(ry));
318 s1.mla(c3, NeonVector::from(rz))
319 }
320 }
321
322 impl<const GRID_SIZE: usize> TetrahedralNeonDouble<GRID_SIZE> {
323 #[inline]
324 fn interpolate(
325 &self,
326 in_r: usize,
327 in_g: usize,
328 in_b: usize,
329 lut: &[BarycentricWeight<f32>],
330 r: impl Fetcher<NeonVectorDouble>,
331 ) -> (NeonVector, NeonVector) {
332 let lut_r = unsafe { *lut.get_unchecked(in_r) };
333 let lut_g = unsafe { *lut.get_unchecked(in_g) };
334 let lut_b = unsafe { *lut.get_unchecked(in_b) };
335
336 let x: i32 = lut_r.x;
337 let y: i32 = lut_g.x;
338 let z: i32 = lut_b.x;
339
340 let x_n: i32 = lut_r.x_n;
341 let y_n: i32 = lut_g.x_n;
342 let z_n: i32 = lut_b.x_n;
343
344 let rx = lut_r.w;
345 let ry = lut_g.w;
346 let rz = lut_b.w;
347
348 let c0 = r.fetch(x, y, z);
349
350 let c2;
351 let c1;
352 let c3;
353 if rx >= ry {
354 if ry >= rz {
355 //rx >= ry && ry >= rz
356 c1 = r.fetch(x_n, y, z) - c0;
357 c2 = r.fetch(x_n, y_n, z) - r.fetch(x_n, y, z);
358 c3 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y_n, z);
359 } else if rx >= rz {
360 //rx >= rz && rz >= ry
361 c1 = r.fetch(x_n, y, z) - c0;
362 c2 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y, z_n);
363 c3 = r.fetch(x_n, y, z_n) - r.fetch(x_n, y, z);
364 } else {
365 //rz > rx && rx >= ry
366 c1 = r.fetch(x_n, y, z_n) - r.fetch(x, y, z_n);
367 c2 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y, z_n);
368 c3 = r.fetch(x, y, z_n) - c0;
369 }
370 } else if rx >= rz {
371 //ry > rx && rx >= rz
372 c1 = r.fetch(x_n, y_n, z) - r.fetch(x, y_n, z);
373 c2 = r.fetch(x, y_n, z) - c0;
374 c3 = r.fetch(x_n, y_n, z_n) - r.fetch(x_n, y_n, z);
375 } else if ry >= rz {
376 //ry >= rz && rz > rx
377 c1 = r.fetch(x_n, y_n, z_n) - r.fetch(x, y_n, z_n);
378 c2 = r.fetch(x, y_n, z) - c0;
379 c3 = r.fetch(x, y_n, z_n) - r.fetch(x, y_n, z);
380 } else {
381 //rz > ry && ry > rx
382 c1 = r.fetch(x_n, y_n, z_n) - r.fetch(x, y_n, z_n);
383 c2 = r.fetch(x, y_n, z_n) - r.fetch(x, y, z_n);
384 c3 = r.fetch(x, y, z_n) - c0;
385 }
386 let s0 = c0.mla(c1, NeonVector::from(rx));
387 let s1 = s0.mla(c2, NeonVector::from(ry));
388 s1.mla(c3, NeonVector::from(rz)).split()
389 }
390 }
391
392 macro_rules! define_md_inter_neon {
393 ($interpolator: ident) => {
394 impl<const GRID_SIZE: usize> NeonMdInterpolation for $interpolator<GRID_SIZE> {
395 fn inter3_neon(
396 &self,
397 cube: &[NeonAlignedF32],
398 in_r: usize,
399 in_g: usize,
400 in_b: usize,
401 lut: &[BarycentricWeight<f32>],
402 ) -> NeonVector {
403 self.interpolate(
404 in_r,
405 in_g,
406 in_b,
407 lut,
408 TetrahedralNeonFetchVector::<GRID_SIZE> { cube },
409 )
410 }
411 }
412 };
413 }
414
415 macro_rules! define_md_inter_neon_d {
416 ($interpolator: ident) => {
417 impl<const GRID_SIZE: usize> NeonMdInterpolationDouble for $interpolator<GRID_SIZE> {
418 fn inter3_neon(
419 &self,
420 table0: &[NeonAlignedF32],
421 table1: &[NeonAlignedF32],
422 in_r: usize,
423 in_g: usize,
424 in_b: usize,
425 lut: &[BarycentricWeight<f32>],
426 ) -> (NeonVector, NeonVector) {
427 self.interpolate(
428 in_r,
429 in_g,
430 in_b,
431 lut,
432 TetrahedralNeonFetchVectorDouble::<GRID_SIZE> {
433 cube0: table0,
434 cube1: table1,
435 },
436 )
437 }
438 }
439 };
440 }
441
442 define_md_inter_neon!(TetrahedralNeon);
443 define_md_inter_neon!(PyramidalNeon);
444 define_md_inter_neon!(PrismaticNeon);
445 define_md_inter_neon!(TrilinearNeon);
446 define_md_inter_neon_d!(PrismaticNeonDouble);
447 define_md_inter_neon_d!(PyramidalNeonDouble);
448 define_md_inter_neon_d!(TetrahedralNeonDouble);
449 define_md_inter_neon_d!(TrilinearNeonDouble);
450
451 impl<const GRID_SIZE: usize> PyramidalNeon<GRID_SIZE> {
452 #[inline]
453 fn interpolate(
454 &self,
455 in_r: usize,
456 in_g: usize,
457 in_b: usize,
458 lut: &[BarycentricWeight<f32>],
459 r: impl Fetcher<NeonVector>,
460 ) -> NeonVector {
461 let lut_r = unsafe { *lut.get_unchecked(in_r) };
462 let lut_g = unsafe { *lut.get_unchecked(in_g) };
463 let lut_b = unsafe { *lut.get_unchecked(in_b) };
464
465 let x: i32 = lut_r.x;
466 let y: i32 = lut_g.x;
467 let z: i32 = lut_b.x;
468
469 let x_n: i32 = lut_r.x_n;
470 let y_n: i32 = lut_g.x_n;
471 let z_n: i32 = lut_b.x_n;
472
473 let dr = lut_r.w;
474 let dg = lut_g.w;
475 let db = lut_b.w;
476
477 let c0 = r.fetch(x, y, z);
478
479 if dr > db && dg > db {
480 let x0 = r.fetch(x_n, y_n, z_n);
481 let x1 = r.fetch(x_n, y_n, z);
482 let x2 = r.fetch(x_n, y, z);
483 let x3 = r.fetch(x, y_n, z);
484
485 let c1 = x0 - x1;
486 let c2 = x2 - c0;
487 let c3 = x3 - c0;
488 let c4 = c0 - x3 - x2 + x1;
489
490 let s0 = c0.mla(c1, NeonVector::from(db));
491 let s1 = s0.mla(c2, NeonVector::from(dr));
492 let s2 = s1.mla(c3, NeonVector::from(dg));
493 s2.mla(c4, NeonVector::from(dr * dg))
494 } else if db > dr && dg > dr {
495 let x0 = r.fetch(x, y, z_n);
496 let x1 = r.fetch(x_n, y_n, z_n);
497 let x2 = r.fetch(x, y_n, z_n);
498 let x3 = r.fetch(x, y_n, z);
499
500 let c1 = x0 - c0;
501 let c2 = x1 - x2;
502 let c3 = x3 - c0;
503 let c4 = c0 - x3 - x0 + x2;
504
505 let s0 = c0.mla(c1, NeonVector::from(db));
506 let s1 = s0.mla(c2, NeonVector::from(dr));
507 let s2 = s1.mla(c3, NeonVector::from(dg));
508 s2.mla(c4, NeonVector::from(dg * db))
509 } else {
510 let x0 = r.fetch(x, y, z_n);
511 let x1 = r.fetch(x_n, y, z);
512 let x2 = r.fetch(x_n, y, z_n);
513 let x3 = r.fetch(x_n, y_n, z_n);
514
515 let c1 = x0 - c0;
516 let c2 = x1 - c0;
517 let c3 = x3 - x2;
518 let c4 = c0 - x1 - x0 + x2;
519
520 let s0 = c0.mla(c1, NeonVector::from(db));
521 let s1 = s0.mla(c2, NeonVector::from(dr));
522 let s2 = s1.mla(c3, NeonVector::from(dg));
523 s2.mla(c4, NeonVector::from(db * dr))
524 }
525 }
526 }
527
528 impl<const GRID_SIZE: usize> PyramidalNeonDouble<GRID_SIZE> {
529 #[inline]
530 fn interpolate(
531 &self,
532 in_r: usize,
533 in_g: usize,
534 in_b: usize,
535 lut: &[BarycentricWeight<f32>],
536 r: impl Fetcher<NeonVectorDouble>,
537 ) -> (NeonVector, NeonVector) {
538 let lut_r = unsafe { *lut.get_unchecked(in_r) };
539 let lut_g = unsafe { *lut.get_unchecked(in_g) };
540 let lut_b = unsafe { *lut.get_unchecked(in_b) };
541
542 let x: i32 = lut_r.x;
543 let y: i32 = lut_g.x;
544 let z: i32 = lut_b.x;
545
546 let x_n: i32 = lut_r.x_n;
547 let y_n: i32 = lut_g.x_n;
548 let z_n: i32 = lut_b.x_n;
549
550 let dr = lut_r.w;
551 let dg = lut_g.w;
552 let db = lut_b.w;
553
554 let c0 = r.fetch(x, y, z);
555
556 let w0 = NeonVector::from(db);
557 let w1 = NeonVector::from(dr);
558 let w2 = NeonVector::from(dg);
559
560 if dr > db && dg > db {
561 let x0 = r.fetch(x_n, y_n, z_n);
562 let x1 = r.fetch(x_n, y_n, z);
563 let x2 = r.fetch(x_n, y, z);
564 let x3 = r.fetch(x, y_n, z);
565
566 let c1 = x0 - x1;
567 let c2 = x2 - c0;
568 let c3 = x3 - c0;
569 let c4 = c0 - x3 - x2 + x1;
570
571 let w3 = NeonVector::from(dr * dg);
572
573 let s0 = c0.mla(c1, w0);
574 let s1 = s0.mla(c2, w1);
575 let s2 = s1.mla(c3, w2);
576 s2.mla(c4, w3).split()
577 } else if db > dr && dg > dr {
578 let x0 = r.fetch(x, y, z_n);
579 let x1 = r.fetch(x_n, y_n, z_n);
580 let x2 = r.fetch(x, y_n, z_n);
581 let x3 = r.fetch(x, y_n, z);
582
583 let c1 = x0 - c0;
584 let c2 = x1 - x2;
585 let c3 = x3 - c0;
586 let c4 = c0 - x3 - x0 + x2;
587
588 let w3 = NeonVector::from(dg * db);
589
590 let s0 = c0.mla(c1, w0);
591 let s1 = s0.mla(c2, w1);
592 let s2 = s1.mla(c3, w2);
593 s2.mla(c4, w3).split()
594 } else {
595 let x0 = r.fetch(x, y, z_n);
596 let x1 = r.fetch(x_n, y, z);
597 let x2 = r.fetch(x_n, y, z_n);
598 let x3 = r.fetch(x_n, y_n, z_n);
599
600 let c1 = x0 - c0;
601 let c2 = x1 - c0;
602 let c3 = x3 - x2;
603 let c4 = c0 - x1 - x0 + x2;
604
605 let w3 = NeonVector::from(db * dr);
606
607 let s0 = c0.mla(c1, w0);
608 let s1 = s0.mla(c2, w1);
609 let s2 = s1.mla(c3, w2);
610 s2.mla(c4, w3).split()
611 }
612 }
613 }
614
615 impl<const GRID_SIZE: usize> PrismaticNeon<GRID_SIZE> {
616 #[inline]
617 fn interpolate(
618 &self,
619 in_r: usize,
620 in_g: usize,
621 in_b: usize,
622 lut: &[BarycentricWeight<f32>],
623 r: impl Fetcher<NeonVector>,
624 ) -> NeonVector {
625 let lut_r = unsafe { *lut.get_unchecked(in_r) };
626 let lut_g = unsafe { *lut.get_unchecked(in_g) };
627 let lut_b = unsafe { *lut.get_unchecked(in_b) };
628
629 let x: i32 = lut_r.x;
630 let y: i32 = lut_g.x;
631 let z: i32 = lut_b.x;
632
633 let x_n: i32 = lut_r.x_n;
634 let y_n: i32 = lut_g.x_n;
635 let z_n: i32 = lut_b.x_n;
636
637 let dr = lut_r.w;
638 let dg = lut_g.w;
639 let db = lut_b.w;
640
641 let c0 = r.fetch(x, y, z);
642
643 if db > dr {
644 let x0 = r.fetch(x, y, z_n);
645 let x1 = r.fetch(x_n, y, z_n);
646 let x2 = r.fetch(x, y_n, z);
647 let x3 = r.fetch(x, y_n, z_n);
648 let x4 = r.fetch(x_n, y_n, z_n);
649
650 let c1 = x0 - c0;
651 let c2 = x1 - x0;
652 let c3 = x2 - c0;
653 let c4 = c0 - x2 - x0 + x3;
654 let c5 = x0 - x3 - x1 + x4;
655
656 let s0 = c0.mla(c1, NeonVector::from(db));
657 let s1 = s0.mla(c2, NeonVector::from(dr));
658 let s2 = s1.mla(c3, NeonVector::from(dg));
659 let s3 = s2.mla(c4, NeonVector::from(dg * db));
660 s3.mla(c5, NeonVector::from(dr * dg))
661 } else {
662 let x0 = r.fetch(x_n, y, z);
663 let x1 = r.fetch(x_n, y, z_n);
664 let x2 = r.fetch(x, y_n, z);
665 let x3 = r.fetch(x_n, y_n, z);
666 let x4 = r.fetch(x_n, y_n, z_n);
667
668 let c1 = x1 - x0;
669 let c2 = x0 - c0;
670 let c3 = x2 - c0;
671 let c4 = x0 - x3 - x1 + x4;
672 let c5 = c0 - x2 - x0 + x3;
673
674 let s0 = c0.mla(c1, NeonVector::from(db));
675 let s1 = s0.mla(c2, NeonVector::from(dr));
676 let s2 = s1.mla(c3, NeonVector::from(dg));
677 let s3 = s2.mla(c4, NeonVector::from(dg * db));
678 s3.mla(c5, NeonVector::from(dr * dg))
679 }
680 }
681 }
682
683 impl<const GRID_SIZE: usize> PrismaticNeonDouble<GRID_SIZE> {
684 #[inline]
685 fn interpolate(
686 &self,
687 in_r: usize,
688 in_g: usize,
689 in_b: usize,
690 lut: &[BarycentricWeight<f32>],
691 rv: impl Fetcher<NeonVectorDouble>,
692 ) -> (NeonVector, NeonVector) {
693 let lut_r = unsafe { *lut.get_unchecked(in_r) };
694 let lut_g = unsafe { *lut.get_unchecked(in_g) };
695 let lut_b = unsafe { *lut.get_unchecked(in_b) };
696
697 let x: i32 = lut_r.x;
698 let y: i32 = lut_g.x;
699 let z: i32 = lut_b.x;
700
701 let x_n: i32 = lut_r.x_n;
702 let y_n: i32 = lut_g.x_n;
703 let z_n: i32 = lut_b.x_n;
704
705 let dr = lut_r.w;
706 let dg = lut_g.w;
707 let db = lut_b.w;
708
709 let c0 = rv.fetch(x, y, z);
710
711 let w0 = NeonVector::from(db);
712 let w1 = NeonVector::from(dr);
713 let w2 = NeonVector::from(dg);
714 let w3 = NeonVector::from(dg * db);
715 let w4 = NeonVector::from(dr * dg);
716
717 if db > dr {
718 let x0 = rv.fetch(x, y, z_n);
719 let x1 = rv.fetch(x_n, y, z_n);
720 let x2 = rv.fetch(x, y_n, z);
721 let x3 = rv.fetch(x, y_n, z_n);
722 let x4 = rv.fetch(x_n, y_n, z_n);
723
724 let c1 = x0 - c0;
725 let c2 = x1 - x0;
726 let c3 = x2 - c0;
727 let c4 = c0 - x2 - x0 + x3;
728 let c5 = x0 - x3 - x1 + x4;
729
730 let s0 = c0.mla(c1, w0);
731 let s1 = s0.mla(c2, w1);
732 let s2 = s1.mla(c3, w2);
733 let s3 = s2.mla(c4, w3);
734 s3.mla(c5, w4).split()
735 } else {
736 let x0 = rv.fetch(x_n, y, z);
737 let x1 = rv.fetch(x_n, y, z_n);
738 let x2 = rv.fetch(x, y_n, z);
739 let x3 = rv.fetch(x_n, y_n, z);
740 let x4 = rv.fetch(x_n, y_n, z_n);
741
742 let c1 = x1 - x0;
743 let c2 = x0 - c0;
744 let c3 = x2 - c0;
745 let c4 = x0 - x3 - x1 + x4;
746 let c5 = c0 - x2 - x0 + x3;
747
748 let s0 = c0.mla(c1, w0);
749 let s1 = s0.mla(c2, w1);
750 let s2 = s1.mla(c3, w2);
751 let s3 = s2.mla(c4, w3);
752 s3.mla(c5, w4).split()
753 }
754 }
755 }
756
757 impl<const GRID_SIZE: usize> TrilinearNeonDouble<GRID_SIZE> {
758 #[inline]
759 fn interpolate(
760 &self,
761 in_r: usize,
762 in_g: usize,
763 in_b: usize,
764 lut: &[BarycentricWeight<f32>],
765 r: impl Fetcher<NeonVectorDouble>,
766 ) -> (NeonVector, NeonVector) {
767 let lut_r = unsafe { *lut.get_unchecked(in_r) };
768 let lut_g = unsafe { *lut.get_unchecked(in_g) };
769 let lut_b = unsafe { *lut.get_unchecked(in_b) };
770
771 let x: i32 = lut_r.x;
772 let y: i32 = lut_g.x;
773 let z: i32 = lut_b.x;
774
775 let x_n: i32 = lut_r.x_n;
776 let y_n: i32 = lut_g.x_n;
777 let z_n: i32 = lut_b.x_n;
778
779 let dr = lut_r.w;
780 let dg = lut_g.w;
781 let db = lut_b.w;
782
783 let w0 = NeonVector::from(dr);
784 let w1 = NeonVector::from(dg);
785 let w2 = NeonVector::from(db);
786
787 let c000 = r.fetch(x, y, z);
788 let c100 = r.fetch(x_n, y, z);
789 let c010 = r.fetch(x, y_n, z);
790 let c110 = r.fetch(x_n, y_n, z);
791 let c001 = r.fetch(x, y, z_n);
792 let c101 = r.fetch(x_n, y, z_n);
793 let c011 = r.fetch(x, y_n, z_n);
794 let c111 = r.fetch(x_n, y_n, z_n);
795
796 let dx = NeonVectorDouble::from(dr);
797
798 let c00 = c000.neg_mla(c000, dx).mla(c100, w0);
799 let c10 = c010.neg_mla(c010, dx).mla(c110, w0);
800 let c01 = c001.neg_mla(c001, dx).mla(c101, w0);
801 let c11 = c011.neg_mla(c011, dx).mla(c111, w0);
802
803 let dy = NeonVectorDouble::from(dg);
804
805 let c0 = c00.neg_mla(c00, dy).mla(c10, w1);
806 let c1 = c01.neg_mla(c01, dy).mla(c11, w1);
807
808 let dz = NeonVectorDouble::from(db);
809
810 c0.neg_mla(c0, dz).mla(c1, w2).split()
811 }
812 }
813
814 impl<const GRID_SIZE: usize> TrilinearNeon<GRID_SIZE> {
815 #[inline]
816 fn interpolate(
817 &self,
818 in_r: usize,
819 in_g: usize,
820 in_b: usize,
821 lut: &[BarycentricWeight<f32>],
822 r: impl Fetcher<NeonVector>,
823 ) -> NeonVector {
824 let lut_r = unsafe { *lut.get_unchecked(in_r) };
825 let lut_g = unsafe { *lut.get_unchecked(in_g) };
826 let lut_b = unsafe { *lut.get_unchecked(in_b) };
827
828 let x: i32 = lut_r.x;
829 let y: i32 = lut_g.x;
830 let z: i32 = lut_b.x;
831
832 let x_n: i32 = lut_r.x_n;
833 let y_n: i32 = lut_g.x_n;
834 let z_n: i32 = lut_b.x_n;
835
836 let dr = lut_r.w;
837 let dg = lut_g.w;
838 let db = lut_b.w;
839
840 let w0 = NeonVector::from(dr);
841 let w1 = NeonVector::from(dg);
842 let w2 = NeonVector::from(db);
843
844 let c000 = r.fetch(x, y, z);
845 let c100 = r.fetch(x_n, y, z);
846 let c010 = r.fetch(x, y_n, z);
847 let c110 = r.fetch(x_n, y_n, z);
848 let c001 = r.fetch(x, y, z_n);
849 let c101 = r.fetch(x_n, y, z_n);
850 let c011 = r.fetch(x, y_n, z_n);
851 let c111 = r.fetch(x_n, y_n, z_n);
852
853 let dx = NeonVector::from(dr);
854
855 let c00 = c000.neg_mla(c000, dx).mla(c100, w0);
856 let c10 = c010.neg_mla(c010, dx).mla(c110, w0);
857 let c01 = c001.neg_mla(c001, dx).mla(c101, w0);
858 let c11 = c011.neg_mla(c011, dx).mla(c111, w0);
859
860 let dy = NeonVector::from(dg);
861
862 let c0 = c00.neg_mla(c00, dy).mla(c10, w1);
863 let c1 = c01.neg_mla(c01, dy).mla(c11, w1);
864
865 let dz = NeonVector::from(db);
866
867 c0.neg_mla(c0, dz).mla(c1, w2)
868 }
869 }