Debian Codegraph @

.cargo/vendor/rav1e/src/asm/x86/lrf.rs

raw
1 // Copyright (c) 2019-2022, The rav1e contributors. All rights reserved
2 //
3 // This source code is subject to the terms of the BSD 2 Clause License and
4 // the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
5 // was not distributed with this source code in the LICENSE file, you can
6 // obtain it at www.aomedia.org/license/software. If the Alliance for Open
7 // Media Patent License 1.0 was not distributed with this source code in the
8 // PATENTS file, you can obtain it at www.aomedia.org/license/patent.
9
10 use crate::cpu_features::CpuFeatureLevel;
11 use crate::frame::PlaneSlice;
12 use crate::lrf::*;
13 use crate::util::Pixel;
14 #[cfg(target_arch = "x86")]
15 use std::arch::x86::*;
16 #[cfg(target_arch = "x86_64")]
17 use std::arch::x86_64::*;
18 use std::mem;
19
20 // computes an intermediate (ab) row for stripe_w + 2 columns at row y
21 #[inline]
22 pub fn sgrproj_box_ab_r1<const BD: usize>(
23 af: &mut [u32], bf: &mut [u32], iimg: &[u32], iimg_sq: &[u32],
24 iimg_stride: usize, y: usize, stripe_w: usize, s: u32, cpu: CpuFeatureLevel,
25 ) {
26 // only use 8-bit AVX2 assembly when bitdepth equals 8
27 if cpu >= CpuFeatureLevel::AVX2 && BD == 8 {
28 // SAFETY: Calls Assembly code.
29 return unsafe {
30 sgrproj_box_ab_r1_avx2::<BD>(
31 af,
32 bf,
33 iimg,
34 iimg_sq,
35 iimg_stride,
36 y,
37 stripe_w,
38 s,
39 );
40 };
41 }
42
43 rust::sgrproj_box_ab_r1::<BD>(
44 af,
45 bf,
46 iimg,
47 iimg_sq,
48 iimg_stride,
49 y,
50 stripe_w,
51 s,
52 cpu,
53 );
54 }
55
56 // computes an intermediate (ab) row for stripe_w + 2 columns at row y
57 #[inline]
58 pub fn sgrproj_box_ab_r2<const BD: usize>(
59 af: &mut [u32], bf: &mut [u32], iimg: &[u32], iimg_sq: &[u32],
60 iimg_stride: usize, y: usize, stripe_w: usize, s: u32, cpu: CpuFeatureLevel,
61 ) {
62 // only use 8-bit AVX2 assembly when bitdepth equals 8
63 if cpu >= CpuFeatureLevel::AVX2 && BD == 8 {
64 // SAFETY: Calls Assembly code.
65 return unsafe {
66 sgrproj_box_ab_r2_avx2::<BD>(
67 af,
68 bf,
69 iimg,
70 iimg_sq,
71 iimg_stride,
72 y,
73 stripe_w,
74 s,
75 );
76 };
77 }
78
79 rust::sgrproj_box_ab_r2::<BD>(
80 af,
81 bf,
82 iimg,
83 iimg_sq,
84 iimg_stride,
85 y,
86 stripe_w,
87 s,
88 cpu,
89 );
90 }
91
92 #[inline]
93 pub fn sgrproj_box_f_r0<T: Pixel>(
94 f: &mut [u32], y: usize, w: usize, cdeffed: &PlaneSlice<T>,
95 cpu: CpuFeatureLevel,
96 ) {
97 if cpu >= CpuFeatureLevel::AVX2 {
98 // SAFETY: Calls Assembly code.
99 return unsafe {
100 sgrproj_box_f_r0_avx2(f, y, w, cdeffed);
101 };
102 }
103
104 rust::sgrproj_box_f_r0(f, y, w, cdeffed, cpu);
105 }
106
107 #[inline]
108 pub fn sgrproj_box_f_r1<T: Pixel>(
109 af: &[&[u32]; 3], bf: &[&[u32]; 3], f: &mut [u32], y: usize, w: usize,
110 cdeffed: &PlaneSlice<T>, cpu: CpuFeatureLevel,
111 ) {
112 if cpu >= CpuFeatureLevel::AVX2 {
113 // SAFETY: Calls Assembly code.
114 return unsafe {
115 sgrproj_box_f_r1_avx2(af, bf, f, y, w, cdeffed);
116 };
117 }
118
119 rust::sgrproj_box_f_r1(af, bf, f, y, w, cdeffed, cpu);
120 }
121
122 #[inline]
123 pub fn sgrproj_box_f_r2<T: Pixel>(
124 af: &[&[u32]; 2], bf: &[&[u32]; 2], f0: &mut [u32], f1: &mut [u32],
125 y: usize, w: usize, cdeffed: &PlaneSlice<T>, cpu: CpuFeatureLevel,
126 ) {
127 if cpu >= CpuFeatureLevel::AVX2 {
128 // SAFETY: Calls Assembly code.
129 return unsafe {
130 sgrproj_box_f_r2_avx2(af, bf, f0, f1, y, w, cdeffed);
131 };
132 }
133
134 rust::sgrproj_box_f_r2(af, bf, f0, f1, y, w, cdeffed, cpu);
135 }
136
137 static X_BY_XPLUS1: [u32; 256] = [
138 // Special case: Map 0 -> 1 (corresponding to a value of 1/256)
139 // instead of 0. See comments in selfguided_restoration_internal() for why
140 1, 128, 171, 192, 205, 213, 219, 224, 228, 230, 233, 235, 236, 238, 239, 240,
141 241, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247, 247, 247, 247, 248,
142 248, 248, 248, 249, 249, 249, 249, 249, 250, 250, 250, 250, 250, 250, 250,
143 251, 251, 251, 251, 251, 251, 251, 251, 251, 251, 252, 252, 252, 252, 252,
144 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 253, 253, 253,
145 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253,
146 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 253, 254, 254, 254, 254,
147 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
148 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
149 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
150 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254,
151 254, 254, 254, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
152 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
153 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
154 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
155 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
156 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 256,
157 ];
158
159 #[inline]
160 #[target_feature(enable = "avx2")]
161 unsafe fn sgrproj_box_ab_8_avx2<const BD: usize>(
162 r: usize, af: &mut [u32], bf: &mut [u32], iimg: &[u32], iimg_sq: &[u32],
163 iimg_stride: usize, x: usize, y: usize, s: u32,
164 ) {
165 let bdm8 = BD - 8;
166 let d: usize = r * 2 + 1;
167 let n: i32 = (d * d) as i32;
168 let one_over_n = if r == 1 { 455 } else { 164 };
169
170 // Using an integral image, compute the sum of a square region
171 #[inline]
172 #[target_feature(enable = "avx2")]
173 unsafe fn get_integral_square_avx2(
174 iimg: &[u32], stride: usize, x: usize, y: usize, size: usize,
175 ) -> __m256i {
176 let iimg = iimg.as_ptr().add(y * stride + x);
177 // Cancel out overflow in iimg by using wrapping arithmetic
178 _mm256_sub_epi32(
179 _mm256_add_epi32(
180 _mm256_loadu_si256(iimg as *const _),
181 _mm256_loadu_si256(iimg.add(size * stride + size) as *const _),
182 ),
183 _mm256_add_epi32(
184 _mm256_loadu_si256(iimg.add(size * stride) as *const _),
185 _mm256_loadu_si256(iimg.add(size) as *const _),
186 ),
187 )
188 }
189
190 let sum = get_integral_square_avx2(iimg, iimg_stride, x, y, d);
191 let ssq = get_integral_square_avx2(iimg_sq, iimg_stride, x, y, d);
192 let scaled_sum = _mm256_srlv_epi32(
193 _mm256_add_epi32(sum, _mm256_set1_epi32(1 << bdm8 as i32 >> 1)),
194 _mm256_set1_epi32(bdm8 as i32),
195 );
196 let scaled_ssq = _mm256_srlv_epi32(
197 _mm256_add_epi32(ssq, _mm256_set1_epi32(1 << (2 * bdm8) as i32 >> 1)),
198 _mm256_set1_epi32(2 * bdm8 as i32),
199 );
200 let p = _mm256_max_epi32(
201 _mm256_setzero_si256(),
202 _mm256_sub_epi32(
203 _mm256_mullo_epi32(scaled_ssq, _mm256_set1_epi32(n)),
204 _mm256_madd_epi16(scaled_sum, scaled_sum),
205 ),
206 );
207 let z = _mm256_srli_epi32(
208 _mm256_add_epi32(
209 _mm256_mullo_epi32(p, _mm256_set1_epi32(s as i32)),
210 _mm256_set1_epi32(1 << SGRPROJ_MTABLE_BITS as i32 >> 1),
211 ),
212 SGRPROJ_MTABLE_BITS as i32,
213 );
214 let a = _mm256_i32gather_epi32(
215 X_BY_XPLUS1.as_ptr() as *const _,
216 _mm256_min_epi32(z, _mm256_set1_epi32(255)),
217 4,
218 );
219 let b = _mm256_mullo_epi32(
220 _mm256_madd_epi16(
221 _mm256_sub_epi32(_mm256_set1_epi32(1 << SGRPROJ_SGR_BITS as i32), a),
222 sum,
223 ),
224 _mm256_set1_epi32(one_over_n),
225 );
226 let b = _mm256_srlv_epi32(
227 _mm256_add_epi32(
228 b,
229 _mm256_set1_epi32(1 << SGRPROJ_RECIP_BITS as i32 >> 1),
230 ),
231 _mm256_set1_epi32(SGRPROJ_RECIP_BITS as i32),
232 );
233 _mm256_storeu_si256(af.as_mut_ptr().add(x) as *mut _, a);
234 _mm256_storeu_si256(bf.as_mut_ptr().add(x) as *mut _, b);
235 }
236
237 #[target_feature(enable = "avx2")]
238 pub(crate) unsafe fn sgrproj_box_ab_r1_avx2<const BD: usize>(
239 af: &mut [u32], bf: &mut [u32], iimg: &[u32], iimg_sq: &[u32],
240 iimg_stride: usize, y: usize, stripe_w: usize, s: u32,
241 ) {
242 for x in (0..stripe_w + 2).step_by(8) {
243 if x + 8 <= stripe_w + 2 {
244 sgrproj_box_ab_8_avx2::<BD>(
245 1,
246 af,
247 bf,
248 iimg,
249 iimg_sq,
250 iimg_stride,
251 x,
252 y,
253 s,
254 );
255 } else {
256 // finish using scalar
257 rust::sgrproj_box_ab_internal::<BD>(
258 1,
259 af,
260 bf,
261 iimg,
262 iimg_sq,
263 iimg_stride,
264 x,
265 y,
266 stripe_w,
267 s,
268 );
269 }
270 }
271
272 #[cfg(feature = "check_asm")]
273 {
274 let mut af_ref: Vec<u32> = vec![0; stripe_w + 2];
275 let mut bf_ref: Vec<u32> = vec![0; stripe_w + 2];
276 rust::sgrproj_box_ab_internal::<BD>(
277 1,
278 &mut af_ref,
279 &mut bf_ref,
280 iimg,
281 iimg_sq,
282 iimg_stride,
283 0,
284 y,
285 stripe_w,
286 s,
287 );
288 assert_eq!(&af[..stripe_w + 2], &af_ref[..]);
289 assert_eq!(&bf[..stripe_w + 2], &bf_ref[..]);
290 }
291 }
292
293 #[target_feature(enable = "avx2")]
294 pub(crate) unsafe fn sgrproj_box_ab_r2_avx2<const BD: usize>(
295 af: &mut [u32], bf: &mut [u32], iimg: &[u32], iimg_sq: &[u32],
296 iimg_stride: usize, y: usize, stripe_w: usize, s: u32,
297 ) {
298 for x in (0..stripe_w + 2).step_by(8) {
299 if x + 8 <= stripe_w + 2 {
300 sgrproj_box_ab_8_avx2::<BD>(
301 2,
302 af,
303 bf,
304 iimg,
305 iimg_sq,
306 iimg_stride,
307 x,
308 y,
309 s,
310 );
311 } else {
312 // finish using scalar
313 rust::sgrproj_box_ab_internal::<BD>(
314 2,
315 af,
316 bf,
317 iimg,
318 iimg_sq,
319 iimg_stride,
320 x,
321 y,
322 stripe_w,
323 s,
324 );
325 }
326 }
327
328 #[cfg(feature = "check_asm")]
329 {
330 let mut af_ref: Vec<u32> = vec![0; stripe_w + 2];
331 let mut bf_ref: Vec<u32> = vec![0; stripe_w + 2];
332 rust::sgrproj_box_ab_internal::<BD>(
333 2,
334 &mut af_ref,
335 &mut bf_ref,
336 iimg,
337 iimg_sq,
338 iimg_stride,
339 0,
340 y,
341 stripe_w,
342 s,
343 );
344 assert_eq!(&af[..stripe_w + 2], &af_ref[..]);
345 assert_eq!(&bf[..stripe_w + 2], &bf_ref[..]);
346 }
347 }
348
349 #[inline]
350 #[target_feature(enable = "avx2")]
351 unsafe fn sgrproj_box_f_r0_8_avx2<T: Pixel>(
352 f: &mut [u32], x: usize, y: usize, cdeffed: &PlaneSlice<T>,
353 ) {
354 _mm256_storeu_si256(
355 f.as_mut_ptr().add(x) as *mut _,
356 _mm256_slli_epi32(
357 if mem::size_of::<T>() == 1 {
358 _mm256_cvtepu8_epi32(_mm_loadl_epi64(
359 cdeffed.subslice(x, y).as_ptr() as *const _
360 ))
361 } else {
362 _mm256_cvtepu16_epi32(_mm_loadu_si128(
363 cdeffed.subslice(x, y).as_ptr() as *const _
364 ))
365 },
366 SGRPROJ_RST_BITS as i32,
367 ),
368 );
369 }
370
371 #[target_feature(enable = "avx2")]
372 pub(crate) unsafe fn sgrproj_box_f_r0_avx2<T: Pixel>(
373 f: &mut [u32], y: usize, w: usize, cdeffed: &PlaneSlice<T>,
374 ) {
375 for x in (0..w).step_by(8) {
376 if x + 8 <= w {
377 sgrproj_box_f_r0_8_avx2(f, x, y, cdeffed);
378 } else {
379 // finish using scalar
380 rust::sgrproj_box_f_r0_internal(f, x, y, w, cdeffed);
381 }
382 }
383
384 #[cfg(feature = "check_asm")]
385 {
386 let mut f_ref: Vec<u32> = vec![0; w];
387 rust::sgrproj_box_f_r0_internal(&mut f_ref, 0, y, w, cdeffed);
388 assert_eq!(&f[..w], &f_ref[..]);
389 }
390 }
391
392 #[inline]
393 #[target_feature(enable = "avx2")]
394 unsafe fn sgrproj_box_f_r1_8_avx2<T: Pixel>(
395 af: &[&[u32]; 3], bf: &[&[u32]; 3], f: &mut [u32], x: usize, y: usize,
396 cdeffed: &PlaneSlice<T>,
397 ) {
398 let three = _mm256_set1_epi32(3);
399 let four = _mm256_set1_epi32(4);
400 let a0 = af[0].as_ptr();
401 let a1 = af[1].as_ptr();
402 let a2 = af[2].as_ptr();
403 let b0 = bf[0].as_ptr();
404 let b1 = bf[1].as_ptr();
405 let b2 = bf[2].as_ptr();
406 let a = _mm256_add_epi32(
407 _mm256_madd_epi16(
408 _mm256_add_epi32(
409 _mm256_add_epi32(
410 _mm256_loadu_si256(a0.add(x) as *const _),
411 _mm256_loadu_si256(a0.add(x + 2) as *const _),
412 ),
413 _mm256_add_epi32(
414 _mm256_loadu_si256(a2.add(x) as *const _),
415 _mm256_loadu_si256(a2.add(x + 2) as *const _),
416 ),
417 ),
418 three,
419 ),
420 _mm256_madd_epi16(
421 _mm256_add_epi32(
422 _mm256_add_epi32(
423 _mm256_loadu_si256(a1.add(x) as *const _),
424 _mm256_loadu_si256(a0.add(x + 1) as *const _),
425 ),
426 _mm256_add_epi32(
427 _mm256_add_epi32(
428 _mm256_loadu_si256(a1.add(x + 1) as *const _),
429 _mm256_loadu_si256(a2.add(x + 1) as *const _),
430 ),
431 _mm256_loadu_si256(a1.add(x + 2) as *const _),
432 ),
433 ),
434 four,
435 ),
436 );
437 let b = _mm256_add_epi32(
438 _mm256_mullo_epi32(
439 _mm256_add_epi32(
440 _mm256_add_epi32(
441 _mm256_loadu_si256(b0.add(x) as *const _),
442 _mm256_loadu_si256(b0.add(x + 2) as *const _),
443 ),
444 _mm256_add_epi32(
445 _mm256_loadu_si256(b2.add(x) as *const _),
446 _mm256_loadu_si256(b2.add(x + 2) as *const _),
447 ),
448 ),
449 three,
450 ),
451 _mm256_mullo_epi32(
452 _mm256_add_epi32(
453 _mm256_add_epi32(
454 _mm256_loadu_si256(b1.add(x) as *const _),
455 _mm256_loadu_si256(b0.add(x + 1) as *const _),
456 ),
457 _mm256_add_epi32(
458 _mm256_add_epi32(
459 _mm256_loadu_si256(b1.add(x + 1) as *const _),
460 _mm256_loadu_si256(b2.add(x + 1) as *const _),
461 ),
462 _mm256_loadu_si256(b1.add(x + 2) as *const _),
463 ),
464 ),
465 four,
466 ),
467 );
468 let v = _mm256_add_epi32(
469 _mm256_madd_epi16(
470 a,
471 if mem::size_of::<T>() == 1 {
472 _mm256_cvtepu8_epi32(_mm_loadl_epi64(
473 cdeffed.subslice(x, y).as_ptr() as *const _
474 ))
475 } else {
476 _mm256_cvtepu16_epi32(_mm_loadu_si128(
477 cdeffed.subslice(x, y).as_ptr() as *const _
478 ))
479 },
480 ),
481 b,
482 );
483 const SHIFT: i32 = (5 + SGRPROJ_SGR_BITS - SGRPROJ_RST_BITS) as i32;
484 _mm256_storeu_si256(
485 f.as_mut_ptr().add(x) as *mut _,
486 _mm256_srli_epi32(
487 _mm256_add_epi32(v, _mm256_set1_epi32(1 << SHIFT >> 1)),
488 SHIFT,
489 ),
490 );
491 }
492
493 #[target_feature(enable = "avx2")]
494 pub(crate) unsafe fn sgrproj_box_f_r1_avx2<T: Pixel>(
495 af: &[&[u32]; 3], bf: &[&[u32]; 3], f: &mut [u32], y: usize, w: usize,
496 cdeffed: &PlaneSlice<T>,
497 ) {
498 for x in (0..w).step_by(8) {
499 if x + 8 <= w {
500 sgrproj_box_f_r1_8_avx2(af, bf, f, x, y, cdeffed);
501 } else {
502 // finish using scalar
503 rust::sgrproj_box_f_r1_internal(af, bf, f, x, y, w, cdeffed);
504 }
505 }
506
507 #[cfg(feature = "check_asm")]
508 {
509 let mut f_ref: Vec<u32> = vec![0; w];
510 rust::sgrproj_box_f_r1_internal(af, bf, &mut f_ref, 0, y, w, cdeffed);
511 assert_eq!(&f[..w], &f_ref[..]);
512 }
513 }
514
515 #[inline]
516 #[target_feature(enable = "avx2")]
517 unsafe fn sgrproj_box_f_r2_8_avx2<T: Pixel>(
518 af: &[&[u32]; 2], bf: &[&[u32]; 2], f0: &mut [u32], f1: &mut [u32],
519 x: usize, y: usize, cdeffed: &PlaneSlice<T>,
520 ) {
521 let five = _mm256_set1_epi32(5);
522 let six = _mm256_set1_epi32(6);
523 let a0 = af[0].as_ptr();
524 let a2 = af[1].as_ptr();
525 let b0 = bf[0].as_ptr();
526 let b2 = bf[1].as_ptr();
527 let a = _mm256_add_epi32(
528 _mm256_madd_epi16(
529 _mm256_add_epi32(
530 _mm256_loadu_si256(a0.add(x) as *const _),
531 _mm256_loadu_si256(a0.add(x + 2) as *const _),
532 ),
533 five,
534 ),
535 _mm256_madd_epi16(_mm256_loadu_si256(a0.add(x + 1) as *const _), six),
536 );
537 let b = _mm256_add_epi32(
538 _mm256_mullo_epi32(
539 _mm256_add_epi32(
540 _mm256_loadu_si256(b0.add(x) as *const _),
541 _mm256_loadu_si256(b0.add(x + 2) as *const _),
542 ),
543 five,
544 ),
545 _mm256_mullo_epi32(_mm256_loadu_si256(b0.add(x + 1) as *const _), six),
546 );
547 let ao = _mm256_add_epi32(
548 _mm256_madd_epi16(
549 _mm256_add_epi32(
550 _mm256_loadu_si256(a2.add(x) as *const _),
551 _mm256_loadu_si256(a2.add(x + 2) as *const _),
552 ),
553 five,
554 ),
555 _mm256_madd_epi16(_mm256_loadu_si256(a2.add(x + 1) as *const _), six),
556 );
557 let bo = _mm256_add_epi32(
558 _mm256_mullo_epi32(
559 _mm256_add_epi32(
560 _mm256_loadu_si256(b2.add(x) as *const _),
561 _mm256_loadu_si256(b2.add(x + 2) as *const _),
562 ),
563 five,
564 ),
565 _mm256_mullo_epi32(_mm256_loadu_si256(b2.add(x + 1) as *const _), six),
566 );
567 let v = _mm256_add_epi32(
568 _mm256_madd_epi16(
569 _mm256_add_epi32(a, ao),
570 if mem::size_of::<T>() == 1 {
571 _mm256_cvtepu8_epi32(_mm_loadl_epi64(
572 cdeffed.subslice(x, y).as_ptr() as *const _
573 ))
574 } else {
575 _mm256_cvtepu16_epi32(_mm_loadu_si128(
576 cdeffed.subslice(x, y).as_ptr() as *const _
577 ))
578 },
579 ),
580 _mm256_add_epi32(b, bo),
581 );
582 let vo = _mm256_add_epi32(
583 _mm256_madd_epi16(
584 ao,
585 if mem::size_of::<T>() == 1 {
586 _mm256_cvtepu8_epi32(_mm_loadl_epi64(
587 cdeffed.subslice(x, y + 1).as_ptr() as *const _,
588 ))
589 } else {
590 _mm256_cvtepu16_epi32(_mm_loadu_si128(
591 cdeffed.subslice(x, y + 1).as_ptr() as *const _,
592 ))
593 },
594 ),
595 bo,
596 );
597 const SHIFT: i32 = (5 + SGRPROJ_SGR_BITS - SGRPROJ_RST_BITS) as i32;
598 _mm256_storeu_si256(
599 f0.as_mut_ptr().add(x) as *mut _,
600 _mm256_srli_epi32(
601 _mm256_add_epi32(v, _mm256_set1_epi32(1 << SHIFT >> 1)),
602 SHIFT,
603 ),
604 );
605 const SHIFTO: i32 = (4 + SGRPROJ_SGR_BITS - SGRPROJ_RST_BITS) as i32;
606 _mm256_storeu_si256(
607 f1.as_mut_ptr().add(x) as *mut _,
608 _mm256_srli_epi32(
609 _mm256_add_epi32(vo, _mm256_set1_epi32(1 << SHIFTO >> 1)),
610 SHIFTO,
611 ),
612 );
613 }
614
615 #[target_feature(enable = "avx2")]
616 pub(crate) unsafe fn sgrproj_box_f_r2_avx2<T: Pixel>(
617 af: &[&[u32]; 2], bf: &[&[u32]; 2], f0: &mut [u32], f1: &mut [u32],
618 y: usize, w: usize, cdeffed: &PlaneSlice<T>,
619 ) {
620 for x in (0..w).step_by(8) {
621 if x + 8 <= w {
622 sgrproj_box_f_r2_8_avx2(af, bf, f0, f1, x, y, cdeffed);
623 } else {
624 // finish using scalar
625 rust::sgrproj_box_f_r2_internal(af, bf, f0, f1, x, y, w, cdeffed);
626 }
627 }
628
629 #[cfg(feature = "check_asm")]
630 {
631 let mut f0_ref: Vec<u32> = vec![0; w];
632 let mut f1_ref: Vec<u32> = vec![0; w];
633 rust::sgrproj_box_f_r2_internal(
634 af,
635 bf,
636 &mut f0_ref,
637 &mut f1_ref,
638 0,
639 y,
640 w,
641 cdeffed,
642 );
643 assert_eq!(&f0[..w], &f0_ref[..]);
644 assert_eq!(&f1[..w], &f1_ref[..]);
645 }
646 }