| 1 |
/* |
| 2 |
* Copyright (c) 2023. |
| 3 |
* |
| 4 |
* This software is free software; |
| 5 |
* |
| 6 |
* You can redistribute it or modify it under terms of the MIT, Apache License or Zlib license |
| 7 |
*/ |
| 8 |
|
| 9 |
use std::simd::prelude::*; |
| 10 |
|
| 11 |
const LANES: usize = 16; |
| 12 |
type V = Simd<i16, LANES>; |
| 13 |
|
| 14 |
pub fn upsample_horizontal_simd( |
| 15 |
input: &[i16], |
| 16 |
in_near: &[i16], |
| 17 |
in_far: &[i16], |
| 18 |
scratch: &mut [i16], |
| 19 |
output: &mut [i16], |
| 20 |
) { |
| 21 |
assert_eq!(input.len() * 2, output.len()); |
| 22 |
assert!(input.len() > 2); |
| 23 |
|
| 24 |
let len = input.len(); |
| 25 |
|
| 26 |
if len < 18 { |
| 27 |
return super::scalar::upsample_horizontal(input, in_near, in_far, scratch, output); |
| 28 |
} |
| 29 |
|
| 30 |
// First two pixels |
| 31 |
output[0] = input[0]; |
| 32 |
output[1] = (input[0] * 3 + input[1] + 2) >> 2; |
| 33 |
|
| 34 |
let v_three = V::splat(3); |
| 35 |
let v_two = V::splat(2); |
| 36 |
|
| 37 |
let upsample16 = |input: &[i16; 18], output: &mut [i16; 32]| { |
| 38 |
let v_prev = V::from_slice(&input[0..LANES]); |
| 39 |
let v_curr = V::from_slice(&input[1..LANES + 1]); |
| 40 |
let v_next = V::from_slice(&input[2..LANES + 2]); |
| 41 |
|
| 42 |
let v_common = v_curr * v_three + v_two; |
| 43 |
|
| 44 |
let v_even = (v_common + v_prev) >> 2; |
| 45 |
let v_odd = (v_common + v_next) >> 2; |
| 46 |
|
| 47 |
let (v_res_1, v_res_2) = v_even.interleave(v_odd); |
| 48 |
|
| 49 |
v_res_1.copy_to_slice(&mut output[0..LANES]); |
| 50 |
v_res_2.copy_to_slice(&mut output[LANES..2 * LANES]); |
| 51 |
}; |
| 52 |
|
| 53 |
for (input, output) in input |
| 54 |
.windows(18) |
| 55 |
.step_by(16) |
| 56 |
.zip(output[2..].chunks_exact_mut(32)) |
| 57 |
{ |
| 58 |
upsample16(input.try_into().unwrap(), output.try_into().unwrap()); |
| 59 |
} |
| 60 |
|
| 61 |
// Upsample the remainder. This may have some overlap, but that's fine. |
| 62 |
if let Some(rest_input) = input.last_chunk::<18>() { |
| 63 |
let end = output.len() - 2; |
| 64 |
if let Some(rest_output) = output[..end].last_chunk_mut::<32>() { |
| 65 |
upsample16(rest_input, rest_output); |
| 66 |
} |
| 67 |
} |
| 68 |
|
| 69 |
// Last two pixels. |
| 70 |
output[output.len() - 2] = (3 * input[len - 1] + input[len - 2] + 2) >> 2; |
| 71 |
output[output.len() - 1] = input[len - 1]; |
| 72 |
} |
| 73 |
|
| 74 |
pub fn upsample_vertical_simd( |
| 75 |
input: &[i16], |
| 76 |
in_near: &[i16], |
| 77 |
in_far: &[i16], |
| 78 |
_scratch_space: &mut [i16], |
| 79 |
output: &mut [i16], |
| 80 |
) { |
| 81 |
assert_eq!(input.len() * 2, output.len()); |
| 82 |
assert_eq!(in_near.len(), input.len()); |
| 83 |
assert_eq!(in_far.len(), input.len()); |
| 84 |
|
| 85 |
let len = input.len(); |
| 86 |
|
| 87 |
if len < 16 { |
| 88 |
return super::scalar::upsample_vertical(input, in_near, in_far, _scratch_space, output); |
| 89 |
} |
| 90 |
|
| 91 |
let middle = output.len() / 2; |
| 92 |
let (out_top, out_bottom) = output.split_at_mut(middle); |
| 93 |
|
| 94 |
let v_three = V::splat(3); |
| 95 |
let v_two = V::splat(2); |
| 96 |
|
| 97 |
let upsample16 = |input: &[i16; 16], |
| 98 |
in_near: &[i16; 16], |
| 99 |
in_far: &[i16; 16], |
| 100 |
out_top: &mut [i16; 16], |
| 101 |
out_bottom: &mut [i16; 16]| { |
| 102 |
let v_in = V::from(*input); |
| 103 |
let v_near = V::from(*in_near); |
| 104 |
let v_far = V::from(*in_far); |
| 105 |
|
| 106 |
let v_common = v_in * v_three + v_two; |
| 107 |
|
| 108 |
let v_out_top = (v_common + v_near) >> 2; |
| 109 |
let v_out_bottom = (v_common + v_far) >> 2; |
| 110 |
|
| 111 |
v_out_top.copy_to_slice(out_top.as_mut_slice()); |
| 112 |
v_out_bottom.copy_to_slice(out_bottom.as_mut_slice()); |
| 113 |
}; |
| 114 |
|
| 115 |
let chunks = input |
| 116 |
.chunks_exact(16) |
| 117 |
.zip(in_near.chunks_exact(16)) |
| 118 |
.zip(in_far.chunks_exact(16)) |
| 119 |
.zip(out_top.chunks_exact_mut(16)) |
| 120 |
.zip(out_bottom.chunks_exact_mut(16)); |
| 121 |
|
| 122 |
for ((((input, in_near), in_far), out_top), out_bottom) in chunks { |
| 123 |
upsample16( |
| 124 |
input.try_into().unwrap(), |
| 125 |
in_near.try_into().unwrap(), |
| 126 |
in_far.try_into().unwrap(), |
| 127 |
out_top.try_into().unwrap(), |
| 128 |
out_bottom.try_into().unwrap(), |
| 129 |
); |
| 130 |
} |
| 131 |
|
| 132 |
// Upsample the remainder. This may have some overlap, but that's fine. |
| 133 |
// Edition upgrade will fix this nested awfulness. |
| 134 |
if let Some(rest) = input.last_chunk::<16>() { |
| 135 |
if let Some( rest_near) = in_near.last_chunk::<16>() { |
| 136 |
if let Some( rest_far) = in_far.last_chunk::<16>() { |
| 137 |
if let Some( rest_top) = out_top.last_chunk_mut::<16>() { |
| 138 |
if let Some( rest_bottom) = out_bottom.last_chunk_mut::<16>() { |
| 139 |
upsample16(rest, rest_near, rest_far, rest_top, rest_bottom); |
| 140 |
} |
| 141 |
} |
| 142 |
} |
| 143 |
} |
| 144 |
} |
| 145 |
} |
| 146 |
|
| 147 |
pub fn upsample_hv_simd( |
| 148 |
input: &[i16], |
| 149 |
in_near: &[i16], |
| 150 |
in_far: &[i16], |
| 151 |
scratch_space: &mut [i16], |
| 152 |
output: &mut [i16], |
| 153 |
) { |
| 154 |
assert_eq!(input.len() * 4, output.len()); |
| 155 |
assert_eq!(input.len() * 2, scratch_space.len()); |
| 156 |
|
| 157 |
upsample_vertical_simd(input, in_near, in_far, &mut [], scratch_space); |
| 158 |
|
| 159 |
let scratch_half = scratch_space.len() / 2; |
| 160 |
let output_half = output.len() / 2; |
| 161 |
|
| 162 |
let (scratch_top, scratch_bottom) = scratch_space.split_at_mut(scratch_half); |
| 163 |
let (out_top, out_bottom) = output.split_at_mut(output_half); |
| 164 |
|
| 165 |
let mut t = [0]; |
| 166 |
upsample_horizontal_simd(scratch_top, &[], &[], &mut t, out_top); |
| 167 |
upsample_horizontal_simd(scratch_bottom, &[], &[], &mut t, out_bottom); |
| 168 |
} |