| 1 |
/* |
| 2 |
* Copyright (c) 2025. |
| 3 |
* |
| 4 |
* This software is free software; |
| 5 |
* |
| 6 |
* You can redistribute it or modify it under terms of the MIT, Apache License or Zlib license |
| 7 |
*/ |
| 8 |
|
| 9 |
#[cfg(target_arch = "aarch64")] |
| 10 |
use core::arch::aarch64::*; |
| 11 |
|
| 12 |
#[cfg(target_arch = "aarch64")] |
| 13 |
#[target_feature(enable = "neon")] |
| 14 |
pub fn upsample_horizontal_neon( |
| 15 |
input: &[i16], |
| 16 |
in_near: &[i16], |
| 17 |
in_far: &[i16], |
| 18 |
scratch: &mut [i16], |
| 19 |
output: &mut [i16], |
| 20 |
) { |
| 21 |
assert_eq!(input.len() * 2, output.len()); |
| 22 |
assert!(input.len() > 2); |
| 23 |
|
| 24 |
let len = input.len(); |
| 25 |
|
| 26 |
if len < 10 { |
| 27 |
return super::scalar::upsample_horizontal(input, in_near, in_far, scratch, output); |
| 28 |
} |
| 29 |
|
| 30 |
// First two pixels |
| 31 |
output[0] = input[0]; |
| 32 |
output[1] = (input[0] * 3 + input[1] + 2) >> 2; |
| 33 |
|
| 34 |
let v_three = vdupq_n_s16(3); |
| 35 |
let v_two = vdupq_n_s16(2); |
| 36 |
|
| 37 |
let upsample8 = |input: &[i16; 10], output: &mut [i16; 16]| { |
| 38 |
let in_ptr = input.as_ptr(); |
| 39 |
let out_ptr = output.as_mut_ptr(); |
| 40 |
|
| 41 |
// SAFETY: The input is 10 * 16 bit long, so the loads are safe. |
| 42 |
let (v_prev, v_curr, v_next) = unsafe { |
| 43 |
( |
| 44 |
vld1q_s16(in_ptr), |
| 45 |
vld1q_s16(in_ptr.add(1)), |
| 46 |
vld1q_s16(in_ptr.add(2)), |
| 47 |
) |
| 48 |
}; |
| 49 |
|
| 50 |
let v_common = vaddq_s16(vmulq_s16(v_curr, v_three), v_two); |
| 51 |
let v_even = vshrq_n_s16::<2>(vaddq_s16(v_common, v_prev)); |
| 52 |
let v_odd = vshrq_n_s16::<2>(vaddq_s16(v_common, v_next)); |
| 53 |
|
| 54 |
let v_res_1 = vzip1q_s16(v_even, v_odd); |
| 55 |
let v_res_2 = vzip2q_s16(v_even, v_odd); |
| 56 |
|
| 57 |
// SAFETY: The output is 16 * 16 bit long, so the stores are safe. |
| 58 |
unsafe { |
| 59 |
vst1q_s16(out_ptr, v_res_1); |
| 60 |
vst1q_s16(out_ptr.add(8), v_res_2); |
| 61 |
} |
| 62 |
}; |
| 63 |
|
| 64 |
for (input, output) in input |
| 65 |
.windows(10) |
| 66 |
.step_by(8) |
| 67 |
.zip(output[2..].chunks_exact_mut(16)) |
| 68 |
{ |
| 69 |
upsample8(input.try_into().unwrap(), output.try_into().unwrap()); |
| 70 |
} |
| 71 |
|
| 72 |
// Upsample the remainder. This may have some overlap, but that's fine. |
| 73 |
if let Some(rest_input) = input.last_chunk::<10>() { |
| 74 |
let end = output.len() - 2; |
| 75 |
if let Some(rest_output) = output[..end].last_chunk_mut::<16>() { |
| 76 |
upsample8(rest_input, rest_output); |
| 77 |
} |
| 78 |
} |
| 79 |
|
| 80 |
// Last two pixels. |
| 81 |
output[output.len() - 2] = (3 * input[len - 1] + input[len - 2] + 2) >> 2; |
| 82 |
output[output.len() - 1] = input[len - 1]; |
| 83 |
} |
| 84 |
|
| 85 |
#[cfg(target_arch = "aarch64")] |
| 86 |
#[target_feature(enable = "neon")] |
| 87 |
pub fn upsample_vertical_neon( |
| 88 |
input: &[i16], |
| 89 |
in_near: &[i16], |
| 90 |
in_far: &[i16], |
| 91 |
scratch: &mut [i16], |
| 92 |
output: &mut [i16], |
| 93 |
) { |
| 94 |
assert_eq!(input.len() * 2, output.len()); |
| 95 |
assert_eq!(in_near.len(), input.len()); |
| 96 |
assert_eq!(in_far.len(), input.len()); |
| 97 |
|
| 98 |
let len = input.len(); |
| 99 |
|
| 100 |
if len < 16 { |
| 101 |
return super::scalar::upsample_vertical(input, in_near, in_far, scratch, output); |
| 102 |
} |
| 103 |
|
| 104 |
let middle = output.len() / 2; |
| 105 |
let (out_top, out_bottom) = output.split_at_mut(middle); |
| 106 |
|
| 107 |
let v_three = vdupq_n_s16(3); |
| 108 |
let v_two = vdupq_n_s16(2); |
| 109 |
|
| 110 |
let upsample8 = |input: &[i16; 8], |
| 111 |
in_near: &[i16; 8], |
| 112 |
in_far: &[i16; 8], |
| 113 |
out_top: &mut [i16; 8], |
| 114 |
out_bottom: &mut [i16; 8]| { |
| 115 |
// SAFETY: Inputs are all 8 * 16 bit long, so the loads are safe. |
| 116 |
let (v_in, v_near, v_far) = unsafe { |
| 117 |
( |
| 118 |
vld1q_s16(input.as_ptr()), |
| 119 |
vld1q_s16(in_near.as_ptr()), |
| 120 |
vld1q_s16(in_far.as_ptr()), |
| 121 |
) |
| 122 |
}; |
| 123 |
|
| 124 |
let v_common = vaddq_s16(vmulq_s16(v_in, v_three), v_two); |
| 125 |
let v_out_top = vshrq_n_s16::<2>(vaddq_s16(v_common, v_near)); |
| 126 |
let v_out_bottom = vshrq_n_s16::<2>(vaddq_s16(v_common, v_far)); |
| 127 |
|
| 128 |
// SAFETY: Outputs are 8 * 16 bit long, so the stores are safe. |
| 129 |
unsafe { |
| 130 |
vst1q_s16(out_top.as_mut_ptr(), v_out_top); |
| 131 |
vst1q_s16(out_bottom.as_mut_ptr(), v_out_bottom); |
| 132 |
} |
| 133 |
}; |
| 134 |
|
| 135 |
let chunks = input |
| 136 |
.chunks_exact(8) |
| 137 |
.zip(in_near.chunks_exact(8)) |
| 138 |
.zip(in_far.chunks_exact(8)) |
| 139 |
.zip(out_top.chunks_exact_mut(8)) |
| 140 |
.zip(out_bottom.chunks_exact_mut(8)); |
| 141 |
|
| 142 |
for ((((input, in_near), in_far), out_top), out_bottom) in chunks { |
| 143 |
upsample8( |
| 144 |
input.try_into().unwrap(), |
| 145 |
in_near.try_into().unwrap(), |
| 146 |
in_far.try_into().unwrap(), |
| 147 |
out_top.try_into().unwrap(), |
| 148 |
out_bottom.try_into().unwrap(), |
| 149 |
); |
| 150 |
} |
| 151 |
|
| 152 |
// Upsample the remainder. |
| 153 |
if let Some(rest) = input.last_chunk::<8>() { |
| 154 |
if let Some(rest_near) = in_near.last_chunk::<8>() { |
| 155 |
if let Some(rest_far) = in_far.last_chunk::<8>() { |
| 156 |
if let Some(mut rest_top) = out_top.last_chunk_mut::<8>() { |
| 157 |
if let Some(mut rest_bottom) = out_bottom.last_chunk_mut::<8>() { |
| 158 |
upsample8(rest, rest_near, rest_far, &mut rest_top, &mut rest_bottom); |
| 159 |
} |
| 160 |
} |
| 161 |
} |
| 162 |
} |
| 163 |
} |
| 164 |
} |
| 165 |
|
| 166 |
#[cfg(target_arch = "aarch64")] |
| 167 |
#[target_feature(enable = "neon")] |
| 168 |
pub fn upsample_hv_neon( |
| 169 |
input: &[i16], |
| 170 |
in_near: &[i16], |
| 171 |
in_far: &[i16], |
| 172 |
scratch_space: &mut [i16], |
| 173 |
output: &mut [i16], |
| 174 |
) { |
| 175 |
assert_eq!(input.len() * 4, output.len()); |
| 176 |
assert_eq!(input.len() * 2, scratch_space.len()); |
| 177 |
|
| 178 |
upsample_vertical_neon(input, in_near, in_far, &mut [], scratch_space); |
| 179 |
|
| 180 |
let scratch_half = scratch_space.len() / 2; |
| 181 |
let output_half = output.len() / 2; |
| 182 |
|
| 183 |
let (scratch_top, scratch_bottom) = scratch_space.split_at_mut(scratch_half); |
| 184 |
let (out_top, out_bottom) = output.split_at_mut(output_half); |
| 185 |
|
| 186 |
let mut t = [0]; |
| 187 |
upsample_horizontal_neon(scratch_top, &[], &[], &mut t, out_top); |
| 188 |
upsample_horizontal_neon(scratch_bottom, &[], &[], &mut t, out_bottom); |
| 189 |
} |