| 1 |
use crate::Complex; |
| 2 |
use crate::FftNum; |
| 3 |
use std::ops::{Deref, DerefMut}; |
| 4 |
|
| 5 |
/// Given an array of size width * height, representing a flattened 2D array, |
| 6 |
/// transpose the rows and columns of that 2D array into the output |
| 7 |
/// benchmarking shows that loop tiling isn't effective for small arrays (in the range of 50x50 or smaller) |
| 8 |
pub unsafe fn transpose_small<T: Copy>(width: usize, height: usize, input: &[T], output: &mut [T]) { |
| 9 |
for x in 0..width { |
| 10 |
for y in 0..height { |
| 11 |
let input_index = x + y * width; |
| 12 |
let output_index = y + x * height; |
| 13 |
|
| 14 |
*output.get_unchecked_mut(output_index) = *input.get_unchecked(input_index); |
| 15 |
} |
| 16 |
} |
| 17 |
} |
| 18 |
|
| 19 |
#[allow(unused)] |
| 20 |
pub unsafe fn workaround_transmute<T, U>(slice: &[T]) -> &[U] { |
| 21 |
let ptr = slice.as_ptr() as *const U; |
| 22 |
let len = slice.len(); |
| 23 |
std::slice::from_raw_parts(ptr, len) |
| 24 |
} |
| 25 |
#[allow(unused)] |
| 26 |
pub unsafe fn workaround_transmute_mut<T, U>(slice: &mut [T]) -> &mut [U] { |
| 27 |
let ptr = slice.as_mut_ptr() as *mut U; |
| 28 |
let len = slice.len(); |
| 29 |
std::slice::from_raw_parts_mut(ptr, len) |
| 30 |
} |
| 31 |
|
| 32 |
pub(crate) trait LoadStore<T: FftNum>: DerefMut { |
| 33 |
unsafe fn load(&self, idx: usize) -> Complex<T>; |
| 34 |
unsafe fn store(&mut self, val: Complex<T>, idx: usize); |
| 35 |
} |
| 36 |
|
| 37 |
impl<T: FftNum> LoadStore<T> for &mut [Complex<T>] { |
| 38 |
#[inline(always)] |
| 39 |
unsafe fn load(&self, idx: usize) -> Complex<T> { |
| 40 |
debug_assert!(idx < self.len()); |
| 41 |
*self.get_unchecked(idx) |
| 42 |
} |
| 43 |
#[inline(always)] |
| 44 |
unsafe fn store(&mut self, val: Complex<T>, idx: usize) { |
| 45 |
debug_assert!(idx < self.len()); |
| 46 |
*self.get_unchecked_mut(idx) = val; |
| 47 |
} |
| 48 |
} |
| 49 |
impl<T: FftNum, const N: usize> LoadStore<T> for &mut [Complex<T>; N] { |
| 50 |
#[inline(always)] |
| 51 |
unsafe fn load(&self, idx: usize) -> Complex<T> { |
| 52 |
debug_assert!(idx < self.len()); |
| 53 |
*self.get_unchecked(idx) |
| 54 |
} |
| 55 |
#[inline(always)] |
| 56 |
unsafe fn store(&mut self, val: Complex<T>, idx: usize) { |
| 57 |
debug_assert!(idx < self.len()); |
| 58 |
*self.get_unchecked_mut(idx) = val; |
| 59 |
} |
| 60 |
} |
| 61 |
|
| 62 |
pub(crate) struct DoubleBuf<'a, T> { |
| 63 |
pub input: &'a [Complex<T>], |
| 64 |
pub output: &'a mut [Complex<T>], |
| 65 |
} |
| 66 |
impl<'a, T> Deref for DoubleBuf<'a, T> { |
| 67 |
type Target = [Complex<T>]; |
| 68 |
fn deref(&self) -> &Self::Target { |
| 69 |
self.input |
| 70 |
} |
| 71 |
} |
| 72 |
impl<'a, T> DerefMut for DoubleBuf<'a, T> { |
| 73 |
fn deref_mut(&mut self) -> &mut Self::Target { |
| 74 |
self.output |
| 75 |
} |
| 76 |
} |
| 77 |
impl<'a, T: FftNum> LoadStore<T> for DoubleBuf<'a, T> { |
| 78 |
#[inline(always)] |
| 79 |
unsafe fn load(&self, idx: usize) -> Complex<T> { |
| 80 |
debug_assert!(idx < self.input.len()); |
| 81 |
*self.input.get_unchecked(idx) |
| 82 |
} |
| 83 |
#[inline(always)] |
| 84 |
unsafe fn store(&mut self, val: Complex<T>, idx: usize) { |
| 85 |
debug_assert!(idx < self.output.len()); |
| 86 |
*self.output.get_unchecked_mut(idx) = val; |
| 87 |
} |
| 88 |
} |
| 89 |
|
| 90 |
#[cfg(test)] |
| 91 |
mod unit_tests { |
| 92 |
use super::*; |
| 93 |
use crate::test_utils::random_signal; |
| 94 |
use num_complex::Complex; |
| 95 |
use num_traits::Zero; |
| 96 |
|
| 97 |
#[test] |
| 98 |
fn test_transpose() { |
| 99 |
let sizes: Vec<usize> = (1..16).collect(); |
| 100 |
|
| 101 |
for &width in &sizes { |
| 102 |
for &height in &sizes { |
| 103 |
let len = width * height; |
| 104 |
|
| 105 |
let input: Vec<Complex<f32>> = random_signal(len); |
| 106 |
let mut output = vec![Zero::zero(); len]; |
| 107 |
|
| 108 |
unsafe { transpose_small(width, height, &input, &mut output) }; |
| 109 |
|
| 110 |
for x in 0..width { |
| 111 |
for y in 0..height { |
| 112 |
assert_eq!( |
| 113 |
input[x + y * width], |
| 114 |
output[y + x * height], |
| 115 |
"x = {}, y = {}", |
| 116 |
x, |
| 117 |
y |
| 118 |
); |
| 119 |
} |
| 120 |
} |
| 121 |
} |
| 122 |
} |
| 123 |
} |
| 124 |
} |
| 125 |
|
| 126 |
// Loop over exact chunks of the provided buffer. Very similar in semantics to ChunksExactMut, but generates smaller code and requires no modulo operations |
| 127 |
// Returns Ok() if every element ended up in a chunk, Err() if there was a remainder |
| 128 |
pub fn iter_chunks<T>( |
| 129 |
mut buffer: &mut [T], |
| 130 |
chunk_size: usize, |
| 131 |
mut chunk_fn: impl FnMut(&mut [T]), |
| 132 |
) -> Result<(), ()> { |
| 133 |
// Loop over the buffer, splicing off chunk_size at a time, and calling chunk_fn on each |
| 134 |
while buffer.len() >= chunk_size { |
| 135 |
let (head, tail) = buffer.split_at_mut(chunk_size); |
| 136 |
buffer = tail; |
| 137 |
|
| 138 |
chunk_fn(head); |
| 139 |
} |
| 140 |
|
| 141 |
// We have a remainder if there's data still in the buffer -- in which case we want to indicate to the caller that there was an unwanted remainder |
| 142 |
if buffer.len() == 0 { |
| 143 |
Ok(()) |
| 144 |
} else { |
| 145 |
Err(()) |
| 146 |
} |
| 147 |
} |
| 148 |
|
| 149 |
// Loop over exact zipped chunks of the 2 provided buffers. Very similar in semantics to ChunksExactMut.zip(ChunksExactMut), but generates smaller code and requires no modulo operations |
| 150 |
// Returns Ok() if every element of both buffers ended up in a chunk, Err() if there was a remainder |
| 151 |
pub fn iter_chunks_zipped<T>( |
| 152 |
mut buffer1: &mut [T], |
| 153 |
mut buffer2: &mut [T], |
| 154 |
chunk_size: usize, |
| 155 |
mut chunk_fn: impl FnMut(&mut [T], &mut [T]), |
| 156 |
) -> Result<(), ()> { |
| 157 |
// If the two buffers aren't the same size, record the fact that they're different, then snip them to be the same size |
| 158 |
let uneven = if buffer1.len() > buffer2.len() { |
| 159 |
buffer1 = &mut buffer1[..buffer2.len()]; |
| 160 |
true |
| 161 |
} else if buffer2.len() < buffer1.len() { |
| 162 |
buffer2 = &mut buffer2[..buffer1.len()]; |
| 163 |
true |
| 164 |
} else { |
| 165 |
false |
| 166 |
}; |
| 167 |
|
| 168 |
// Now that we know the two slices are the same length, loop over each one, splicing off chunk_size at a time, and calling chunk_fn on each |
| 169 |
while buffer1.len() >= chunk_size && buffer2.len() >= chunk_size { |
| 170 |
let (head1, tail1) = buffer1.split_at_mut(chunk_size); |
| 171 |
buffer1 = tail1; |
| 172 |
|
| 173 |
let (head2, tail2) = buffer2.split_at_mut(chunk_size); |
| 174 |
buffer2 = tail2; |
| 175 |
|
| 176 |
chunk_fn(head1, head2); |
| 177 |
} |
| 178 |
|
| 179 |
// We have a remainder if the 2 chunks were uneven to start with, or if there's still data in the buffers -- in which case we want to indicate to the caller that there was an unwanted remainder |
| 180 |
if !uneven && buffer1.len() == 0 { |
| 181 |
Ok(()) |
| 182 |
} else { |
| 183 |
Err(()) |
| 184 |
} |
| 185 |
} |