Debian Codegraph @

.cargo/vendor/zune-jpeg-0.4.21/src/idct/neon.rs

raw
1 /*
2 * Copyright (c) 2023.
3 *
4 * This software is free software;
5 *
6 * You can redistribute it or modify it under terms of the MIT, Apache License or Zlib license
7 */
8
9 #![cfg(target_arch = "aarch64")]
10 //! AVX optimised IDCT.
11 //!
12 //! Okay not thaat optimised.
13 //!
14 //!
15 //! # The implementation
16 //! The implementation is neatly broken down into two operations.
17 //!
18 //! 1. Test for zeroes
19 //! > There is a shortcut method for idct where when all AC values are zero, we can get the answer really quickly.
20 //! by scaling the 1/8th of the DCT coefficient of the block to the whole block and level shifting.
21 //!
22 //! 2. If above fails, we proceed to carry out IDCT as a two pass one dimensional algorithm.
23 //! IT does two whole scans where it carries out IDCT on all items
24 //! After each successive scan, data is transposed in register(thank you x86 SIMD powers). and the second
25 //! pass is carried out.
26 //!
27 //! The code is not super optimized, it produces bit identical results with scalar code hence it's
28 //! `mm256_add_epi16`
29 //! and it also has the advantage of making this implementation easy to maintain.
30
31 #![cfg(feature = "neon")]
32
33 use core::arch::aarch64::*;
34
35 use crate::unsafe_utils::{transpose, YmmRegister};
36
37 const SCALE_BITS: i32 = 512 + 65536 + (128 << 17);
38
39 /// SAFETY
40 /// ------
41 ///
42 /// It is the responsibility of the CALLER to ensure that this function is
43 /// called in contexts where the CPU supports it
44 ///
45 ///
46 /// For documentation see module docs.
47
48 pub fn idct_neon(in_vector: &mut [i32; 64], out_vector: &mut [i16], stride: usize) {
49 unsafe {
50 // We don't call this method directly because we need to flag the code function
51 // with #[target_feature] so that the compiler does do weird stuff with
52 // it
53 idct_int_neon_inner(in_vector, out_vector, stride);
54 }
55 }
56
57 #[inline]
58 #[target_feature(enable = "neon")]
59 unsafe fn pack_16(a: int32x4x2_t) -> int16x8_t {
60 vcombine_s16(vqmovn_s32(a.0), vqmovn_s32(a.1))
61 }
62
63 #[inline]
64 #[target_feature(enable = "neon")]
65 unsafe fn condense_bottom_16(a: int32x4x2_t, b: int32x4x2_t) -> int16x8x2_t {
66 int16x8x2_t(pack_16(a), pack_16(b))
67 }
68
69 #[target_feature(enable = "neon")]
70 #[allow(
71 clippy::too_many_lines,
72 clippy::cast_possible_truncation,
73 clippy::similar_names,
74 clippy::op_ref,
75 unused_assignments,
76 clippy::zero_prefixed_literal
77 )]
78 pub unsafe fn idct_int_neon_inner(
79 in_vector: &mut [i32; 64], out_vector: &mut [i16], stride: usize
80 ) {
81 let mut pos = 0;
82
83 // load into registers
84 //
85 // We sign extend i16's to i32's and calculate them with extended precision and
86 // later reduce them to i16's when we are done carrying out IDCT
87
88 let mut row0 = YmmRegister::load(in_vector[00..].as_ptr().cast());
89 let mut row1 = YmmRegister::load(in_vector[08..].as_ptr().cast());
90 let mut row2 = YmmRegister::load(in_vector[16..].as_ptr().cast());
91 let mut row3 = YmmRegister::load(in_vector[24..].as_ptr().cast());
92 let mut row4 = YmmRegister::load(in_vector[32..].as_ptr().cast());
93 let mut row5 = YmmRegister::load(in_vector[40..].as_ptr().cast());
94 let mut row6 = YmmRegister::load(in_vector[48..].as_ptr().cast());
95 let mut row7 = YmmRegister::load(in_vector[56..].as_ptr().cast());
96
97 // Forward DCT and quantization may cause all the AC terms to be zero, for such
98 // cases we can try to accelerate it
99
100 // Basically the poop is that whenever the array has 63 zeroes, its idct is
101 // (arr[0]>>3)or (arr[0]/8) propagated to all the elements.
102 // We first test to see if the array contains zero elements and if it does, we go the
103 // short way.
104 //
105 // This reduces IDCT overhead from about 39% to 18 %, almost half
106
107 // Do another load for the first row, we don't want to check DC value, because
108 // we only care about AC terms
109 // TODO this should be a shift/shuffle, not a likely unaligned load
110 let row8 = YmmRegister::load(in_vector[1..].as_ptr().cast());
111
112 let or_tree = (((row1 | row8) | (row2 | row3)) | ((row4 | row5) | (row6 | row7)));
113
114 if or_tree.all_zero() {
115 // AC terms all zero, idct of the block is is ( coeff[0] * qt[0] )/8 + 128 (bias)
116 // (and clamped to 255)
117 let coeff = ((in_vector[0] + 4 + 1024) >> 3).clamp(0, 255) as i16;
118 let idct_value = vdupq_n_s16(coeff);
119 macro_rules! store {
120 ($pos:tt,$value:tt) => {
121 // store
122 vst1q_s16(
123 out_vector
124 .get_mut($pos..$pos + 8)
125 .unwrap()
126 .as_mut_ptr()
127 .cast(),
128 $value
129 );
130 $pos += stride;
131 };
132 }
133 store!(pos, idct_value);
134 store!(pos, idct_value);
135 store!(pos, idct_value);
136 store!(pos, idct_value);
137
138 store!(pos, idct_value);
139 store!(pos, idct_value);
140 store!(pos, idct_value);
141 store!(pos, idct_value);
142
143 return;
144 }
145
146 macro_rules! dct_pass {
147 ($SCALE_BITS:tt,$scale:tt) => {
148 // There are a lot of ways to do this
149 // but to keep it simple(and beautiful), ill make a direct translation of the
150 // scalar code to also make this code fully transparent(this version and the non
151 // avx one should produce identical code.)
152
153 // Compiler does a pretty good job of optimizing add + mul pairs
154 // into multiply-acumulate pairs
155
156 // even part
157 let p1 = (row2 + row6) * 2217;
158
159 let mut t2 = p1 + row6 * -7567;
160 let mut t3 = p1 + row2 * 3135;
161
162 let mut t0 = (row0 + row4).const_shl::<12>();
163 let mut t1 = (row0 - row4).const_shl::<12>();
164
165 let x0 = t0 + t3 + $SCALE_BITS;
166 let x3 = t0 - t3 + $SCALE_BITS;
167 let x1 = t1 + t2 + $SCALE_BITS;
168 let x2 = t1 - t2 + $SCALE_BITS;
169
170 let p3 = row7 + row3;
171 let p4 = row5 + row1;
172 let p1 = row7 + row1;
173 let p2 = row5 + row3;
174 let p5 = (p3 + p4) * 4816;
175
176 t0 = row7 * 1223;
177 t1 = row5 * 8410;
178 t2 = row3 * 12586;
179 t3 = row1 * 6149;
180
181 let p1 = p5 + p1 * -3685;
182 let p2 = p5 + (p2 * -10497);
183 let p3 = p3 * -8034;
184 let p4 = p4 * -1597;
185
186 t3 += p1 + p4;
187 t2 += p2 + p3;
188 t1 += p2 + p4;
189 t0 += p1 + p3;
190
191 row0 = (x0 + t3).const_shra::<$scale>();
192 row1 = (x1 + t2).const_shra::<$scale>();
193 row2 = (x2 + t1).const_shra::<$scale>();
194 row3 = (x3 + t0).const_shra::<$scale>();
195
196 row4 = (x3 - t0).const_shra::<$scale>();
197 row5 = (x2 - t1).const_shra::<$scale>();
198 row6 = (x1 - t2).const_shra::<$scale>();
199 row7 = (x0 - t3).const_shra::<$scale>();
200 };
201 }
202
203 // Process rows
204 dct_pass!(512, 10);
205 transpose(
206 &mut row0, &mut row1, &mut row2, &mut row3, &mut row4, &mut row5, &mut row6, &mut row7
207 );
208
209 // process columns
210 dct_pass!(SCALE_BITS, 17);
211 transpose(
212 &mut row0, &mut row1, &mut row2, &mut row3, &mut row4, &mut row5, &mut row6, &mut row7
213 );
214
215 // Pack i32 to i16's,
216 // clamp them to be between 0-255
217 // Undo shuffling
218 // Store back to array
219
220 // This could potentially be reorganized to take advantage of the multi-register stores
221 macro_rules! permute_store {
222 ($x:tt,$y:tt,$index:tt,$out:tt) => {
223 let a = condense_bottom_16($x, $y);
224
225 // Clamp the values after packing, we can clamp more values at once
226 let b = clamp256_neon(a);
227
228 // store first vector
229 vst1q_s16(
230 ($out)
231 .get_mut($index..$index + 8)
232 .unwrap()
233 .as_mut_ptr()
234 .cast(),
235 b.0
236 );
237 $index += stride;
238 // second vector
239 vst1q_s16(
240 ($out)
241 .get_mut($index..$index + 8)
242 .unwrap()
243 .as_mut_ptr()
244 .cast(),
245 b.1
246 );
247 $index += stride;
248 };
249 }
250 // Pack and write the values back to the array
251 permute_store!((row0.mm256), (row1.mm256), pos, out_vector);
252 permute_store!((row2.mm256), (row3.mm256), pos, out_vector);
253 permute_store!((row4.mm256), (row5.mm256), pos, out_vector);
254 permute_store!((row6.mm256), (row7.mm256), pos, out_vector);
255 }
256
257 #[inline]
258 #[target_feature(enable = "neon")]
259 unsafe fn clamp_neon(reg: int16x8_t) -> int16x8_t {
260 let min_s = vdupq_n_s16(0);
261 let max_s = vdupq_n_s16(255);
262
263 let max_v = vmaxq_s16(reg, min_s); //max(a,0)
264 let min_v = vminq_s16(max_v, max_s); //min(max(a,0),255)
265 min_v
266 }
267
268 #[inline]
269 #[target_feature(enable = "neon")]
270 unsafe fn clamp256_neon(reg: int16x8x2_t) -> int16x8x2_t {
271 int16x8x2_t(clamp_neon(reg.0), clamp_neon(reg.1))
272 }
273
274 #[cfg(test)]
275 mod test {
276 use super::*;
277
278 #[test]
279 fn test_neon_clamp_256() {
280 unsafe {
281 let vals: [i16; 16] = [-1, -2, -3, 4, 256, 257, 258, 240, -1, 290, 2, 3, 4, 5, 6, 7];
282 let loaded = vld1q_s16_x2(vals.as_ptr().cast());
283 let shuffled = clamp256_neon(loaded);
284
285 let mut result: [i16; 16] = [0; 16];
286
287 vst1q_s16_x2(result.as_mut_ptr().cast(), shuffled);
288
289 assert_eq!(
290 result,
291 [0, 0, 0, 4, 255, 255, 255, 240, 0, 255, 2, 3, 4, 5, 6, 7]
292 )
293 }
294 }
295 }