Debian Codegraph @

.cargo/vendor/zune-jpeg/src/upsampler/portable_simd.rs

raw
1 /*
2 * Copyright (c) 2023.
3 *
4 * This software is free software;
5 *
6 * You can redistribute it or modify it under terms of the MIT, Apache License or Zlib license
7 */
8
9 use std::simd::prelude::*;
10
11 const LANES: usize = 16;
12 type V = Simd<i16, LANES>;
13
14 pub fn upsample_horizontal_simd(
15 input: &[i16],
16 in_near: &[i16],
17 in_far: &[i16],
18 scratch: &mut [i16],
19 output: &mut [i16],
20 ) {
21 assert_eq!(input.len() * 2, output.len());
22 assert!(input.len() > 2);
23
24 let len = input.len();
25
26 if len < 18 {
27 return super::scalar::upsample_horizontal(input, in_near, in_far, scratch, output);
28 }
29
30 // First two pixels
31 output[0] = input[0];
32 output[1] = (input[0] * 3 + input[1] + 2) >> 2;
33
34 let v_three = V::splat(3);
35 let v_two = V::splat(2);
36
37 let upsample16 = |input: &[i16; 18], output: &mut [i16; 32]| {
38 let v_prev = V::from_slice(&input[0..LANES]);
39 let v_curr = V::from_slice(&input[1..LANES + 1]);
40 let v_next = V::from_slice(&input[2..LANES + 2]);
41
42 let v_common = v_curr * v_three + v_two;
43
44 let v_even = (v_common + v_prev) >> 2;
45 let v_odd = (v_common + v_next) >> 2;
46
47 let (v_res_1, v_res_2) = v_even.interleave(v_odd);
48
49 v_res_1.copy_to_slice(&mut output[0..LANES]);
50 v_res_2.copy_to_slice(&mut output[LANES..2 * LANES]);
51 };
52
53 for (input, output) in input
54 .windows(18)
55 .step_by(16)
56 .zip(output[2..].chunks_exact_mut(32))
57 {
58 upsample16(input.try_into().unwrap(), output.try_into().unwrap());
59 }
60
61 // Upsample the remainder. This may have some overlap, but that's fine.
62 if let Some(rest_input) = input.last_chunk::<18>() {
63 let end = output.len() - 2;
64 if let Some(rest_output) = output[..end].last_chunk_mut::<32>() {
65 upsample16(rest_input, rest_output);
66 }
67 }
68
69 // Last two pixels.
70 output[output.len() - 2] = (3 * input[len - 1] + input[len - 2] + 2) >> 2;
71 output[output.len() - 1] = input[len - 1];
72 }
73
74 pub fn upsample_vertical_simd(
75 input: &[i16],
76 in_near: &[i16],
77 in_far: &[i16],
78 _scratch_space: &mut [i16],
79 output: &mut [i16],
80 ) {
81 assert_eq!(input.len() * 2, output.len());
82 assert_eq!(in_near.len(), input.len());
83 assert_eq!(in_far.len(), input.len());
84
85 let len = input.len();
86
87 if len < 16 {
88 return super::scalar::upsample_vertical(input, in_near, in_far, _scratch_space, output);
89 }
90
91 let middle = output.len() / 2;
92 let (out_top, out_bottom) = output.split_at_mut(middle);
93
94 let v_three = V::splat(3);
95 let v_two = V::splat(2);
96
97 let upsample16 = |input: &[i16; 16],
98 in_near: &[i16; 16],
99 in_far: &[i16; 16],
100 out_top: &mut [i16; 16],
101 out_bottom: &mut [i16; 16]| {
102 let v_in = V::from(*input);
103 let v_near = V::from(*in_near);
104 let v_far = V::from(*in_far);
105
106 let v_common = v_in * v_three + v_two;
107
108 let v_out_top = (v_common + v_near) >> 2;
109 let v_out_bottom = (v_common + v_far) >> 2;
110
111 v_out_top.copy_to_slice(out_top.as_mut_slice());
112 v_out_bottom.copy_to_slice(out_bottom.as_mut_slice());
113 };
114
115 let chunks = input
116 .chunks_exact(16)
117 .zip(in_near.chunks_exact(16))
118 .zip(in_far.chunks_exact(16))
119 .zip(out_top.chunks_exact_mut(16))
120 .zip(out_bottom.chunks_exact_mut(16));
121
122 for ((((input, in_near), in_far), out_top), out_bottom) in chunks {
123 upsample16(
124 input.try_into().unwrap(),
125 in_near.try_into().unwrap(),
126 in_far.try_into().unwrap(),
127 out_top.try_into().unwrap(),
128 out_bottom.try_into().unwrap(),
129 );
130 }
131
132 // Upsample the remainder. This may have some overlap, but that's fine.
133 // Edition upgrade will fix this nested awfulness.
134 if let Some(rest) = input.last_chunk::<16>() {
135 if let Some( rest_near) = in_near.last_chunk::<16>() {
136 if let Some( rest_far) = in_far.last_chunk::<16>() {
137 if let Some( rest_top) = out_top.last_chunk_mut::<16>() {
138 if let Some( rest_bottom) = out_bottom.last_chunk_mut::<16>() {
139 upsample16(rest, rest_near, rest_far, rest_top, rest_bottom);
140 }
141 }
142 }
143 }
144 }
145 }
146
147 pub fn upsample_hv_simd(
148 input: &[i16],
149 in_near: &[i16],
150 in_far: &[i16],
151 scratch_space: &mut [i16],
152 output: &mut [i16],
153 ) {
154 assert_eq!(input.len() * 4, output.len());
155 assert_eq!(input.len() * 2, scratch_space.len());
156
157 upsample_vertical_simd(input, in_near, in_far, &mut [], scratch_space);
158
159 let scratch_half = scratch_space.len() / 2;
160 let output_half = output.len() / 2;
161
162 let (scratch_top, scratch_bottom) = scratch_space.split_at_mut(scratch_half);
163 let (out_top, out_bottom) = output.split_at_mut(output_half);
164
165 let mut t = [0];
166 upsample_horizontal_simd(scratch_top, &[], &[], &mut t, out_top);
167 upsample_horizontal_simd(scratch_bottom, &[], &[], &mut t, out_bottom);
168 }