Debian Codegraph @

.cargo/vendor/zune-jpeg/src/upsampler/neon.rs

raw
1 /*
2 * Copyright (c) 2025.
3 *
4 * This software is free software;
5 *
6 * You can redistribute it or modify it under terms of the MIT, Apache License or Zlib license
7 */
8
9 #[cfg(target_arch = "aarch64")]
10 use core::arch::aarch64::*;
11
12 #[cfg(target_arch = "aarch64")]
13 #[target_feature(enable = "neon")]
14 pub fn upsample_horizontal_neon(
15 input: &[i16],
16 in_near: &[i16],
17 in_far: &[i16],
18 scratch: &mut [i16],
19 output: &mut [i16],
20 ) {
21 assert_eq!(input.len() * 2, output.len());
22 assert!(input.len() > 2);
23
24 let len = input.len();
25
26 if len < 10 {
27 return super::scalar::upsample_horizontal(input, in_near, in_far, scratch, output);
28 }
29
30 // First two pixels
31 output[0] = input[0];
32 output[1] = (input[0] * 3 + input[1] + 2) >> 2;
33
34 let v_three = vdupq_n_s16(3);
35 let v_two = vdupq_n_s16(2);
36
37 let upsample8 = |input: &[i16; 10], output: &mut [i16; 16]| {
38 let in_ptr = input.as_ptr();
39 let out_ptr = output.as_mut_ptr();
40
41 // SAFETY: The input is 10 * 16 bit long, so the loads are safe.
42 let (v_prev, v_curr, v_next) = unsafe {
43 (
44 vld1q_s16(in_ptr),
45 vld1q_s16(in_ptr.add(1)),
46 vld1q_s16(in_ptr.add(2)),
47 )
48 };
49
50 let v_common = vaddq_s16(vmulq_s16(v_curr, v_three), v_two);
51 let v_even = vshrq_n_s16::<2>(vaddq_s16(v_common, v_prev));
52 let v_odd = vshrq_n_s16::<2>(vaddq_s16(v_common, v_next));
53
54 let v_res_1 = vzip1q_s16(v_even, v_odd);
55 let v_res_2 = vzip2q_s16(v_even, v_odd);
56
57 // SAFETY: The output is 16 * 16 bit long, so the stores are safe.
58 unsafe {
59 vst1q_s16(out_ptr, v_res_1);
60 vst1q_s16(out_ptr.add(8), v_res_2);
61 }
62 };
63
64 for (input, output) in input
65 .windows(10)
66 .step_by(8)
67 .zip(output[2..].chunks_exact_mut(16))
68 {
69 upsample8(input.try_into().unwrap(), output.try_into().unwrap());
70 }
71
72 // Upsample the remainder. This may have some overlap, but that's fine.
73 if let Some(rest_input) = input.last_chunk::<10>() {
74 let end = output.len() - 2;
75 if let Some(rest_output) = output[..end].last_chunk_mut::<16>() {
76 upsample8(rest_input, rest_output);
77 }
78 }
79
80 // Last two pixels.
81 output[output.len() - 2] = (3 * input[len - 1] + input[len - 2] + 2) >> 2;
82 output[output.len() - 1] = input[len - 1];
83 }
84
85 #[cfg(target_arch = "aarch64")]
86 #[target_feature(enable = "neon")]
87 pub fn upsample_vertical_neon(
88 input: &[i16],
89 in_near: &[i16],
90 in_far: &[i16],
91 scratch: &mut [i16],
92 output: &mut [i16],
93 ) {
94 assert_eq!(input.len() * 2, output.len());
95 assert_eq!(in_near.len(), input.len());
96 assert_eq!(in_far.len(), input.len());
97
98 let len = input.len();
99
100 if len < 16 {
101 return super::scalar::upsample_vertical(input, in_near, in_far, scratch, output);
102 }
103
104 let middle = output.len() / 2;
105 let (out_top, out_bottom) = output.split_at_mut(middle);
106
107 let v_three = vdupq_n_s16(3);
108 let v_two = vdupq_n_s16(2);
109
110 let upsample8 = |input: &[i16; 8],
111 in_near: &[i16; 8],
112 in_far: &[i16; 8],
113 out_top: &mut [i16; 8],
114 out_bottom: &mut [i16; 8]| {
115 // SAFETY: Inputs are all 8 * 16 bit long, so the loads are safe.
116 let (v_in, v_near, v_far) = unsafe {
117 (
118 vld1q_s16(input.as_ptr()),
119 vld1q_s16(in_near.as_ptr()),
120 vld1q_s16(in_far.as_ptr()),
121 )
122 };
123
124 let v_common = vaddq_s16(vmulq_s16(v_in, v_three), v_two);
125 let v_out_top = vshrq_n_s16::<2>(vaddq_s16(v_common, v_near));
126 let v_out_bottom = vshrq_n_s16::<2>(vaddq_s16(v_common, v_far));
127
128 // SAFETY: Outputs are 8 * 16 bit long, so the stores are safe.
129 unsafe {
130 vst1q_s16(out_top.as_mut_ptr(), v_out_top);
131 vst1q_s16(out_bottom.as_mut_ptr(), v_out_bottom);
132 }
133 };
134
135 let chunks = input
136 .chunks_exact(8)
137 .zip(in_near.chunks_exact(8))
138 .zip(in_far.chunks_exact(8))
139 .zip(out_top.chunks_exact_mut(8))
140 .zip(out_bottom.chunks_exact_mut(8));
141
142 for ((((input, in_near), in_far), out_top), out_bottom) in chunks {
143 upsample8(
144 input.try_into().unwrap(),
145 in_near.try_into().unwrap(),
146 in_far.try_into().unwrap(),
147 out_top.try_into().unwrap(),
148 out_bottom.try_into().unwrap(),
149 );
150 }
151
152 // Upsample the remainder.
153 if let Some(rest) = input.last_chunk::<8>() {
154 if let Some(rest_near) = in_near.last_chunk::<8>() {
155 if let Some(rest_far) = in_far.last_chunk::<8>() {
156 if let Some(mut rest_top) = out_top.last_chunk_mut::<8>() {
157 if let Some(mut rest_bottom) = out_bottom.last_chunk_mut::<8>() {
158 upsample8(rest, rest_near, rest_far, &mut rest_top, &mut rest_bottom);
159 }
160 }
161 }
162 }
163 }
164 }
165
166 #[cfg(target_arch = "aarch64")]
167 #[target_feature(enable = "neon")]
168 pub fn upsample_hv_neon(
169 input: &[i16],
170 in_near: &[i16],
171 in_far: &[i16],
172 scratch_space: &mut [i16],
173 output: &mut [i16],
174 ) {
175 assert_eq!(input.len() * 4, output.len());
176 assert_eq!(input.len() * 2, scratch_space.len());
177
178 upsample_vertical_neon(input, in_near, in_far, &mut [], scratch_space);
179
180 let scratch_half = scratch_space.len() / 2;
181 let output_half = output.len() / 2;
182
183 let (scratch_top, scratch_bottom) = scratch_space.split_at_mut(scratch_half);
184 let (out_top, out_bottom) = output.split_at_mut(output_half);
185
186 let mut t = [0];
187 upsample_horizontal_neon(scratch_top, &[], &[], &mut t, out_top);
188 upsample_horizontal_neon(scratch_bottom, &[], &[], &mut t, out_bottom);
189 }