Debian Codegraph @

src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp

raw
1 /*
2 * Copyright (c) 2021-2023 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include "arm_gemm_local.hpp"
26
27 #include "depthwise_implementation.hpp"
28 #include "depthwise_depthfirst.hpp"
29 #include "depthwise_depthfirst_generic.hpp"
30 #include "depthwise_depthfirst_multiplier.hpp"
31 #include "depthwise_planar.hpp"
32
33 #include "depthwise_implementation_constraints.hpp"
34
35 #if defined(__aarch64__)
36 #if defined(ARM_COMPUTE_ENABLE_SVE)
37 #if defined(ARM_COMPUTE_ENABLE_SME2)
38 #include "kernels/sme2_s8q_planar_3x3_s1_4rows_dot_za.hpp"
39 #include "kernels/sme2_s8q_planar_3x3_s2_4rows_dot_za.hpp"
40 #include "kernels/sme2_s8q_planar_5x5_s1_4rows_dot_za.hpp"
41 #include "kernels/sme2_s8q_planar_5x5_s2_4rows_dot_za.hpp"
42 #endif // defined(ARM_COMPUTE_ENABLE_SME2)
43
44 #include "kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp"
45 #include "kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp"
46 #include "kernels/sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp"
47 #include "kernels/sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp"
48 #include "kernels/sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp"
49 #include "kernels/sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp"
50 #include "kernels/sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp"
51 #endif // defined(ARM_COMPUTE_ENABLE_SVE)
52 #include "kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp"
53 #include "kernels/a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp"
54 #include "kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp"
55 #include "kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp"
56 #include "kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp"
57 #include "kernels/a64_s8q_nhwc_generic_output9_mla_depthfirst.hpp"
58 #include "kernels/a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst.hpp"
59 #include "kernels/a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst.hpp"
60 #include "kernels/a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp"
61 #endif // defined(__aarch64__)
62
63 #include <cstdint>
64
65 using arm_gemm::Requantize32;
66
67 namespace arm_conv {
68 namespace depthwise {
69
70 namespace
71 {
72 #if defined(__aarch64__)
73 bool qp_weights_are_symmetric(const DepthwiseArgs &, const void *_qp)
74 {
75 const auto qp = static_cast<const arm_gemm::Requantize32 *>(_qp);
76 return qp->b_offset == 0;
77 }
78
79 uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &)
80 {
81 return std::numeric_limits<uint64_t>::max();
82 }
83 #endif // defined(__aarch64__)
84 }
85
86 static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depthwise_s8q_methods[] = {
87 #if defined(__aarch64__)
88 #if defined(ARM_COMPUTE_ENABLE_SVE)
89 #if defined(ARM_COMPUTE_ENABLE_SME2)
90 {
91 DepthwiseMethod::PLANAR,
92 "sme2_s8q_planar_3x3_s1_4rows_dot_za",
93 constraint<Requantize32>(cpu_has_sme, cpu_has_sme2,
94 is_supported<sme2_s8q_planar_3x3_s1_4rows_dot_za>,
95 has_no_channel_multiplier,
96 qp_has_no_left_shift, no_prime_right_pad),
97 nullptr,
98 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
99 auto strat = new sme2_s8q_planar_3x3_s1_4rows_dot_za(args.cpu_info);
100 return new DepthwisePlanar<int8_t>(strat, args, qp);
101 },
102 },
103 {
104 DepthwiseMethod::PLANAR,
105 "sme2_s8q_planar_3x3_s2_4rows_dot_za",
106 constraint<Requantize32>(cpu_has_sme, cpu_has_sme2,
107 is_supported<sme2_s8q_planar_3x3_s2_4rows_dot_za>,
108 has_no_channel_multiplier,
109 qp_has_no_left_shift, no_prime_right_pad),
110 nullptr,
111 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
112 auto strat = new sme2_s8q_planar_3x3_s2_4rows_dot_za(args.cpu_info);
113 return new DepthwisePlanar<int8_t>(strat, args, qp);
114 },
115 },
116 {
117 DepthwiseMethod::PLANAR,
118 "sme2_s8q_planar_5x5_s1_4rows_dot_za",
119 constraint<Requantize32>(cpu_has_sme, cpu_has_sme2,
120 is_supported<sme2_s8q_planar_5x5_s1_4rows_dot_za>,
121 has_no_channel_multiplier,
122 qp_has_no_left_shift, no_prime_right_pad),
123 nullptr,
124 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
125 auto strat = new sme2_s8q_planar_5x5_s1_4rows_dot_za(args.cpu_info);
126 return new DepthwisePlanar<int8_t>(strat, args, qp);
127 },
128 },
129 {
130 DepthwiseMethod::PLANAR,
131 "sme2_s8q_planar_5x5_s2_4rows_dot_za",
132 constraint<Requantize32>(cpu_has_sme, cpu_has_sme2,
133 is_supported<sme2_s8q_planar_5x5_s2_4rows_dot_za>,
134 has_no_channel_multiplier,
135 qp_has_no_left_shift, no_prime_right_pad),
136 nullptr,
137 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
138 auto strat = new sme2_s8q_planar_5x5_s2_4rows_dot_za(args.cpu_info);
139 return new DepthwisePlanar<int8_t>(strat, args, qp);
140 },
141 },
142 #endif // defined(ARM_COMPUTE_ENABLE_SME2)
143 {
144 DepthwiseMethod::DEPTHFIRST,
145 "sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst",
146 constraint<Requantize32>(is_supported<sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst>,
147 qp_has_no_left_shift,
148 qp_weights_are_symmetric,
149 cpu_has_sve2),
150 nullptr,
151 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
152 auto strat = new sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst(args.cpu_info);
153 return new DepthwiseDepthfirst<int8_t>(strat, args, qp);
154 },
155 },
156 {
157 DepthwiseMethod::DEPTHFIRST,
158 "sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst",
159 constraint<Requantize32>(is_supported<sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst>,
160 qp_has_no_left_shift,
161 cpu_has_sve2),
162 nullptr,
163 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
164 auto strat = new sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst(args.cpu_info);
165 return new DepthwiseDepthfirst<int8_t>(strat, args, qp);
166 },
167 },
168 {
169 DepthwiseMethod::DEPTHFIRST,
170 "sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst",
171 constraint<Requantize32>(is_supported<sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst>,
172 qp_has_no_left_shift,
173 cpu_has_sve2),
174 nullptr,
175 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
176 auto strat = new sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
177 return new DepthwiseDepthfirst<int8_t>(strat, args, qp);
178 },
179 },
180 {
181 DepthwiseMethod::DEPTHFIRST,
182 "sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst",
183 constraint<Requantize32>(is_supported<sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst>,
184 qp_has_no_left_shift,
185 cpu_has_sve2),
186 nullptr,
187 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
188 auto strat = new sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
189 return new DepthwiseDepthfirst<int8_t>(strat, args, qp);
190 },
191 },
192 {
193 DepthwiseMethod::DEPTHFIRST,
194 "sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst",
195 constraint<Requantize32>(is_supported<sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst>,
196 qp_has_no_left_shift,
197 cpu_has_sve2),
198 nullptr,
199 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
200 auto strat = new sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
201 return new DepthwiseDepthfirst<int8_t>(strat, args, qp);
202 },
203 },
204 {
205 DepthwiseMethod::DEPTHFIRST,
206 "sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst",
207 constraint<Requantize32>(is_supported<sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst>,
208 qp_has_no_left_shift,
209 has_channel_multiplier,
210 cpu_has_sve2),
211 not_preferred,
212 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
213 auto strat = new sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info);
214 return new DepthwiseDepthfirstMultiplier<int8_t, int8_t, int8_t, int32_t, false>(strat, args, qp);
215 },
216 },
217 {
218 DepthwiseMethod::DEPTHFIRST,
219 "sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst",
220 constraint<Requantize32>(is_supported<sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst>,
221 qp_has_no_left_shift,
222 has_channel_multiplier,
223 cpu_has_sve2),
224 not_preferred,
225 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
226 auto strat = new sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info);
227 return new DepthwiseDepthfirstMultiplier<int8_t, int8_t, int8_t, int32_t, false>(strat, args, qp);
228 },
229 },
230 #endif // defined(ARM_COMPUTE_ENABLE_SVE)
231 {
232 DepthwiseMethod::DEPTHFIRST,
233 "a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst",
234 constraint<Requantize32>(is_supported<a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst>,
235 qp_weights_are_symmetric,
236 qp_has_no_left_shift,
237 cpu_has_dot_product),
238 nullptr,
239 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
240 auto strat = new a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst(args.cpu_info);
241 return new DepthwiseDepthfirst<int8_t>(strat, args, qp);
242 },
243 },
244 {
245 DepthwiseMethod::DEPTHFIRST,
246 "a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst",
247 constraint<Requantize32>(is_supported<a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst>,
248 qp_has_no_left_shift,
249 cpu_has_dot_product),
250 nullptr,
251 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
252 auto strat = new a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst(args.cpu_info);
253 return new DepthwiseDepthfirst<int8_t>(strat, args, qp);
254 },
255 },
256 {
257 DepthwiseMethod::DEPTHFIRST,
258 "a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst",
259 constraint<Requantize32>(is_supported<a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst>,
260 qp_has_no_left_shift),
261 nullptr,
262 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
263 auto strat = new a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
264 return new DepthwiseDepthfirst<int8_t>(strat, args, qp);
265 },
266 },
267 {
268 DepthwiseMethod::DEPTHFIRST,
269 "a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst",
270 constraint<Requantize32>(is_supported<a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst>,
271 qp_has_no_left_shift),
272 nullptr,
273 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
274 auto strat = new a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
275 return new DepthwiseDepthfirst<int8_t>(strat, args, qp);
276 },
277 },
278 {
279 DepthwiseMethod::DEPTHFIRST,
280 "a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst",
281 constraint<Requantize32>(is_supported<a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst>,
282 qp_has_no_left_shift),
283 nullptr,
284 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
285 auto strat = new a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
286 return new DepthwiseDepthfirst<int8_t>(strat, args, qp);
287 },
288 },
289 {
290 DepthwiseMethod::DEPTHFIRST,
291 "a64_s8q_nhwc_generic_output3x3_mla_depthfirst",
292 nullptr,
293 nullptr,
294 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
295 auto kernel = new a64_s8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info);
296 auto strat = new GenericDepthfirstStrategy<int8_t>(kernel, 3, 3, args);
297 return new DepthwiseDepthfirstGeneric<int8_t>(strat, args, qp);
298 },
299 },
300 {
301 DepthwiseMethod::DEPTHFIRST,
302 "a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst",
303 constraint<Requantize32>(is_supported<a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst>,
304 qp_has_no_left_shift,
305 has_channel_multiplier,
306 cpu_has_dot_product),
307 not_preferred,
308 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
309 auto strat = new a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info);
310 return new DepthwiseDepthfirstMultiplier<int8_t, int8_t, int8_t, int32_t, false>(strat, args, qp);
311 },
312 },
313 {
314 DepthwiseMethod::DEPTHFIRST,
315 "a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst",
316 constraint<Requantize32>(is_supported<a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst>,
317 qp_has_no_left_shift,
318 has_channel_multiplier,
319 cpu_has_dot_product),
320 not_preferred,
321 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
322 auto strat = new a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info);
323 return new DepthwiseDepthfirstMultiplier<int8_t, int8_t, int8_t, int32_t, false>(strat, args, qp);
324 },
325 },
326 {
327 DepthwiseMethod::DEPTHFIRST,
328 "a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst",
329 constraint<Requantize32>(has_channel_multiplier),
330 not_preferred,
331 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * {
332 auto kern = new a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info);
333 auto strat = new GenericDepthfirstMultiplierStrategy<int8_t>(kern, args);
334 return new DepthwiseDepthfirstMultiplier<int8_t, int8_t, int8_t, int32_t, true>(strat, args, qp);
335 },
336 },
337 #endif // defined(__aarch64__)
338 { DepthwiseMethod::DEFAULT, "", nullptr, nullptr, nullptr }, // End of list
339 };
340
341 template <>
342 const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> *depthwise_implementation_list()
343 {
344 return depthwise_s8q_methods;
345 }
346
347 template UniqueDepthwiseCommon<int8_t, int8_t, int8_t> depthwise(const DepthwiseArgs &, const Requantize32 &);
348 template std::vector<KernelDescription> get_compatible_kernels<int8_t, int8_t, int8_t, Requantize32>(const DepthwiseArgs &, const Requantize32 &);
349
350 } // namespace depthwise
351 } // namespace arm_conv