Debian Codegraph @

src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp

raw
1 /*
2 * Copyright (c) 2021-2023 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include "arm_gemm_local.hpp"
26
27 #include "depthwise_implementation.hpp"
28 #include "depthwise_depthfirst.hpp"
29 #include "depthwise_depthfirst_generic.hpp"
30 #include "depthwise_depthfirst_multiplier.hpp"
31 #include "depthwise_planar.hpp"
32
33 #include "depthwise_implementation_constraints.hpp"
34
35 // This can only be built if the target/compiler supports FP16 arguments.
36 #if defined(__ARM_FP16_ARGS)
37
38 #if defined(__aarch64__)
39 #if defined(ARM_COMPUTE_ENABLE_SME2)
40 #include "kernels/sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp"
41 #include "kernels/sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp"
42 #include "kernels/sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp"
43 #include "kernels/sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp"
44 #include "kernels/sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp"
45 #endif // defined(ARM_COMPUTE_ENABLE_SME2)
46 #if defined(ARM_COMPUTE_ENABLE_SVE)
47 #include "kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp"
48 #include "kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp"
49 #include "kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp"
50 #include "kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp"
51 #include "kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp"
52 #endif // defined(ARM_COMPUTE_ENABLE_SVE)
53 #if defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
54 #include "kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp"
55 #include "kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp"
56 #include "kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp"
57 #include "kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp"
58 #include "kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp"
59 #include "kernels/a64_fp16_nhwc_generic_output9_mla_depthfirst.hpp"
60 #include "kernels/a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp"
61 #endif // defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
62 #endif // defined(__aarch64__)
63
64 namespace arm_conv {
65 namespace depthwise {
66
67 namespace
68 {
69 #if defined(__aarch64__)
70 #if defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
71 bool prefer_premultiply(const DepthwiseArgs &args) {
72 if ((args.stride_rows != args.stride_cols) || (args.kernel_rows != args.kernel_cols))
73 {
74 return false;
75 }
76
77 unsigned int threshold;
78
79 if (args.stride_rows == 1 && args.kernel_rows == 3)
80 {
81 threshold = 30;
82 }
83 else if (args.stride_rows == 1 && args.kernel_rows == 5)
84 {
85 threshold = 31;
86 }
87 else if (args.stride_rows == 2 && args.kernel_rows == 3)
88 {
89 threshold = 11;
90 }
91 else if (args.stride_rows == 2 && args.kernel_rows == 5)
92 {
93 threshold = 19;
94 } else
95 {
96 return false;
97 }
98
99 return args.channel_multiplier <= threshold;
100 }
101
102 template <class Strategy>
103 unsigned int cycle_estimate(const DepthwiseArgs &args, const Nothing &)
104 {
105 if (args.channel_multiplier > 1 && !prefer_premultiply(args))
106 {
107 return std::numeric_limits<unsigned int>::max();
108 }
109
110 // First-pass: compute the number of output pixels which will be computed.
111 return arm_gemm::roundup(args.output_rows, Strategy::output_rows) *
112 arm_gemm::roundup(args.output_cols, Strategy::output_cols) *
113 arm_gemm::iceildiv(
114 (long unsigned) args.input_channels * args.channel_multiplier,
115 arm_gemm::utils::get_vector_length<typename Strategy::return_type>(Strategy::vl_type)
116 );
117 }
118
119 template <class Strategy>
120 unsigned int planar_cycle_estimate(const DepthwiseArgs &args, const Nothing &)
121 {
122 // First-pass: compute the number of output pixels which will be computed.
123 return arm_gemm::roundup(args.output_rows, Strategy::output_rows) *
124 args.output_cols *
125 arm_gemm::iceildiv(
126 (long unsigned) args.input_channels * args.channel_multiplier,
127 arm_gemm::utils::get_vector_length<typename Strategy::return_type>(Strategy::vl_type)
128 );
129 }
130
131 unsigned int multiplier_cycle_estimate(const DepthwiseArgs &args, const Nothing &)
132 {
133 return prefer_premultiply(args)? std::numeric_limits<unsigned int>::max() : 0;
134 }
135
136 unsigned int not_preferred(const DepthwiseArgs &, const Nothing &) __attribute__ ((unused));
137 unsigned int not_preferred(const DepthwiseArgs &, const Nothing &)
138 {
139 return std::numeric_limits<unsigned int>::max();
140 }
141 #endif // defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
142 #endif // defined(__aarch64__)
143 }
144
145 static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = {
146 #if defined(__aarch64__)
147 #if defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
148 #if defined(ARM_COMPUTE_ENABLE_SVE)
149 #if defined(ARM_COMPUTE_ENABLE_SME2)
150 {
151 DepthwiseMethod::DEPTHFIRST,
152 "sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst",
153 constraint(is_supported<sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>,
154 cpu_has_sme2),
155 cycle_estimate<sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>,
156 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
157 auto strat = new sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info);
158 return new DepthwiseDepthfirst<__fp16>(strat, args);
159 },
160 },
161 {
162 DepthwiseMethod::DEPTHFIRST,
163 "sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst",
164 constraint(is_supported<sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>,
165 cpu_has_sme2),
166 cycle_estimate<sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>,
167 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
168 auto strat = new sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info);
169 return new DepthwiseDepthfirst<__fp16>(strat, args);
170 },
171 },
172 {
173 DepthwiseMethod::DEPTHFIRST,
174 "sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst",
175 constraint(is_supported<sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>,
176 cpu_has_sme2),
177 cycle_estimate<sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>,
178 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
179 auto strat = new sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
180 return new DepthwiseDepthfirst<__fp16>(strat, args);
181 },
182 },
183 {
184 DepthwiseMethod::DEPTHFIRST,
185 "sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst",
186 constraint(is_supported<sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>,
187 cpu_has_sme2),
188 cycle_estimate<sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>,
189 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
190 auto strat = new sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
191 return new DepthwiseDepthfirst<__fp16>(strat, args);
192 },
193 },
194 {
195 DepthwiseMethod::DEPTHFIRST,
196 "sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst",
197 constraint(is_supported<sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>,
198 cpu_has_sme2),
199 cycle_estimate<sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>,
200 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
201 auto strat = new sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
202 return new DepthwiseDepthfirst<__fp16>(strat, args);
203 },
204 },
205 #endif // defined(ARM_COMPUTE_ENABLE_SME2)
206 {
207 DepthwiseMethod::DEPTHFIRST,
208 "sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst",
209 constraint(is_supported<sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>,
210 cpu_has_sve),
211 cycle_estimate<sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>,
212 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
213 auto strat = new sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info);
214 return new DepthwiseDepthfirst<__fp16>(strat, args);
215 },
216 },
217 {
218 DepthwiseMethod::DEPTHFIRST,
219 "sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst",
220 constraint(is_supported<sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>,
221 cpu_has_sve),
222 cycle_estimate<sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>,
223 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
224 auto strat = new sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info);
225 return new DepthwiseDepthfirst<__fp16>(strat, args);
226 },
227 },
228 {
229 DepthwiseMethod::DEPTHFIRST,
230 "sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst",
231 constraint(is_supported<sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>,
232 cpu_has_sve),
233 cycle_estimate<sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>,
234 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
235 auto strat = new sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
236 return new DepthwiseDepthfirst<__fp16>(strat, args);
237 },
238 },
239 {
240 DepthwiseMethod::DEPTHFIRST,
241 "sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst",
242 constraint(is_supported<sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>,
243 cpu_has_sve),
244 cycle_estimate<sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>,
245 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
246 auto strat = new sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
247 return new DepthwiseDepthfirst<__fp16>(strat, args);
248 },
249 },
250 {
251 DepthwiseMethod::DEPTHFIRST,
252 "sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst",
253 constraint(is_supported<sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>,
254 cpu_has_sve),
255 cycle_estimate<sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>,
256 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
257 auto strat = new sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
258 return new DepthwiseDepthfirst<__fp16>(strat, args);
259 },
260 },
261 #endif // defined(ARM_COMPUTE_ENABLE_SVE)
262 {
263 DepthwiseMethod::DEPTHFIRST,
264 "a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst",
265 constraint(is_supported<a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>,
266 cpu_has_fp16),
267 cycle_estimate<a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>,
268 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
269 auto strat = new a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info);
270 return new DepthwiseDepthfirst<__fp16>(strat, args);
271 },
272 },
273 {
274 DepthwiseMethod::DEPTHFIRST,
275 "a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst",
276 constraint(is_supported<a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>,
277 cpu_has_fp16),
278 cycle_estimate<a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>,
279 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
280 auto strat = new a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info);
281 return new DepthwiseDepthfirst<__fp16>(strat, args);
282 },
283 },
284 {
285 DepthwiseMethod::DEPTHFIRST,
286 "a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst",
287 constraint(is_supported<a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>,
288 cpu_has_fp16),
289 cycle_estimate<a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>,
290 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
291 auto strat = new a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
292 return new DepthwiseDepthfirst<__fp16>(strat, args);
293 },
294 },
295 {
296 DepthwiseMethod::DEPTHFIRST,
297 "a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst",
298 constraint(is_supported<a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>,
299 cpu_has_fp16),
300 cycle_estimate<a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>,
301 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
302 auto strat = new a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
303 return new DepthwiseDepthfirst<__fp16>(strat, args);
304 },
305 },
306 {
307 DepthwiseMethod::DEPTHFIRST,
308 "a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst",
309 constraint(is_supported<a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>,
310 cpu_has_fp16),
311 cycle_estimate<a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>,
312 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
313 auto strat = new a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
314 return new DepthwiseDepthfirst<__fp16>(strat, args);
315 },
316 },
317 {
318 DepthwiseMethod::DEPTHFIRST,
319 "a64_fp16_nhwc_generic_output3x3_mla_depthfirst",
320 constraint(cpu_has_fp16),
321 not_preferred,
322 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
323 auto kern = new a64_fp16_nhwc_generic_output9_mla_depthfirst(args.cpu_info);
324 auto strat = new GenericDepthfirstStrategy<__fp16>(kern, 3, 3, args);
325 return new DepthwiseDepthfirstGeneric<__fp16>(strat, args);
326 },
327 },
328 {
329 DepthwiseMethod::DEPTHFIRST,
330 "a64_fp16_nhwc_generic_with_multiplier_output2x8_mla_depthfirst",
331 constraint(cpu_has_fp16, has_channel_multiplier),
332 multiplier_cycle_estimate,
333 [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * {
334 auto kern = new a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info);
335 auto strat = new GenericDepthfirstMultiplierStrategy<__fp16>(kern, args);
336 return new DepthwiseDepthfirstMultiplier<__fp16, __fp16, __fp16, __fp16, true>(strat, args);
337 },
338 },
339 #endif // defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
340 #endif // defined(__aarch64__)
341 { DepthwiseMethod::DEFAULT, "", nullptr, nullptr, nullptr }, // End of list
342 };
343
344 template <>
345 const DepthwiseImplementation<__fp16> *depthwise_implementation_list()
346 {
347 return depthwise_fp16_methods;
348 }
349
350 template UniqueDepthwiseCommon<__fp16> depthwise(const DepthwiseArgs &, const Nothing &);
351 template std::vector<KernelDescription> get_compatible_kernels<__fp16>(const DepthwiseArgs &, const Nothing &);
352
353 } // namespace depthwise
354 } // namespace arm_conv
355
356 #endif // defined(__ARM_FP16_ARGS)