Debian Codegraph @

src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp

raw
1 /*
2 * Copyright (c) 2021-2023 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include "arm_gemm_local.hpp"
26
27 #include "depthwise_implementation.hpp"
28 #include "depthwise_depthfirst.hpp"
29 #include "depthwise_depthfirst_generic.hpp"
30 #include "depthwise_depthfirst_multiplier.hpp"
31 #include "depthwise_planar.hpp"
32
33 #include "depthwise_implementation_constraints.hpp"
34
35 #if defined(__aarch64__)
36 #if defined(ARM_COMPUTE_ENABLE_SVE)
37 #if defined(ARM_COMPUTE_ENABLE_SME2)
38 #include "kernels/sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za.hpp"
39 #include "kernels/sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za.hpp"
40 #include "kernels/sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za.hpp"
41 #include "kernels/sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za.hpp"
42 #endif // defined(ARM_COMPUTE_ENABLE_SME2)
43 #include "kernels/sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp"
44 #include "kernels/sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp"
45 #include "kernels/sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp"
46 #endif // defined(ARM_COMPUTE_ENABLE_SVE)
47 #include "kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp"
48 #include "kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp"
49 #include "kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp"
50 #include "kernels/a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst.hpp"
51 #include "kernels/a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst.hpp"
52 #endif // defined(__aarch64__)
53
54 #include <cstdint>
55
56 using arm_gemm::Requantize32;
57
58 namespace arm_conv {
59 namespace depthwise {
60
61 namespace
62 {
63 #if defined(__aarch64__)
64 uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &)
65 {
66 return std::numeric_limits<uint64_t>::max();
67 }
68 #endif // defined(__aarch64__)
69 }
70
71 static const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> depthwise_u8q_methods[] = {
72 #if defined(__aarch64__)
73 #if defined(ARM_COMPUTE_ENABLE_SVE)
74 #if defined(ARM_COMPUTE_ENABLE_SME2)
75 {
76 DepthwiseMethod::PLANAR,
77 "sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za",
78 constraint<Requantize32>(cpu_has_sme, cpu_has_sme2,
79 is_supported<sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za>,
80 has_no_channel_multiplier,
81 qp_has_no_left_shift),
82 nullptr,
83 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
84 auto strat = new sme2_u8s8u8q_planar_3x3_s1_4rows_dot_za(args.cpu_info);
85 return new DepthwisePlanar<uint8_t, int8_t>(strat, args, qp);
86 },
87 },
88 {
89 DepthwiseMethod::PLANAR,
90 "sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za",
91 constraint<Requantize32>(cpu_has_sme, cpu_has_sme2,
92 is_supported<sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za>,
93 has_no_channel_multiplier,
94 qp_has_no_left_shift),
95 nullptr,
96 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
97 auto strat = new sme2_u8s8u8q_planar_3x3_s2_4rows_dot_za(args.cpu_info);
98 return new DepthwisePlanar<uint8_t, int8_t>(strat, args, qp);
99 },
100 },
101 {
102 DepthwiseMethod::PLANAR,
103 "sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za",
104 constraint<Requantize32>(cpu_has_sme, cpu_has_sme2,
105 is_supported<sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za>,
106 has_no_channel_multiplier,
107 qp_has_no_left_shift),
108 nullptr,
109 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
110 auto strat = new sme2_u8s8u8q_planar_5x5_s1_4rows_dot_za(args.cpu_info);
111 return new DepthwisePlanar<uint8_t, int8_t>(strat, args, qp);
112 },
113 },
114 {
115 DepthwiseMethod::PLANAR,
116 "sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za",
117 constraint<Requantize32>(cpu_has_sme, cpu_has_sme2,
118 is_supported<sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za>,
119 has_no_channel_multiplier,
120 qp_has_no_left_shift),
121 nullptr,
122 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
123 auto strat = new sme2_u8s8u8q_planar_5x5_s2_4rows_dot_za(args.cpu_info);
124 return new DepthwisePlanar<uint8_t, int8_t>(strat, args, qp);
125 },
126 },
127 #endif // defined(ARM_COMPUTE_ENABLE_SME2)
128 {
129 DepthwiseMethod::DEPTHFIRST,
130 "sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst",
131 constraint<Requantize32>(is_supported<sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst>,
132 qp_has_no_left_shift,
133 cpu_has_sve2),
134 nullptr,
135 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
136 auto strat = new sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
137 return new DepthwiseDepthfirst<uint8_t, int8_t>(strat, args, qp);
138 },
139 },
140 {
141 DepthwiseMethod::DEPTHFIRST,
142 "sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst",
143 constraint<Requantize32>(is_supported<sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst>,
144 qp_has_no_left_shift,
145 cpu_has_sve2),
146 nullptr,
147 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
148 auto strat = new sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
149 return new DepthwiseDepthfirst<uint8_t, int8_t>(strat, args, qp);
150 },
151 },
152 {
153 DepthwiseMethod::DEPTHFIRST,
154 "sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst",
155 constraint<Requantize32>(is_supported<sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst>,
156 qp_has_no_left_shift,
157 cpu_has_sve2),
158 nullptr,
159 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
160 auto strat = new sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
161 return new DepthwiseDepthfirst<uint8_t, int8_t>(strat, args, qp);
162 },
163 },
164 #endif // defined(ARM_COMPUTE_ENABLE_SVE)
165 {
166 DepthwiseMethod::DEPTHFIRST,
167 "a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst",
168 constraint<Requantize32>(is_supported<a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst>,
169 qp_has_no_left_shift),
170 nullptr,
171 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
172 auto strat = new a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info);
173 return new DepthwiseDepthfirst<uint8_t, int8_t>(strat, args, qp);
174 },
175 },
176 {
177 DepthwiseMethod::DEPTHFIRST,
178 "a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst",
179 constraint<Requantize32>(is_supported<a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst>,
180 qp_has_no_left_shift),
181 nullptr,
182 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
183 auto strat = new a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info);
184 return new DepthwiseDepthfirst<uint8_t, int8_t>(strat, args, qp);
185 },
186 },
187 {
188 DepthwiseMethod::DEPTHFIRST,
189 "a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst",
190 constraint<Requantize32>(is_supported<a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst>,
191 qp_has_no_left_shift),
192 nullptr,
193 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
194 auto strat = new a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info);
195 return new DepthwiseDepthfirst<uint8_t, int8_t>(strat, args, qp);
196 },
197 },
198 {
199 DepthwiseMethod::DEPTHFIRST,
200 "a64_u8s8u8q_nhwc_generic_output3x3_mla_depthfirst",
201 nullptr,
202 nullptr,
203 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
204 auto kernel = new a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info);
205 auto strat = new GenericDepthfirstStrategy<uint8_t, int8_t>(kernel, 3, 3, args);
206 return new DepthwiseDepthfirstGeneric<uint8_t, int8_t>(strat, args, qp);
207 },
208 },
209 {
210 DepthwiseMethod::DEPTHFIRST,
211 "a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst",
212 constraint<Requantize32>(has_channel_multiplier),
213 not_preferred,
214 [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * {
215 auto kern = new a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info);
216 auto strat = new GenericDepthfirstMultiplierStrategy<uint8_t, int8_t>(kern, args);
217 return new DepthwiseDepthfirstMultiplier<uint8_t, int8_t, uint8_t, int32_t, true>(strat, args, qp);
218 },
219 },
220 #endif // defined(__aarch64__)
221 { DepthwiseMethod::DEFAULT, "", nullptr, nullptr, nullptr }, // End of list
222 };
223
224 template <>
225 const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> *depthwise_implementation_list()
226 {
227 return depthwise_u8q_methods;
228 }
229
230 template UniqueDepthwiseCommon<uint8_t, int8_t, uint8_t> depthwise(const DepthwiseArgs &, const Requantize32 &);
231 template std::vector<KernelDescription> get_compatible_kernels<uint8_t, int8_t, uint8_t, Requantize32>(const DepthwiseArgs &, const Requantize32 &);
232
233 } // namespace depthwise
234 } // namespace arm_conv