Debian Codegraph @

src/core/NEON/kernels/arm_conv/depthwise/depthwise_planar.hpp

raw
1 /*
2 * Copyright (c) 2022-2024 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #pragma once
26
27 #include "depthfirst_driver.hpp"
28 #include "interleaves/generic.hpp"
29
30 namespace arm_conv {
31 namespace depthwise {
32
33 template <typename OutputStage>
34 class IPlanarStrategy
35 {
36 public:
37 virtual ~IPlanarStrategy() = default;
38 virtual unsigned int get_output_rows(void) const = 0;
39 virtual arm_gemm::VLType get_vl_type(void) const = 0;
40
41 virtual size_t get_storage_size(const DepthwiseArgs &) const = 0;
42 virtual void pack_parameters(
43 const DepthwiseArgs &args, void *buffer,
44 const void *biases, const OutputStage &,
45 const void *weights, size_t ld_weight_col, size_t ld_weight_row
46 ) const = 0;
47 };
48
49
50 template <typename TInput, typename TWeight, typename TOutput, typename TAccum,
51 typename OutputStage>
52 struct PlanarKernelType;
53
54 template <typename TInput, typename TWeight, typename TOutput, typename TAccum>
55 struct PlanarKernelType<TInput, TWeight, TOutput, TAccum, Nothing>
56 {
57 typedef void (*Type)(
58 const TInput *, size_t ld_in_row, size_t ld_in_col, size_t ld_in_vl,
59 unsigned int pad_top, unsigned int valid_input_rows,
60 unsigned int pad_left, unsigned int valid_input_cols,
61 const TWeight *, const TAccum *,
62 TOutput **, const size_t *, const size_t *, unsigned int output_cols,
63 unsigned int start_channels, unsigned int valid_channels,
64 TAccum act_min, TAccum act_max
65 );
66
67 template <typename WorkspaceType>
68 static inline void execute(
69 const Type fn,
70 const TInput *inptr, size_t ld_in_row, size_t ld_in_col, size_t ld_in_vl,
71 unsigned int pad_top, unsigned int valid_input_rows,
72 unsigned int pad_left, unsigned int valid_input_cols,
73 const TWeight *weights, const TAccum *bias,
74 TOutput **outptrs, const size_t *outlds, const size_t *outvllds, unsigned int output_cols,
75 unsigned int start_channel, unsigned int valid_channels,
76 const Nothing &, const WorkspaceType *ws
77 )
78 {
79 fn(
80 inptr, ld_in_row, ld_in_col, ld_in_vl,
81 pad_top, valid_input_rows,
82 pad_left, valid_input_cols,
83 weights, bias,
84 outptrs, outlds, outvllds, output_cols,
85 start_channel, valid_channels,
86 ws->activation_min, ws->activation_max
87 );
88 }
89 };
90
91 template <typename TInput, typename TWeight, typename TOutput>
92 struct PlanarKernelType<TInput, TWeight, TOutput, int32_t, arm_gemm::Requantize32>
93 {
94 typedef void (*Type)(
95 const TInput *, size_t ld_in_row, size_t ld_in_col, size_t ld_in_vl,
96 unsigned int pad_top, unsigned int valid_input_rows,
97 unsigned int pad_left, unsigned int valid_input_cols,
98 const TWeight *,
99 TOutput **, const size_t *, const size_t *, unsigned int output_cols,
100 unsigned int start_channel, unsigned int valid_channels,
101 const arm_gemm::Requantize32 &
102 );
103
104 template <typename WorkspaceType>
105 static inline void execute(
106 const Type fn,
107 const TInput *inptr, size_t ld_in_row, size_t ld_in_col, size_t ld_in_vl,
108 unsigned int pad_top, unsigned int valid_input_rows,
109 unsigned int pad_left, unsigned int valid_input_cols,
110 const TWeight *weights, const int32_t *,
111 TOutput **outptrs, const size_t *outlds, const size_t *outldvls, unsigned int output_cols,
112 unsigned int first_channel, unsigned int valid_channels,
113 const arm_gemm::Requantize32 &qp, const WorkspaceType *
114 )
115 {
116 fn(
117 inptr, ld_in_row, ld_in_col, ld_in_vl,
118 pad_top, valid_input_rows,
119 pad_left, valid_input_cols,
120 weights,
121 outptrs, outlds, outldvls, output_cols,
122 first_channel, valid_channels,
123 qp
124 );
125 }
126 };
127
128
129 template <typename TInput, typename TWeight=TInput, typename TOutput=TInput,
130 typename TAccum=typename DefaultTAccum<TOutput>::Type,
131 typename OutputStage=typename DefaultOutputStage<TOutput>::Type>
132 class PlanarStrategy : public IPlanarStrategy<OutputStage>
133 {
134 unsigned int m_kernel_rows, m_kernel_cols;
135 unsigned int m_stride_rows, m_stride_cols;
136 unsigned int m_output_rows;
137 arm_gemm::VLType m_vl_type;
138
139 protected:
140 virtual bool get_kernel_packing_point(const unsigned int index, unsigned int &x, unsigned int &y) const
141 {
142 // Get the kernel point to pack at the given index; return false to
143 // indicate that this index (and all greater indices) is out of range.
144 if (m_kernel_rows * m_kernel_cols <= index)
145 return false;
146
147 y = index % m_kernel_cols;
148 x = index / m_kernel_cols;
149 return true;
150 }
151
152 virtual interleaves::PackingArguments get_kernel_packing_arguments(void) const
153 {
154 return interleaves::PackingArguments(
155 m_kernel_rows, m_kernel_cols, sizeof(TWeight),
156 false, sizeof(TAccum), true, // Don't pack the bias
157 m_vl_type, sizeof(TAccum), 1, // Accumulator depth of 1 TODO
158 [this] (unsigned int idx, unsigned int &x, unsigned int &y) -> bool
159 { return this->get_kernel_packing_point(idx, x, y); }
160 );
161 }
162
163 public:
164 PlanarStrategy(
165 unsigned int kernel_rows, unsigned int kernel_cols,
166 unsigned int stride_rows, unsigned int stride_cols,
167 unsigned int output_rows,
168 arm_gemm::VLType vl_type
169 ) : m_kernel_rows(kernel_rows), m_kernel_cols(kernel_cols),
170 m_stride_rows(stride_rows), m_stride_cols(stride_cols),
171 m_output_rows(output_rows), m_vl_type(vl_type)
172 {
173 }
174
175 unsigned int get_output_rows(void) const override { return m_output_rows; }
176 arm_gemm::VLType get_vl_type(void) const override { return m_vl_type; }
177
178 size_t get_storage_size(const DepthwiseArgs &args) const override
179 {
180 return interleaves::get_storage_size_generic(this->get_kernel_packing_arguments(), args);
181 }
182
183 void pack_parameters(
184 const DepthwiseArgs &args, void *buffer,
185 const void *biases, const OutputStage &,
186 const void *weights, size_t ld_weight_col, size_t ld_weight_row
187 ) const override
188 {
189 interleaves::pack_parameters_generic(
190 this->get_kernel_packing_arguments(), args,
191 buffer, biases, weights, ld_weight_col, ld_weight_row
192 );
193 }
194
195 using KernelType = typename PlanarKernelType<TInput, TWeight, TOutput, TAccum, OutputStage>::Type;
196 virtual KernelType get_kernel(void) const = 0;
197 };
198
199
200 namespace {
201
202 template <typename T>
203 struct OutputRowPtrsElement
204 {
205 struct Workspace
206 {
207 T **output_row_ptrs;
208 size_t *output_ld_cols;
209 size_t *output_ld_vls; // Stride between vectors of channels
210 T *output_padding_buffer;
211 };
212
213 // On some implementations there is a significant performance benefit to
214 // aligning the padding buffer to a 1024 byte boundary. This routine
215 // adds as much padding as needed to an arbitrary input pointer and
216 // returns an aligned void *.
217 static constexpr intptr_t BUFFER_ALIGNMENT=1024;
218
219 template <typename ptr_T>
220 static void *do_align(ptr_T in)
221 {
222 intptr_t v = reinterpret_cast<intptr_t>(in);
223 intptr_t odds = v & (BUFFER_ALIGNMENT-1);
224 intptr_t pad = odds ? (BUFFER_ALIGNMENT - odds) : 0;
225
226 return reinterpret_cast<void *>(v + pad);
227 }
228
229 template <typename OutputStage>
230 static size_t get_element_size(const WorkspaceArgs<IPlanarStrategy<OutputStage>, OutputStage> &args)
231 {
232 // We need one pointer and stride for each row of output, and an
233 // additional blob of memory into which padded stores can go. Allow
234 // extra space so that this padding buffer can be aligned at both ends.
235 return args.strategy->get_output_rows() * (sizeof(T *) + 2*sizeof(size_t)) +
236 get_vector_length<char>(args.strategy->get_vl_type()) + BUFFER_ALIGNMENT*2;
237 }
238
239 template <typename WorkspaceType, typename OutputStage>
240 static void *initialise(WorkspaceType *ws, void *buffer,
241 const WorkspaceArgs<IPlanarStrategy<OutputStage>, OutputStage> &args)
242 {
243 const auto n_rows = args.strategy->get_output_rows();
244 ws->output_row_ptrs = reinterpret_cast<T **>(buffer);
245 ws->output_ld_cols = reinterpret_cast<size_t *>(ws->output_row_ptrs + n_rows);
246 ws->output_ld_vls = ws->output_ld_cols + n_rows;
247 ws->output_padding_buffer = reinterpret_cast<T *>(do_align(ws->output_ld_vls + n_rows));
248 return do_align(ws->output_padding_buffer + get_vector_length<T>(args.strategy->get_vl_type()));
249 }
250 };
251
252 } // namespace {anonymous}
253
254
255 template <typename TInput, typename TWeight=TInput, typename TOutput=TInput,
256 typename TAccum=typename DefaultTAccum<TOutput>::Type,
257 typename OutputStage=typename DefaultOutputStage<TOutput>::Type>
258 class DepthwisePlanar : public DepthwiseCommon<TInput, TWeight, TOutput>
259 {
260 using Parent = DepthwiseCommon<TInput, TWeight, TOutput>;
261 using StrategyType = IPlanarStrategy<OutputStage>;
262 using WorkspaceManager = Workspace<
263 OutputRowPtrsElement<TOutput>,
264 ActivationsElement<TAccum, OutputStage>
265 >;
266 using WorkspaceType = typename WorkspaceManager::WorkspaceType;
267
268 std::unique_ptr<StrategyType> m_strat;
269 const TAccum *m_bias;
270 OutputStage m_os;
271
272 public:
273 DepthwisePlanar(StrategyType *const strat, const DepthwiseArgs &args, const OutputStage &os = {})
274 : Parent(args), m_strat(strat), m_bias(nullptr), m_os(os)
275 {
276 }
277
278 DepthwisePlanar(DepthwisePlanar &) = delete;
279 DepthwisePlanar &operator=(DepthwisePlanar &) = delete;
280
281 size_t get_storage_size(void) const override
282 {
283 return m_strat->get_storage_size(this->m_args);
284 }
285
286 void pack_parameters(
287 void *buffer, const void *biases,
288 const void *weights, size_t ld_weight_col, size_t ld_weight_row
289 ) override
290 {
291 m_strat->pack_parameters(this->m_args, buffer, biases, {}, weights, ld_weight_col, ld_weight_row);
292 this->m_bias = reinterpret_cast<const TAccum *>(biases);
293 depthwise_depthfirst::stash_bias(this->m_os, biases);
294 }
295
296 size_t get_working_size(unsigned int n_threads) const override
297 {
298 return this->get_working_size_per_thread() * n_threads;
299 }
300
301 protected:
302 /* Compute the amount of working space required for a single thread. */
303 virtual size_t get_working_size_per_thread(void) const
304 {
305 return WorkspaceManager::get_sizeof_workspace(
306 WorkspaceArgs<IPlanarStrategy<OutputStage>, OutputStage>(m_strat.get(), this->m_args, m_os));
307 }
308
309 /* Initialise the working space for a thread. */
310 virtual void initialise_working_space(void *buffer) const
311 {
312 WorkspaceManager::initialise(
313 buffer,
314 WorkspaceArgs<IPlanarStrategy<OutputStage>, OutputStage>(m_strat.get(), this->m_args, m_os)
315 );
316 }
317
318 /* Execute the kernel for a given chunk of work. */
319 virtual void execute_kernel(
320 const TInput *inptr, size_t ld_in_row, size_t ld_in_col, size_t ld_in_vl,
321 unsigned int pad_top, unsigned int valid_input_rows,
322 unsigned int pad_left, unsigned int valid_input_cols,
323 const TWeight *weights, const TAccum *bias,
324 TOutput *outptr, size_t ld_out_row, size_t ld_out_col, size_t ld_out_vl,
325 unsigned int valid_output_rows, unsigned int valid_output_cols,
326 unsigned int first_channel, unsigned int valid_channels,
327 WorkspaceType *ws
328 ) const
329 {
330 // Initialise the output pointers
331 for (auto i = 0u; i < m_strat->get_output_rows(); i++)
332 {
333 // Point at the output tensor for all valid rows; otherwise point at the
334 // padding buffer.
335 ws->output_row_ptrs[i] = i < valid_output_rows ? outptr : ws->output_padding_buffer;
336 ws->output_ld_cols[i] = i < valid_output_rows ? ld_out_col : 0;
337 ws->output_ld_vls[i] = i < valid_output_rows ? ld_out_vl : 0;
338 outptr += ld_out_row;
339 }
340
341 // Execute the kernel
342 PlanarKernelType<TInput, TWeight, TOutput, TAccum, OutputStage>::template execute<WorkspaceType>(
343 reinterpret_cast<const PlanarStrategy<TInput, TWeight, TOutput, TAccum, OutputStage> *>(m_strat.get())->get_kernel(),
344 inptr, ld_in_row, ld_in_col, ld_in_vl,
345 pad_top, valid_input_rows, pad_left, valid_input_cols,
346 weights, bias,
347 ws->output_row_ptrs, ws->output_ld_cols, ws->output_ld_vls,
348 valid_output_cols, first_channel, valid_channels,
349 this->m_os, ws
350 );
351 }
352
353 void execute_internal(
354 const DepthwiseArgs &args,
355 const void *input,
356 size_t ld_input_col,
357 size_t ld_input_row,
358 size_t ld_input_batch,
359 const void *parameters,
360 void *output,
361 size_t ld_output_col,
362 size_t ld_output_row,
363 size_t ld_output_batch,
364 void *working_space,
365 unsigned int thread_id,
366 unsigned int n_threads
367 ) const override
368 {
369 // Get and initialise the working space for this thread.
370 void *thread_working_space =
371 static_cast<uint8_t *>(working_space) + thread_id * this->get_working_size_per_thread();
372 this->initialise_working_space(thread_working_space);
373 auto ws = reinterpret_cast<WorkspaceType *>(thread_working_space);
374
375 const auto n_output_channels = args.input_channels * args.channel_multiplier;
376 const auto vl = get_vector_length<TAccum>(m_strat->get_vl_type());
377
378 // Get typed pointers
379 auto input_batch = reinterpret_cast<const TInput *>(input);
380 auto output_batch = reinterpret_cast<TOutput *>(output);
381 auto weights = reinterpret_cast<const TWeight *>(parameters);
382
383 // Iterate over batches
384 for (auto batches = args.n_batches; batches; batches--)
385 {
386 // NOTE: Other loop orderings are possible and it would be worth
387 // investigating them.
388
389 // Within a batch, stripe threads across rows.
390 for (auto start_output_i = thread_id * m_strat->get_output_rows();
391 start_output_i < args.output_rows;
392 start_output_i += n_threads * m_strat->get_output_rows())
393 {
394 // Determine what (if any padding) is required on the top/bottom of
395 // this row of the convolution.
396 const int start_input_i = start_output_i * args.stride_rows - args.padding.top;
397 const unsigned int input_pad_top = start_input_i < 0 ? -start_input_i : 0;
398 const unsigned int input_i = start_input_i < 0 ? 0 : start_input_i;
399 const unsigned int valid_input_rows = input_i > args.input_rows ? 0 : args.input_rows - input_i;
400 const unsigned int valid_output_rows = args.output_rows - start_output_i;
401
402 auto inptr_row = input_batch + input_i*ld_input_row;
403 auto outptr_row = output_batch + start_output_i * ld_output_row;
404
405 // Execute the kernel
406 this->execute_kernel(
407 inptr_row, ld_input_row, ld_input_col, vl,
408 input_pad_top, valid_input_rows, args.padding.left, args.input_cols,
409 weights, this->m_bias,
410 outptr_row, ld_output_row, ld_output_col, vl,
411 valid_output_rows, args.output_cols,
412 0 /* first channel */, n_output_channels,
413 ws
414 );
415 }
416
417 // Update the input and output pointers to account for batch
418 input_batch += ld_input_batch;
419 output_batch += ld_output_batch;
420 }
421 }
422 };
423
424 } // namespace depthwise
425 } // namespace arm_conv