Debian Codegraph @

src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp

raw
1 /*
2 * Copyright (c) 2021-2023 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #pragma once
26
27 #include "src/core/NEON/kernels/arm_conv/addressing.hpp"
28 #include "depthwise_strategies_common.hpp"
29 #include "working_space.hpp"
30
31 #ifdef CYCLE_PROFILING
32 #include "profiler.hpp"
33 #endif
34
35 #include <limits>
36
37 namespace arm_conv {
38 namespace depthwise {
39
40 template <typename TInput, typename TWeight, typename TOutput, typename TAccum,
41 typename OutputStage>
42 class DepthwiseDepthfirstStrategyCommon
43 : public DepthfirstStrategy<TInput, TWeight, TOutput, TAccum, OutputStage>
44 {
45 protected:
46 unsigned int m_output_rows, m_output_cols;
47 unsigned int m_kernel_rows, m_kernel_cols;
48 unsigned int m_stride_rows, m_stride_cols;
49
50 public:
51 DepthwiseDepthfirstStrategyCommon(
52 unsigned int output_rows, unsigned int output_cols,
53 unsigned int kernel_rows, unsigned int kernel_cols,
54 unsigned int stride_rows=1, unsigned int stride_cols=1
55 ) : m_output_rows(output_rows), m_output_cols(output_cols),
56 m_kernel_rows(kernel_rows), m_kernel_cols(kernel_cols),
57 m_stride_rows(stride_rows), m_stride_cols(stride_cols)
58 {
59 }
60
61 DepthwiseDepthfirstStrategyCommon(unsigned int output_size, unsigned int kernel_size, unsigned int stride=1)
62 : DepthwiseDepthfirstStrategyCommon(output_size, output_size, kernel_size, kernel_size, stride, stride)
63 {
64 }
65
66 virtual ~DepthwiseDepthfirstStrategyCommon() {}
67
68 unsigned int get_output_rows() const override { return m_output_rows; }
69 unsigned int get_output_cols() const override { return m_output_cols; }
70
71 unsigned int get_kernel_rows() const override { return m_kernel_rows; }
72 unsigned int get_kernel_cols() const override { return m_kernel_cols; }
73
74 unsigned int get_stride_rows() const override { return m_stride_rows; }
75 unsigned int get_stride_cols() const override { return m_stride_cols; }
76 };
77
78 template <typename TInput, typename TWeight, typename TOutput, typename TAccum, typename OutputStage=typename DefaultOutputStage<TOutput>::Type>
79 class DepthwiseDepthfirstStrategy : public DepthwiseDepthfirstStrategyCommon<TInput, TWeight, TOutput, TAccum, OutputStage>
80 {
81 using Parent = DepthwiseDepthfirstStrategyCommon<TInput, TWeight, TOutput, TAccum, OutputStage>;
82
83 public:
84 using Parent::Parent;
85
86 typedef void (*IndirectKernelType)(
87 const TInput *const *input_ptrs,
88 TOutput *const *output_ptrs,
89 const void *params,
90 unsigned int n_channels,
91 const TAccum activation_min,
92 const TAccum activation_max
93 );
94 virtual IndirectKernelType get_indirect_kernel(void) const = 0;
95
96 typedef void (*DirectKernelType)(
97 const unsigned int n_tile_rows, const unsigned int n_tile_cols,
98 const TInput *inptr_base, int64_t ld_input_row, int64_t ld_input_col,
99 TOutput *outptr_base, int64_t ld_output_row, int64_t ld_output_col,
100 const void *params, unsigned int n_channels,
101 const TAccum activation_min,
102 const TAccum activation_max
103 );
104 virtual DirectKernelType get_direct_kernel(void) const = 0;
105 };
106
107 template <typename TInput, typename TWeight, typename TOutput>
108 class DepthwiseDepthfirstStrategy<TInput, TWeight, TOutput, int32_t>
109 : public DepthwiseDepthfirstStrategyCommon<TInput, TWeight, TOutput, int32_t, arm_gemm::Requantize32>
110 {
111 using Parent = DepthwiseDepthfirstStrategyCommon<TInput, TWeight, TOutput, int32_t, arm_gemm::Requantize32>;
112
113 protected:
114 interleaves::PackingArguments get_packing_args(void) const
115 {
116 return interleaves::PackingArguments(
117 this->get_kernel_rows(), this->get_kernel_cols(), sizeof(TWeight),
118 false, sizeof(int32_t), this->uses_premultiply(), // Don't pack the bias
119 this->get_vl_type(), sizeof(int32_t), this->get_accumulator_depth_vl(),
120 [this] (unsigned int idx, unsigned int &x, unsigned int &y) -> bool
121 { return this->get_kernel_packing_point(idx, x, y); }
122 );
123 }
124
125 public:
126 using Parent::Parent;
127
128 typedef void (*KernelType)(
129 unsigned int, // n_channels,
130 const TInput *const *, // inptrs
131 const TWeight *, // weights
132 const int32_t *, // bias,
133 const arm_gemm::Requantize32 &,
134 const int32_t *, const int32_t *, // requant_muls and requant_shifts
135 TOutput *const * // outptrs
136 );
137 virtual KernelType get_kernel() const = 0;
138
139 size_t get_storage_size(const DepthwiseArgs &args) const override
140 {
141 return interleaves::get_storage_size_generic(get_packing_args(), args);
142 }
143
144 void pack_parameters(
145 const DepthwiseArgs &args, void *buffer,
146 const void *biases, const arm_gemm::Requantize32 &,
147 const void *weights, size_t ld_weight_col, size_t ld_weight_row
148 ) const override
149 {
150 interleaves::pack_parameters_generic(
151 get_packing_args(), args, buffer, biases, weights, ld_weight_col, ld_weight_row);
152 }
153 };
154
155 template <typename TInput, typename TWeight, typename TOutput, typename TAccum, typename OutputStage>
156 class DepthwiseDepthfirstCommon : public DepthfirstDriver<TInput, TWeight, TOutput>
157 {
158 using StratType = DepthwiseDepthfirstStrategyCommon<TInput, TWeight, TOutput, TAccum, OutputStage>;
159 OutputStage m_os;
160
161 protected:
162 inline OutputStage &get_output_stage(void) { return m_os; }
163 inline const OutputStage &get_output_stage(void) const { return m_os; }
164
165 bool uses_intermediate_array() const
166 {
167 return this->m_args.channel_multiplier != 1 && this->uses_premultiply();
168 }
169
170 virtual void fill_inptr_array(const DepthwiseArgs &args,
171 const TensorSpec<const TInput *> &input,
172 const TInput **inptr_array, TInput *input_buffer,
173 const unsigned int input_i, const unsigned int input_j,
174 const unsigned int input_pad_top, const unsigned int input_pad_left) const = 0;
175
176 void initialise_inptr_array(const DepthwiseArgs &args,
177 unsigned int output_channel_start, unsigned int output_channel_end,
178 const TensorSpec<const TInput *> &input,
179 const TInput **inptr_array, TInput *input_buffer, TInput *intermediate_buffer,
180 const unsigned int input_i, const unsigned int input_j,
181 const unsigned int input_pad_top, const unsigned int input_pad_left,
182 Tile<TInput> &multiplied_input
183 ) const
184 {
185 // Compute the input pointer array
186 const auto input_channel_start = output_channel_start / args.channel_multiplier;
187
188 const auto last_valid_row = std::min(input_pad_top + args.input_rows - input_i, this->m_strat->get_input_rows());
189 const auto last_valid_col = std::min(input_pad_left + args.input_cols - input_j, this->m_strat->get_input_cols());
190
191 const auto tile_rows = last_valid_row - input_pad_top;
192 const auto tile_cols = last_valid_col - input_pad_left;
193
194 const auto tile_channels = output_channel_end - output_channel_start;
195
196 TensorSpec<const TInput *> tile_tensor(0, 0, 0);
197 if (this->uses_intermediate_array()) {
198 multiplied_input = Tile<TInput>(intermediate_buffer, tile_rows, tile_cols, tile_channels);
199 multiplied_input.load_from(input.base, input.ld_row, input.ld_col,
200 args.input_rows, args.input_cols,
201 input_i, input_j, args.channel_multiplier);
202
203 tile_tensor = TensorSpec<const TInput *>(
204 multiplied_input.array,
205 tile_cols * tile_channels, tile_channels
206 );
207 } else {
208 tile_tensor = TensorSpec<const TInput *>(
209 input.base + input_i*input.ld_row + input_j*input.ld_col + input_channel_start,
210 input.ld_row, input.ld_col
211 );
212 }
213
214 fill_inptr_array(args,
215 tile_tensor,
216 inptr_array, input_buffer,
217 input_i, input_j,
218 input_pad_top,
219 input_pad_left
220 );
221 }
222
223 public:
224 DepthwiseDepthfirstCommon(StratType *const strat, const DepthwiseArgs &args, const OutputStage &os)
225 : DepthfirstDriver<TInput, TWeight, TOutput>(strat, args), m_os(os)
226 {
227 }
228
229 DepthwiseDepthfirstCommon(DepthwiseDepthfirstCommon &) = delete;
230 DepthwiseDepthfirstCommon &operator=(DepthwiseDepthfirstCommon &) = delete;
231
232 size_t get_storage_size(void) const override
233 {
234 return reinterpret_cast<const StratType *>(this->m_strat.get())->
235 get_storage_size(this->m_args);
236 }
237
238 void pack_parameters(void *buffer, const void *biases, const void *weights, size_t ld_weight_col, size_t ld_weight_row) override
239 {
240 reinterpret_cast<const StratType *>(this->m_strat.get())->
241 pack_parameters(this->m_args, buffer, biases, m_os, weights, ld_weight_col, ld_weight_row);
242 }
243 };
244
245 namespace depthwise_depthfirst {
246
247 /* Workspace Element for an array of input pointers as consumed by the
248 * specialised depthwise kernels.
249 */
250 template <typename T>
251 class InputArrayElement
252 {
253 public:
254 struct Workspace
255 {
256 const T **inptr_array;
257 };
258
259 template <class OutputStage>
260 static size_t get_element_size(const WorkspaceArgs<IDepthfirstStrategy, OutputStage> &args)
261 {
262 return sizeof(T **) * args.strategy->get_input_rows() * args.strategy->get_input_cols();
263 }
264
265 template <class WorkspaceType, class OutputStage>
266 static void *initialise(WorkspaceType *ws, void *buffer, const WorkspaceArgs<IDepthfirstStrategy, OutputStage> &args)
267 {
268 ws->inptr_array = reinterpret_cast<const T**>(buffer);
269 return reinterpret_cast<char *>(buffer) + get_element_size(args);
270 }
271 };
272
273 template <typename TAccum, typename OutputStage, bool IsDot=false>
274 struct WorkspaceFinalElement
275 {
276 using Element = ActivationsElement<TAccum, OutputStage>;
277 };
278
279 template <>
280 struct WorkspaceFinalElement<int32_t, arm_gemm::Requantize32, false>
281 {
282 using Element = RequantizationParametersElement;
283 };
284
285 template <typename TInput, typename TWeight, typename TOutput, typename TAccum, typename OutputStage>
286 struct Invoke
287 {
288 constexpr static bool supports_direct_kernel = true;
289
290 template <typename Strat, typename Workspace>
291 static inline void indirect(const Strat *strat, const Workspace *ws, const OutputStage &, const void *params, const TAccum *, unsigned int n_channels)
292 {
293 strat->get_indirect_kernel()(
294 ws->inptr_array,
295 ws->outptr_array,
296 params, n_channels,
297 ws->activation_min, ws->activation_max
298 );
299 }
300
301 template <typename Strat, typename Workspace>
302 static void direct(
303 const Strat *strat, const Workspace *ws, const OutputStage &,
304 unsigned int n_tile_rows, unsigned int n_tile_cols,
305 const TInput *inptr, size_t ld_in_row, size_t ld_in_col,
306 TOutput *outptr, size_t ld_out_row, size_t ld_out_col,
307 const void *params, unsigned int n_channels
308 )
309 {
310 strat->get_direct_kernel()(
311 n_tile_rows, n_tile_cols,
312 inptr, ld_in_row, ld_in_col,
313 outptr, ld_out_row, ld_out_col,
314 params, n_channels, ws->activation_min, ws->activation_max
315 );
316 }
317 };
318
319 template <typename TInput, typename TWeight, typename TOutput, typename TAccum>
320 struct Invoke<TInput, TWeight, TOutput, TAccum, arm_gemm::Requantize32>
321 {
322 constexpr static bool supports_direct_kernel = false;
323
324 template <typename Strat, typename Workspace>
325 static inline void indirect(const Strat *strat, const Workspace *ws, const arm_gemm::Requantize32 &qp, const void *params, const TAccum *, unsigned int n_channels)
326 {
327 strat->get_kernel()(
328 n_channels, ws->inptr_array,
329 reinterpret_cast<const TWeight *>(params), ws->bias,
330 qp, ws->requant_muls, ws->requant_shifts,
331 ws->outptr_array
332 );
333 }
334
335 template <typename Strat, typename Workspace>
336 static inline void direct(
337 const Strat *, const Workspace *, const arm_gemm::Requantize32 &,
338 unsigned int, unsigned int, // n_tile_rows, n_tile_cols
339 const TInput *, size_t, size_t, // Input pointer, row stride, column stride
340 TOutput *, size_t, size_t, // Output pointer, row stride, column stride
341 const void *, unsigned int // Parameters, number of channels
342 )
343 {
344 // Do nothing - this should never be reached because entry to it is guarded
345 // by an `if` on a `constexpr static bool`.
346 }
347 };
348
349 namespace
350 {
351
352 template <typename OutputStage>
353 inline void stash_bias(OutputStage &, const void *) {}
354
355 template <>
356 inline void stash_bias(arm_gemm::Requantize32 &qp, const void *bias) __attribute__ ((unused));
357
358 template <>
359 inline void stash_bias(arm_gemm::Requantize32 &qp, const void *bias)
360 {
361 qp.bias = reinterpret_cast<const int32_t *>(bias);
362 }
363
364 }
365
366 } // namespace depthwise_depthfirst
367
368 template <typename TInput,
369 typename TWeight=TInput,
370 typename TOutput=TInput,
371 typename TAccum=typename DefaultTAccum<TInput>::Type,
372 typename OutputStage=typename DefaultOutputStage<TOutput>::Type>
373 class DepthwiseDepthfirst
374 : public DepthwiseDepthfirstCommon<TInput, TWeight, TOutput, TAccum, OutputStage>
375 {
376 using StratType = DepthwiseDepthfirstStrategy<TInput, TWeight, TOutput, TAccum>;
377 using Parent = DepthwiseDepthfirstCommon<TInput, TWeight, TOutput, TAccum, OutputStage>;
378 using WorkspaceManager = Workspace<
379 OutputArrayElement<TOutput>,
380 depthwise_depthfirst::InputArrayElement<TInput>,
381 InputBufferElement<TInput>,
382 IntermediateBufferElement<TInput>,
383 typename depthwise_depthfirst::WorkspaceFinalElement<TAccum, OutputStage>::Element
384 >;
385 using WorkingSpace = typename WorkspaceManager::WorkspaceType;
386
387 // We keep a copy of the bias and output stage
388 const TAccum *m_bias;
389
390 public:
391 DepthwiseDepthfirst(StratType *const strat, const DepthwiseArgs &args, const OutputStage &os = {})
392 : Parent(strat, args, os), m_bias(nullptr)
393 {
394 }
395
396 DepthwiseDepthfirst(DepthwiseDepthfirst &) = delete;
397 DepthwiseDepthfirst &operator=(DepthwiseDepthfirst &) = delete;
398
399 void pack_parameters(void *buffer, const void *biases, const void *weights, size_t ld_weight_col, size_t ld_weight_row) override
400 {
401 reinterpret_cast<const StratType *>(this->m_strat.get())->pack_parameters(
402 this->m_args, buffer, biases, this->get_output_stage(),
403 weights, ld_weight_col, ld_weight_row
404 );
405 m_bias = reinterpret_cast<const TAccum *>(biases);
406 depthwise_depthfirst::stash_bias(this->get_output_stage(), biases);
407 }
408
409 size_t get_working_size_per_thread() const override
410 {
411 DepthwiseArgs args(this->m_args);
412 return WorkspaceManager::get_sizeof_workspace(
413 WorkspaceArgs<IDepthfirstStrategy, OutputStage>(this->m_strat.get(), args, this->get_output_stage())
414 );
415 }
416
417 void initialise_working_space(void *buffer) const override
418 {
419 DepthwiseArgs args(this->m_args);
420 WorkspaceManager::initialise(
421 buffer, WorkspaceArgs<IDepthfirstStrategy, OutputStage>(this->m_strat.get(), args, this->get_output_stage())
422 );
423 }
424
425 virtual bool supports_direct_padding() const override
426 {
427 using Invoker = depthwise_depthfirst::Invoke<TInput, TWeight, TOutput, TAccum, OutputStage>;
428 return Invoker::supports_direct_kernel && this->uses_intermediate_array();
429 }
430
431 protected:
432
433 void fill_inptr_array(const DepthwiseArgs &args,
434 const TensorSpec<const TInput *> &input,
435 const TInput **inptr_array, TInput *input_buffer,
436 const unsigned int input_i, const unsigned int input_j,
437 const unsigned int input_pad_top, const unsigned int input_pad_left) const override
438 {
439 fill_pointer_array<const TInput>(
440 inptr_array, this->m_strat->get_input_rows(), this->m_strat->get_input_cols(),
441 input.base,
442 input.ld_row, input.ld_col,
443 input_buffer,
444 input_pad_top, args.input_rows - input_i,
445 input_pad_left, args.input_cols - input_j
446 );
447 }
448
449 void compute_tile_padded(
450 const DepthwiseArgs &args,
451 unsigned int output_i, unsigned int output_j,
452 unsigned int output_channel_start, unsigned int output_channel_end,
453 const TensorSpec<const TInput *> &input,
454 const TensorSpec<TOutput *> &output,
455 const void *parameters,
456 void *working_space_raw
457 ) const override
458 {
459 // Get the working space
460 auto ws = reinterpret_cast<WorkingSpace *>(working_space_raw);
461
462 // Compute the input pointer array
463 const int ii = static_cast<int>(output_i * args.stride_rows) - args.padding.top;
464 const auto input_pad_top = static_cast<unsigned int>(ii < 0 ? -ii : 0);
465 const auto input_i = static_cast<unsigned int>(ii < 0 ? 0 : ii);
466
467 const int ij = static_cast<int>(output_j * args.stride_cols) - args.padding.left;
468 const auto input_pad_left = static_cast<unsigned int>(ij < 0 ? -ij : 0);
469 const auto input_j = static_cast<unsigned int>(ij < 0 ? 0 : ij);
470
471 Tile<TInput> multiplied_input;
472 this->initialise_inptr_array(args, output_channel_start, output_channel_end, input,
473 ws->inptr_array, ws->input_buffer, ws->intermediate_buffer,
474 input_i, input_j, input_pad_top, input_pad_left, multiplied_input);
475
476 // Compute the output pointer array
477 fill_pointer_array(
478 ws->outptr_array, this->m_strat->get_output_rows(), this->m_strat->get_output_cols(),
479 output.base + output_i*output.ld_row + output_j*output.ld_col + output_channel_start,
480 output.ld_row, output.ld_col,
481 ws->output_buffer,
482 0, args.output_rows - output_i, // Top padding, # valid rows
483 0, args.output_cols - output_j // Left padding, # valid columns
484 );
485
486 // Execute the kernel
487 depthwise_depthfirst::Invoke<TInput, TWeight, TOutput, TAccum, OutputStage>::indirect(
488 reinterpret_cast<const StratType *>(this->m_strat.get()),
489 ws, this->get_output_stage(), parameters, m_bias, output_channel_end - output_channel_start
490 );
491 }
492
493 void compute_row_padded_tile_row(
494 const DepthwiseArgs &args,
495 const unsigned int output_i, unsigned int output_j, unsigned int n_tile_cols,
496 const unsigned int output_channel_start, const unsigned int output_channel_end,
497 const TensorSpec<const TInput *> &input,
498 const TensorSpec<TOutput *> &output,
499 const void *parameters,
500 void *working_space
501 ) const override
502 {
503 using Invoker = depthwise_depthfirst::Invoke<TInput, TWeight, TOutput, TAccum, OutputStage>;
504 auto ws = reinterpret_cast<WorkingSpace *>(working_space);
505 const auto strat = reinterpret_cast<const StratType *>(this->m_strat.get());
506 const auto os = this->get_output_stage();
507
508 // Compute top and bottom padding; hence fill in the initial pointer arrays.
509 const int ii = static_cast<int>(output_i * args.stride_rows) - args.padding.top;
510 const auto input_pad_top = static_cast<unsigned int>(ii < 0 ? -ii : 0);
511
512 const auto input_i = static_cast<unsigned int>(ii < 0 ? 0 : ii);
513 auto input_j = output_j * args.stride_cols - args.padding.left;
514
515 // Valid input rows is the smallest of the input rows that aren't padding for this tile, and the number of rows
516 // available.
517 const auto valid_input_rows = std::min(strat->get_input_rows() - input_pad_top, args.input_rows - input_i);
518 const auto valid_output_rows = std::min(strat->get_output_rows(), args.output_rows - output_i);
519
520 const auto input_point_stride = input.ld_col * this->m_strat->get_output_cols() * args.stride_cols;
521 const auto output_point_stride = output.ld_col * this->m_strat->get_output_cols();
522
523 Tile<TInput> multiplied_input;
524 this->initialise_inptr_array(args, output_channel_start, output_channel_end, input,
525 ws->inptr_array, ws->input_buffer, ws->intermediate_buffer,
526 input_i, input_j, input_pad_top, 0, multiplied_input);
527
528 fill_pointer_array(
529 ws->outptr_array, this->m_strat->get_output_rows(), this->m_strat->get_output_cols(),
530 output.base + output_i*output.ld_row + output_j*output.ld_col + output_channel_start,
531 output.ld_row, output.ld_col,
532 ws->output_buffer,
533 0, args.output_rows - output_i, // Top padding, # valid rows
534 0, args.output_cols - output_j // Left padding, # valid columns
535 );
536
537 for (; n_tile_cols; n_tile_cols--)
538 {
539 // Execute the kernel
540 Invoker::indirect(
541 strat, ws, os, parameters, m_bias, output_channel_end - output_channel_start
542 );
543
544 // Update all unpadded pointers
545 if (this->uses_intermediate_array()) {
546 input_j += input_point_stride / input.ld_col;
547 multiplied_input.load_from(input.base,
548 input.ld_row, input.ld_col,
549 args.input_rows, args.input_cols,
550 input_i, input_j, args.channel_multiplier);
551 } else {
552 {
553 auto ptr = ws->inptr_array + strat->get_input_cols() * input_pad_top;
554 for (auto n = input_pad_top; n < (valid_input_rows + input_pad_top); n++)
555 {
556 for (auto m = 0u; m < strat->get_input_cols(); m++)
557 {
558 *(ptr++) += input_point_stride;
559 }
560 }
561 }
562 }
563
564 {
565 auto ptr = ws->outptr_array;
566 for (auto n = 0u; n < valid_output_rows * strat->get_output_cols(); n++)
567 {
568 *(ptr++) += output_point_stride;
569 }
570 }
571 }
572 }
573
574 void compute_tiles_unpadded(
575 const DepthwiseArgs &args,
576 unsigned int output_i, const unsigned int output_j,
577 unsigned int n_tile_rows, unsigned int n_tile_cols,
578 unsigned int output_channel_start, unsigned int output_channel_end,
579 const TensorSpec<const TInput *> &input,
580 const TensorSpec<TOutput *> &output,
581 const void *parameters,
582 void *working_space_raw
583 ) const override
584 {
585 using Invoker = depthwise_depthfirst::Invoke<TInput, TWeight, TOutput, TAccum, OutputStage>;
586 auto ws = reinterpret_cast<WorkingSpace *>(working_space_raw);
587 const auto strat = reinterpret_cast<const StratType *>(this->m_strat.get());
588 const auto os = this->get_output_stage();
589
590 if (Invoker::supports_direct_kernel)
591 {
592 PaddingValues tile_padding = {
593 args.kernel_cols / 2,
594 args.kernel_rows / 2,
595 args.kernel_cols / 2,
596 args.kernel_rows / 2
597 };
598
599 // If the direct kernel is supported, then use it.
600 // Compute the base pointers we'll use in the tile.
601 auto outptr = output.base + output_channel_start + output_i * output.ld_row + output_j * output.ld_col;
602 const int start_input_i = output_i * args.stride_rows - args.padding.top;
603 const int start_input_j = output_j * args.stride_cols - args.padding.left;
604 auto inptr = input.base + output_channel_start + start_input_i * input.ld_row + start_input_j * input.ld_col;
605
606 auto ld_row = input.ld_row;
607 auto ld_col = input.ld_col;
608
609 const auto tile_rows = this->m_strat->get_output_rows() * args.stride_rows * n_tile_rows + tile_padding.top + tile_padding.bottom;
610 const auto tile_cols = this->m_strat->get_output_cols() * args.stride_cols * n_tile_cols + tile_padding.left + tile_padding.right;
611 const auto tile_channels = output_channel_end - output_channel_start;
612
613 Tile<TInput> multiplied_input;
614 if (this->uses_intermediate_array()) {
615 multiplied_input = Tile<TInput>(ws->intermediate_buffer, tile_rows, tile_cols, tile_channels);
616 multiplied_input.load_from(input.base,
617 input.ld_row, input.ld_col,
618 args.input_rows, args.input_cols,
619 start_input_i, start_input_j, args.channel_multiplier);
620
621 ld_row = tile_cols * tile_channels;
622 ld_col = tile_channels;
623 inptr = multiplied_input.array;
624 }
625
626 // Execute the kernel
627 Invoker::direct(
628 strat, ws, os,
629 n_tile_rows, n_tile_cols,
630 inptr, ld_row, ld_col,
631 outptr, output.ld_row, output.ld_col,
632 parameters, output_channel_end - output_channel_start
633 );
634 }
635 else
636 {
637 // Otherwise, we repeatedly call the padded kernel but use our knowledge
638 // of the tensor structure to avoid recomputing the pointer array.
639
640 const auto n_input_pointers = this->m_strat->get_input_rows() * this->m_strat->get_input_cols();
641 const auto input_point_stride = input.ld_col * this->m_strat->get_output_cols() * args.stride_cols;
642 const auto n_output_pointers = this->m_strat->get_output_rows() * this->m_strat->get_output_cols();
643 const auto output_point_stride = output.ld_col * this->m_strat->get_output_cols();
644
645 // For each tile row, initialise the input and output pointer arrays. For
646 // each subsequent tile we simply update the pointers.
647 for (unsigned int tile_i = 0; tile_i < n_tile_rows; tile_i++)
648 {
649 const int input_i = static_cast<int>(output_i * args.stride_rows) - args.padding.top;
650 int input_j = static_cast<int>(output_j * args.stride_cols) - args.padding.left;
651
652 Tile<TInput> multiplied_input;
653 this->initialise_inptr_array(args, output_channel_start, output_channel_end, input,
654 ws->inptr_array, ws->input_buffer, ws->intermediate_buffer,
655 input_i, input_j, 0, 0, multiplied_input);
656
657 // Compute the output pointer array
658 fill_pointer_array(
659 ws->outptr_array, this->m_strat->get_output_rows(), this->m_strat->get_output_cols(),
660 output.base + output_i*output.ld_row + output_j*output.ld_col + output_channel_start,
661 output.ld_row, output.ld_col,
662 ws->output_buffer,
663 0, args.output_rows,
664 0, args.output_cols
665 );
666
667 for (unsigned int tile_j = 0; tile_j < n_tile_cols; tile_j++)
668 {
669 // Invoke the indirect kernel for this tile
670 depthwise_depthfirst::Invoke<TInput, TWeight, TOutput, TAccum, OutputStage>::indirect(
671 strat, ws, os, parameters, m_bias, output_channel_end - output_channel_start
672 );
673
674 // Progress the pointers
675 if (this->uses_intermediate_array()) {
676 input_j += input_point_stride / input.ld_col;
677 multiplied_input.load_from(input.base,
678 input.ld_row, input.ld_col,
679 args.input_rows, args.input_cols, input_i, input_j, args.channel_multiplier);
680 } else {
681 for (auto i = 0u; i < n_input_pointers; i++)
682 {
683 ws->inptr_array[i] += input_point_stride;
684 }
685 }
686
687 for (auto i = 0u; i < n_output_pointers; i++)
688 {
689 ws->outptr_array[i] += output_point_stride;
690 }
691 }
692
693 output_i += this->m_strat->get_output_rows();
694 }
695 }
696 }
697 };
698
699 } // namespace depthwise
700 } // namespace arm_conv