Debian Codegraph @

src/x32-packx/gen/x32-packx-4x-neon-st4-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packx/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/packx.h"
16 #include "xnnpack/prefetch.h"
17
18
19 void xnn_x32_packx_ukernel_4x__neon_st4_x8_prfm(
20 size_t m,
21 size_t k,
22 const uint32_t* x,
23 size_t x_stride,
24 uint32_t* restrict y)
25 {
26 assert(m != 0);
27 assert(m <= 4);
28 assert(k != 0);
29 assert(x != NULL);
30 assert(y != NULL);
31
32 const uint32_t* x0 = x;
33 const uint32_t* x1 = (const uint32_t*) ((uintptr_t) x0 + x_stride);
34 if XNN_UNPREDICTABLE(m < 2) {
35 x1 = x0;
36 }
37 const uint32_t* x2 = (const uint32_t*) ((uintptr_t) x1 + x_stride);
38 if XNN_UNPREDICTABLE(m <= 2) {
39 x2 = x1;
40 }
41 const uint32_t* x3 = (const uint32_t*) ((uintptr_t) x2 + x_stride);
42 if XNN_UNPREDICTABLE(m != 4) {
43 x3 = x2;
44 }
45
46 for (; k >= 8; k -= 8) {
47 uint32x4x4_t vx0123x0123;
48 uint32x4x4_t vx4567x0123;
49 vx0123x0123.val[0] = vld1q_u32(x0); x0 += 4;
50 vx0123x0123.val[1] = vld1q_u32(x1); x1 += 4;
51 vx0123x0123.val[2] = vld1q_u32(x2); x2 += 4;
52 vx0123x0123.val[3] = vld1q_u32(x3); x3 += 4;
53 vx4567x0123.val[0] = vld1q_u32(x0); x0 += 4;
54 vx4567x0123.val[1] = vld1q_u32(x1); x1 += 4;
55 vx4567x0123.val[2] = vld1q_u32(x2); x2 += 4;
56 vx4567x0123.val[3] = vld1q_u32(x3); x3 += 4;
57 xnn_prefetch_to_l1((const int8_t*) x0 + 128);
58 xnn_prefetch_to_l1((const int8_t*) x1 + 128);
59 xnn_prefetch_to_l1((const int8_t*) x2 + 128);
60 xnn_prefetch_to_l1((const int8_t*) x3 + 128);
61 vst4q_u32(y, vx0123x0123); y += 16;
62 vst4q_u32(y, vx4567x0123); y += 16;
63 }
64
65 if XNN_UNLIKELY(k != 0) {
66 uint32x4_t vt0123 = vdupq_n_u32(0);
67 do {
68 vt0123 = vld1q_lane_u32(x0, vt0123, 0); x0 += 1;
69 vt0123 = vld1q_lane_u32(x1, vt0123, 1); x1 += 1;
70 vt0123 = vld1q_lane_u32(x2, vt0123, 2); x2 += 1;
71 vt0123 = vld1q_lane_u32(x3, vt0123, 3); x3 += 1;
72 xnn_prefetch_to_l1((const int8_t*) x0 + 128);
73 xnn_prefetch_to_l1((const int8_t*) x1 + 128);
74 xnn_prefetch_to_l1((const int8_t*) x2 + 128);
75 xnn_prefetch_to_l1((const int8_t*) x3 + 128);
76 vst1q_u32(y, vt0123); y += 4;
77 } while (--k != 0);
78 }
79 }