Debian Codegraph @

src/x32-packx/gen/x32-packx-8x-neon-st4-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packx/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2019 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/packx.h"
16 #include "xnnpack/prefetch.h"
17
18
19 void xnn_x32_packx_ukernel_8x__neon_st4_x8_prfm(
20 size_t m,
21 size_t k,
22 const uint32_t* x,
23 size_t x_stride,
24 uint32_t* restrict y)
25 {
26 assert(m != 0);
27 assert(m <= 8);
28 assert(k != 0);
29 assert(x != NULL);
30 assert(y != NULL);
31
32 const uint32_t* x0 = x;
33 const uint32_t* x1 = (const uint32_t*) ((uintptr_t) x0 + x_stride);
34 if XNN_UNPREDICTABLE(m < 2) {
35 x1 = x0;
36 }
37 const uint32_t* x2 = (const uint32_t*) ((uintptr_t) x1 + x_stride);
38 if XNN_UNPREDICTABLE(m <= 2) {
39 x2 = x1;
40 }
41 const uint32_t* x3 = (const uint32_t*) ((uintptr_t) x2 + x_stride);
42 if XNN_UNPREDICTABLE(m < 4) {
43 x3 = x2;
44 }
45 const uint32_t* x4 = (const uint32_t*) ((uintptr_t) x3 + x_stride);
46 if XNN_UNPREDICTABLE(m <= 4) {
47 x4 = x3;
48 }
49 const uint32_t* x5 = (const uint32_t*) ((uintptr_t) x4 + x_stride);
50 if XNN_UNPREDICTABLE(m < 6) {
51 x5 = x4;
52 }
53 const uint32_t* x6 = (const uint32_t*) ((uintptr_t) x5 + x_stride);
54 if XNN_UNPREDICTABLE(m <= 6) {
55 x6 = x5;
56 }
57 const uint32_t* x7 = (const uint32_t*) ((uintptr_t) x6 + x_stride);
58 if XNN_UNPREDICTABLE(m != 8) {
59 x7 = x6;
60 }
61
62 for (; k >= 8; k -= 8) {
63 const uint32x4_t vx0123x0 = vld1q_u32(x0); x0 += 4;
64 const uint32x4_t vx0123x1 = vld1q_u32(x1); x1 += 4;
65 const uint32x4_t vx0123x2 = vld1q_u32(x2); x2 += 4;
66 const uint32x4_t vx0123x3 = vld1q_u32(x3); x3 += 4;
67 const uint32x4_t vx0123x4 = vld1q_u32(x4); x4 += 4;
68 const uint32x4_t vx0123x5 = vld1q_u32(x5); x5 += 4;
69 const uint32x4_t vx0123x6 = vld1q_u32(x6); x6 += 4;
70 const uint32x4_t vx0123x7 = vld1q_u32(x7); x7 += 4;
71 const uint32x4_t vx4567x0 = vld1q_u32(x0); x0 += 4;
72 const uint32x4_t vx4567x1 = vld1q_u32(x1); x1 += 4;
73 const uint32x4_t vx4567x2 = vld1q_u32(x2); x2 += 4;
74 const uint32x4_t vx4567x3 = vld1q_u32(x3); x3 += 4;
75 const uint32x4_t vx4567x4 = vld1q_u32(x4); x4 += 4;
76 const uint32x4_t vx4567x5 = vld1q_u32(x5); x5 += 4;
77 const uint32x4_t vx4567x6 = vld1q_u32(x6); x6 += 4;
78 const uint32x4_t vx4567x7 = vld1q_u32(x7); x7 += 4;
79 xnn_prefetch_to_l1((const int8_t*) x0 + 128);
80 xnn_prefetch_to_l1((const int8_t*) x1 + 128);
81 xnn_prefetch_to_l1((const int8_t*) x2 + 128);
82 xnn_prefetch_to_l1((const int8_t*) x3 + 128);
83 xnn_prefetch_to_l1((const int8_t*) x4 + 128);
84 xnn_prefetch_to_l1((const int8_t*) x5 + 128);
85 xnn_prefetch_to_l1((const int8_t*) x6 + 128);
86 xnn_prefetch_to_l1((const int8_t*) x7 + 128);
87 const uint32x4x2_t vz0123x0 = vzipq_u32(vx0123x0, vx0123x4);
88 const uint32x4x2_t vz0123x1 = vzipq_u32(vx0123x1, vx0123x5);
89 const uint32x4x2_t vz0123x2 = vzipq_u32(vx0123x2, vx0123x6);
90 const uint32x4x2_t vz0123x3 = vzipq_u32(vx0123x3, vx0123x7);
91 const uint32x4x2_t vz4567x0 = vzipq_u32(vx4567x0, vx4567x4);
92 const uint32x4x2_t vz4567x1 = vzipq_u32(vx4567x1, vx4567x5);
93 const uint32x4x2_t vz4567x2 = vzipq_u32(vx4567x2, vx4567x6);
94 const uint32x4x2_t vz4567x3 = vzipq_u32(vx4567x3, vx4567x7);
95
96 const uint32x4x4_t vy0123x0 = { vz0123x0.val[0], vz0123x1.val[0], vz0123x2.val[0], vz0123x3.val[0] };
97 const uint32x4x4_t vy0123x1 = { vz0123x0.val[1], vz0123x1.val[1], vz0123x2.val[1], vz0123x3.val[1] };
98 const uint32x4x4_t vy4567x0 = { vz4567x0.val[0], vz4567x1.val[0], vz4567x2.val[0], vz4567x3.val[0] };
99 const uint32x4x4_t vy4567x1 = { vz4567x0.val[1], vz4567x1.val[1], vz4567x2.val[1], vz4567x3.val[1] };
100 vst4q_u32(y, vy0123x0); y += 16;
101 vst4q_u32(y, vy0123x1); y += 16;
102 vst4q_u32(y, vy4567x0); y += 16;
103 vst4q_u32(y, vy4567x1); y += 16;
104 }
105
106 if XNN_UNLIKELY(k != 0) {
107 uint32x4_t vt0123 = vdupq_n_u32(0);
108 uint32x4_t vt4567 = vdupq_n_u32(0);
109 do {
110 vt0123 = vld1q_lane_u32(x0, vt0123, 0); x0 += 1;
111 vt0123 = vld1q_lane_u32(x1, vt0123, 1); x1 += 1;
112 vt0123 = vld1q_lane_u32(x2, vt0123, 2); x2 += 1;
113 vt0123 = vld1q_lane_u32(x3, vt0123, 3); x3 += 1;
114 vt4567 = vld1q_lane_u32(x4, vt4567, 0); x4 += 1;
115 vt4567 = vld1q_lane_u32(x5, vt4567, 1); x5 += 1;
116 vt4567 = vld1q_lane_u32(x6, vt4567, 2); x6 += 1;
117 vt4567 = vld1q_lane_u32(x7, vt4567, 3); x7 += 1;
118 xnn_prefetch_to_l1((const int8_t*) x0 + 128);
119 xnn_prefetch_to_l1((const int8_t*) x1 + 128);
120 xnn_prefetch_to_l1((const int8_t*) x2 + 128);
121 xnn_prefetch_to_l1((const int8_t*) x3 + 128);
122 xnn_prefetch_to_l1((const int8_t*) x4 + 128);
123 xnn_prefetch_to_l1((const int8_t*) x5 + 128);
124 xnn_prefetch_to_l1((const int8_t*) x6 + 128);
125 xnn_prefetch_to_l1((const int8_t*) x7 + 128);
126 vst1q_u32(y, vt0123); y += 4;
127 vst1q_u32(y, vt4567); y += 4;
128 } while (--k != 0);
129 }
130 }