Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-neon-ld4lane-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/neon.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <arm_neon.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 8);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42 uint32x4x4_t vtmp0123x0123;
43 vtmp0123x0123.val[0] = vdupq_n_u32(0);
44 vtmp0123x0123.val[1] = vdupq_n_u32(0);
45 vtmp0123x0123.val[2] = vdupq_n_u32(0);
46 vtmp0123x0123.val[3] = vdupq_n_u32(0);
47 uint32x4x4_t vtmp0123x4567;
48 vtmp0123x4567.val[0] = vdupq_n_u32(0);
49 vtmp0123x4567.val[1] = vdupq_n_u32(0);
50 vtmp0123x4567.val[2] = vdupq_n_u32(0);
51 vtmp0123x4567.val[3] = vdupq_n_u32(0);
52
53 do {
54 // NC main loop multiple of 8
55 const uint32_t* w0 = weights;
56 size_t n = nc;
57
58 for (; n >= 8; n -= 8) {
59 if XNN_LIKELY(bias != NULL) {
60 uint32x4_t vb0 = vld1q_u32(bias); bias += 4;
61 uint32x4_t vb4 = vld1q_u32(bias); bias += 4;
62 vst1q_u32(packed_weights, vb0); packed_weights += 4;
63 vst1q_u32(packed_weights, vb4); packed_weights += 4;
64 } else {
65 const uint32x4_t vzero = vmovq_n_u32(0);
66 vst1q_u32(packed_weights, vzero); packed_weights += 4;
67 vst1q_u32(packed_weights, vzero); packed_weights += 4;
68 }
69
70 const uint32_t* w1 = w0 + kc;
71 const uint32_t* w2 = w1 + kc;
72 const uint32_t* w3 = w2 + kc;
73 const uint32_t* w4 = w3 + kc;
74 const uint32_t* w5 = w4 + kc;
75 const uint32_t* w6 = w5 + kc;
76 const uint32_t* w7 = w6 + kc;
77
78 // KC main loop multiple of 4
79 size_t k = kc;
80 for (; k >= 4; k -= 4) {
81 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
82 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
83 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
84 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
85 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
86 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
87 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
88 vtmp0123x4567 = vld4q_lane_u32(w7, vtmp0123x4567, 3); w7 += 4;
89 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
90 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
91 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
92 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
93 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
94 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
95 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
96 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
97 }
98
99 // KC remainder of 1..3
100 // Same as main loop but ld1, ld2 or ld3
101 if XNN_UNLIKELY(k != 0) {
102 assert(k >= 1);
103 assert(k <= 3);
104 switch (k) {
105 // KC remainder of 1
106 case 1:
107 {
108 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
109 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
110 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0); w0 += 1;
111 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1); w1 += 1;
112 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2); w2 += 1;
113 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3); w3 += 1;
114 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0); w4 += 1;
115 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1); w5 += 1;
116 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2); w6 += 1;
117 vtmp0x4567 = vld1q_lane_u32(w7, vtmp0x4567, 3); w7 += 1;
118 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
119 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
120 break;
121 }
122 // KC remainder of 2
123 case 2:
124 {
125 uint32x4x2_t vtmp01x0123;
126 vtmp01x0123.val[0] = vdupq_n_u32(0);
127 vtmp01x0123.val[1] = vdupq_n_u32(0);
128 uint32x4x2_t vtmp01x4567;
129 vtmp01x4567.val[0] = vdupq_n_u32(0);
130 vtmp01x4567.val[1] = vdupq_n_u32(0);
131 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0); w0 += 2;
132 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1); w1 += 2;
133 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2); w2 += 2;
134 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3); w3 += 2;
135 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0); w4 += 2;
136 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1); w5 += 2;
137 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2); w6 += 2;
138 vtmp01x4567 = vld2q_lane_u32(w7, vtmp01x4567, 3); w7 += 2;
139 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
140 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
141 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
142 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
143 break;
144 }
145 // KC remainder of 3
146 case 3:
147 {
148 uint32x4x3_t vtmp012x0123;
149 vtmp012x0123.val[0] = vdupq_n_u32(0);
150 vtmp012x0123.val[1] = vdupq_n_u32(0);
151 vtmp012x0123.val[2] = vdupq_n_u32(0);
152 uint32x4x3_t vtmp012x4567;
153 vtmp012x4567.val[0] = vdupq_n_u32(0);
154 vtmp012x4567.val[1] = vdupq_n_u32(0);
155 vtmp012x4567.val[2] = vdupq_n_u32(0);
156 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0); w0 += 3;
157 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1); w1 += 3;
158 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2); w2 += 3;
159 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3); w3 += 3;
160 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0); w4 += 3;
161 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1); w5 += 3;
162 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2); w6 += 3;
163 vtmp012x4567 = vld3q_lane_u32(w7, vtmp012x4567, 3); w7 += 3;
164 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
165 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
166 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
167 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
168 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
169 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
170 break;
171 }
172 default:
173 XNN_UNREACHABLE;
174 }
175 }
176 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
177 w0 = w7;
178 }
179
180 // NC remainder (1..7)
181 if XNN_UNLIKELY(n != 0) {
182 assert(n >= 1);
183 assert(n <= 7);
184 if XNN_LIKELY(bias != NULL) {
185 size_t nb = n;
186 do {
187 *packed_weights++ = *bias++;
188 } while (--nb != 0);
189 packed_weights += (8 - n);
190 } else {
191 const uint32x4_t vzero = vmovq_n_u32(0);
192 vst1q_u32(packed_weights, vzero); packed_weights += 4;
193 vst1q_u32(packed_weights, vzero); packed_weights += 4;
194 }
195
196 // NR remainder has less than 8 rows so last row is not loaded
197 const uint32_t* w1 = w0 + kc;
198 if XNN_UNPREDICTABLE(n < 2) {
199 w1 = w0;
200 }
201 const uint32_t* w2 = w1 + kc;
202 if XNN_UNPREDICTABLE(n <= 2) {
203 w2 = w1;
204 }
205 const uint32_t* w3 = w2 + kc;
206 if XNN_UNPREDICTABLE(n < 4) {
207 w3 = w2;
208 }
209 const uint32_t* w4 = w3 + kc;
210 if XNN_UNPREDICTABLE(n <= 4) {
211 w4 = w3;
212 }
213 const uint32_t* w5 = w4 + kc;
214 if XNN_UNPREDICTABLE(n < 6) {
215 w5 = w4;
216 }
217 const uint32_t* w6 = w5 + kc;
218 if XNN_UNPREDICTABLE(n <= 6) {
219 w6 = w5;
220 }
221
222 // KC main loop multiple of 4
223 size_t k = kc;
224 for (; k >= 4; k -= 4) {
225 vtmp0123x0123 = vld4q_lane_u32(w0, vtmp0123x0123, 0); w0 += 4;
226 vtmp0123x0123 = vld4q_lane_u32(w1, vtmp0123x0123, 1); w1 += 4;
227 vtmp0123x0123 = vld4q_lane_u32(w2, vtmp0123x0123, 2); w2 += 4;
228 vtmp0123x0123 = vld4q_lane_u32(w3, vtmp0123x0123, 3); w3 += 4;
229 vtmp0123x4567 = vld4q_lane_u32(w4, vtmp0123x4567, 0); w4 += 4;
230 vtmp0123x4567 = vld4q_lane_u32(w5, vtmp0123x4567, 1); w5 += 4;
231 vtmp0123x4567 = vld4q_lane_u32(w6, vtmp0123x4567, 2); w6 += 4;
232 vst1q_u32(packed_weights, vtmp0123x0123.val[0]); packed_weights += 4;
233 vst1q_u32(packed_weights, vtmp0123x4567.val[0]); packed_weights += 4;
234 vst1q_u32(packed_weights, vtmp0123x0123.val[1]); packed_weights += 4;
235 vst1q_u32(packed_weights, vtmp0123x4567.val[1]); packed_weights += 4;
236 vst1q_u32(packed_weights, vtmp0123x0123.val[2]); packed_weights += 4;
237 vst1q_u32(packed_weights, vtmp0123x4567.val[2]); packed_weights += 4;
238 vst1q_u32(packed_weights, vtmp0123x0123.val[3]); packed_weights += 4;
239 vst1q_u32(packed_weights, vtmp0123x4567.val[3]); packed_weights += 4;
240 }
241
242 // KC remainder of 1..3
243 // Same as main loop but ld1, ld2 or ld3
244 if XNN_UNLIKELY(k != 0) {
245 assert(k >= 1);
246 assert(k <= 3);
247 switch (k) {
248 // KC remainder of 1
249 case 1:
250 {
251 uint32x4_t vtmp0x0123 = vdupq_n_u32(0);
252 uint32x4_t vtmp0x4567 = vdupq_n_u32(0);
253 vtmp0x0123 = vld1q_lane_u32(w0, vtmp0x0123, 0);
254 vtmp0x0123 = vld1q_lane_u32(w1, vtmp0x0123, 1);
255 vtmp0x0123 = vld1q_lane_u32(w2, vtmp0x0123, 2);
256 vtmp0x0123 = vld1q_lane_u32(w3, vtmp0x0123, 3);
257 vtmp0x4567 = vld1q_lane_u32(w4, vtmp0x4567, 0);
258 vtmp0x4567 = vld1q_lane_u32(w5, vtmp0x4567, 1);
259 vtmp0x4567 = vld1q_lane_u32(w6, vtmp0x4567, 2);
260 vst1q_u32(packed_weights, vtmp0x0123); packed_weights += 4;
261 vst1q_u32(packed_weights, vtmp0x4567); packed_weights += 4;
262 break;
263 }
264 // KC remainder of 2
265 case 2:
266 {
267 uint32x4x2_t vtmp01x0123;
268 vtmp01x0123.val[0] = vdupq_n_u32(0);
269 vtmp01x0123.val[1] = vdupq_n_u32(0);
270 uint32x4x2_t vtmp01x4567;
271 vtmp01x4567.val[0] = vdupq_n_u32(0);
272 vtmp01x4567.val[1] = vdupq_n_u32(0);
273 vtmp01x0123 = vld2q_lane_u32(w0, vtmp01x0123, 0);
274 vtmp01x0123 = vld2q_lane_u32(w1, vtmp01x0123, 1);
275 vtmp01x0123 = vld2q_lane_u32(w2, vtmp01x0123, 2);
276 vtmp01x0123 = vld2q_lane_u32(w3, vtmp01x0123, 3);
277 vtmp01x4567 = vld2q_lane_u32(w4, vtmp01x4567, 0);
278 vtmp01x4567 = vld2q_lane_u32(w5, vtmp01x4567, 1);
279 vtmp01x4567 = vld2q_lane_u32(w6, vtmp01x4567, 2);
280 vst1q_u32(packed_weights, vtmp01x0123.val[0]); packed_weights += 4;
281 vst1q_u32(packed_weights, vtmp01x4567.val[0]); packed_weights += 4;
282 vst1q_u32(packed_weights, vtmp01x0123.val[1]); packed_weights += 4;
283 vst1q_u32(packed_weights, vtmp01x4567.val[1]); packed_weights += 4;
284 break;
285 }
286 // KC remainder of 3
287 case 3:
288 {
289 uint32x4x3_t vtmp012x0123;
290 vtmp012x0123.val[0] = vdupq_n_u32(0);
291 vtmp012x0123.val[1] = vdupq_n_u32(0);
292 vtmp012x0123.val[2] = vdupq_n_u32(0);
293 uint32x4x3_t vtmp012x4567;
294 vtmp012x4567.val[0] = vdupq_n_u32(0);
295 vtmp012x4567.val[1] = vdupq_n_u32(0);
296 vtmp012x4567.val[2] = vdupq_n_u32(0);
297 vtmp012x0123 = vld3q_lane_u32(w0, vtmp012x0123, 0);
298 vtmp012x0123 = vld3q_lane_u32(w1, vtmp012x0123, 1);
299 vtmp012x0123 = vld3q_lane_u32(w2, vtmp012x0123, 2);
300 vtmp012x0123 = vld3q_lane_u32(w3, vtmp012x0123, 3);
301 vtmp012x4567 = vld3q_lane_u32(w4, vtmp012x4567, 0);
302 vtmp012x4567 = vld3q_lane_u32(w5, vtmp012x4567, 1);
303 vtmp012x4567 = vld3q_lane_u32(w6, vtmp012x4567, 2);
304 vst1q_u32(packed_weights, vtmp012x0123.val[0]); packed_weights += 4;
305 vst1q_u32(packed_weights, vtmp012x4567.val[0]); packed_weights += 4;
306 vst1q_u32(packed_weights, vtmp012x0123.val[1]); packed_weights += 4;
307 vst1q_u32(packed_weights, vtmp012x4567.val[1]); packed_weights += 4;
308 vst1q_u32(packed_weights, vtmp012x0123.val[2]); packed_weights += 4;
309 vst1q_u32(packed_weights, vtmp012x4567.val[2]); packed_weights += 4;
310 break;
311 }
312 default:
313 XNN_UNREACHABLE;
314 }
315 }
316 packed_weights = (uint32_t*) ((uintptr_t) packed_weights + extra_bytes);
317 }
318
319 weights += nc * kc;
320 } while (--g != 0);
321 }