Debian Codegraph @

src/x32-packw/gen/x32-packw-x16s4-gemm-goi-avx-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x16s4__avx_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 16);
38 assert(kr == 1);
39 assert(sr == 4);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 const float* b = (const float*) bias;
44 float* packed_w = (float*) packed_weights;
45 do {
46 // NC main loop multiple of 16
47 const float* w0 = (const float*) weights;
48 size_t n = nc;
49
50 for (; n >= 16; n -= 16) {
51 if XNN_LIKELY(b != NULL) {
52 const __m256 vb0 = _mm256_loadu_ps(b);
53 const __m256 vb8 = _mm256_loadu_ps(b + 8);
54 _mm256_store_ps(packed_w, vb0);
55 _mm256_store_ps(packed_w + 8, vb8);
56 b += 16;
57 } else {
58 const __m256 vzero = _mm256_setzero_ps();
59 _mm256_store_ps(packed_w, vzero);
60 _mm256_store_ps(packed_w + 8, vzero);
61 }
62 packed_w += 16;
63
64 const float* w1 = w0 + kc;
65 const float* w2 = w1 + kc;
66 const float* w3 = w2 + kc;
67 const float* w4 = w3 + kc;
68 const float* w5 = w4 + kc;
69 const float* w6 = w5 + kc;
70 const float* w7 = w6 + kc;
71 const float* w8 = w7 + kc;
72 const float* w9 = w8 + kc;
73 const float* w10 = w9 + kc;
74 const float* w11 = w10 + kc;
75 const float* w12 = w11 + kc;
76 const float* w13 = w12 + kc;
77 const float* w14 = w13 + kc;
78 const float* w15 = w14 + kc;
79
80 // KC main loop multiple of 4
81 size_t k = kc;
82 for (; k >= 4; k -= 4) {
83 // Read blocks of 4x4
84 // a b c d
85 // e f g h
86 // i j k l
87 // m n o p
88 // Load first 4 rows of N into low part of each register
89 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
90 w0 += 4;
91 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
92 w1 += 4;
93 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
94 w2 += 4;
95 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
96 w3 += 4;
97 __m256 v8x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w8));
98 w8 += 4;
99 __m256 v9x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w9));
100 w9 += 4;
101 __m256 v10x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w10));
102 w10 += 4;
103 __m256 v11x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w11));
104 w11 += 4;
105 // Load next 4 rows of N into the high part of each register
106 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
107 w4 += 4;
108 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
109 w5 += 4;
110 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
111 w6 += 4;
112 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
113 w7 += 4;
114 v8x0123 = _mm256_insertf128_ps(v8x0123, _mm_loadu_ps(w12), 1);
115 w12 += 4;
116 v9x0123 = _mm256_insertf128_ps(v9x0123, _mm_loadu_ps(w13), 1);
117 w13 += 4;
118 v10x0123 = _mm256_insertf128_ps(v10x0123, _mm_loadu_ps(w14), 1);
119 w14 += 4;
120 v11x0123 = _mm256_insertf128_ps(v11x0123, _mm_loadu_ps(w15), 1);
121 w15 += 4;
122
123 // Apply SR4 shuffle
124 v1x0123 = _mm256_permute_ps(v1x0123, _MM_SHUFFLE(0, 3, 2, 1));
125 v2x0123 = _mm256_permute_ps(v2x0123, _MM_SHUFFLE(1, 0, 3, 2));
126 v3x0123 = _mm256_permute_ps(v3x0123, _MM_SHUFFLE(2, 1, 0, 3));
127 v9x0123 = _mm256_permute_ps(v9x0123, _MM_SHUFFLE(0, 3, 2, 1));
128 v10x0123 = _mm256_permute_ps(v10x0123, _MM_SHUFFLE(1, 0, 3, 2));
129 v11x0123 = _mm256_permute_ps(v11x0123, _MM_SHUFFLE(2, 1, 0, 3));
130
131 // Transpose 2x2
132 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
133 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
134 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
135 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
136 const __m256 vtmp8x0123 = _mm256_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
137 const __m256 vtmp9x0123 = _mm256_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
138 const __m256 vtmp10x0123 = _mm256_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
139 const __m256 vtmp11x0123 = _mm256_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
140 // Transpose 4x4
141 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
142 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
143 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
144 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
145 v8x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // a e i m from row 0, 1
146 v9x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // b f j n from row 0, 1
147 v10x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // c g k o from row 2, 3
148 v11x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // d h l p from row 2, 3
149
150 _mm256_store_ps(packed_w, v0x0123);
151 _mm256_store_ps(packed_w + 8, v8x0123);
152 _mm256_store_ps(packed_w + 16, v1x0123);
153 _mm256_store_ps(packed_w + 24, v9x0123);
154 _mm256_store_ps(packed_w + 32, v2x0123);
155 _mm256_store_ps(packed_w + 40, v10x0123);
156 _mm256_store_ps(packed_w + 48, v3x0123);
157 _mm256_store_ps(packed_w + 56, v11x0123);
158 packed_w += 64;
159 }
160
161 // KC remainder (1..3)
162 if XNN_UNLIKELY(k != 0) {
163 assert(k >= 1);
164 assert(k <= 3);
165 __m128 v0 = _mm_undefined_ps();
166 __m128 v1 = _mm_undefined_ps();
167 __m128 v2 = _mm_undefined_ps();
168 __m128 v3 = _mm_undefined_ps();
169 __m128 v4 = _mm_undefined_ps();
170 __m128 v5 = _mm_undefined_ps();
171 __m128 v6 = _mm_undefined_ps();
172 __m128 v7 = _mm_undefined_ps();
173 __m128 v8 = _mm_undefined_ps();
174 __m128 v9 = _mm_undefined_ps();
175 __m128 v10 = _mm_undefined_ps();
176 __m128 v11 = _mm_undefined_ps();
177 __m128 v12 = _mm_undefined_ps();
178 __m128 v13 = _mm_undefined_ps();
179 __m128 v14 = _mm_undefined_ps();
180 __m128 v15 = _mm_undefined_ps();
181
182 switch (k) {
183 case 1:
184 // Read blocks of 4x1
185 // a
186 // e
187 // i
188 // m
189 v0 = _mm_load_ss(w0);
190 w0 += 1;
191 v1 = _mm_load_ss(w1);
192 w1 += 1;
193 v2 = _mm_load_ss(w2);
194 w2 += 1;
195 v3 = _mm_load_ss(w3);
196 w3 += 1;
197 v4 = _mm_load_ss(w4);
198 w4 += 1;
199 v5 = _mm_load_ss(w5);
200 w5 += 1;
201 v6 = _mm_load_ss(w6);
202 w6 += 1;
203 v7 = _mm_load_ss(w7);
204 w7 += 1;
205 v8 = _mm_load_ss(w8);
206 w8 += 1;
207 v9 = _mm_load_ss(w9);
208 w9 += 1;
209 v10 = _mm_load_ss(w10);
210 w10 += 1;
211 v11 = _mm_load_ss(w11);
212 w11 += 1;
213 v12 = _mm_load_ss(w12);
214 w12 += 1;
215 v13 = _mm_load_ss(w13);
216 w13 += 1;
217 v14 = _mm_load_ss(w14);
218 w14 += 1;
219 v15 = _mm_load_ss(w15);
220 w15 += 1;
221 break;
222 case 2:
223 // Read blocks of 4x2
224 // a b
225 // e f
226 // i j
227 // m n
228 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
229 w0 += 2;
230 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
231 w1 += 2;
232 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
233 w2 += 2;
234 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
235 w3 += 2;
236 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
237 w4 += 2;
238 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
239 w5 += 2;
240 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
241 w6 += 2;
242 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
243 w7 += 2;
244 v8 = _mm_castpd_ps(_mm_load_sd((const double*) w8));
245 w8 += 2;
246 v9 = _mm_castpd_ps(_mm_load_sd((const double*) w9));
247 w9 += 2;
248 v10 = _mm_castpd_ps(_mm_load_sd((const double*) w10));
249 w10 += 2;
250 v11 = _mm_castpd_ps(_mm_load_sd((const double*) w11));
251 w11 += 2;
252 v12 = _mm_castpd_ps(_mm_load_sd((const double*) w12));
253 w12 += 2;
254 v13 = _mm_castpd_ps(_mm_load_sd((const double*) w13));
255 w13 += 2;
256 v14 = _mm_castpd_ps(_mm_load_sd((const double*) w14));
257 w14 += 2;
258 v15 = _mm_castpd_ps(_mm_load_sd((const double*) w15));
259 w15 += 2;
260 break;
261 case 3:
262 {
263 // Read blocks of 4x3
264 // a b c
265 // e f g
266 // i j k
267 // m n o
268 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
269 const __m128 v0hi = _mm_load_ss(w0 + 2);
270 v0 = _mm_movelh_ps(v0lo, v0hi);
271 w0 += 3;
272 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
273 const __m128 v1hi = _mm_load_ss(w1 + 2);
274 v1 = _mm_movelh_ps(v1lo, v1hi);
275 w1 += 3;
276 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
277 const __m128 v2hi = _mm_load_ss(w2 + 2);
278 v2 = _mm_movelh_ps(v2lo, v2hi);
279 w2 += 3;
280 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
281 const __m128 v3hi = _mm_load_ss(w3 + 2);
282 v3 = _mm_movelh_ps(v3lo, v3hi);
283 w3 += 3;
284 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
285 const __m128 v4hi = _mm_load_ss(w4 + 2);
286 v4 = _mm_movelh_ps(v4lo, v4hi);
287 w4 += 3;
288 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
289 const __m128 v5hi = _mm_load_ss(w5 + 2);
290 v5 = _mm_movelh_ps(v5lo, v5hi);
291 w5 += 3;
292 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
293 const __m128 v6hi = _mm_load_ss(w6 + 2);
294 v6 = _mm_movelh_ps(v6lo, v6hi);
295 w6 += 3;
296 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
297 const __m128 v7hi = _mm_load_ss(w7 + 2);
298 v7 = _mm_movelh_ps(v7lo, v7hi);
299 w7 += 3;
300 const __m128 v8lo = _mm_castpd_ps(_mm_load_sd((const double*) w8));
301 const __m128 v8hi = _mm_load_ss(w8 + 2);
302 v8 = _mm_movelh_ps(v8lo, v8hi);
303 w8 += 3;
304 const __m128 v9lo = _mm_castpd_ps(_mm_load_sd((const double*) w9));
305 const __m128 v9hi = _mm_load_ss(w9 + 2);
306 v9 = _mm_movelh_ps(v9lo, v9hi);
307 w9 += 3;
308 const __m128 v10lo = _mm_castpd_ps(_mm_load_sd((const double*) w10));
309 const __m128 v10hi = _mm_load_ss(w10 + 2);
310 v10 = _mm_movelh_ps(v10lo, v10hi);
311 w10 += 3;
312 const __m128 v11lo = _mm_castpd_ps(_mm_load_sd((const double*) w11));
313 const __m128 v11hi = _mm_load_ss(w11 + 2);
314 v11 = _mm_movelh_ps(v11lo, v11hi);
315 w11 += 3;
316 const __m128 v12lo = _mm_castpd_ps(_mm_load_sd((const double*) w12));
317 const __m128 v12hi = _mm_load_ss(w12 + 2);
318 v12 = _mm_movelh_ps(v12lo, v12hi);
319 w12 += 3;
320 const __m128 v13lo = _mm_castpd_ps(_mm_load_sd((const double*) w13));
321 const __m128 v13hi = _mm_load_ss(w13 + 2);
322 v13 = _mm_movelh_ps(v13lo, v13hi);
323 w13 += 3;
324 const __m128 v14lo = _mm_castpd_ps(_mm_load_sd((const double*) w14));
325 const __m128 v14hi = _mm_load_ss(w14 + 2);
326 v14 = _mm_movelh_ps(v14lo, v14hi);
327 w14 += 3;
328 const __m128 v15lo = _mm_castpd_ps(_mm_load_sd((const double*) w15));
329 const __m128 v15hi = _mm_load_ss(w15 + 2);
330 v15 = _mm_movelh_ps(v15lo, v15hi);
331 w15 += 3;
332 break;
333 }
334 default:
335 XNN_UNREACHABLE;
336 }
337
338 // Apply SR4 shuffle
339 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
340 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
341 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
342 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
343 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
344 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
345 v9 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9), _MM_SHUFFLE(0, 3, 2, 1)));
346 v10 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10), _MM_SHUFFLE(1, 0, 3, 2)));
347 v11 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11), _MM_SHUFFLE(2, 1, 0, 3)));
348 v13 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13), _MM_SHUFFLE(0, 3, 2, 1)));
349 v14 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14), _MM_SHUFFLE(1, 0, 3, 2)));
350 v15 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15), _MM_SHUFFLE(2, 1, 0, 3)));
351 // Transpose 2x2
352 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
353 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
354 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
355 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
356 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
357 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
358 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
359 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
360 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a e b f from row 0, 1
361 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // i m j n from row 2, 3
362 const __m128 vtmp10 = _mm_unpackhi_ps(v8, v9); // c g d h from row 0, 1
363 const __m128 vtmp11 = _mm_unpackhi_ps(v10, v11); // k o l p from row 2, 3
364 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a e b f from row 0, 1
365 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // i m j n from row 2, 3
366 const __m128 vtmp14 = _mm_unpackhi_ps(v12, v13); // c g d h from row 0, 1
367 const __m128 vtmp15 = _mm_unpackhi_ps(v14, v15); // k o l p from row 2, 3
368 // Transpose 4x4
369 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
370 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
371 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
372 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
373 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
374 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
375 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
376 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
377 v8 = _mm_movelh_ps(vtmp8, vtmp9); // a e i m from row 0, 1
378 v9 = _mm_movehl_ps(vtmp9, vtmp8); // b f j n from row 0, 1
379 v10 = _mm_movelh_ps(vtmp10, vtmp11); // c g k o from row 2, 3
380 v11 = _mm_movehl_ps(vtmp11, vtmp10); // d h l p from row 2, 3
381 v12 = _mm_movelh_ps(vtmp12, vtmp13); // a e i m from row 0, 1
382 v13 = _mm_movehl_ps(vtmp13, vtmp12); // b f j n from row 0, 1
383 v14 = _mm_movelh_ps(vtmp14, vtmp15); // c g k o from row 2, 3
384 v15 = _mm_movehl_ps(vtmp15, vtmp14); // d h l p from row 2, 3
385 _mm_store_ps(packed_w, v0);
386 _mm_store_ps(packed_w + 4, v4);
387 _mm_store_ps(packed_w + 8, v8);
388 _mm_store_ps(packed_w + 12, v12);
389 _mm_store_ps(packed_w + 16, v1);
390 _mm_store_ps(packed_w + 20, v5);
391 _mm_store_ps(packed_w + 24, v9);
392 _mm_store_ps(packed_w + 28, v13);
393 _mm_store_ps(packed_w + 32, v2);
394 _mm_store_ps(packed_w + 36, v6);
395 _mm_store_ps(packed_w + 40, v10);
396 _mm_store_ps(packed_w + 44, v14);
397 _mm_store_ps(packed_w + 48, v3);
398 _mm_store_ps(packed_w + 52, v7);
399 _mm_store_ps(packed_w + 56, v11);
400 _mm_store_ps(packed_w + 60, v15);
401 packed_w += 64;
402 }
403 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
404 w0 = w15;
405 }
406
407 // NC remainder (1..15)
408 if XNN_UNLIKELY(n != 0) {
409 assert(n >= 1);
410 assert(n <= 15);
411 if XNN_LIKELY(b != NULL) {
412 size_t nb = n;
413 do {
414 *packed_w++ = *b++;
415 } while (--nb != 0);
416 packed_w += (16 - n);
417 } else {
418 const __m128 vzero = _mm_setzero_ps();
419 _mm_store_ps(packed_w, vzero);
420 _mm_store_ps(packed_w + 4, vzero);
421 _mm_store_ps(packed_w + 8, vzero);
422 _mm_store_ps(packed_w + 12, vzero);
423 packed_w += 16;
424 }
425
426 // NR remainder has less than 16 rows so last row is not loaded
427 const float* w1 = w0 + kc;
428 if XNN_UNPREDICTABLE(n < 2) {
429 w1 = w0;
430 }
431 const float* w2 = w1 + kc;
432 if XNN_UNPREDICTABLE(n <= 2) {
433 w2 = w1;
434 }
435 const float* w3 = w2 + kc;
436 if XNN_UNPREDICTABLE(n < 4) {
437 w3 = w2;
438 }
439 const float* w4 = w3 + kc;
440 if XNN_UNPREDICTABLE(n <= 4) {
441 w4 = w3;
442 }
443 const float* w5 = w4 + kc;
444 if XNN_UNPREDICTABLE(n < 6) {
445 w5 = w4;
446 }
447 const float* w6 = w5 + kc;
448 if XNN_UNPREDICTABLE(n <= 6) {
449 w6 = w5;
450 }
451 const float* w7 = w6 + kc;
452 if XNN_UNPREDICTABLE(n < 8) {
453 w7 = w6;
454 }
455 const float* w8 = w7 + kc;
456 if XNN_UNPREDICTABLE(n <= 8) {
457 w8 = w7;
458 }
459 const float* w9 = w8 + kc;
460 if XNN_UNPREDICTABLE(n < 10) {
461 w9 = w8;
462 }
463 const float* w10 = w9 + kc;
464 if XNN_UNPREDICTABLE(n <= 10) {
465 w10 = w9;
466 }
467 const float* w11 = w10 + kc;
468 if XNN_UNPREDICTABLE(n < 12) {
469 w11 = w10;
470 }
471 const float* w12 = w11 + kc;
472 if XNN_UNPREDICTABLE(n <= 12) {
473 w12 = w11;
474 }
475 const float* w13 = w12 + kc;
476 if XNN_UNPREDICTABLE(n < 14) {
477 w13 = w12;
478 }
479 const float* w14 = w13 + kc;
480 if XNN_UNPREDICTABLE(n <= 14) {
481 w14 = w13;
482 }
483
484 // KC main loop multiple of 4
485 size_t k = kc;
486 for (; k >= 4; k -= 4) {
487 // Read blocks of 4x4
488 // a b c d
489 // e f g h
490 // i j k l
491 // m n o p
492 // Load first 4 rows of N into low part of each register
493 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
494 w0 += 4;
495 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
496 w1 += 4;
497 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
498 w2 += 4;
499 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
500 w3 += 4;
501 __m256 v8x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w8));
502 w8 += 4;
503 __m256 v9x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w9));
504 w9 += 4;
505 __m256 v10x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w10));
506 w10 += 4;
507 __m256 v11x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w11));
508 w11 += 4;
509 // Load next 4 rows of N into the high part of each register
510 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
511 w4 += 4;
512 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
513 w5 += 4;
514 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
515 w6 += 4;
516 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
517 w7 += 4;
518 v8x0123 = _mm256_insertf128_ps(v8x0123, _mm_loadu_ps(w12), 1);
519 w12 += 4;
520 v9x0123 = _mm256_insertf128_ps(v9x0123, _mm_loadu_ps(w13), 1);
521 w13 += 4;
522 v10x0123 = _mm256_insertf128_ps(v10x0123, _mm_loadu_ps(w14), 1);
523 w14 += 4;
524 // Apply SR4 shuffle
525 v1x0123 = _mm256_permute_ps(v1x0123, _MM_SHUFFLE(0, 3, 2, 1));
526 v2x0123 = _mm256_permute_ps(v2x0123, _MM_SHUFFLE(1, 0, 3, 2));
527 v3x0123 = _mm256_permute_ps(v3x0123, _MM_SHUFFLE(2, 1, 0, 3));
528 v9x0123 = _mm256_permute_ps(v9x0123, _MM_SHUFFLE(0, 3, 2, 1));
529 v10x0123 = _mm256_permute_ps(v10x0123, _MM_SHUFFLE(1, 0, 3, 2));
530 v11x0123 = _mm256_permute_ps(v11x0123, _MM_SHUFFLE(2, 1, 0, 3));
531 // Transpose 2x2
532 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
533 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
534 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
535 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
536 const __m256 vtmp8x0123 = _mm256_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
537 const __m256 vtmp9x0123 = _mm256_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
538 const __m256 vtmp10x0123 = _mm256_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
539 const __m256 vtmp11x0123 = _mm256_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
540 // Transpose 4x4
541 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
542 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
543 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
544 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
545 v8x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // a e i m from row 0, 1
546 v9x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // b f j n from row 0, 1
547 v10x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // c g k o from row 2, 3
548 v11x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // d h l p from row 2, 3
549
550 _mm256_store_ps(packed_w, v0x0123);
551 _mm256_store_ps(packed_w + 8, v8x0123);
552 _mm256_store_ps(packed_w + 16, v1x0123);
553 _mm256_store_ps(packed_w + 24, v9x0123);
554 _mm256_store_ps(packed_w + 32, v2x0123);
555 _mm256_store_ps(packed_w + 40, v10x0123);
556 _mm256_store_ps(packed_w + 48, v3x0123);
557 _mm256_store_ps(packed_w + 56, v11x0123);
558 packed_w += 64;
559 }
560
561 // KC remainder (1..3)
562 if XNN_UNLIKELY(k != 0) {
563 assert(k >= 1);
564 assert(k <= 3);
565 __m128 v0 = _mm_undefined_ps();
566 __m128 v1 = _mm_undefined_ps();
567 __m128 v2 = _mm_undefined_ps();
568 __m128 v3 = _mm_undefined_ps();
569 __m128 v4 = _mm_undefined_ps();
570 __m128 v5 = _mm_undefined_ps();
571 __m128 v6 = _mm_undefined_ps();
572 __m128 v7 = _mm_undefined_ps();
573 __m128 v8 = _mm_undefined_ps();
574 __m128 v9 = _mm_undefined_ps();
575 __m128 v10 = _mm_undefined_ps();
576 __m128 v11 = _mm_undefined_ps();
577 __m128 v12 = _mm_undefined_ps();
578 __m128 v13 = _mm_undefined_ps();
579 __m128 v14 = _mm_undefined_ps();
580 __m128 v15 = _mm_undefined_ps();
581
582 switch (k) {
583 case 1:
584 // Read blocks of 4x1
585 // a
586 // e
587 // i
588 // m
589 v0 = _mm_load_ss(w0);
590 w0 += 1;
591 v1 = _mm_load_ss(w1);
592 w1 += 1;
593 v2 = _mm_load_ss(w2);
594 w2 += 1;
595 v3 = _mm_load_ss(w3);
596 w3 += 1;
597 v4 = _mm_load_ss(w4);
598 w4 += 1;
599 v5 = _mm_load_ss(w5);
600 w5 += 1;
601 v6 = _mm_load_ss(w6);
602 w6 += 1;
603 v7 = _mm_load_ss(w7);
604 w7 += 1;
605 v8 = _mm_load_ss(w8);
606 w8 += 1;
607 v9 = _mm_load_ss(w9);
608 w9 += 1;
609 v10 = _mm_load_ss(w10);
610 w10 += 1;
611 v11 = _mm_load_ss(w11);
612 w11 += 1;
613 v12 = _mm_load_ss(w12);
614 w12 += 1;
615 v13 = _mm_load_ss(w13);
616 w13 += 1;
617 v14 = _mm_load_ss(w14);
618 w14 += 1;
619 break;
620 case 2:
621 // Read blocks of 4x2
622 // a b
623 // e f
624 // i j
625 // m n
626 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
627 w0 += 2;
628 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
629 w1 += 2;
630 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
631 w2 += 2;
632 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
633 w3 += 2;
634 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
635 w4 += 2;
636 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
637 w5 += 2;
638 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
639 w6 += 2;
640 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
641 w7 += 2;
642 v8 = _mm_castpd_ps(_mm_load_sd((const double*) w8));
643 w8 += 2;
644 v9 = _mm_castpd_ps(_mm_load_sd((const double*) w9));
645 w9 += 2;
646 v10 = _mm_castpd_ps(_mm_load_sd((const double*) w10));
647 w10 += 2;
648 v11 = _mm_castpd_ps(_mm_load_sd((const double*) w11));
649 w11 += 2;
650 v12 = _mm_castpd_ps(_mm_load_sd((const double*) w12));
651 w12 += 2;
652 v13 = _mm_castpd_ps(_mm_load_sd((const double*) w13));
653 w13 += 2;
654 v14 = _mm_castpd_ps(_mm_load_sd((const double*) w14));
655 w14 += 2;
656 break;
657 case 3:
658 {
659 // Read blocks of 4x3
660 // a b c
661 // e f g
662 // i j k
663 // m n o
664 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
665 const __m128 v0hi = _mm_load_ss(w0 + 2);
666 v0 = _mm_movelh_ps(v0lo, v0hi);
667 w0 += 3;
668 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
669 const __m128 v1hi = _mm_load_ss(w1 + 2);
670 v1 = _mm_movelh_ps(v1lo, v1hi);
671 w1 += 3;
672 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
673 const __m128 v2hi = _mm_load_ss(w2 + 2);
674 v2 = _mm_movelh_ps(v2lo, v2hi);
675 w2 += 3;
676 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
677 const __m128 v3hi = _mm_load_ss(w3 + 2);
678 v3 = _mm_movelh_ps(v3lo, v3hi);
679 w3 += 3;
680 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
681 const __m128 v4hi = _mm_load_ss(w4 + 2);
682 v4 = _mm_movelh_ps(v4lo, v4hi);
683 w4 += 3;
684 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
685 const __m128 v5hi = _mm_load_ss(w5 + 2);
686 v5 = _mm_movelh_ps(v5lo, v5hi);
687 w5 += 3;
688 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
689 const __m128 v6hi = _mm_load_ss(w6 + 2);
690 v6 = _mm_movelh_ps(v6lo, v6hi);
691 w6 += 3;
692 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
693 const __m128 v7hi = _mm_load_ss(w7 + 2);
694 v7 = _mm_movelh_ps(v7lo, v7hi);
695 w7 += 3;
696 const __m128 v8lo = _mm_castpd_ps(_mm_load_sd((const double*) w8));
697 const __m128 v8hi = _mm_load_ss(w8 + 2);
698 v8 = _mm_movelh_ps(v8lo, v8hi);
699 w8 += 3;
700 const __m128 v9lo = _mm_castpd_ps(_mm_load_sd((const double*) w9));
701 const __m128 v9hi = _mm_load_ss(w9 + 2);
702 v9 = _mm_movelh_ps(v9lo, v9hi);
703 w9 += 3;
704 const __m128 v10lo = _mm_castpd_ps(_mm_load_sd((const double*) w10));
705 const __m128 v10hi = _mm_load_ss(w10 + 2);
706 v10 = _mm_movelh_ps(v10lo, v10hi);
707 w10 += 3;
708 const __m128 v11lo = _mm_castpd_ps(_mm_load_sd((const double*) w11));
709 const __m128 v11hi = _mm_load_ss(w11 + 2);
710 v11 = _mm_movelh_ps(v11lo, v11hi);
711 w11 += 3;
712 const __m128 v12lo = _mm_castpd_ps(_mm_load_sd((const double*) w12));
713 const __m128 v12hi = _mm_load_ss(w12 + 2);
714 v12 = _mm_movelh_ps(v12lo, v12hi);
715 w12 += 3;
716 const __m128 v13lo = _mm_castpd_ps(_mm_load_sd((const double*) w13));
717 const __m128 v13hi = _mm_load_ss(w13 + 2);
718 v13 = _mm_movelh_ps(v13lo, v13hi);
719 w13 += 3;
720 const __m128 v14lo = _mm_castpd_ps(_mm_load_sd((const double*) w14));
721 const __m128 v14hi = _mm_load_ss(w14 + 2);
722 v14 = _mm_movelh_ps(v14lo, v14hi);
723 w14 += 3;
724 break;
725 }
726 default:
727 XNN_UNREACHABLE;
728 }
729
730 // Apply SR4 shuffle
731 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
732 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
733 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
734 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
735 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
736 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
737 v9 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9), _MM_SHUFFLE(0, 3, 2, 1)));
738 v10 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10), _MM_SHUFFLE(1, 0, 3, 2)));
739 v11 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11), _MM_SHUFFLE(2, 1, 0, 3)));
740 v13 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13), _MM_SHUFFLE(0, 3, 2, 1)));
741 v14 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14), _MM_SHUFFLE(1, 0, 3, 2)));
742 // Transpose 2x2
743 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
744 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
745 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
746 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
747 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
748 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
749 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
750 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
751 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a e b f from row 0, 1
752 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // i m j n from row 2, 3
753 const __m128 vtmp10 = _mm_unpackhi_ps(v8, v9); // c g d h from row 0, 1
754 const __m128 vtmp11 = _mm_unpackhi_ps(v10, v11); // k o l p from row 2, 3
755 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a e b f from row 0, 1
756 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // i m j n from row 2, 3
757 const __m128 vtmp14 = _mm_unpackhi_ps(v12, v13); // c g d h from row 0, 1
758 const __m128 vtmp15 = _mm_unpackhi_ps(v14, v14); // k o l p from row 2, 3
759 // Transpose 4x4
760 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
761 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
762 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
763 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
764 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
765 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
766 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
767 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
768 v8 = _mm_movelh_ps(vtmp8, vtmp9); // a e i m from row 0, 1
769 v9 = _mm_movehl_ps(vtmp9, vtmp8); // b f j n from row 0, 1
770 v10 = _mm_movelh_ps(vtmp10, vtmp11); // c g k o from row 2, 3
771 v11 = _mm_movehl_ps(vtmp11, vtmp10); // d h l p from row 2, 3
772 v12 = _mm_movelh_ps(vtmp12, vtmp13); // a e i m from row 0, 1
773 v13 = _mm_movehl_ps(vtmp13, vtmp12); // b f j n from row 0, 1
774 v14 = _mm_movelh_ps(vtmp14, vtmp15); // c g k o from row 2, 3
775 v15 = _mm_movehl_ps(vtmp15, vtmp14); // d h l p from row 2, 3
776 _mm_store_ps(packed_w, v0);
777 _mm_store_ps(packed_w + 4, v4);
778 _mm_store_ps(packed_w + 8, v8);
779 _mm_store_ps(packed_w + 12, v12);
780 _mm_store_ps(packed_w + 16, v1);
781 _mm_store_ps(packed_w + 20, v5);
782 _mm_store_ps(packed_w + 24, v9);
783 _mm_store_ps(packed_w + 28, v13);
784 _mm_store_ps(packed_w + 32, v2);
785 _mm_store_ps(packed_w + 36, v6);
786 _mm_store_ps(packed_w + 40, v10);
787 _mm_store_ps(packed_w + 44, v14);
788 _mm_store_ps(packed_w + 48, v3);
789 _mm_store_ps(packed_w + 52, v7);
790 _mm_store_ps(packed_w + 56, v11);
791 _mm_store_ps(packed_w + 60, v15);
792 packed_w += 64;
793 }
794 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
795 }
796 weights += nc * kc;
797 } while (--g != 0);
798 }