Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-avx512f-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/avx512.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/intrinsics-polyfill.h"
18 #include "xnnpack/packw.h"
19 #include "xnnpack/prefetch.h"
20
21
22 void xnn_x32_packw_gemm_goi_ukernel_x16__avx512f_u4_prfm(
23 size_t g,
24 size_t nc,
25 size_t kc,
26 size_t nr,
27 size_t kr,
28 size_t sr,
29 const uint32_t* weights,
30 const uint32_t* bias,
31 const void* scale,
32 uint32_t* packed_weights,
33 size_t extra_bytes,
34 const void* params)
35 {
36 assert(g != 0);
37 assert(nc != 0);
38 assert(kc != 0);
39 assert(nr == 16); // This kernel is for NR=16
40 assert(kr == 1);
41 assert(sr == 1);
42 assert(weights != NULL);
43 assert(packed_weights != NULL);
44
45 const float* b = (const float*) bias;
46 float* packed_w = (float*) packed_weights;
47 do {
48 // NC main loop multiple of 16
49 const float* w0 = (const float*) weights;
50 size_t n = nc;
51
52 for (; n >= 16; n -= 16) {
53 if XNN_LIKELY(b != NULL) {
54 const __m512 vb0 = _mm512_loadu_ps(b);
55 _mm512_store_ps(packed_w, vb0);
56 b += 16;
57 } else {
58 const __m512 vzero = _mm512_setzero_ps();
59 _mm512_store_ps(packed_w, vzero);
60 }
61 packed_w += 16;
62
63 const float* w1 = w0 + kc;
64 const float* w2 = w1 + kc;
65 const float* w3 = w2 + kc;
66 const float* w4 = w3 + kc;
67 const float* w5 = w4 + kc;
68 const float* w6 = w5 + kc;
69 const float* w7 = w6 + kc;
70 const float* w8 = w7 + kc;
71 const float* w9 = w8 + kc;
72 const float* w10 = w9 + kc;
73 const float* w11 = w10 + kc;
74 const float* w12 = w11 + kc;
75 const float* w13 = w12 + kc;
76 const float* w14 = w13 + kc;
77 const float* w15 = w14 + kc;
78 xnn_prefetch_to_l1((const int8_t*) w0);
79 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
80 xnn_prefetch_to_l1((const int8_t*) w1);
81 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
82 xnn_prefetch_to_l1((const int8_t*) w2);
83 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
84 xnn_prefetch_to_l1((const int8_t*) w3);
85 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
86 xnn_prefetch_to_l1((const int8_t*) w4);
87 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
88 xnn_prefetch_to_l1((const int8_t*) w5);
89 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
90 xnn_prefetch_to_l1((const int8_t*) w6);
91 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
92 xnn_prefetch_to_l1((const int8_t*) w7);
93 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
94 xnn_prefetch_to_l1((const int8_t*) w8);
95 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
96 xnn_prefetch_to_l1((const int8_t*) w9);
97 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
98 xnn_prefetch_to_l1((const int8_t*) w10);
99 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
100 xnn_prefetch_to_l1((const int8_t*) w11);
101 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
102 xnn_prefetch_to_l1((const int8_t*) w12);
103 xnn_prefetch_to_l1((const int8_t*) w12 + 64);
104 xnn_prefetch_to_l1((const int8_t*) w13);
105 xnn_prefetch_to_l1((const int8_t*) w13 + 64);
106 xnn_prefetch_to_l1((const int8_t*) w14);
107 xnn_prefetch_to_l1((const int8_t*) w14 + 64);
108 xnn_prefetch_to_l1((const int8_t*) w15);
109 xnn_prefetch_to_l1((const int8_t*) w15 + 64);
110
111 // KC main loop multiple of 4
112 size_t k = kc;
113 for (; k >= 4; k -= 4) {
114 // Read blocks of 4x4
115 // a b c d
116 // e f g h
117 // i j k l
118 // m n o p
119 // Load first 4 rows of N into low part of each register
120 __m512 v0x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w0));
121 w0 += 4;
122 __m512 v1x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w1));
123 w1 += 4;
124 __m512 v2x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w2));
125 w2 += 4;
126 __m512 v3x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w3));
127 w3 += 4;
128 // Load next 4 rows of N into the high part of each register
129 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w4), 1);
130 w4 += 4;
131 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w5), 1);
132 w5 += 4;
133 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w6), 1);
134 w6 += 4;
135 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w7), 1);
136 w7 += 4;
137 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w8), 2);
138 w8 += 4;
139 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w9), 2);
140 w9 += 4;
141 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w10), 2);
142 w10 += 4;
143 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w11), 2);
144 w11 += 4;
145 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w12), 3);
146 w12 += 4;
147 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w13), 3);
148 w13 += 4;
149 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w14), 3);
150 w14 += 4;
151 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w15), 3);
152 w15 += 4;
153
154 // Transpose 2x2
155 const __m512 vtmp0x0123 = _mm512_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
156 const __m512 vtmp1x0123 = _mm512_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
157 const __m512 vtmp2x0123 = _mm512_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
158 const __m512 vtmp3x0123 = _mm512_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
159 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
160 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
161 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
162 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
163 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
164 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
165 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
166 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
167 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
168 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
169 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
170 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
171 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
172 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
173 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
174 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
175 // Transpose 4x4
176 v0x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
177 v1x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
178 v2x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
179 v3x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
180
181 _mm512_store_ps(packed_w, v0x0123);
182 _mm512_store_ps(packed_w + 16, v1x0123);
183 _mm512_store_ps(packed_w + 32, v2x0123);
184 _mm512_store_ps(packed_w + 48, v3x0123);
185 packed_w += 64;
186 }
187
188 // KC remainder (1..3)
189 if XNN_UNLIKELY(k != 0) {
190 assert(k >= 1);
191 assert(k <= 3);
192 if (k & 2) {
193 // Read blocks of 4x2
194 // a b
195 // c d
196 // e f
197 // g h
198 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
199 w0 += 2;
200 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
201 w1 += 2;
202 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
203 w2 += 2;
204 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
205 w3 += 2;
206 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
207 w4 += 2;
208 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
209 w5 += 2;
210 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
211 w6 += 2;
212 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
213 w7 += 2;
214 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
215 w8 += 2;
216 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
217 w9 += 2;
218 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
219 w10 += 2;
220 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
221 w11 += 2;
222 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
223 w12 += 2;
224 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
225 w13 += 2;
226 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
227 w14 += 2;
228 __m128 v15 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
229 w15 += 2;
230
231 // Transpose 2x2
232 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
233 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
234 // Transpose 4x4
235 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
236 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
237 // Transpose 2x2
238 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
239 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
240 // Transpose 4x4
241 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
242 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
243 // Transpose 2x2
244 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
245 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
246 // Transpose 4x4
247 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
248 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
249 // Transpose 2x2
250 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
251 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // e g f h from row 2, 3
252 // Transpose 4x4
253 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
254 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
255
256 _mm_store_ps(packed_w, v0);
257 _mm_store_ps(packed_w + 4, v4);
258 _mm_store_ps(packed_w + 8, v8);
259 _mm_store_ps(packed_w + 12, v12);
260 _mm_store_ps(packed_w + 16, v1);
261 _mm_store_ps(packed_w + 20, v5);
262 _mm_store_ps(packed_w + 24, v9);
263 _mm_store_ps(packed_w + 28, v13);
264 packed_w += 32;
265 }
266 if (k & 1) {
267 // Read blocks of 4x1
268 // a
269 // b
270 // c
271 // d
272 __m128 v0 = _mm_load_ss(w0);
273 w0 += 1;
274 __m128 v1 = _mm_load_ss(w1);
275 w1 += 1;
276 __m128 v2 = _mm_load_ss(w2);
277 w2 += 1;
278 __m128 v3 = _mm_load_ss(w3);
279 w3 += 1;
280 __m128 v4 = _mm_load_ss(w4);
281 w4 += 1;
282 __m128 v5 = _mm_load_ss(w5);
283 w5 += 1;
284 __m128 v6 = _mm_load_ss(w6);
285 w6 += 1;
286 __m128 v7 = _mm_load_ss(w7);
287 w7 += 1;
288 __m128 v8 = _mm_load_ss(w8);
289 w8 += 1;
290 __m128 v9 = _mm_load_ss(w9);
291 w9 += 1;
292 __m128 v10 = _mm_load_ss(w10);
293 w10 += 1;
294 __m128 v11 = _mm_load_ss(w11);
295 w11 += 1;
296 __m128 v12 = _mm_load_ss(w12);
297 w12 += 1;
298 __m128 v13 = _mm_load_ss(w13);
299 w13 += 1;
300 __m128 v14 = _mm_load_ss(w14);
301 w14 += 1;
302 __m128 v15 = _mm_load_ss(w15);
303 w15 += 1;
304
305 // Transpose 2x2
306 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
307 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
308 // Transpose 4x4
309 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
310 // Transpose 2x2
311 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
312 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
313 // Transpose 4x4
314 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
315 // Transpose 2x2
316 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
317 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
318 // Transpose 4x4
319 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
320 // Transpose 2x2
321 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
322 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // c d from row 2, 3
323 // Transpose 4x4
324 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
325
326 _mm_store_ps(packed_w, v0);
327 _mm_store_ps(packed_w + 4, v4);
328 _mm_store_ps(packed_w + 8, v8);
329 _mm_store_ps(packed_w + 12, v12);
330 packed_w += 16;
331 }
332 }
333 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
334 w0 = w15;
335 }
336
337 // NC remainder (1..15)
338 if XNN_UNLIKELY(n != 0) {
339 assert(n >= 1);
340 assert(n <= 15);
341 if XNN_LIKELY(b != NULL) {
342 size_t nb = n;
343 do {
344 *packed_w++ = *b++;
345 } while (--nb != 0);
346 packed_w += (16 - n);
347 } else {
348 const __m512 vzero = _mm512_setzero_ps();
349 _mm512_store_ps(packed_w, vzero);
350 packed_w += 16;
351 }
352
353 // NR remainder has less than 16 rows so last row is not loaded
354 // For SR=4 the
355 const float* w1 = w0 + kc;
356 if XNN_UNPREDICTABLE(n < 2) {
357 w1 = w0;
358 }
359 const float* w2 = w1 + kc;
360 if XNN_UNPREDICTABLE(n <= 2) {
361 w2 = w1;
362 }
363 const float* w3 = w2 + kc;
364 if XNN_UNPREDICTABLE(n < 4) {
365 w3 = w2;
366 }
367 const float* w4 = w3 + kc;
368 if XNN_UNPREDICTABLE(n <= 4) {
369 w4 = w3;
370 }
371 const float* w5 = w4 + kc;
372 if XNN_UNPREDICTABLE(n < 6) {
373 w5 = w4;
374 }
375 const float* w6 = w5 + kc;
376 if XNN_UNPREDICTABLE(n <= 6) {
377 w6 = w5;
378 }
379 const float* w7 = w6 + kc;
380 if XNN_UNPREDICTABLE(n < 8) {
381 w7 = w6;
382 }
383 const float* w8 = w7 + kc;
384 if XNN_UNPREDICTABLE(n <= 8) {
385 w8 = w7;
386 }
387 const float* w9 = w8 + kc;
388 if XNN_UNPREDICTABLE(n < 10) {
389 w9 = w8;
390 }
391 const float* w10 = w9 + kc;
392 if XNN_UNPREDICTABLE(n <= 10) {
393 w10 = w9;
394 }
395 const float* w11 = w10 + kc;
396 if XNN_UNPREDICTABLE(n < 12) {
397 w11 = w10;
398 }
399 const float* w12 = w11 + kc;
400 if XNN_UNPREDICTABLE(n <= 12) {
401 w12 = w11;
402 }
403 const float* w13 = w12 + kc;
404 if XNN_UNPREDICTABLE(n < 14) {
405 w13 = w12;
406 }
407 const float* w14 = w13 + kc;
408 if XNN_UNPREDICTABLE(n <= 14) {
409 w14 = w13;
410 }
411
412 // KC main loop multiple of 4
413 size_t k = kc;
414 for (; k >= 4; k -= 4) {
415 // Read blocks of 4x4
416 // a b c d
417 // e f g h
418 // i j k l
419 // m n o p
420 // Load first 4 rows of N into low part of each register
421 __m512 v0x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w0));
422 w0 += 4;
423 __m512 v1x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w1));
424 w1 += 4;
425 __m512 v2x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w2));
426 w2 += 4;
427 // castps leaves upper 128 bits undefined, so zero them.
428 __m512 v3x0123 = _mm512_zextps128_ps512(_mm_loadu_ps(w3));
429 w3 += 4;
430 // Load next 4 rows of N into the high part of each register
431 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w4), 1);
432 w4 += 4;
433 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w5), 1);
434 w5 += 4;
435 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w6), 1);
436 w6 += 4;
437 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w7), 1);
438 w7 += 4;
439 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w8), 2);
440 w8 += 4;
441 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w9), 2);
442 w9 += 4;
443 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w10), 2);
444 w10 += 4;
445 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w11), 2);
446 w11 += 4;
447 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w12), 3);
448 w12 += 4;
449 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w13), 3);
450 w13 += 4;
451 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w14), 3);
452 w14 += 4;
453
454 // Transpose 2x2
455 const __m512 vtmp0x0123 = _mm512_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
456 const __m512 vtmp1x0123 = _mm512_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
457 const __m512 vtmp2x0123 = _mm512_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
458 const __m512 vtmp3x0123 = _mm512_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
459 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
460 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
461 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
462 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
463 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
464 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
465 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
466 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
467 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
468 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
469 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
470 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
471 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
472 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
473 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
474 // Transpose 4x4
475 v0x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
476 v1x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
477 v2x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
478 v3x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
479
480 _mm512_store_ps(packed_w, v0x0123);
481 _mm512_store_ps(packed_w + 16, v1x0123);
482 _mm512_store_ps(packed_w + 32, v2x0123);
483 _mm512_store_ps(packed_w + 48, v3x0123);
484 packed_w += 64;
485 }
486
487 // KC remainder (1..3)
488 if XNN_UNLIKELY(k != 0) {
489 assert(k >= 1);
490 assert(k <= 3);
491 if (k & 2) {
492 // Read blocks of 4x2
493 // a b
494 // c d
495 // e f
496 // g h
497 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
498 w0 += 2;
499 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
500 w1 += 2;
501 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
502 w2 += 2;
503 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
504 w3 += 2;
505 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
506 w4 += 2;
507 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
508 w5 += 2;
509 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
510 w6 += 2;
511 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
512 w7 += 2;
513 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
514 w8 += 2;
515 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
516 w9 += 2;
517 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
518 w10 += 2;
519 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
520 w11 += 2;
521 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
522 w12 += 2;
523 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
524 w13 += 2;
525 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
526 w14 += 2;
527
528 // Transpose 2x2
529 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
530 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
531 // Transpose 4x4
532 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
533 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
534 // Transpose 2x2
535 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
536 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
537 // Transpose 4x4
538 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
539 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
540 // Transpose 2x2
541 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
542 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
543 // Transpose 4x4
544 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
545 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
546 // Transpose 2x2
547 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
548 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // e g f h from row 2, 3
549 // Transpose 4x4
550 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
551 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
552
553 _mm_store_ps(packed_w, v0);
554 _mm_store_ps(packed_w + 4, v4);
555 _mm_store_ps(packed_w + 8, v8);
556 _mm_store_ps(packed_w + 12, v12);
557 _mm_store_ps(packed_w + 16, v1);
558 _mm_store_ps(packed_w + 20, v5);
559 _mm_store_ps(packed_w + 24, v9);
560 _mm_store_ps(packed_w + 28, v13);
561 packed_w += 32;
562 }
563 if (k & 1) {
564 // Read blocks of 4x1
565 // a
566 // b
567 // c
568 // d
569 __m128 v0 = _mm_load_ss(w0);
570 w0 += 1;
571 __m128 v1 = _mm_load_ss(w1);
572 w1 += 1;
573 __m128 v2 = _mm_load_ss(w2);
574 w2 += 1;
575 __m128 v3 = _mm_load_ss(w3);
576 w3 += 1;
577 __m128 v4 = _mm_load_ss(w4);
578 w4 += 1;
579 __m128 v5 = _mm_load_ss(w5);
580 w5 += 1;
581 __m128 v6 = _mm_load_ss(w6);
582 w6 += 1;
583 __m128 v7 = _mm_load_ss(w7);
584 w7 += 1;
585 __m128 v8 = _mm_load_ss(w8);
586 w8 += 1;
587 __m128 v9 = _mm_load_ss(w9);
588 w9 += 1;
589 __m128 v10 = _mm_load_ss(w10);
590 w10 += 1;
591 __m128 v11 = _mm_load_ss(w11);
592 w11 += 1;
593 __m128 v12 = _mm_load_ss(w12);
594 w12 += 1;
595 __m128 v13 = _mm_load_ss(w13);
596 w13 += 1;
597 __m128 v14 = _mm_load_ss(w14);
598 w14 += 1;
599
600 // Transpose 2x2
601 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
602 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
603 // Transpose 4x4
604 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
605 // Transpose 2x2
606 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
607 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
608 // Transpose 4x4
609 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
610 // Transpose 2x2
611 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
612 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
613 // Transpose 4x4
614 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
615 // Transpose 2x2
616 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
617 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // c d from row 2, 3
618 // Transpose 4x4
619 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
620
621 _mm_store_ps(packed_w, v0);
622 _mm_store_ps(packed_w + 4, v4);
623 _mm_store_ps(packed_w + 8, v8);
624 _mm_store_ps(packed_w + 12, v12);
625 packed_w += 16;
626 }
627 }
628 }
629 weights += nc * kc;
630 } while (--g != 0);
631 }