Debian Codegraph @

src/x32-packw/gen/x32-packw-x32-gemm-goi-avx512f-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/avx512.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/intrinsics-polyfill.h"
18 #include "xnnpack/packw.h"
19 #include "xnnpack/prefetch.h"
20
21
22 void xnn_x32_packw_gemm_goi_ukernel_x32__avx512f_u4_prfm(
23 size_t g,
24 size_t nc,
25 size_t kc,
26 size_t nr,
27 size_t kr,
28 size_t sr,
29 const uint32_t* weights,
30 const uint32_t* bias,
31 const void* scale,
32 uint32_t* packed_weights,
33 size_t extra_bytes,
34 const void* params)
35 {
36 assert(g != 0);
37 assert(nc != 0);
38 assert(kc != 0);
39 assert(nr == 32); // This kernel is for NR=32
40 assert(kr == 1);
41 assert(sr == 1);
42 assert(weights != NULL);
43 assert(packed_weights != NULL);
44
45 const float* b = (const float*) bias;
46 float* packed_w = (float*) packed_weights;
47 do {
48 // NC main loop multiple of 32
49 const float* w0 = (const float*) weights;
50 size_t n = nc;
51
52 for (; n >= 32; n -= 32) {
53 if XNN_LIKELY(b != NULL) {
54 const __m512 vb0 = _mm512_loadu_ps(b);
55 const __m512 vb16 = _mm512_loadu_ps(b + 16);
56 _mm512_store_ps(packed_w, vb0);
57 _mm512_store_ps(packed_w + 16, vb16);
58 b += 32;
59 } else {
60 const __m512 vzero = _mm512_setzero_ps();
61 _mm512_store_ps(packed_w, vzero);
62 _mm512_store_ps(packed_w + 16, vzero);
63 }
64 packed_w += 32;
65
66 const float* w1 = w0 + kc;
67 const float* w2 = w1 + kc;
68 const float* w3 = w2 + kc;
69 const float* w4 = w3 + kc;
70 const float* w5 = w4 + kc;
71 const float* w6 = w5 + kc;
72 const float* w7 = w6 + kc;
73 const float* w8 = w7 + kc;
74 const float* w9 = w8 + kc;
75 const float* w10 = w9 + kc;
76 const float* w11 = w10 + kc;
77 const float* w12 = w11 + kc;
78 const float* w13 = w12 + kc;
79 const float* w14 = w13 + kc;
80 const float* w15 = w14 + kc;
81 const float* w16 = w15 + kc;
82 const float* w17 = w16 + kc;
83 const float* w18 = w17 + kc;
84 const float* w19 = w18 + kc;
85 const float* w20 = w19 + kc;
86 const float* w21 = w20 + kc;
87 const float* w22 = w21 + kc;
88 const float* w23 = w22 + kc;
89 const float* w24 = w23 + kc;
90 const float* w25 = w24 + kc;
91 const float* w26 = w25 + kc;
92 const float* w27 = w26 + kc;
93 const float* w28 = w27 + kc;
94 const float* w29 = w28 + kc;
95 const float* w30 = w29 + kc;
96 const float* w31 = w30 + kc;
97 xnn_prefetch_to_l1((const int8_t*) w0);
98 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
99 xnn_prefetch_to_l1((const int8_t*) w1);
100 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
101 xnn_prefetch_to_l1((const int8_t*) w2);
102 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
103 xnn_prefetch_to_l1((const int8_t*) w3);
104 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
105 xnn_prefetch_to_l1((const int8_t*) w4);
106 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
107 xnn_prefetch_to_l1((const int8_t*) w5);
108 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
109 xnn_prefetch_to_l1((const int8_t*) w6);
110 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
111 xnn_prefetch_to_l1((const int8_t*) w7);
112 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
113 xnn_prefetch_to_l1((const int8_t*) w8);
114 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
115 xnn_prefetch_to_l1((const int8_t*) w9);
116 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
117 xnn_prefetch_to_l1((const int8_t*) w10);
118 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
119 xnn_prefetch_to_l1((const int8_t*) w11);
120 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
121 xnn_prefetch_to_l1((const int8_t*) w12);
122 xnn_prefetch_to_l1((const int8_t*) w12 + 64);
123 xnn_prefetch_to_l1((const int8_t*) w13);
124 xnn_prefetch_to_l1((const int8_t*) w13 + 64);
125 xnn_prefetch_to_l1((const int8_t*) w14);
126 xnn_prefetch_to_l1((const int8_t*) w14 + 64);
127 xnn_prefetch_to_l1((const int8_t*) w15);
128 xnn_prefetch_to_l1((const int8_t*) w15 + 64);
129 xnn_prefetch_to_l1((const int8_t*) w16);
130 xnn_prefetch_to_l1((const int8_t*) w16 + 64);
131 xnn_prefetch_to_l1((const int8_t*) w17);
132 xnn_prefetch_to_l1((const int8_t*) w17 + 64);
133 xnn_prefetch_to_l1((const int8_t*) w18);
134 xnn_prefetch_to_l1((const int8_t*) w18 + 64);
135 xnn_prefetch_to_l1((const int8_t*) w19);
136 xnn_prefetch_to_l1((const int8_t*) w19 + 64);
137 xnn_prefetch_to_l1((const int8_t*) w20);
138 xnn_prefetch_to_l1((const int8_t*) w20 + 64);
139 xnn_prefetch_to_l1((const int8_t*) w21);
140 xnn_prefetch_to_l1((const int8_t*) w21 + 64);
141 xnn_prefetch_to_l1((const int8_t*) w22);
142 xnn_prefetch_to_l1((const int8_t*) w22 + 64);
143 xnn_prefetch_to_l1((const int8_t*) w23);
144 xnn_prefetch_to_l1((const int8_t*) w23 + 64);
145 xnn_prefetch_to_l1((const int8_t*) w24);
146 xnn_prefetch_to_l1((const int8_t*) w24 + 64);
147 xnn_prefetch_to_l1((const int8_t*) w25);
148 xnn_prefetch_to_l1((const int8_t*) w25 + 64);
149 xnn_prefetch_to_l1((const int8_t*) w26);
150 xnn_prefetch_to_l1((const int8_t*) w26 + 64);
151 xnn_prefetch_to_l1((const int8_t*) w27);
152 xnn_prefetch_to_l1((const int8_t*) w27 + 64);
153 xnn_prefetch_to_l1((const int8_t*) w28);
154 xnn_prefetch_to_l1((const int8_t*) w28 + 64);
155 xnn_prefetch_to_l1((const int8_t*) w29);
156 xnn_prefetch_to_l1((const int8_t*) w29 + 64);
157 xnn_prefetch_to_l1((const int8_t*) w30);
158 xnn_prefetch_to_l1((const int8_t*) w30 + 64);
159 xnn_prefetch_to_l1((const int8_t*) w31);
160 xnn_prefetch_to_l1((const int8_t*) w31 + 64);
161
162 // KC main loop multiple of 4
163 size_t k = kc;
164 for (; k >= 4; k -= 4) {
165 // Read blocks of 4x4
166 // a b c d
167 // e f g h
168 // i j k l
169 // m n o p
170 // Load first 4 rows of N into low part of each register
171 __m512 v0x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w0));
172 w0 += 4;
173 __m512 v1x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w1));
174 w1 += 4;
175 __m512 v2x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w2));
176 w2 += 4;
177 __m512 v3x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w3));
178 w3 += 4;
179 __m512 v16x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w16));
180 w16 += 4;
181 __m512 v17x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w17));
182 w17 += 4;
183 __m512 v18x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w18));
184 w18 += 4;
185 __m512 v19x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w19));
186 w19 += 4;
187 // Load next 4 rows of N into the high part of each register
188 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w4), 1);
189 w4 += 4;
190 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w5), 1);
191 w5 += 4;
192 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w6), 1);
193 w6 += 4;
194 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w7), 1);
195 w7 += 4;
196 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w8), 2);
197 w8 += 4;
198 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w9), 2);
199 w9 += 4;
200 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w10), 2);
201 w10 += 4;
202 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w11), 2);
203 w11 += 4;
204 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w12), 3);
205 w12 += 4;
206 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w13), 3);
207 w13 += 4;
208 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w14), 3);
209 w14 += 4;
210 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w15), 3);
211 w15 += 4;
212 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w20), 1);
213 w20 += 4;
214 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w21), 1);
215 w21 += 4;
216 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w22), 1);
217 w22 += 4;
218 v19x0123 = _mm512_insertf32x4(v19x0123, _mm_loadu_ps(w23), 1);
219 w23 += 4;
220 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w24), 2);
221 w24 += 4;
222 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w25), 2);
223 w25 += 4;
224 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w26), 2);
225 w26 += 4;
226 v19x0123 = _mm512_insertf32x4(v19x0123, _mm_loadu_ps(w27), 2);
227 w27 += 4;
228 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w28), 3);
229 w28 += 4;
230 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w29), 3);
231 w29 += 4;
232 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w30), 3);
233 w30 += 4;
234 v19x0123 = _mm512_insertf32x4(v19x0123, _mm_loadu_ps(w31), 3);
235 w31 += 4;
236
237 // Transpose 2x2
238 const __m512 vtmp0x0123 = _mm512_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
239 const __m512 vtmp1x0123 = _mm512_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
240 const __m512 vtmp2x0123 = _mm512_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
241 const __m512 vtmp3x0123 = _mm512_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
242 const __m512 vtmp16x0123 = _mm512_unpacklo_ps(v16x0123, v17x0123); // a e b f from row 0, 1
243 const __m512 vtmp17x0123 = _mm512_unpacklo_ps(v18x0123, v19x0123); // i m j n from row 2, 3
244 const __m512 vtmp18x0123 = _mm512_unpackhi_ps(v16x0123, v17x0123); // c g d h from row 0, 1
245 const __m512 vtmp19x0123 = _mm512_unpackhi_ps(v18x0123, v19x0123); // k o l p from row 2, 3
246 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
247 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
248 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
249 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
250 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
251 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
252 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
253 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
254 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
255 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
256 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
257 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
258 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
259 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
260 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
261 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
262 xnn_prefetch_to_l1((const int8_t*) w16 + 128);
263 xnn_prefetch_to_l1((const int8_t*) w17 + 128);
264 xnn_prefetch_to_l1((const int8_t*) w18 + 128);
265 xnn_prefetch_to_l1((const int8_t*) w19 + 128);
266 xnn_prefetch_to_l1((const int8_t*) w20 + 128);
267 xnn_prefetch_to_l1((const int8_t*) w21 + 128);
268 xnn_prefetch_to_l1((const int8_t*) w22 + 128);
269 xnn_prefetch_to_l1((const int8_t*) w23 + 128);
270 xnn_prefetch_to_l1((const int8_t*) w24 + 128);
271 xnn_prefetch_to_l1((const int8_t*) w25 + 128);
272 xnn_prefetch_to_l1((const int8_t*) w26 + 128);
273 xnn_prefetch_to_l1((const int8_t*) w27 + 128);
274 xnn_prefetch_to_l1((const int8_t*) w28 + 128);
275 xnn_prefetch_to_l1((const int8_t*) w29 + 128);
276 xnn_prefetch_to_l1((const int8_t*) w30 + 128);
277 xnn_prefetch_to_l1((const int8_t*) w31 + 128);
278 // Transpose 4x4
279 v0x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
280 v1x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
281 v2x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
282 v3x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
283 v16x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp16x0123), _mm512_castps_pd(vtmp17x0123))); // a e i m from row 0, 1
284 v17x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp16x0123), _mm512_castps_pd(vtmp17x0123))); // b f j n from row 0, 1
285 v18x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp18x0123), _mm512_castps_pd(vtmp19x0123))); // c g k o from row 2, 3
286 v19x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp18x0123), _mm512_castps_pd(vtmp19x0123))); // d h l p from row 2, 3
287
288 _mm512_store_ps(packed_w, v0x0123);
289 _mm512_store_ps(packed_w + 16, v16x0123);
290 _mm512_store_ps(packed_w + 32, v1x0123);
291 _mm512_store_ps(packed_w + 48, v17x0123);
292 _mm512_store_ps(packed_w + 64, v2x0123);
293 _mm512_store_ps(packed_w + 80, v18x0123);
294 _mm512_store_ps(packed_w + 96, v3x0123);
295 _mm512_store_ps(packed_w + 112, v19x0123);
296 packed_w += 128;
297 }
298
299 // KC remainder (1..3)
300 if XNN_UNLIKELY(k != 0) {
301 assert(k >= 1);
302 assert(k <= 3);
303 if (k & 2) {
304 // Read blocks of 4x2
305 // a b
306 // c d
307 // e f
308 // g h
309 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
310 w0 += 2;
311 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
312 w1 += 2;
313 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
314 w2 += 2;
315 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
316 w3 += 2;
317 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
318 w4 += 2;
319 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
320 w5 += 2;
321 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
322 w6 += 2;
323 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
324 w7 += 2;
325 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
326 w8 += 2;
327 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
328 w9 += 2;
329 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
330 w10 += 2;
331 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
332 w11 += 2;
333 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
334 w12 += 2;
335 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
336 w13 += 2;
337 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
338 w14 += 2;
339 __m128 v15 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
340 w15 += 2;
341 __m128 v16 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w16));
342 w16 += 2;
343 __m128 v17 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w17));
344 w17 += 2;
345 __m128 v18 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w18));
346 w18 += 2;
347 __m128 v19 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w19));
348 w19 += 2;
349 __m128 v20 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w20));
350 w20 += 2;
351 __m128 v21 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w21));
352 w21 += 2;
353 __m128 v22 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w22));
354 w22 += 2;
355 __m128 v23 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w23));
356 w23 += 2;
357 __m128 v24 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w24));
358 w24 += 2;
359 __m128 v25 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w25));
360 w25 += 2;
361 __m128 v26 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w26));
362 w26 += 2;
363 __m128 v27 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w27));
364 w27 += 2;
365 __m128 v28 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w28));
366 w28 += 2;
367 __m128 v29 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w29));
368 w29 += 2;
369 __m128 v30 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w30));
370 w30 += 2;
371 __m128 v31 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w31));
372 w31 += 2;
373
374 // Transpose 2x2
375 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
376 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
377 // Transpose 4x4
378 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
379 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
380 // Transpose 2x2
381 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
382 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
383 // Transpose 4x4
384 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
385 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
386 // Transpose 2x2
387 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
388 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
389 // Transpose 4x4
390 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
391 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
392 // Transpose 2x2
393 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
394 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // e g f h from row 2, 3
395 // Transpose 4x4
396 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
397 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
398 // Transpose 2x2
399 const __m128 vtmp16 = _mm_unpacklo_ps(v16, v17); // a c b d from row 0, 1
400 const __m128 vtmp17 = _mm_unpacklo_ps(v18, v19); // e g f h from row 2, 3
401 // Transpose 4x4
402 v16 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // a c e g from row 0, 1
403 v17 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // b d f h from row 0, 1
404 // Transpose 2x2
405 const __m128 vtmp20 = _mm_unpacklo_ps(v20, v21); // a c b d from row 0, 1
406 const __m128 vtmp21 = _mm_unpacklo_ps(v22, v23); // e g f h from row 2, 3
407 // Transpose 4x4
408 v20 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // a c e g from row 0, 1
409 v21 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // b d f h from row 0, 1
410 // Transpose 2x2
411 const __m128 vtmp24 = _mm_unpacklo_ps(v24, v25); // a c b d from row 0, 1
412 const __m128 vtmp25 = _mm_unpacklo_ps(v26, v27); // e g f h from row 2, 3
413 // Transpose 4x4
414 v24 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // a c e g from row 0, 1
415 v25 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // b d f h from row 0, 1
416 // Transpose 2x2
417 const __m128 vtmp28 = _mm_unpacklo_ps(v28, v29); // a c b d from row 0, 1
418 const __m128 vtmp29 = _mm_unpacklo_ps(v30, v31); // e g f h from row 2, 3
419 // Transpose 4x4
420 v28 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // a c e g from row 0, 1
421 v29 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // b d f h from row 0, 1
422
423 _mm_store_ps(packed_w, v0);
424 _mm_store_ps(packed_w + 4, v4);
425 _mm_store_ps(packed_w + 8, v8);
426 _mm_store_ps(packed_w + 12, v12);
427 _mm_store_ps(packed_w + 16, v16);
428 _mm_store_ps(packed_w + 20, v20);
429 _mm_store_ps(packed_w + 24, v24);
430 _mm_store_ps(packed_w + 28, v28);
431 _mm_store_ps(packed_w + 32, v1);
432 _mm_store_ps(packed_w + 36, v5);
433 _mm_store_ps(packed_w + 40, v9);
434 _mm_store_ps(packed_w + 44, v13);
435 _mm_store_ps(packed_w + 48, v17);
436 _mm_store_ps(packed_w + 52, v21);
437 _mm_store_ps(packed_w + 56, v25);
438 _mm_store_ps(packed_w + 60, v29);
439 packed_w += 64;
440 }
441 if (k & 1) {
442 // Read blocks of 4x1
443 // a
444 // b
445 // c
446 // d
447 __m128 v0 = _mm_load_ss(w0);
448 w0 += 1;
449 __m128 v1 = _mm_load_ss(w1);
450 w1 += 1;
451 __m128 v2 = _mm_load_ss(w2);
452 w2 += 1;
453 __m128 v3 = _mm_load_ss(w3);
454 w3 += 1;
455 __m128 v4 = _mm_load_ss(w4);
456 w4 += 1;
457 __m128 v5 = _mm_load_ss(w5);
458 w5 += 1;
459 __m128 v6 = _mm_load_ss(w6);
460 w6 += 1;
461 __m128 v7 = _mm_load_ss(w7);
462 w7 += 1;
463 __m128 v8 = _mm_load_ss(w8);
464 w8 += 1;
465 __m128 v9 = _mm_load_ss(w9);
466 w9 += 1;
467 __m128 v10 = _mm_load_ss(w10);
468 w10 += 1;
469 __m128 v11 = _mm_load_ss(w11);
470 w11 += 1;
471 __m128 v12 = _mm_load_ss(w12);
472 w12 += 1;
473 __m128 v13 = _mm_load_ss(w13);
474 w13 += 1;
475 __m128 v14 = _mm_load_ss(w14);
476 w14 += 1;
477 __m128 v15 = _mm_load_ss(w15);
478 w15 += 1;
479 __m128 v16 = _mm_load_ss(w16);
480 w16 += 1;
481 __m128 v17 = _mm_load_ss(w17);
482 w17 += 1;
483 __m128 v18 = _mm_load_ss(w18);
484 w18 += 1;
485 __m128 v19 = _mm_load_ss(w19);
486 w19 += 1;
487 __m128 v20 = _mm_load_ss(w20);
488 w20 += 1;
489 __m128 v21 = _mm_load_ss(w21);
490 w21 += 1;
491 __m128 v22 = _mm_load_ss(w22);
492 w22 += 1;
493 __m128 v23 = _mm_load_ss(w23);
494 w23 += 1;
495 __m128 v24 = _mm_load_ss(w24);
496 w24 += 1;
497 __m128 v25 = _mm_load_ss(w25);
498 w25 += 1;
499 __m128 v26 = _mm_load_ss(w26);
500 w26 += 1;
501 __m128 v27 = _mm_load_ss(w27);
502 w27 += 1;
503 __m128 v28 = _mm_load_ss(w28);
504 w28 += 1;
505 __m128 v29 = _mm_load_ss(w29);
506 w29 += 1;
507 __m128 v30 = _mm_load_ss(w30);
508 w30 += 1;
509 __m128 v31 = _mm_load_ss(w31);
510 w31 += 1;
511
512 // Transpose 2x2
513 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
514 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
515 // Transpose 4x4
516 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
517 // Transpose 2x2
518 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
519 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
520 // Transpose 4x4
521 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
522 // Transpose 2x2
523 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
524 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
525 // Transpose 4x4
526 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
527 // Transpose 2x2
528 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
529 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // c d from row 2, 3
530 // Transpose 4x4
531 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
532 // Transpose 2x2
533 const __m128 vtmp16 = _mm_unpacklo_ps(v16, v17); // a b from row 0, 1
534 const __m128 vtmp17 = _mm_unpacklo_ps(v18, v19); // c d from row 2, 3
535 // Transpose 4x4
536 v16 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // a b c d from row 0, 1
537 // Transpose 2x2
538 const __m128 vtmp20 = _mm_unpacklo_ps(v20, v21); // a b from row 0, 1
539 const __m128 vtmp21 = _mm_unpacklo_ps(v22, v23); // c d from row 2, 3
540 // Transpose 4x4
541 v20 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // a b c d from row 0, 1
542 // Transpose 2x2
543 const __m128 vtmp24 = _mm_unpacklo_ps(v24, v25); // a b from row 0, 1
544 const __m128 vtmp25 = _mm_unpacklo_ps(v26, v27); // c d from row 2, 3
545 // Transpose 4x4
546 v24 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // a b c d from row 0, 1
547 // Transpose 2x2
548 const __m128 vtmp28 = _mm_unpacklo_ps(v28, v29); // a b from row 0, 1
549 const __m128 vtmp29 = _mm_unpacklo_ps(v30, v31); // c d from row 2, 3
550 // Transpose 4x4
551 v28 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // a b c d from row 0, 1
552
553 _mm_store_ps(packed_w, v0);
554 _mm_store_ps(packed_w + 4, v4);
555 _mm_store_ps(packed_w + 8, v8);
556 _mm_store_ps(packed_w + 12, v12);
557 _mm_store_ps(packed_w + 16, v16);
558 _mm_store_ps(packed_w + 20, v20);
559 _mm_store_ps(packed_w + 24, v24);
560 _mm_store_ps(packed_w + 28, v28);
561 packed_w += 32;
562 }
563 }
564 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
565 w0 = w31;
566 }
567
568 // NC remainder (1..31)
569 if XNN_UNLIKELY(n != 0) {
570 assert(n >= 1);
571 assert(n <= 31);
572 if XNN_LIKELY(b != NULL) {
573 size_t nb = n;
574 do {
575 *packed_w++ = *b++;
576 } while (--nb != 0);
577 packed_w += (32 - n);
578 } else {
579 const __m512 vzero = _mm512_setzero_ps();
580 _mm512_store_ps(packed_w, vzero);
581 _mm512_store_ps(packed_w + 16, vzero);
582 packed_w += 32;
583 }
584
585 // NR remainder has less than 32 rows so last row is not loaded
586 // For SR=4 the
587 const float* w1 = w0 + kc;
588 if XNN_UNPREDICTABLE(n < 2) {
589 w1 = w0;
590 }
591 const float* w2 = w1 + kc;
592 if XNN_UNPREDICTABLE(n <= 2) {
593 w2 = w1;
594 }
595 const float* w3 = w2 + kc;
596 if XNN_UNPREDICTABLE(n < 4) {
597 w3 = w2;
598 }
599 const float* w4 = w3 + kc;
600 if XNN_UNPREDICTABLE(n <= 4) {
601 w4 = w3;
602 }
603 const float* w5 = w4 + kc;
604 if XNN_UNPREDICTABLE(n < 6) {
605 w5 = w4;
606 }
607 const float* w6 = w5 + kc;
608 if XNN_UNPREDICTABLE(n <= 6) {
609 w6 = w5;
610 }
611 const float* w7 = w6 + kc;
612 if XNN_UNPREDICTABLE(n < 8) {
613 w7 = w6;
614 }
615 const float* w8 = w7 + kc;
616 if XNN_UNPREDICTABLE(n <= 8) {
617 w8 = w7;
618 }
619 const float* w9 = w8 + kc;
620 if XNN_UNPREDICTABLE(n < 10) {
621 w9 = w8;
622 }
623 const float* w10 = w9 + kc;
624 if XNN_UNPREDICTABLE(n <= 10) {
625 w10 = w9;
626 }
627 const float* w11 = w10 + kc;
628 if XNN_UNPREDICTABLE(n < 12) {
629 w11 = w10;
630 }
631 const float* w12 = w11 + kc;
632 if XNN_UNPREDICTABLE(n <= 12) {
633 w12 = w11;
634 }
635 const float* w13 = w12 + kc;
636 if XNN_UNPREDICTABLE(n < 14) {
637 w13 = w12;
638 }
639 const float* w14 = w13 + kc;
640 if XNN_UNPREDICTABLE(n <= 14) {
641 w14 = w13;
642 }
643 const float* w15 = w14 + kc;
644 if XNN_UNPREDICTABLE(n < 16) {
645 w15 = w14;
646 }
647 const float* w16 = w15 + kc;
648 if XNN_UNPREDICTABLE(n <= 16) {
649 w16 = w15;
650 }
651 const float* w17 = w16 + kc;
652 if XNN_UNPREDICTABLE(n < 18) {
653 w17 = w16;
654 }
655 const float* w18 = w17 + kc;
656 if XNN_UNPREDICTABLE(n <= 18) {
657 w18 = w17;
658 }
659 const float* w19 = w18 + kc;
660 if XNN_UNPREDICTABLE(n < 20) {
661 w19 = w18;
662 }
663 const float* w20 = w19 + kc;
664 if XNN_UNPREDICTABLE(n <= 20) {
665 w20 = w19;
666 }
667 const float* w21 = w20 + kc;
668 if XNN_UNPREDICTABLE(n < 22) {
669 w21 = w20;
670 }
671 const float* w22 = w21 + kc;
672 if XNN_UNPREDICTABLE(n <= 22) {
673 w22 = w21;
674 }
675 const float* w23 = w22 + kc;
676 if XNN_UNPREDICTABLE(n < 24) {
677 w23 = w22;
678 }
679 const float* w24 = w23 + kc;
680 if XNN_UNPREDICTABLE(n <= 24) {
681 w24 = w23;
682 }
683 const float* w25 = w24 + kc;
684 if XNN_UNPREDICTABLE(n < 26) {
685 w25 = w24;
686 }
687 const float* w26 = w25 + kc;
688 if XNN_UNPREDICTABLE(n <= 26) {
689 w26 = w25;
690 }
691 const float* w27 = w26 + kc;
692 if XNN_UNPREDICTABLE(n < 28) {
693 w27 = w26;
694 }
695 const float* w28 = w27 + kc;
696 if XNN_UNPREDICTABLE(n <= 28) {
697 w28 = w27;
698 }
699 const float* w29 = w28 + kc;
700 if XNN_UNPREDICTABLE(n < 30) {
701 w29 = w28;
702 }
703 const float* w30 = w29 + kc;
704 if XNN_UNPREDICTABLE(n <= 30) {
705 w30 = w29;
706 }
707
708 // KC main loop multiple of 4
709 size_t k = kc;
710 for (; k >= 4; k -= 4) {
711 // Read blocks of 4x4
712 // a b c d
713 // e f g h
714 // i j k l
715 // m n o p
716 // Load first 4 rows of N into low part of each register
717 __m512 v0x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w0));
718 w0 += 4;
719 __m512 v1x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w1));
720 w1 += 4;
721 __m512 v2x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w2));
722 w2 += 4;
723 __m512 v3x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w3));
724 w3 += 4;
725 __m512 v16x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w16));
726 w16 += 4;
727 __m512 v17x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w17));
728 w17 += 4;
729 __m512 v18x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w18));
730 w18 += 4;
731 // castps leaves upper 128 bits undefined, so zero them.
732 __m512 v19x0123 = _mm512_zextps128_ps512(_mm_loadu_ps(w19));
733 w19 += 4;
734 // Load next 4 rows of N into the high part of each register
735 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w4), 1);
736 w4 += 4;
737 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w5), 1);
738 w5 += 4;
739 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w6), 1);
740 w6 += 4;
741 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w7), 1);
742 w7 += 4;
743 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w8), 2);
744 w8 += 4;
745 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w9), 2);
746 w9 += 4;
747 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w10), 2);
748 w10 += 4;
749 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w11), 2);
750 w11 += 4;
751 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w12), 3);
752 w12 += 4;
753 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w13), 3);
754 w13 += 4;
755 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w14), 3);
756 w14 += 4;
757 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w15), 3);
758 w15 += 4;
759 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w20), 1);
760 w20 += 4;
761 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w21), 1);
762 w21 += 4;
763 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w22), 1);
764 w22 += 4;
765 v19x0123 = _mm512_insertf32x4(v19x0123, _mm_loadu_ps(w23), 1);
766 w23 += 4;
767 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w24), 2);
768 w24 += 4;
769 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w25), 2);
770 w25 += 4;
771 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w26), 2);
772 w26 += 4;
773 v19x0123 = _mm512_insertf32x4(v19x0123, _mm_loadu_ps(w27), 2);
774 w27 += 4;
775 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w28), 3);
776 w28 += 4;
777 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w29), 3);
778 w29 += 4;
779 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w30), 3);
780 w30 += 4;
781
782 // Transpose 2x2
783 const __m512 vtmp0x0123 = _mm512_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
784 const __m512 vtmp1x0123 = _mm512_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
785 const __m512 vtmp2x0123 = _mm512_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
786 const __m512 vtmp3x0123 = _mm512_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
787 const __m512 vtmp16x0123 = _mm512_unpacklo_ps(v16x0123, v17x0123); // a e b f from row 0, 1
788 const __m512 vtmp17x0123 = _mm512_unpacklo_ps(v18x0123, v19x0123); // i m j n from row 2, 3
789 const __m512 vtmp18x0123 = _mm512_unpackhi_ps(v16x0123, v17x0123); // c g d h from row 0, 1
790 const __m512 vtmp19x0123 = _mm512_unpackhi_ps(v18x0123, v19x0123); // k o l p from row 2, 3
791 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
792 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
793 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
794 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
795 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
796 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
797 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
798 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
799 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
800 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
801 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
802 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
803 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
804 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
805 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
806 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
807 xnn_prefetch_to_l1((const int8_t*) w16 + 128);
808 xnn_prefetch_to_l1((const int8_t*) w17 + 128);
809 xnn_prefetch_to_l1((const int8_t*) w18 + 128);
810 xnn_prefetch_to_l1((const int8_t*) w19 + 128);
811 xnn_prefetch_to_l1((const int8_t*) w20 + 128);
812 xnn_prefetch_to_l1((const int8_t*) w21 + 128);
813 xnn_prefetch_to_l1((const int8_t*) w22 + 128);
814 xnn_prefetch_to_l1((const int8_t*) w23 + 128);
815 xnn_prefetch_to_l1((const int8_t*) w24 + 128);
816 xnn_prefetch_to_l1((const int8_t*) w25 + 128);
817 xnn_prefetch_to_l1((const int8_t*) w26 + 128);
818 xnn_prefetch_to_l1((const int8_t*) w27 + 128);
819 xnn_prefetch_to_l1((const int8_t*) w28 + 128);
820 xnn_prefetch_to_l1((const int8_t*) w29 + 128);
821 xnn_prefetch_to_l1((const int8_t*) w30 + 128);
822 // Transpose 4x4
823 v0x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
824 v1x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
825 v2x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
826 v3x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
827 v16x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp16x0123), _mm512_castps_pd(vtmp17x0123))); // a e i m from row 0, 1
828 v17x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp16x0123), _mm512_castps_pd(vtmp17x0123))); // b f j n from row 0, 1
829 v18x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp18x0123), _mm512_castps_pd(vtmp19x0123))); // c g k o from row 2, 3
830 v19x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp18x0123), _mm512_castps_pd(vtmp19x0123))); // d h l p from row 2, 3
831
832 _mm512_store_ps(packed_w, v0x0123);
833 _mm512_store_ps(packed_w + 16, v16x0123);
834 _mm512_store_ps(packed_w + 32, v1x0123);
835 _mm512_store_ps(packed_w + 48, v17x0123);
836 _mm512_store_ps(packed_w + 64, v2x0123);
837 _mm512_store_ps(packed_w + 80, v18x0123);
838 _mm512_store_ps(packed_w + 96, v3x0123);
839 _mm512_store_ps(packed_w + 112, v19x0123);
840 packed_w += 128;
841 }
842
843 // KC remainder (1..3)
844 if XNN_UNLIKELY(k != 0) {
845 assert(k >= 1);
846 assert(k <= 3);
847 if (k & 2) {
848 // Read blocks of 4x2
849 // a b
850 // c d
851 // e f
852 // g h
853 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
854 w0 += 2;
855 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
856 w1 += 2;
857 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
858 w2 += 2;
859 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
860 w3 += 2;
861 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
862 w4 += 2;
863 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
864 w5 += 2;
865 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
866 w6 += 2;
867 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
868 w7 += 2;
869 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
870 w8 += 2;
871 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
872 w9 += 2;
873 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
874 w10 += 2;
875 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
876 w11 += 2;
877 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
878 w12 += 2;
879 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
880 w13 += 2;
881 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
882 w14 += 2;
883 __m128 v15 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
884 w15 += 2;
885 __m128 v16 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w16));
886 w16 += 2;
887 __m128 v17 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w17));
888 w17 += 2;
889 __m128 v18 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w18));
890 w18 += 2;
891 __m128 v19 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w19));
892 w19 += 2;
893 __m128 v20 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w20));
894 w20 += 2;
895 __m128 v21 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w21));
896 w21 += 2;
897 __m128 v22 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w22));
898 w22 += 2;
899 __m128 v23 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w23));
900 w23 += 2;
901 __m128 v24 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w24));
902 w24 += 2;
903 __m128 v25 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w25));
904 w25 += 2;
905 __m128 v26 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w26));
906 w26 += 2;
907 __m128 v27 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w27));
908 w27 += 2;
909 __m128 v28 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w28));
910 w28 += 2;
911 __m128 v29 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w29));
912 w29 += 2;
913 __m128 v30 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w30));
914 w30 += 2;
915
916 // Transpose 2x2
917 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
918 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
919 // Transpose 4x4
920 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
921 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
922 // Transpose 2x2
923 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
924 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
925 // Transpose 4x4
926 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
927 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
928 // Transpose 2x2
929 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
930 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
931 // Transpose 4x4
932 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
933 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
934 // Transpose 2x2
935 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
936 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // e g f h from row 2, 3
937 // Transpose 4x4
938 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
939 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
940 // Transpose 2x2
941 const __m128 vtmp16 = _mm_unpacklo_ps(v16, v17); // a c b d from row 0, 1
942 const __m128 vtmp17 = _mm_unpacklo_ps(v18, v19); // e g f h from row 2, 3
943 // Transpose 4x4
944 v16 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // a c e g from row 0, 1
945 v17 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // b d f h from row 0, 1
946 // Transpose 2x2
947 const __m128 vtmp20 = _mm_unpacklo_ps(v20, v21); // a c b d from row 0, 1
948 const __m128 vtmp21 = _mm_unpacklo_ps(v22, v23); // e g f h from row 2, 3
949 // Transpose 4x4
950 v20 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // a c e g from row 0, 1
951 v21 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // b d f h from row 0, 1
952 // Transpose 2x2
953 const __m128 vtmp24 = _mm_unpacklo_ps(v24, v25); // a c b d from row 0, 1
954 const __m128 vtmp25 = _mm_unpacklo_ps(v26, v27); // e g f h from row 2, 3
955 // Transpose 4x4
956 v24 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // a c e g from row 0, 1
957 v25 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // b d f h from row 0, 1
958 // Transpose 2x2
959 const __m128 vtmp28 = _mm_unpacklo_ps(v28, v29); // a c b d from row 0, 1
960 const __m128 vtmp29 = _mm_unpacklo_ps(v30, v30); // e g f h from row 2, 3
961 // Transpose 4x4
962 v28 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // a c e g from row 0, 1
963 v29 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // b d f h from row 0, 1
964
965 _mm_store_ps(packed_w, v0);
966 _mm_store_ps(packed_w + 4, v4);
967 _mm_store_ps(packed_w + 8, v8);
968 _mm_store_ps(packed_w + 12, v12);
969 _mm_store_ps(packed_w + 16, v16);
970 _mm_store_ps(packed_w + 20, v20);
971 _mm_store_ps(packed_w + 24, v24);
972 _mm_store_ps(packed_w + 28, v28);
973 _mm_store_ps(packed_w + 32, v1);
974 _mm_store_ps(packed_w + 36, v5);
975 _mm_store_ps(packed_w + 40, v9);
976 _mm_store_ps(packed_w + 44, v13);
977 _mm_store_ps(packed_w + 48, v17);
978 _mm_store_ps(packed_w + 52, v21);
979 _mm_store_ps(packed_w + 56, v25);
980 _mm_store_ps(packed_w + 60, v29);
981 packed_w += 64;
982 }
983 if (k & 1) {
984 // Read blocks of 4x1
985 // a
986 // b
987 // c
988 // d
989 __m128 v0 = _mm_load_ss(w0);
990 w0 += 1;
991 __m128 v1 = _mm_load_ss(w1);
992 w1 += 1;
993 __m128 v2 = _mm_load_ss(w2);
994 w2 += 1;
995 __m128 v3 = _mm_load_ss(w3);
996 w3 += 1;
997 __m128 v4 = _mm_load_ss(w4);
998 w4 += 1;
999 __m128 v5 = _mm_load_ss(w5);
1000 w5 += 1;
1001 __m128 v6 = _mm_load_ss(w6);
1002 w6 += 1;
1003 __m128 v7 = _mm_load_ss(w7);
1004 w7 += 1;
1005 __m128 v8 = _mm_load_ss(w8);
1006 w8 += 1;
1007 __m128 v9 = _mm_load_ss(w9);
1008 w9 += 1;
1009 __m128 v10 = _mm_load_ss(w10);
1010 w10 += 1;
1011 __m128 v11 = _mm_load_ss(w11);
1012 w11 += 1;
1013 __m128 v12 = _mm_load_ss(w12);
1014 w12 += 1;
1015 __m128 v13 = _mm_load_ss(w13);
1016 w13 += 1;
1017 __m128 v14 = _mm_load_ss(w14);
1018 w14 += 1;
1019 __m128 v15 = _mm_load_ss(w15);
1020 w15 += 1;
1021 __m128 v16 = _mm_load_ss(w16);
1022 w16 += 1;
1023 __m128 v17 = _mm_load_ss(w17);
1024 w17 += 1;
1025 __m128 v18 = _mm_load_ss(w18);
1026 w18 += 1;
1027 __m128 v19 = _mm_load_ss(w19);
1028 w19 += 1;
1029 __m128 v20 = _mm_load_ss(w20);
1030 w20 += 1;
1031 __m128 v21 = _mm_load_ss(w21);
1032 w21 += 1;
1033 __m128 v22 = _mm_load_ss(w22);
1034 w22 += 1;
1035 __m128 v23 = _mm_load_ss(w23);
1036 w23 += 1;
1037 __m128 v24 = _mm_load_ss(w24);
1038 w24 += 1;
1039 __m128 v25 = _mm_load_ss(w25);
1040 w25 += 1;
1041 __m128 v26 = _mm_load_ss(w26);
1042 w26 += 1;
1043 __m128 v27 = _mm_load_ss(w27);
1044 w27 += 1;
1045 __m128 v28 = _mm_load_ss(w28);
1046 w28 += 1;
1047 __m128 v29 = _mm_load_ss(w29);
1048 w29 += 1;
1049 __m128 v30 = _mm_load_ss(w30);
1050 w30 += 1;
1051
1052 // Transpose 2x2
1053 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
1054 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
1055 // Transpose 4x4
1056 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
1057 // Transpose 2x2
1058 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
1059 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
1060 // Transpose 4x4
1061 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
1062 // Transpose 2x2
1063 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
1064 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
1065 // Transpose 4x4
1066 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
1067 // Transpose 2x2
1068 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
1069 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // c d from row 2, 3
1070 // Transpose 4x4
1071 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
1072 // Transpose 2x2
1073 const __m128 vtmp16 = _mm_unpacklo_ps(v16, v17); // a b from row 0, 1
1074 const __m128 vtmp17 = _mm_unpacklo_ps(v18, v19); // c d from row 2, 3
1075 // Transpose 4x4
1076 v16 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // a b c d from row 0, 1
1077 // Transpose 2x2
1078 const __m128 vtmp20 = _mm_unpacklo_ps(v20, v21); // a b from row 0, 1
1079 const __m128 vtmp21 = _mm_unpacklo_ps(v22, v23); // c d from row 2, 3
1080 // Transpose 4x4
1081 v20 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // a b c d from row 0, 1
1082 // Transpose 2x2
1083 const __m128 vtmp24 = _mm_unpacklo_ps(v24, v25); // a b from row 0, 1
1084 const __m128 vtmp25 = _mm_unpacklo_ps(v26, v27); // c d from row 2, 3
1085 // Transpose 4x4
1086 v24 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // a b c d from row 0, 1
1087 // Transpose 2x2
1088 const __m128 vtmp28 = _mm_unpacklo_ps(v28, v29); // a b from row 0, 1
1089 const __m128 vtmp29 = _mm_unpacklo_ps(v30, v30); // c d from row 2, 3
1090 // Transpose 4x4
1091 v28 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // a b c d from row 0, 1
1092
1093 _mm_store_ps(packed_w, v0);
1094 _mm_store_ps(packed_w + 4, v4);
1095 _mm_store_ps(packed_w + 8, v8);
1096 _mm_store_ps(packed_w + 12, v12);
1097 _mm_store_ps(packed_w + 16, v16);
1098 _mm_store_ps(packed_w + 20, v20);
1099 _mm_store_ps(packed_w + 24, v24);
1100 _mm_store_ps(packed_w + 28, v28);
1101 packed_w += 32;
1102 }
1103 }
1104 }
1105 weights += nc * kc;
1106 } while (--g != 0);
1107 }