Debian Codegraph @

src/x32-packw/gen/x32-packw-x32-gemm-goi-avx512f-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/avx512.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/intrinsics-polyfill.h"
18 #include "xnnpack/packw.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x32__avx512f_u4(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 32); // This kernel is for NR=32
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 const float* b = (const float*) bias;
45 float* packed_w = (float*) packed_weights;
46 do {
47 // NC main loop multiple of 32
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50
51 for (; n >= 32; n -= 32) {
52 if XNN_LIKELY(b != NULL) {
53 const __m512 vb0 = _mm512_loadu_ps(b);
54 const __m512 vb16 = _mm512_loadu_ps(b + 16);
55 _mm512_store_ps(packed_w, vb0);
56 _mm512_store_ps(packed_w + 16, vb16);
57 b += 32;
58 } else {
59 const __m512 vzero = _mm512_setzero_ps();
60 _mm512_store_ps(packed_w, vzero);
61 _mm512_store_ps(packed_w + 16, vzero);
62 }
63 packed_w += 32;
64
65 const float* w1 = w0 + kc;
66 const float* w2 = w1 + kc;
67 const float* w3 = w2 + kc;
68 const float* w4 = w3 + kc;
69 const float* w5 = w4 + kc;
70 const float* w6 = w5 + kc;
71 const float* w7 = w6 + kc;
72 const float* w8 = w7 + kc;
73 const float* w9 = w8 + kc;
74 const float* w10 = w9 + kc;
75 const float* w11 = w10 + kc;
76 const float* w12 = w11 + kc;
77 const float* w13 = w12 + kc;
78 const float* w14 = w13 + kc;
79 const float* w15 = w14 + kc;
80 const float* w16 = w15 + kc;
81 const float* w17 = w16 + kc;
82 const float* w18 = w17 + kc;
83 const float* w19 = w18 + kc;
84 const float* w20 = w19 + kc;
85 const float* w21 = w20 + kc;
86 const float* w22 = w21 + kc;
87 const float* w23 = w22 + kc;
88 const float* w24 = w23 + kc;
89 const float* w25 = w24 + kc;
90 const float* w26 = w25 + kc;
91 const float* w27 = w26 + kc;
92 const float* w28 = w27 + kc;
93 const float* w29 = w28 + kc;
94 const float* w30 = w29 + kc;
95 const float* w31 = w30 + kc;
96
97 // KC main loop multiple of 4
98 size_t k = kc;
99 for (; k >= 4; k -= 4) {
100 // Read blocks of 4x4
101 // a b c d
102 // e f g h
103 // i j k l
104 // m n o p
105 // Load first 4 rows of N into low part of each register
106 __m512 v0x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w0));
107 w0 += 4;
108 __m512 v1x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w1));
109 w1 += 4;
110 __m512 v2x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w2));
111 w2 += 4;
112 __m512 v3x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w3));
113 w3 += 4;
114 __m512 v16x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w16));
115 w16 += 4;
116 __m512 v17x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w17));
117 w17 += 4;
118 __m512 v18x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w18));
119 w18 += 4;
120 __m512 v19x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w19));
121 w19 += 4;
122 // Load next 4 rows of N into the high part of each register
123 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w4), 1);
124 w4 += 4;
125 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w5), 1);
126 w5 += 4;
127 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w6), 1);
128 w6 += 4;
129 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w7), 1);
130 w7 += 4;
131 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w8), 2);
132 w8 += 4;
133 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w9), 2);
134 w9 += 4;
135 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w10), 2);
136 w10 += 4;
137 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w11), 2);
138 w11 += 4;
139 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w12), 3);
140 w12 += 4;
141 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w13), 3);
142 w13 += 4;
143 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w14), 3);
144 w14 += 4;
145 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w15), 3);
146 w15 += 4;
147 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w20), 1);
148 w20 += 4;
149 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w21), 1);
150 w21 += 4;
151 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w22), 1);
152 w22 += 4;
153 v19x0123 = _mm512_insertf32x4(v19x0123, _mm_loadu_ps(w23), 1);
154 w23 += 4;
155 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w24), 2);
156 w24 += 4;
157 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w25), 2);
158 w25 += 4;
159 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w26), 2);
160 w26 += 4;
161 v19x0123 = _mm512_insertf32x4(v19x0123, _mm_loadu_ps(w27), 2);
162 w27 += 4;
163 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w28), 3);
164 w28 += 4;
165 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w29), 3);
166 w29 += 4;
167 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w30), 3);
168 w30 += 4;
169 v19x0123 = _mm512_insertf32x4(v19x0123, _mm_loadu_ps(w31), 3);
170 w31 += 4;
171
172 // Transpose 2x2
173 const __m512 vtmp0x0123 = _mm512_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
174 const __m512 vtmp1x0123 = _mm512_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
175 const __m512 vtmp2x0123 = _mm512_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
176 const __m512 vtmp3x0123 = _mm512_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
177 const __m512 vtmp16x0123 = _mm512_unpacklo_ps(v16x0123, v17x0123); // a e b f from row 0, 1
178 const __m512 vtmp17x0123 = _mm512_unpacklo_ps(v18x0123, v19x0123); // i m j n from row 2, 3
179 const __m512 vtmp18x0123 = _mm512_unpackhi_ps(v16x0123, v17x0123); // c g d h from row 0, 1
180 const __m512 vtmp19x0123 = _mm512_unpackhi_ps(v18x0123, v19x0123); // k o l p from row 2, 3
181 // Transpose 4x4
182 v0x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
183 v1x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
184 v2x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
185 v3x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
186 v16x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp16x0123), _mm512_castps_pd(vtmp17x0123))); // a e i m from row 0, 1
187 v17x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp16x0123), _mm512_castps_pd(vtmp17x0123))); // b f j n from row 0, 1
188 v18x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp18x0123), _mm512_castps_pd(vtmp19x0123))); // c g k o from row 2, 3
189 v19x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp18x0123), _mm512_castps_pd(vtmp19x0123))); // d h l p from row 2, 3
190
191 _mm512_store_ps(packed_w, v0x0123);
192 _mm512_store_ps(packed_w + 16, v16x0123);
193 _mm512_store_ps(packed_w + 32, v1x0123);
194 _mm512_store_ps(packed_w + 48, v17x0123);
195 _mm512_store_ps(packed_w + 64, v2x0123);
196 _mm512_store_ps(packed_w + 80, v18x0123);
197 _mm512_store_ps(packed_w + 96, v3x0123);
198 _mm512_store_ps(packed_w + 112, v19x0123);
199 packed_w += 128;
200 }
201
202 // KC remainder (1..3)
203 if XNN_UNLIKELY(k != 0) {
204 assert(k >= 1);
205 assert(k <= 3);
206 if (k & 2) {
207 // Read blocks of 4x2
208 // a b
209 // c d
210 // e f
211 // g h
212 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
213 w0 += 2;
214 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
215 w1 += 2;
216 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
217 w2 += 2;
218 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
219 w3 += 2;
220 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
221 w4 += 2;
222 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
223 w5 += 2;
224 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
225 w6 += 2;
226 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
227 w7 += 2;
228 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
229 w8 += 2;
230 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
231 w9 += 2;
232 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
233 w10 += 2;
234 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
235 w11 += 2;
236 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
237 w12 += 2;
238 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
239 w13 += 2;
240 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
241 w14 += 2;
242 __m128 v15 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
243 w15 += 2;
244 __m128 v16 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w16));
245 w16 += 2;
246 __m128 v17 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w17));
247 w17 += 2;
248 __m128 v18 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w18));
249 w18 += 2;
250 __m128 v19 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w19));
251 w19 += 2;
252 __m128 v20 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w20));
253 w20 += 2;
254 __m128 v21 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w21));
255 w21 += 2;
256 __m128 v22 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w22));
257 w22 += 2;
258 __m128 v23 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w23));
259 w23 += 2;
260 __m128 v24 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w24));
261 w24 += 2;
262 __m128 v25 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w25));
263 w25 += 2;
264 __m128 v26 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w26));
265 w26 += 2;
266 __m128 v27 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w27));
267 w27 += 2;
268 __m128 v28 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w28));
269 w28 += 2;
270 __m128 v29 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w29));
271 w29 += 2;
272 __m128 v30 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w30));
273 w30 += 2;
274 __m128 v31 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w31));
275 w31 += 2;
276
277 // Transpose 2x2
278 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
279 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
280 // Transpose 4x4
281 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
282 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
283 // Transpose 2x2
284 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
285 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
286 // Transpose 4x4
287 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
288 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
289 // Transpose 2x2
290 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
291 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
292 // Transpose 4x4
293 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
294 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
295 // Transpose 2x2
296 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
297 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // e g f h from row 2, 3
298 // Transpose 4x4
299 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
300 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
301 // Transpose 2x2
302 const __m128 vtmp16 = _mm_unpacklo_ps(v16, v17); // a c b d from row 0, 1
303 const __m128 vtmp17 = _mm_unpacklo_ps(v18, v19); // e g f h from row 2, 3
304 // Transpose 4x4
305 v16 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // a c e g from row 0, 1
306 v17 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // b d f h from row 0, 1
307 // Transpose 2x2
308 const __m128 vtmp20 = _mm_unpacklo_ps(v20, v21); // a c b d from row 0, 1
309 const __m128 vtmp21 = _mm_unpacklo_ps(v22, v23); // e g f h from row 2, 3
310 // Transpose 4x4
311 v20 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // a c e g from row 0, 1
312 v21 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // b d f h from row 0, 1
313 // Transpose 2x2
314 const __m128 vtmp24 = _mm_unpacklo_ps(v24, v25); // a c b d from row 0, 1
315 const __m128 vtmp25 = _mm_unpacklo_ps(v26, v27); // e g f h from row 2, 3
316 // Transpose 4x4
317 v24 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // a c e g from row 0, 1
318 v25 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // b d f h from row 0, 1
319 // Transpose 2x2
320 const __m128 vtmp28 = _mm_unpacklo_ps(v28, v29); // a c b d from row 0, 1
321 const __m128 vtmp29 = _mm_unpacklo_ps(v30, v31); // e g f h from row 2, 3
322 // Transpose 4x4
323 v28 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // a c e g from row 0, 1
324 v29 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // b d f h from row 0, 1
325
326 _mm_store_ps(packed_w, v0);
327 _mm_store_ps(packed_w + 4, v4);
328 _mm_store_ps(packed_w + 8, v8);
329 _mm_store_ps(packed_w + 12, v12);
330 _mm_store_ps(packed_w + 16, v16);
331 _mm_store_ps(packed_w + 20, v20);
332 _mm_store_ps(packed_w + 24, v24);
333 _mm_store_ps(packed_w + 28, v28);
334 _mm_store_ps(packed_w + 32, v1);
335 _mm_store_ps(packed_w + 36, v5);
336 _mm_store_ps(packed_w + 40, v9);
337 _mm_store_ps(packed_w + 44, v13);
338 _mm_store_ps(packed_w + 48, v17);
339 _mm_store_ps(packed_w + 52, v21);
340 _mm_store_ps(packed_w + 56, v25);
341 _mm_store_ps(packed_w + 60, v29);
342 packed_w += 64;
343 }
344 if (k & 1) {
345 // Read blocks of 4x1
346 // a
347 // b
348 // c
349 // d
350 __m128 v0 = _mm_load_ss(w0);
351 w0 += 1;
352 __m128 v1 = _mm_load_ss(w1);
353 w1 += 1;
354 __m128 v2 = _mm_load_ss(w2);
355 w2 += 1;
356 __m128 v3 = _mm_load_ss(w3);
357 w3 += 1;
358 __m128 v4 = _mm_load_ss(w4);
359 w4 += 1;
360 __m128 v5 = _mm_load_ss(w5);
361 w5 += 1;
362 __m128 v6 = _mm_load_ss(w6);
363 w6 += 1;
364 __m128 v7 = _mm_load_ss(w7);
365 w7 += 1;
366 __m128 v8 = _mm_load_ss(w8);
367 w8 += 1;
368 __m128 v9 = _mm_load_ss(w9);
369 w9 += 1;
370 __m128 v10 = _mm_load_ss(w10);
371 w10 += 1;
372 __m128 v11 = _mm_load_ss(w11);
373 w11 += 1;
374 __m128 v12 = _mm_load_ss(w12);
375 w12 += 1;
376 __m128 v13 = _mm_load_ss(w13);
377 w13 += 1;
378 __m128 v14 = _mm_load_ss(w14);
379 w14 += 1;
380 __m128 v15 = _mm_load_ss(w15);
381 w15 += 1;
382 __m128 v16 = _mm_load_ss(w16);
383 w16 += 1;
384 __m128 v17 = _mm_load_ss(w17);
385 w17 += 1;
386 __m128 v18 = _mm_load_ss(w18);
387 w18 += 1;
388 __m128 v19 = _mm_load_ss(w19);
389 w19 += 1;
390 __m128 v20 = _mm_load_ss(w20);
391 w20 += 1;
392 __m128 v21 = _mm_load_ss(w21);
393 w21 += 1;
394 __m128 v22 = _mm_load_ss(w22);
395 w22 += 1;
396 __m128 v23 = _mm_load_ss(w23);
397 w23 += 1;
398 __m128 v24 = _mm_load_ss(w24);
399 w24 += 1;
400 __m128 v25 = _mm_load_ss(w25);
401 w25 += 1;
402 __m128 v26 = _mm_load_ss(w26);
403 w26 += 1;
404 __m128 v27 = _mm_load_ss(w27);
405 w27 += 1;
406 __m128 v28 = _mm_load_ss(w28);
407 w28 += 1;
408 __m128 v29 = _mm_load_ss(w29);
409 w29 += 1;
410 __m128 v30 = _mm_load_ss(w30);
411 w30 += 1;
412 __m128 v31 = _mm_load_ss(w31);
413 w31 += 1;
414
415 // Transpose 2x2
416 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
417 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
418 // Transpose 4x4
419 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
420 // Transpose 2x2
421 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
422 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
423 // Transpose 4x4
424 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
425 // Transpose 2x2
426 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
427 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
428 // Transpose 4x4
429 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
430 // Transpose 2x2
431 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
432 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // c d from row 2, 3
433 // Transpose 4x4
434 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
435 // Transpose 2x2
436 const __m128 vtmp16 = _mm_unpacklo_ps(v16, v17); // a b from row 0, 1
437 const __m128 vtmp17 = _mm_unpacklo_ps(v18, v19); // c d from row 2, 3
438 // Transpose 4x4
439 v16 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // a b c d from row 0, 1
440 // Transpose 2x2
441 const __m128 vtmp20 = _mm_unpacklo_ps(v20, v21); // a b from row 0, 1
442 const __m128 vtmp21 = _mm_unpacklo_ps(v22, v23); // c d from row 2, 3
443 // Transpose 4x4
444 v20 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // a b c d from row 0, 1
445 // Transpose 2x2
446 const __m128 vtmp24 = _mm_unpacklo_ps(v24, v25); // a b from row 0, 1
447 const __m128 vtmp25 = _mm_unpacklo_ps(v26, v27); // c d from row 2, 3
448 // Transpose 4x4
449 v24 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // a b c d from row 0, 1
450 // Transpose 2x2
451 const __m128 vtmp28 = _mm_unpacklo_ps(v28, v29); // a b from row 0, 1
452 const __m128 vtmp29 = _mm_unpacklo_ps(v30, v31); // c d from row 2, 3
453 // Transpose 4x4
454 v28 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // a b c d from row 0, 1
455
456 _mm_store_ps(packed_w, v0);
457 _mm_store_ps(packed_w + 4, v4);
458 _mm_store_ps(packed_w + 8, v8);
459 _mm_store_ps(packed_w + 12, v12);
460 _mm_store_ps(packed_w + 16, v16);
461 _mm_store_ps(packed_w + 20, v20);
462 _mm_store_ps(packed_w + 24, v24);
463 _mm_store_ps(packed_w + 28, v28);
464 packed_w += 32;
465 }
466 }
467 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
468 w0 = w31;
469 }
470
471 // NC remainder (1..31)
472 if XNN_UNLIKELY(n != 0) {
473 assert(n >= 1);
474 assert(n <= 31);
475 if XNN_LIKELY(b != NULL) {
476 size_t nb = n;
477 do {
478 *packed_w++ = *b++;
479 } while (--nb != 0);
480 packed_w += (32 - n);
481 } else {
482 const __m512 vzero = _mm512_setzero_ps();
483 _mm512_store_ps(packed_w, vzero);
484 _mm512_store_ps(packed_w + 16, vzero);
485 packed_w += 32;
486 }
487
488 // NR remainder has less than 32 rows so last row is not loaded
489 // For SR=4 the
490 const float* w1 = w0 + kc;
491 if XNN_UNPREDICTABLE(n < 2) {
492 w1 = w0;
493 }
494 const float* w2 = w1 + kc;
495 if XNN_UNPREDICTABLE(n <= 2) {
496 w2 = w1;
497 }
498 const float* w3 = w2 + kc;
499 if XNN_UNPREDICTABLE(n < 4) {
500 w3 = w2;
501 }
502 const float* w4 = w3 + kc;
503 if XNN_UNPREDICTABLE(n <= 4) {
504 w4 = w3;
505 }
506 const float* w5 = w4 + kc;
507 if XNN_UNPREDICTABLE(n < 6) {
508 w5 = w4;
509 }
510 const float* w6 = w5 + kc;
511 if XNN_UNPREDICTABLE(n <= 6) {
512 w6 = w5;
513 }
514 const float* w7 = w6 + kc;
515 if XNN_UNPREDICTABLE(n < 8) {
516 w7 = w6;
517 }
518 const float* w8 = w7 + kc;
519 if XNN_UNPREDICTABLE(n <= 8) {
520 w8 = w7;
521 }
522 const float* w9 = w8 + kc;
523 if XNN_UNPREDICTABLE(n < 10) {
524 w9 = w8;
525 }
526 const float* w10 = w9 + kc;
527 if XNN_UNPREDICTABLE(n <= 10) {
528 w10 = w9;
529 }
530 const float* w11 = w10 + kc;
531 if XNN_UNPREDICTABLE(n < 12) {
532 w11 = w10;
533 }
534 const float* w12 = w11 + kc;
535 if XNN_UNPREDICTABLE(n <= 12) {
536 w12 = w11;
537 }
538 const float* w13 = w12 + kc;
539 if XNN_UNPREDICTABLE(n < 14) {
540 w13 = w12;
541 }
542 const float* w14 = w13 + kc;
543 if XNN_UNPREDICTABLE(n <= 14) {
544 w14 = w13;
545 }
546 const float* w15 = w14 + kc;
547 if XNN_UNPREDICTABLE(n < 16) {
548 w15 = w14;
549 }
550 const float* w16 = w15 + kc;
551 if XNN_UNPREDICTABLE(n <= 16) {
552 w16 = w15;
553 }
554 const float* w17 = w16 + kc;
555 if XNN_UNPREDICTABLE(n < 18) {
556 w17 = w16;
557 }
558 const float* w18 = w17 + kc;
559 if XNN_UNPREDICTABLE(n <= 18) {
560 w18 = w17;
561 }
562 const float* w19 = w18 + kc;
563 if XNN_UNPREDICTABLE(n < 20) {
564 w19 = w18;
565 }
566 const float* w20 = w19 + kc;
567 if XNN_UNPREDICTABLE(n <= 20) {
568 w20 = w19;
569 }
570 const float* w21 = w20 + kc;
571 if XNN_UNPREDICTABLE(n < 22) {
572 w21 = w20;
573 }
574 const float* w22 = w21 + kc;
575 if XNN_UNPREDICTABLE(n <= 22) {
576 w22 = w21;
577 }
578 const float* w23 = w22 + kc;
579 if XNN_UNPREDICTABLE(n < 24) {
580 w23 = w22;
581 }
582 const float* w24 = w23 + kc;
583 if XNN_UNPREDICTABLE(n <= 24) {
584 w24 = w23;
585 }
586 const float* w25 = w24 + kc;
587 if XNN_UNPREDICTABLE(n < 26) {
588 w25 = w24;
589 }
590 const float* w26 = w25 + kc;
591 if XNN_UNPREDICTABLE(n <= 26) {
592 w26 = w25;
593 }
594 const float* w27 = w26 + kc;
595 if XNN_UNPREDICTABLE(n < 28) {
596 w27 = w26;
597 }
598 const float* w28 = w27 + kc;
599 if XNN_UNPREDICTABLE(n <= 28) {
600 w28 = w27;
601 }
602 const float* w29 = w28 + kc;
603 if XNN_UNPREDICTABLE(n < 30) {
604 w29 = w28;
605 }
606 const float* w30 = w29 + kc;
607 if XNN_UNPREDICTABLE(n <= 30) {
608 w30 = w29;
609 }
610
611 // KC main loop multiple of 4
612 size_t k = kc;
613 for (; k >= 4; k -= 4) {
614 // Read blocks of 4x4
615 // a b c d
616 // e f g h
617 // i j k l
618 // m n o p
619 // Load first 4 rows of N into low part of each register
620 __m512 v0x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w0));
621 w0 += 4;
622 __m512 v1x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w1));
623 w1 += 4;
624 __m512 v2x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w2));
625 w2 += 4;
626 __m512 v3x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w3));
627 w3 += 4;
628 __m512 v16x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w16));
629 w16 += 4;
630 __m512 v17x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w17));
631 w17 += 4;
632 __m512 v18x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w18));
633 w18 += 4;
634 // castps leaves upper 128 bits undefined, so zero them.
635 __m512 v19x0123 = _mm512_zextps128_ps512(_mm_loadu_ps(w19));
636 w19 += 4;
637 // Load next 4 rows of N into the high part of each register
638 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w4), 1);
639 w4 += 4;
640 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w5), 1);
641 w5 += 4;
642 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w6), 1);
643 w6 += 4;
644 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w7), 1);
645 w7 += 4;
646 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w8), 2);
647 w8 += 4;
648 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w9), 2);
649 w9 += 4;
650 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w10), 2);
651 w10 += 4;
652 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w11), 2);
653 w11 += 4;
654 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w12), 3);
655 w12 += 4;
656 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w13), 3);
657 w13 += 4;
658 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w14), 3);
659 w14 += 4;
660 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w15), 3);
661 w15 += 4;
662 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w20), 1);
663 w20 += 4;
664 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w21), 1);
665 w21 += 4;
666 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w22), 1);
667 w22 += 4;
668 v19x0123 = _mm512_insertf32x4(v19x0123, _mm_loadu_ps(w23), 1);
669 w23 += 4;
670 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w24), 2);
671 w24 += 4;
672 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w25), 2);
673 w25 += 4;
674 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w26), 2);
675 w26 += 4;
676 v19x0123 = _mm512_insertf32x4(v19x0123, _mm_loadu_ps(w27), 2);
677 w27 += 4;
678 v16x0123 = _mm512_insertf32x4(v16x0123, _mm_loadu_ps(w28), 3);
679 w28 += 4;
680 v17x0123 = _mm512_insertf32x4(v17x0123, _mm_loadu_ps(w29), 3);
681 w29 += 4;
682 v18x0123 = _mm512_insertf32x4(v18x0123, _mm_loadu_ps(w30), 3);
683 w30 += 4;
684
685 // Transpose 2x2
686 const __m512 vtmp0x0123 = _mm512_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
687 const __m512 vtmp1x0123 = _mm512_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
688 const __m512 vtmp2x0123 = _mm512_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
689 const __m512 vtmp3x0123 = _mm512_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
690 const __m512 vtmp16x0123 = _mm512_unpacklo_ps(v16x0123, v17x0123); // a e b f from row 0, 1
691 const __m512 vtmp17x0123 = _mm512_unpacklo_ps(v18x0123, v19x0123); // i m j n from row 2, 3
692 const __m512 vtmp18x0123 = _mm512_unpackhi_ps(v16x0123, v17x0123); // c g d h from row 0, 1
693 const __m512 vtmp19x0123 = _mm512_unpackhi_ps(v18x0123, v19x0123); // k o l p from row 2, 3
694 // Transpose 4x4
695 v0x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
696 v1x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
697 v2x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
698 v3x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
699 v16x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp16x0123), _mm512_castps_pd(vtmp17x0123))); // a e i m from row 0, 1
700 v17x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp16x0123), _mm512_castps_pd(vtmp17x0123))); // b f j n from row 0, 1
701 v18x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp18x0123), _mm512_castps_pd(vtmp19x0123))); // c g k o from row 2, 3
702 v19x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp18x0123), _mm512_castps_pd(vtmp19x0123))); // d h l p from row 2, 3
703
704 _mm512_store_ps(packed_w, v0x0123);
705 _mm512_store_ps(packed_w + 16, v16x0123);
706 _mm512_store_ps(packed_w + 32, v1x0123);
707 _mm512_store_ps(packed_w + 48, v17x0123);
708 _mm512_store_ps(packed_w + 64, v2x0123);
709 _mm512_store_ps(packed_w + 80, v18x0123);
710 _mm512_store_ps(packed_w + 96, v3x0123);
711 _mm512_store_ps(packed_w + 112, v19x0123);
712 packed_w += 128;
713 }
714
715 // KC remainder (1..3)
716 if XNN_UNLIKELY(k != 0) {
717 assert(k >= 1);
718 assert(k <= 3);
719 if (k & 2) {
720 // Read blocks of 4x2
721 // a b
722 // c d
723 // e f
724 // g h
725 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
726 w0 += 2;
727 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
728 w1 += 2;
729 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
730 w2 += 2;
731 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
732 w3 += 2;
733 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
734 w4 += 2;
735 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
736 w5 += 2;
737 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
738 w6 += 2;
739 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
740 w7 += 2;
741 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
742 w8 += 2;
743 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
744 w9 += 2;
745 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
746 w10 += 2;
747 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
748 w11 += 2;
749 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
750 w12 += 2;
751 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
752 w13 += 2;
753 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
754 w14 += 2;
755 __m128 v15 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
756 w15 += 2;
757 __m128 v16 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w16));
758 w16 += 2;
759 __m128 v17 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w17));
760 w17 += 2;
761 __m128 v18 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w18));
762 w18 += 2;
763 __m128 v19 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w19));
764 w19 += 2;
765 __m128 v20 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w20));
766 w20 += 2;
767 __m128 v21 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w21));
768 w21 += 2;
769 __m128 v22 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w22));
770 w22 += 2;
771 __m128 v23 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w23));
772 w23 += 2;
773 __m128 v24 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w24));
774 w24 += 2;
775 __m128 v25 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w25));
776 w25 += 2;
777 __m128 v26 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w26));
778 w26 += 2;
779 __m128 v27 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w27));
780 w27 += 2;
781 __m128 v28 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w28));
782 w28 += 2;
783 __m128 v29 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w29));
784 w29 += 2;
785 __m128 v30 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w30));
786 w30 += 2;
787
788 // Transpose 2x2
789 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
790 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
791 // Transpose 4x4
792 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
793 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
794 // Transpose 2x2
795 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
796 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
797 // Transpose 4x4
798 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
799 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
800 // Transpose 2x2
801 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
802 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
803 // Transpose 4x4
804 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
805 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
806 // Transpose 2x2
807 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
808 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // e g f h from row 2, 3
809 // Transpose 4x4
810 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
811 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
812 // Transpose 2x2
813 const __m128 vtmp16 = _mm_unpacklo_ps(v16, v17); // a c b d from row 0, 1
814 const __m128 vtmp17 = _mm_unpacklo_ps(v18, v19); // e g f h from row 2, 3
815 // Transpose 4x4
816 v16 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // a c e g from row 0, 1
817 v17 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // b d f h from row 0, 1
818 // Transpose 2x2
819 const __m128 vtmp20 = _mm_unpacklo_ps(v20, v21); // a c b d from row 0, 1
820 const __m128 vtmp21 = _mm_unpacklo_ps(v22, v23); // e g f h from row 2, 3
821 // Transpose 4x4
822 v20 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // a c e g from row 0, 1
823 v21 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // b d f h from row 0, 1
824 // Transpose 2x2
825 const __m128 vtmp24 = _mm_unpacklo_ps(v24, v25); // a c b d from row 0, 1
826 const __m128 vtmp25 = _mm_unpacklo_ps(v26, v27); // e g f h from row 2, 3
827 // Transpose 4x4
828 v24 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // a c e g from row 0, 1
829 v25 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // b d f h from row 0, 1
830 // Transpose 2x2
831 const __m128 vtmp28 = _mm_unpacklo_ps(v28, v29); // a c b d from row 0, 1
832 const __m128 vtmp29 = _mm_unpacklo_ps(v30, v30); // e g f h from row 2, 3
833 // Transpose 4x4
834 v28 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // a c e g from row 0, 1
835 v29 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // b d f h from row 0, 1
836
837 _mm_store_ps(packed_w, v0);
838 _mm_store_ps(packed_w + 4, v4);
839 _mm_store_ps(packed_w + 8, v8);
840 _mm_store_ps(packed_w + 12, v12);
841 _mm_store_ps(packed_w + 16, v16);
842 _mm_store_ps(packed_w + 20, v20);
843 _mm_store_ps(packed_w + 24, v24);
844 _mm_store_ps(packed_w + 28, v28);
845 _mm_store_ps(packed_w + 32, v1);
846 _mm_store_ps(packed_w + 36, v5);
847 _mm_store_ps(packed_w + 40, v9);
848 _mm_store_ps(packed_w + 44, v13);
849 _mm_store_ps(packed_w + 48, v17);
850 _mm_store_ps(packed_w + 52, v21);
851 _mm_store_ps(packed_w + 56, v25);
852 _mm_store_ps(packed_w + 60, v29);
853 packed_w += 64;
854 }
855 if (k & 1) {
856 // Read blocks of 4x1
857 // a
858 // b
859 // c
860 // d
861 __m128 v0 = _mm_load_ss(w0);
862 w0 += 1;
863 __m128 v1 = _mm_load_ss(w1);
864 w1 += 1;
865 __m128 v2 = _mm_load_ss(w2);
866 w2 += 1;
867 __m128 v3 = _mm_load_ss(w3);
868 w3 += 1;
869 __m128 v4 = _mm_load_ss(w4);
870 w4 += 1;
871 __m128 v5 = _mm_load_ss(w5);
872 w5 += 1;
873 __m128 v6 = _mm_load_ss(w6);
874 w6 += 1;
875 __m128 v7 = _mm_load_ss(w7);
876 w7 += 1;
877 __m128 v8 = _mm_load_ss(w8);
878 w8 += 1;
879 __m128 v9 = _mm_load_ss(w9);
880 w9 += 1;
881 __m128 v10 = _mm_load_ss(w10);
882 w10 += 1;
883 __m128 v11 = _mm_load_ss(w11);
884 w11 += 1;
885 __m128 v12 = _mm_load_ss(w12);
886 w12 += 1;
887 __m128 v13 = _mm_load_ss(w13);
888 w13 += 1;
889 __m128 v14 = _mm_load_ss(w14);
890 w14 += 1;
891 __m128 v15 = _mm_load_ss(w15);
892 w15 += 1;
893 __m128 v16 = _mm_load_ss(w16);
894 w16 += 1;
895 __m128 v17 = _mm_load_ss(w17);
896 w17 += 1;
897 __m128 v18 = _mm_load_ss(w18);
898 w18 += 1;
899 __m128 v19 = _mm_load_ss(w19);
900 w19 += 1;
901 __m128 v20 = _mm_load_ss(w20);
902 w20 += 1;
903 __m128 v21 = _mm_load_ss(w21);
904 w21 += 1;
905 __m128 v22 = _mm_load_ss(w22);
906 w22 += 1;
907 __m128 v23 = _mm_load_ss(w23);
908 w23 += 1;
909 __m128 v24 = _mm_load_ss(w24);
910 w24 += 1;
911 __m128 v25 = _mm_load_ss(w25);
912 w25 += 1;
913 __m128 v26 = _mm_load_ss(w26);
914 w26 += 1;
915 __m128 v27 = _mm_load_ss(w27);
916 w27 += 1;
917 __m128 v28 = _mm_load_ss(w28);
918 w28 += 1;
919 __m128 v29 = _mm_load_ss(w29);
920 w29 += 1;
921 __m128 v30 = _mm_load_ss(w30);
922 w30 += 1;
923
924 // Transpose 2x2
925 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
926 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
927 // Transpose 4x4
928 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
929 // Transpose 2x2
930 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
931 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
932 // Transpose 4x4
933 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
934 // Transpose 2x2
935 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
936 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
937 // Transpose 4x4
938 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
939 // Transpose 2x2
940 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
941 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // c d from row 2, 3
942 // Transpose 4x4
943 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
944 // Transpose 2x2
945 const __m128 vtmp16 = _mm_unpacklo_ps(v16, v17); // a b from row 0, 1
946 const __m128 vtmp17 = _mm_unpacklo_ps(v18, v19); // c d from row 2, 3
947 // Transpose 4x4
948 v16 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp16), _mm_castps_pd(vtmp17))); // a b c d from row 0, 1
949 // Transpose 2x2
950 const __m128 vtmp20 = _mm_unpacklo_ps(v20, v21); // a b from row 0, 1
951 const __m128 vtmp21 = _mm_unpacklo_ps(v22, v23); // c d from row 2, 3
952 // Transpose 4x4
953 v20 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp20), _mm_castps_pd(vtmp21))); // a b c d from row 0, 1
954 // Transpose 2x2
955 const __m128 vtmp24 = _mm_unpacklo_ps(v24, v25); // a b from row 0, 1
956 const __m128 vtmp25 = _mm_unpacklo_ps(v26, v27); // c d from row 2, 3
957 // Transpose 4x4
958 v24 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp24), _mm_castps_pd(vtmp25))); // a b c d from row 0, 1
959 // Transpose 2x2
960 const __m128 vtmp28 = _mm_unpacklo_ps(v28, v29); // a b from row 0, 1
961 const __m128 vtmp29 = _mm_unpacklo_ps(v30, v30); // c d from row 2, 3
962 // Transpose 4x4
963 v28 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp28), _mm_castps_pd(vtmp29))); // a b c d from row 0, 1
964
965 _mm_store_ps(packed_w, v0);
966 _mm_store_ps(packed_w + 4, v4);
967 _mm_store_ps(packed_w + 8, v8);
968 _mm_store_ps(packed_w + 12, v12);
969 _mm_store_ps(packed_w + 16, v16);
970 _mm_store_ps(packed_w + 20, v20);
971 _mm_store_ps(packed_w + 24, v24);
972 _mm_store_ps(packed_w + 28, v28);
973 packed_w += 32;
974 }
975 }
976 }
977 weights += nc * kc;
978 } while (--g != 0);
979 }