Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-avx512f-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/avx512.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/intrinsics-polyfill.h"
18 #include "xnnpack/packw.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x16__avx512f_u4(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 16); // This kernel is for NR=16
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 const float* b = (const float*) bias;
45 float* packed_w = (float*) packed_weights;
46 do {
47 // NC main loop multiple of 16
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50
51 for (; n >= 16; n -= 16) {
52 if XNN_LIKELY(b != NULL) {
53 const __m512 vb0 = _mm512_loadu_ps(b);
54 _mm512_store_ps(packed_w, vb0);
55 b += 16;
56 } else {
57 const __m512 vzero = _mm512_setzero_ps();
58 _mm512_store_ps(packed_w, vzero);
59 }
60 packed_w += 16;
61
62 const float* w1 = w0 + kc;
63 const float* w2 = w1 + kc;
64 const float* w3 = w2 + kc;
65 const float* w4 = w3 + kc;
66 const float* w5 = w4 + kc;
67 const float* w6 = w5 + kc;
68 const float* w7 = w6 + kc;
69 const float* w8 = w7 + kc;
70 const float* w9 = w8 + kc;
71 const float* w10 = w9 + kc;
72 const float* w11 = w10 + kc;
73 const float* w12 = w11 + kc;
74 const float* w13 = w12 + kc;
75 const float* w14 = w13 + kc;
76 const float* w15 = w14 + kc;
77
78 // KC main loop multiple of 4
79 size_t k = kc;
80 for (; k >= 4; k -= 4) {
81 // Read blocks of 4x4
82 // a b c d
83 // e f g h
84 // i j k l
85 // m n o p
86 // Load first 4 rows of N into low part of each register
87 __m512 v0x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w0));
88 w0 += 4;
89 __m512 v1x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w1));
90 w1 += 4;
91 __m512 v2x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w2));
92 w2 += 4;
93 __m512 v3x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w3));
94 w3 += 4;
95 // Load next 4 rows of N into the high part of each register
96 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w4), 1);
97 w4 += 4;
98 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w5), 1);
99 w5 += 4;
100 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w6), 1);
101 w6 += 4;
102 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w7), 1);
103 w7 += 4;
104 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w8), 2);
105 w8 += 4;
106 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w9), 2);
107 w9 += 4;
108 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w10), 2);
109 w10 += 4;
110 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w11), 2);
111 w11 += 4;
112 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w12), 3);
113 w12 += 4;
114 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w13), 3);
115 w13 += 4;
116 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w14), 3);
117 w14 += 4;
118 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w15), 3);
119 w15 += 4;
120
121 // Transpose 2x2
122 const __m512 vtmp0x0123 = _mm512_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
123 const __m512 vtmp1x0123 = _mm512_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
124 const __m512 vtmp2x0123 = _mm512_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
125 const __m512 vtmp3x0123 = _mm512_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
126 // Transpose 4x4
127 v0x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
128 v1x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
129 v2x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
130 v3x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
131
132 _mm512_store_ps(packed_w, v0x0123);
133 _mm512_store_ps(packed_w + 16, v1x0123);
134 _mm512_store_ps(packed_w + 32, v2x0123);
135 _mm512_store_ps(packed_w + 48, v3x0123);
136 packed_w += 64;
137 }
138
139 // KC remainder (1..3)
140 if XNN_UNLIKELY(k != 0) {
141 assert(k >= 1);
142 assert(k <= 3);
143 if (k & 2) {
144 // Read blocks of 4x2
145 // a b
146 // c d
147 // e f
148 // g h
149 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
150 w0 += 2;
151 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
152 w1 += 2;
153 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
154 w2 += 2;
155 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
156 w3 += 2;
157 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
158 w4 += 2;
159 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
160 w5 += 2;
161 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
162 w6 += 2;
163 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
164 w7 += 2;
165 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
166 w8 += 2;
167 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
168 w9 += 2;
169 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
170 w10 += 2;
171 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
172 w11 += 2;
173 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
174 w12 += 2;
175 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
176 w13 += 2;
177 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
178 w14 += 2;
179 __m128 v15 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
180 w15 += 2;
181
182 // Transpose 2x2
183 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
184 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
185 // Transpose 4x4
186 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
187 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
188 // Transpose 2x2
189 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
190 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
191 // Transpose 4x4
192 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
193 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
194 // Transpose 2x2
195 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
196 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
197 // Transpose 4x4
198 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
199 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
200 // Transpose 2x2
201 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
202 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // e g f h from row 2, 3
203 // Transpose 4x4
204 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
205 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
206
207 _mm_store_ps(packed_w, v0);
208 _mm_store_ps(packed_w + 4, v4);
209 _mm_store_ps(packed_w + 8, v8);
210 _mm_store_ps(packed_w + 12, v12);
211 _mm_store_ps(packed_w + 16, v1);
212 _mm_store_ps(packed_w + 20, v5);
213 _mm_store_ps(packed_w + 24, v9);
214 _mm_store_ps(packed_w + 28, v13);
215 packed_w += 32;
216 }
217 if (k & 1) {
218 // Read blocks of 4x1
219 // a
220 // b
221 // c
222 // d
223 __m128 v0 = _mm_load_ss(w0);
224 w0 += 1;
225 __m128 v1 = _mm_load_ss(w1);
226 w1 += 1;
227 __m128 v2 = _mm_load_ss(w2);
228 w2 += 1;
229 __m128 v3 = _mm_load_ss(w3);
230 w3 += 1;
231 __m128 v4 = _mm_load_ss(w4);
232 w4 += 1;
233 __m128 v5 = _mm_load_ss(w5);
234 w5 += 1;
235 __m128 v6 = _mm_load_ss(w6);
236 w6 += 1;
237 __m128 v7 = _mm_load_ss(w7);
238 w7 += 1;
239 __m128 v8 = _mm_load_ss(w8);
240 w8 += 1;
241 __m128 v9 = _mm_load_ss(w9);
242 w9 += 1;
243 __m128 v10 = _mm_load_ss(w10);
244 w10 += 1;
245 __m128 v11 = _mm_load_ss(w11);
246 w11 += 1;
247 __m128 v12 = _mm_load_ss(w12);
248 w12 += 1;
249 __m128 v13 = _mm_load_ss(w13);
250 w13 += 1;
251 __m128 v14 = _mm_load_ss(w14);
252 w14 += 1;
253 __m128 v15 = _mm_load_ss(w15);
254 w15 += 1;
255
256 // Transpose 2x2
257 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
258 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
259 // Transpose 4x4
260 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
261 // Transpose 2x2
262 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
263 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
264 // Transpose 4x4
265 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
266 // Transpose 2x2
267 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
268 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
269 // Transpose 4x4
270 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
271 // Transpose 2x2
272 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
273 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // c d from row 2, 3
274 // Transpose 4x4
275 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
276
277 _mm_store_ps(packed_w, v0);
278 _mm_store_ps(packed_w + 4, v4);
279 _mm_store_ps(packed_w + 8, v8);
280 _mm_store_ps(packed_w + 12, v12);
281 packed_w += 16;
282 }
283 }
284 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
285 w0 = w15;
286 }
287
288 // NC remainder (1..15)
289 if XNN_UNLIKELY(n != 0) {
290 assert(n >= 1);
291 assert(n <= 15);
292 if XNN_LIKELY(b != NULL) {
293 size_t nb = n;
294 do {
295 *packed_w++ = *b++;
296 } while (--nb != 0);
297 packed_w += (16 - n);
298 } else {
299 const __m512 vzero = _mm512_setzero_ps();
300 _mm512_store_ps(packed_w, vzero);
301 packed_w += 16;
302 }
303
304 // NR remainder has less than 16 rows so last row is not loaded
305 // For SR=4 the
306 const float* w1 = w0 + kc;
307 if XNN_UNPREDICTABLE(n < 2) {
308 w1 = w0;
309 }
310 const float* w2 = w1 + kc;
311 if XNN_UNPREDICTABLE(n <= 2) {
312 w2 = w1;
313 }
314 const float* w3 = w2 + kc;
315 if XNN_UNPREDICTABLE(n < 4) {
316 w3 = w2;
317 }
318 const float* w4 = w3 + kc;
319 if XNN_UNPREDICTABLE(n <= 4) {
320 w4 = w3;
321 }
322 const float* w5 = w4 + kc;
323 if XNN_UNPREDICTABLE(n < 6) {
324 w5 = w4;
325 }
326 const float* w6 = w5 + kc;
327 if XNN_UNPREDICTABLE(n <= 6) {
328 w6 = w5;
329 }
330 const float* w7 = w6 + kc;
331 if XNN_UNPREDICTABLE(n < 8) {
332 w7 = w6;
333 }
334 const float* w8 = w7 + kc;
335 if XNN_UNPREDICTABLE(n <= 8) {
336 w8 = w7;
337 }
338 const float* w9 = w8 + kc;
339 if XNN_UNPREDICTABLE(n < 10) {
340 w9 = w8;
341 }
342 const float* w10 = w9 + kc;
343 if XNN_UNPREDICTABLE(n <= 10) {
344 w10 = w9;
345 }
346 const float* w11 = w10 + kc;
347 if XNN_UNPREDICTABLE(n < 12) {
348 w11 = w10;
349 }
350 const float* w12 = w11 + kc;
351 if XNN_UNPREDICTABLE(n <= 12) {
352 w12 = w11;
353 }
354 const float* w13 = w12 + kc;
355 if XNN_UNPREDICTABLE(n < 14) {
356 w13 = w12;
357 }
358 const float* w14 = w13 + kc;
359 if XNN_UNPREDICTABLE(n <= 14) {
360 w14 = w13;
361 }
362
363 // KC main loop multiple of 4
364 size_t k = kc;
365 for (; k >= 4; k -= 4) {
366 // Read blocks of 4x4
367 // a b c d
368 // e f g h
369 // i j k l
370 // m n o p
371 // Load first 4 rows of N into low part of each register
372 __m512 v0x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w0));
373 w0 += 4;
374 __m512 v1x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w1));
375 w1 += 4;
376 __m512 v2x0123 = _mm512_castps128_ps512(_mm_loadu_ps(w2));
377 w2 += 4;
378 // castps leaves upper 128 bits undefined, so zero them.
379 __m512 v3x0123 = _mm512_zextps128_ps512(_mm_loadu_ps(w3));
380 w3 += 4;
381 // Load next 4 rows of N into the high part of each register
382 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w4), 1);
383 w4 += 4;
384 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w5), 1);
385 w5 += 4;
386 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w6), 1);
387 w6 += 4;
388 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w7), 1);
389 w7 += 4;
390 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w8), 2);
391 w8 += 4;
392 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w9), 2);
393 w9 += 4;
394 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w10), 2);
395 w10 += 4;
396 v3x0123 = _mm512_insertf32x4(v3x0123, _mm_loadu_ps(w11), 2);
397 w11 += 4;
398 v0x0123 = _mm512_insertf32x4(v0x0123, _mm_loadu_ps(w12), 3);
399 w12 += 4;
400 v1x0123 = _mm512_insertf32x4(v1x0123, _mm_loadu_ps(w13), 3);
401 w13 += 4;
402 v2x0123 = _mm512_insertf32x4(v2x0123, _mm_loadu_ps(w14), 3);
403 w14 += 4;
404
405 // Transpose 2x2
406 const __m512 vtmp0x0123 = _mm512_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
407 const __m512 vtmp1x0123 = _mm512_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
408 const __m512 vtmp2x0123 = _mm512_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
409 const __m512 vtmp3x0123 = _mm512_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
410 // Transpose 4x4
411 v0x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
412 v1x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp0x0123), _mm512_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
413 v2x0123 = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
414 v3x0123 = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(vtmp2x0123), _mm512_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
415
416 _mm512_store_ps(packed_w, v0x0123);
417 _mm512_store_ps(packed_w + 16, v1x0123);
418 _mm512_store_ps(packed_w + 32, v2x0123);
419 _mm512_store_ps(packed_w + 48, v3x0123);
420 packed_w += 64;
421 }
422
423 // KC remainder (1..3)
424 if XNN_UNLIKELY(k != 0) {
425 assert(k >= 1);
426 assert(k <= 3);
427 if (k & 2) {
428 // Read blocks of 4x2
429 // a b
430 // c d
431 // e f
432 // g h
433 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
434 w0 += 2;
435 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
436 w1 += 2;
437 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
438 w2 += 2;
439 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
440 w3 += 2;
441 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
442 w4 += 2;
443 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
444 w5 += 2;
445 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
446 w6 += 2;
447 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
448 w7 += 2;
449 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
450 w8 += 2;
451 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
452 w9 += 2;
453 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
454 w10 += 2;
455 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
456 w11 += 2;
457 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
458 w12 += 2;
459 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
460 w13 += 2;
461 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
462 w14 += 2;
463
464 // Transpose 2x2
465 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
466 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
467 // Transpose 4x4
468 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
469 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
470 // Transpose 2x2
471 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
472 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
473 // Transpose 4x4
474 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
475 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
476 // Transpose 2x2
477 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
478 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
479 // Transpose 4x4
480 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
481 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
482 // Transpose 2x2
483 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
484 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // e g f h from row 2, 3
485 // Transpose 4x4
486 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
487 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
488
489 _mm_store_ps(packed_w, v0);
490 _mm_store_ps(packed_w + 4, v4);
491 _mm_store_ps(packed_w + 8, v8);
492 _mm_store_ps(packed_w + 12, v12);
493 _mm_store_ps(packed_w + 16, v1);
494 _mm_store_ps(packed_w + 20, v5);
495 _mm_store_ps(packed_w + 24, v9);
496 _mm_store_ps(packed_w + 28, v13);
497 packed_w += 32;
498 }
499 if (k & 1) {
500 // Read blocks of 4x1
501 // a
502 // b
503 // c
504 // d
505 __m128 v0 = _mm_load_ss(w0);
506 w0 += 1;
507 __m128 v1 = _mm_load_ss(w1);
508 w1 += 1;
509 __m128 v2 = _mm_load_ss(w2);
510 w2 += 1;
511 __m128 v3 = _mm_load_ss(w3);
512 w3 += 1;
513 __m128 v4 = _mm_load_ss(w4);
514 w4 += 1;
515 __m128 v5 = _mm_load_ss(w5);
516 w5 += 1;
517 __m128 v6 = _mm_load_ss(w6);
518 w6 += 1;
519 __m128 v7 = _mm_load_ss(w7);
520 w7 += 1;
521 __m128 v8 = _mm_load_ss(w8);
522 w8 += 1;
523 __m128 v9 = _mm_load_ss(w9);
524 w9 += 1;
525 __m128 v10 = _mm_load_ss(w10);
526 w10 += 1;
527 __m128 v11 = _mm_load_ss(w11);
528 w11 += 1;
529 __m128 v12 = _mm_load_ss(w12);
530 w12 += 1;
531 __m128 v13 = _mm_load_ss(w13);
532 w13 += 1;
533 __m128 v14 = _mm_load_ss(w14);
534 w14 += 1;
535
536 // Transpose 2x2
537 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
538 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
539 // Transpose 4x4
540 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
541 // Transpose 2x2
542 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
543 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
544 // Transpose 4x4
545 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
546 // Transpose 2x2
547 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
548 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
549 // Transpose 4x4
550 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
551 // Transpose 2x2
552 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
553 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // c d from row 2, 3
554 // Transpose 4x4
555 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
556
557 _mm_store_ps(packed_w, v0);
558 _mm_store_ps(packed_w + 4, v4);
559 _mm_store_ps(packed_w + 8, v8);
560 _mm_store_ps(packed_w + 12, v12);
561 packed_w += 16;
562 }
563 }
564 }
565 weights += nc * kc;
566 } while (--g != 0);
567 }