Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-sse2-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x8s4__sse2_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 8);
38 assert(kr == 1);
39 assert(sr == 4);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 const float* b = (const float*) bias;
44 float* packed_w = (float*) packed_weights;
45 do {
46 // NC main loop multiple of 8
47 const float* w0 = (const float*) weights;
48 size_t n = nc;
49
50 for (; n >= 8; n -= 8) {
51 if XNN_LIKELY(b != NULL) {
52 const __m128 vb0 = _mm_loadu_ps(b);
53 const __m128 vb4 = _mm_loadu_ps(b + 4);
54 _mm_store_ps(packed_w, vb0);
55 _mm_store_ps(packed_w + 4, vb4);
56 b += 8;
57 } else {
58 const __m128 vzero = _mm_setzero_ps();
59 _mm_store_ps(packed_w, vzero);
60 _mm_store_ps(packed_w + 4, vzero);
61 }
62 packed_w += 8;
63
64 const float* w1 = w0 + kc;
65 const float* w2 = w1 + kc;
66 const float* w3 = w2 + kc;
67 const float* w4 = w3 + kc;
68 const float* w5 = w4 + kc;
69 const float* w6 = w5 + kc;
70 const float* w7 = w6 + kc;
71
72 size_t k = kc;
73
74 // KC multiple of 4
75 for (; k >= 4; k -= 4) {
76 // Read blocks of 4x4
77 // a b c d
78 // e f g h
79 // i j k l
80 // m n o p
81 __m128 v0x0123 = _mm_loadu_ps(w0);
82 w0 += 4;
83 __m128 v1x0123 = _mm_loadu_ps(w1);
84 w1 += 4;
85 __m128 v2x0123 = _mm_loadu_ps(w2);
86 w2 += 4;
87 __m128 v3x0123 = _mm_loadu_ps(w3);
88 w3 += 4;
89 __m128 v4x0123 = _mm_loadu_ps(w4);
90 w4 += 4;
91 __m128 v5x0123 = _mm_loadu_ps(w5);
92 w5 += 4;
93 __m128 v6x0123 = _mm_loadu_ps(w6);
94 w6 += 4;
95 __m128 v7x0123 = _mm_loadu_ps(w7);
96 w7 += 4;
97
98 // Apply SR4 shuffle
99 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
100 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
101 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
102 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
103 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
104 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
105
106 // Transpose 2x2
107 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
108 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
109 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
110 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
111 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
112 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
113 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
114 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
115 // Transpose 4x4
116 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
117 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
118 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
119 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
120 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
121 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
122 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
123 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
124
125 _mm_store_ps(packed_w, v0x0123);
126 _mm_store_ps(packed_w + 4, v4x0123);
127 _mm_store_ps(packed_w + 8, v1x0123);
128 _mm_store_ps(packed_w + 12, v5x0123);
129 _mm_store_ps(packed_w + 16, v2x0123);
130 _mm_store_ps(packed_w + 20, v6x0123);
131 _mm_store_ps(packed_w + 24, v3x0123);
132 _mm_store_ps(packed_w + 28, v7x0123);
133 packed_w += 32;
134 }
135
136 // KC remainder (1..3)
137 if XNN_UNLIKELY(k != 0) {
138 assert(k >= 1);
139 assert(k <= 3);
140 __m128 v0 = _mm_undefined_ps();
141 __m128 v1 = _mm_undefined_ps();
142 __m128 v2 = _mm_undefined_ps();
143 __m128 v3 = _mm_undefined_ps();
144 __m128 v4 = _mm_undefined_ps();
145 __m128 v5 = _mm_undefined_ps();
146 __m128 v6 = _mm_undefined_ps();
147 __m128 v7 = _mm_undefined_ps();
148
149 switch (k) {
150 case 1:
151 // Read blocks of 4x1
152 // a
153 // e
154 // i
155 // m
156 v0 = _mm_load_ss(w0);
157 w0 += 1;
158 v1 = _mm_load_ss(w1);
159 w1 += 1;
160 v2 = _mm_load_ss(w2);
161 w2 += 1;
162 v3 = _mm_load_ss(w3);
163 w3 += 1;
164 v4 = _mm_load_ss(w4);
165 w4 += 1;
166 v5 = _mm_load_ss(w5);
167 w5 += 1;
168 v6 = _mm_load_ss(w6);
169 w6 += 1;
170 v7 = _mm_load_ss(w7);
171 w7 += 1;
172 break;
173 case 2:
174 // Read blocks of 4x2
175 // a b
176 // e f
177 // i j
178 // m n
179 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
180 w0 += 2;
181 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
182 w1 += 2;
183 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
184 w2 += 2;
185 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
186 w3 += 2;
187 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
188 w4 += 2;
189 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
190 w5 += 2;
191 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
192 w6 += 2;
193 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
194 w7 += 2;
195 break;
196 case 3:
197 {
198 // Read blocks of 4x3
199 // a b c
200 // e f g
201 // i j k
202 // m n o
203 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
204 const __m128 v0hi = _mm_load_ss(w0 + 2);
205 v0 = _mm_movelh_ps(v0lo, v0hi);
206 w0 += 3;
207 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
208 const __m128 v1hi = _mm_load_ss(w1 + 2);
209 v1 = _mm_movelh_ps(v1lo, v1hi);
210 w1 += 3;
211 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
212 const __m128 v2hi = _mm_load_ss(w2 + 2);
213 v2 = _mm_movelh_ps(v2lo, v2hi);
214 w2 += 3;
215 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
216 const __m128 v3hi = _mm_load_ss(w3 + 2);
217 v3 = _mm_movelh_ps(v3lo, v3hi);
218 w3 += 3;
219 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
220 const __m128 v4hi = _mm_load_ss(w4 + 2);
221 v4 = _mm_movelh_ps(v4lo, v4hi);
222 w4 += 3;
223 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
224 const __m128 v5hi = _mm_load_ss(w5 + 2);
225 v5 = _mm_movelh_ps(v5lo, v5hi);
226 w5 += 3;
227 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
228 const __m128 v6hi = _mm_load_ss(w6 + 2);
229 v6 = _mm_movelh_ps(v6lo, v6hi);
230 w6 += 3;
231 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
232 const __m128 v7hi = _mm_load_ss(w7 + 2);
233 v7 = _mm_movelh_ps(v7lo, v7hi);
234 w7 += 3;
235 break;
236 }
237 default:
238 XNN_UNREACHABLE;
239 }
240
241 // Apply SR4 shuffle
242 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
243 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
244 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
245 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
246 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
247 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
248 // Transpose 2x2
249 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
250 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
251 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
252 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
253 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
254 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
255 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
256 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
257 // Transpose 4x4
258 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
259 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
260 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
261 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
262 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
263 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
264 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
265 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
266 _mm_store_ps(packed_w, v0);
267 _mm_store_ps(packed_w + 4, v4);
268 _mm_store_ps(packed_w + 8, v1);
269 _mm_store_ps(packed_w + 12, v5);
270 _mm_store_ps(packed_w + 16, v2);
271 _mm_store_ps(packed_w + 20, v6);
272 _mm_store_ps(packed_w + 24, v3);
273 _mm_store_ps(packed_w + 28, v7);
274 packed_w += 32;
275 }
276 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
277 w0 = w7;
278 }
279
280 // NC remainder (1..7)
281 if XNN_UNLIKELY(n != 0) {
282 assert(n >= 1);
283 assert(n <= 7);
284 if XNN_LIKELY(b != NULL) {
285 size_t nb = n;
286 do {
287 *packed_w++ = *b++;
288 } while (--nb != 0);
289 packed_w += (8 - n);
290 } else {
291 const __m128 vzero = _mm_setzero_ps();
292 _mm_store_ps(packed_w, vzero);
293 _mm_store_ps(packed_w + 4, vzero);
294 packed_w += 8;
295 }
296
297 // NR remainder has less than 8 rows so last row is not loaded
298 const float* w1 = w0 + kc;
299 if XNN_UNPREDICTABLE(n < 2) {
300 w1 = w0;
301 }
302 const float* w2 = w1 + kc;
303 if XNN_UNPREDICTABLE(n <= 2) {
304 w2 = w1;
305 }
306 const float* w3 = w2 + kc;
307 if XNN_UNPREDICTABLE(n < 4) {
308 w3 = w2;
309 }
310 const float* w4 = w3 + kc;
311 if XNN_UNPREDICTABLE(n <= 4) {
312 w4 = w3;
313 }
314 const float* w5 = w4 + kc;
315 if XNN_UNPREDICTABLE(n < 6) {
316 w5 = w4;
317 }
318 const float* w6 = w5 + kc;
319 if XNN_UNPREDICTABLE(n <= 6) {
320 w6 = w5;
321 }
322
323 size_t k = kc;
324
325 // KC multiple of 4
326 for (; k >= 4; k -= 4) {
327 // Read blocks of 4x4
328 // a b c d
329 // e f g h
330 // i j k l
331 // m n o p
332 __m128 v0x0123 = _mm_loadu_ps(w0);
333 w0 += 4;
334 __m128 v1x0123 = _mm_loadu_ps(w1);
335 w1 += 4;
336 __m128 v2x0123 = _mm_loadu_ps(w2);
337 w2 += 4;
338 __m128 v3x0123 = _mm_loadu_ps(w3);
339 w3 += 4;
340 __m128 v4x0123 = _mm_loadu_ps(w4);
341 w4 += 4;
342 __m128 v5x0123 = _mm_loadu_ps(w5);
343 w5 += 4;
344 __m128 v6x0123 = _mm_loadu_ps(w6);
345 w6 += 4;
346 __m128 v7x0123 = _mm_undefined_ps();
347
348 // Apply SR4 shuffle
349 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
350 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
351 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
352 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
353 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
354
355 // Transpose 2x2
356 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
357 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
358 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
359 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
360 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
361 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v6x0123); // i m j n from row 2, 3
362 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
363 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v6x0123); // k o l p from row 2, 3
364 // Transpose 4x4
365 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
366 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
367 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
368 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
369 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
370 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
371 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
372 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
373
374 _mm_store_ps(packed_w, v0x0123);
375 _mm_store_ps(packed_w + 4, v4x0123);
376 _mm_store_ps(packed_w + 8, v1x0123);
377 _mm_store_ps(packed_w + 12, v5x0123);
378 _mm_store_ps(packed_w + 16, v2x0123);
379 _mm_store_ps(packed_w + 20, v6x0123);
380 _mm_store_ps(packed_w + 24, v3x0123);
381 _mm_store_ps(packed_w + 28, v7x0123);
382 packed_w += 32;
383 }
384
385 // KC remainder (1..3)
386 if XNN_UNLIKELY(k != 0) {
387 assert(k >= 1);
388 assert(k <= 3);
389 __m128 v0 = _mm_undefined_ps();
390 __m128 v1 = _mm_undefined_ps();
391 __m128 v2 = _mm_undefined_ps();
392 __m128 v3 = _mm_undefined_ps();
393 __m128 v4 = _mm_undefined_ps();
394 __m128 v5 = _mm_undefined_ps();
395 __m128 v6 = _mm_undefined_ps();
396 __m128 v7 = _mm_undefined_ps();
397
398 switch (k) {
399 case 1:
400 // Read blocks of 4x1
401 // a
402 // e
403 // i
404 // m
405 v0 = _mm_load_ss(w0);
406 w0 += 1;
407 v1 = _mm_load_ss(w1);
408 w1 += 1;
409 v2 = _mm_load_ss(w2);
410 w2 += 1;
411 v3 = _mm_load_ss(w3);
412 w3 += 1;
413 v4 = _mm_load_ss(w4);
414 w4 += 1;
415 v5 = _mm_load_ss(w5);
416 w5 += 1;
417 v6 = _mm_load_ss(w6);
418 w6 += 1;
419 break;
420 case 2:
421 // Read blocks of 4x2
422 // a b
423 // e f
424 // i j
425 // m n
426 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
427 w0 += 2;
428 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
429 w1 += 2;
430 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
431 w2 += 2;
432 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
433 w3 += 2;
434 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
435 w4 += 2;
436 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
437 w5 += 2;
438 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
439 w6 += 2;
440 break;
441 case 3:
442 {
443 // Read blocks of 4x3
444 // a b c
445 // e f g
446 // i j k
447 // m n o
448 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
449 const __m128 v0hi = _mm_load_ss(w0 + 2);
450 v0 = _mm_movelh_ps(v0lo, v0hi);
451 w0 += 3;
452 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
453 const __m128 v1hi = _mm_load_ss(w1 + 2);
454 v1 = _mm_movelh_ps(v1lo, v1hi);
455 w1 += 3;
456 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
457 const __m128 v2hi = _mm_load_ss(w2 + 2);
458 v2 = _mm_movelh_ps(v2lo, v2hi);
459 w2 += 3;
460 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
461 const __m128 v3hi = _mm_load_ss(w3 + 2);
462 v3 = _mm_movelh_ps(v3lo, v3hi);
463 w3 += 3;
464 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
465 const __m128 v4hi = _mm_load_ss(w4 + 2);
466 v4 = _mm_movelh_ps(v4lo, v4hi);
467 w4 += 3;
468 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
469 const __m128 v5hi = _mm_load_ss(w5 + 2);
470 v5 = _mm_movelh_ps(v5lo, v5hi);
471 w5 += 3;
472 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
473 const __m128 v6hi = _mm_load_ss(w6 + 2);
474 v6 = _mm_movelh_ps(v6lo, v6hi);
475 w6 += 3;
476 break;
477 }
478 default:
479 XNN_UNREACHABLE;
480 }
481
482 // Apply SR4 shuffle
483 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
484 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
485 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
486 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
487 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
488 // Transpose 2x2
489 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
490 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
491 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
492 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
493 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
494 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v6); // i m j n from row 2, 3
495 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
496 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v6); // k o l p from row 2, 3
497 // Transpose 4x4
498 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
499 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
500 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
501 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
502 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
503 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
504 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
505 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
506 _mm_store_ps(packed_w, v0);
507 _mm_store_ps(packed_w + 4, v4);
508 _mm_store_ps(packed_w + 8, v1);
509 _mm_store_ps(packed_w + 12, v5);
510 _mm_store_ps(packed_w + 16, v2);
511 _mm_store_ps(packed_w + 20, v6);
512 _mm_store_ps(packed_w + 24, v3);
513 _mm_store_ps(packed_w + 28, v7);
514 packed_w += 32;
515 }
516 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
517 }
518 weights += nc * kc;
519 } while (--g != 0);
520 }