Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-avx-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x8s4__avx_u4_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8);
39 assert(kr == 1);
40 assert(sr == 4);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 const float* b = (const float*) bias;
45 float* packed_w = (float*) packed_weights;
46 do {
47 // NC main loop multiple of 8
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50
51 for (; n >= 8; n -= 8) {
52 if XNN_LIKELY(b != NULL) {
53 const __m256 vb0 = _mm256_loadu_ps(b);
54 _mm256_store_ps(packed_w, vb0);
55 b += 8;
56 } else {
57 const __m256 vzero = _mm256_setzero_ps();
58 _mm256_store_ps(packed_w, vzero);
59 }
60 packed_w += 8;
61
62 const float* w1 = w0 + kc;
63 const float* w2 = w1 + kc;
64 const float* w3 = w2 + kc;
65 const float* w4 = w3 + kc;
66 const float* w5 = w4 + kc;
67 const float* w6 = w5 + kc;
68 const float* w7 = w6 + kc;
69 xnn_prefetch_to_l1((const int8_t*) w0);
70 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
71 xnn_prefetch_to_l1((const int8_t*) w1);
72 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
73 xnn_prefetch_to_l1((const int8_t*) w2);
74 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
75 xnn_prefetch_to_l1((const int8_t*) w3);
76 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
77 xnn_prefetch_to_l1((const int8_t*) w4);
78 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
79 xnn_prefetch_to_l1((const int8_t*) w5);
80 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
81 xnn_prefetch_to_l1((const int8_t*) w6);
82 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
83 xnn_prefetch_to_l1((const int8_t*) w7);
84 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
85
86 // KC main loop multiple of 4
87 size_t k = kc;
88 for (; k >= 4; k -= 4) {
89 // Read blocks of 4x4
90 // a b c d
91 // e f g h
92 // i j k l
93 // m n o p
94 // Load first 4 rows of N into low part of each register
95 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
96 w0 += 4;
97 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
98 w1 += 4;
99 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
100 w2 += 4;
101 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
102 w3 += 4;
103 // Load next 4 rows of N into the high part of each register
104 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
105 w4 += 4;
106 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
107 w5 += 4;
108 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
109 w6 += 4;
110 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
111 w7 += 4;
112
113 // Apply SR4 shuffle
114 v1x0123 = _mm256_permute_ps(v1x0123, _MM_SHUFFLE(0, 3, 2, 1));
115 v2x0123 = _mm256_permute_ps(v2x0123, _MM_SHUFFLE(1, 0, 3, 2));
116 v3x0123 = _mm256_permute_ps(v3x0123, _MM_SHUFFLE(2, 1, 0, 3));
117
118 // Transpose 2x2
119 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
120 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
121 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
122 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
123 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
124 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
125 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
126 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
127 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
128 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
129 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
130 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
131 // Transpose 4x4
132 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
133 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
134 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
135 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
136
137 _mm256_store_ps(packed_w, v0x0123);
138 _mm256_store_ps(packed_w + 8, v1x0123);
139 _mm256_store_ps(packed_w + 16, v2x0123);
140 _mm256_store_ps(packed_w + 24, v3x0123);
141 packed_w += 32;
142 }
143
144 // KC remainder (1..3)
145 if XNN_UNLIKELY(k != 0) {
146 assert(k >= 1);
147 assert(k <= 3);
148 __m128 v0 = _mm_undefined_ps();
149 __m128 v1 = _mm_undefined_ps();
150 __m128 v2 = _mm_undefined_ps();
151 __m128 v3 = _mm_undefined_ps();
152 __m128 v4 = _mm_undefined_ps();
153 __m128 v5 = _mm_undefined_ps();
154 __m128 v6 = _mm_undefined_ps();
155 __m128 v7 = _mm_undefined_ps();
156
157 switch (k) {
158 case 1:
159 // Read blocks of 4x1
160 // a
161 // e
162 // i
163 // m
164 v0 = _mm_load_ss(w0);
165 w0 += 1;
166 v1 = _mm_load_ss(w1);
167 w1 += 1;
168 v2 = _mm_load_ss(w2);
169 w2 += 1;
170 v3 = _mm_load_ss(w3);
171 w3 += 1;
172 v4 = _mm_load_ss(w4);
173 w4 += 1;
174 v5 = _mm_load_ss(w5);
175 w5 += 1;
176 v6 = _mm_load_ss(w6);
177 w6 += 1;
178 v7 = _mm_load_ss(w7);
179 w7 += 1;
180 break;
181 case 2:
182 // Read blocks of 4x2
183 // a b
184 // e f
185 // i j
186 // m n
187 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
188 w0 += 2;
189 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
190 w1 += 2;
191 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
192 w2 += 2;
193 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
194 w3 += 2;
195 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
196 w4 += 2;
197 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
198 w5 += 2;
199 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
200 w6 += 2;
201 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
202 w7 += 2;
203 break;
204 case 3:
205 {
206 // Read blocks of 4x3
207 // a b c
208 // e f g
209 // i j k
210 // m n o
211 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
212 const __m128 v0hi = _mm_load_ss(w0 + 2);
213 v0 = _mm_movelh_ps(v0lo, v0hi);
214 w0 += 3;
215 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
216 const __m128 v1hi = _mm_load_ss(w1 + 2);
217 v1 = _mm_movelh_ps(v1lo, v1hi);
218 w1 += 3;
219 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
220 const __m128 v2hi = _mm_load_ss(w2 + 2);
221 v2 = _mm_movelh_ps(v2lo, v2hi);
222 w2 += 3;
223 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
224 const __m128 v3hi = _mm_load_ss(w3 + 2);
225 v3 = _mm_movelh_ps(v3lo, v3hi);
226 w3 += 3;
227 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
228 const __m128 v4hi = _mm_load_ss(w4 + 2);
229 v4 = _mm_movelh_ps(v4lo, v4hi);
230 w4 += 3;
231 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
232 const __m128 v5hi = _mm_load_ss(w5 + 2);
233 v5 = _mm_movelh_ps(v5lo, v5hi);
234 w5 += 3;
235 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
236 const __m128 v6hi = _mm_load_ss(w6 + 2);
237 v6 = _mm_movelh_ps(v6lo, v6hi);
238 w6 += 3;
239 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
240 const __m128 v7hi = _mm_load_ss(w7 + 2);
241 v7 = _mm_movelh_ps(v7lo, v7hi);
242 w7 += 3;
243 break;
244 }
245 default:
246 XNN_UNREACHABLE;
247 }
248
249 // Apply SR4 shuffle
250 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
251 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
252 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
253 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
254 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
255 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
256 // Transpose 2x2
257 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
258 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
259 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
260 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
261 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
262 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
263 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
264 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
265 // Transpose 4x4
266 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
267 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
268 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
269 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
270 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
271 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
272 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
273 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
274 _mm_store_ps(packed_w, v0);
275 _mm_store_ps(packed_w + 4, v4);
276 _mm_store_ps(packed_w + 8, v1);
277 _mm_store_ps(packed_w + 12, v5);
278 _mm_store_ps(packed_w + 16, v2);
279 _mm_store_ps(packed_w + 20, v6);
280 _mm_store_ps(packed_w + 24, v3);
281 _mm_store_ps(packed_w + 28, v7);
282 packed_w += 32;
283 }
284 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
285 w0 = w7;
286 }
287
288 // NC remainder (1..7)
289 if XNN_UNLIKELY(n != 0) {
290 assert(n >= 1);
291 assert(n <= 7);
292 if XNN_LIKELY(b != NULL) {
293 size_t nb = n;
294 do {
295 *packed_w++ = *b++;
296 } while (--nb != 0);
297 packed_w += (8 - n);
298 } else {
299 const __m128 vzero = _mm_setzero_ps();
300 _mm_store_ps(packed_w, vzero);
301 _mm_store_ps(packed_w + 4, vzero);
302 packed_w += 8;
303 }
304
305 // NR remainder has less than 8 rows so last row is not loaded
306 const float* w1 = w0 + kc;
307 if XNN_UNPREDICTABLE(n < 2) {
308 w1 = w0;
309 }
310 const float* w2 = w1 + kc;
311 if XNN_UNPREDICTABLE(n <= 2) {
312 w2 = w1;
313 }
314 const float* w3 = w2 + kc;
315 if XNN_UNPREDICTABLE(n < 4) {
316 w3 = w2;
317 }
318 const float* w4 = w3 + kc;
319 if XNN_UNPREDICTABLE(n <= 4) {
320 w4 = w3;
321 }
322 const float* w5 = w4 + kc;
323 if XNN_UNPREDICTABLE(n < 6) {
324 w5 = w4;
325 }
326 const float* w6 = w5 + kc;
327 if XNN_UNPREDICTABLE(n <= 6) {
328 w6 = w5;
329 }
330
331 // KC main loop multiple of 4
332 size_t k = kc;
333 for (; k >= 4; k -= 4) {
334 // Read blocks of 4x4
335 // a b c d
336 // e f g h
337 // i j k l
338 // m n o p
339 // Load first 4 rows of N into low part of each register
340 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
341 w0 += 4;
342 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
343 w1 += 4;
344 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
345 w2 += 4;
346 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
347 w3 += 4;
348 // Load next 4 rows of N into the high part of each register
349 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
350 w4 += 4;
351 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
352 w5 += 4;
353 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
354 w6 += 4;
355 // Apply SR4 shuffle
356 v1x0123 = _mm256_permute_ps(v1x0123, _MM_SHUFFLE(0, 3, 2, 1));
357 v2x0123 = _mm256_permute_ps(v2x0123, _MM_SHUFFLE(1, 0, 3, 2));
358 v3x0123 = _mm256_permute_ps(v3x0123, _MM_SHUFFLE(2, 1, 0, 3));
359 // Transpose 2x2
360 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
361 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
362 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
363 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
364 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
365 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
366 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
367 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
368 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
369 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
370 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
371 // Transpose 4x4
372 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
373 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
374 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
375 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
376
377 _mm256_store_ps(packed_w, v0x0123);
378 _mm256_store_ps(packed_w + 8, v1x0123);
379 _mm256_store_ps(packed_w + 16, v2x0123);
380 _mm256_store_ps(packed_w + 24, v3x0123);
381 packed_w += 32;
382 }
383
384 // KC remainder (1..3)
385 if XNN_UNLIKELY(k != 0) {
386 assert(k >= 1);
387 assert(k <= 3);
388 __m128 v0 = _mm_undefined_ps();
389 __m128 v1 = _mm_undefined_ps();
390 __m128 v2 = _mm_undefined_ps();
391 __m128 v3 = _mm_undefined_ps();
392 __m128 v4 = _mm_undefined_ps();
393 __m128 v5 = _mm_undefined_ps();
394 __m128 v6 = _mm_undefined_ps();
395 __m128 v7 = _mm_undefined_ps();
396
397 switch (k) {
398 case 1:
399 // Read blocks of 4x1
400 // a
401 // e
402 // i
403 // m
404 v0 = _mm_load_ss(w0);
405 w0 += 1;
406 v1 = _mm_load_ss(w1);
407 w1 += 1;
408 v2 = _mm_load_ss(w2);
409 w2 += 1;
410 v3 = _mm_load_ss(w3);
411 w3 += 1;
412 v4 = _mm_load_ss(w4);
413 w4 += 1;
414 v5 = _mm_load_ss(w5);
415 w5 += 1;
416 v6 = _mm_load_ss(w6);
417 w6 += 1;
418 break;
419 case 2:
420 // Read blocks of 4x2
421 // a b
422 // e f
423 // i j
424 // m n
425 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
426 w0 += 2;
427 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
428 w1 += 2;
429 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
430 w2 += 2;
431 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
432 w3 += 2;
433 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
434 w4 += 2;
435 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
436 w5 += 2;
437 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
438 w6 += 2;
439 break;
440 case 3:
441 {
442 // Read blocks of 4x3
443 // a b c
444 // e f g
445 // i j k
446 // m n o
447 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
448 const __m128 v0hi = _mm_load_ss(w0 + 2);
449 v0 = _mm_movelh_ps(v0lo, v0hi);
450 w0 += 3;
451 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
452 const __m128 v1hi = _mm_load_ss(w1 + 2);
453 v1 = _mm_movelh_ps(v1lo, v1hi);
454 w1 += 3;
455 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
456 const __m128 v2hi = _mm_load_ss(w2 + 2);
457 v2 = _mm_movelh_ps(v2lo, v2hi);
458 w2 += 3;
459 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
460 const __m128 v3hi = _mm_load_ss(w3 + 2);
461 v3 = _mm_movelh_ps(v3lo, v3hi);
462 w3 += 3;
463 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
464 const __m128 v4hi = _mm_load_ss(w4 + 2);
465 v4 = _mm_movelh_ps(v4lo, v4hi);
466 w4 += 3;
467 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
468 const __m128 v5hi = _mm_load_ss(w5 + 2);
469 v5 = _mm_movelh_ps(v5lo, v5hi);
470 w5 += 3;
471 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
472 const __m128 v6hi = _mm_load_ss(w6 + 2);
473 v6 = _mm_movelh_ps(v6lo, v6hi);
474 w6 += 3;
475 break;
476 }
477 default:
478 XNN_UNREACHABLE;
479 }
480
481 // Apply SR4 shuffle
482 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
483 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
484 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
485 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
486 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
487 // Transpose 2x2
488 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
489 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
490 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
491 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
492 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
493 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v6); // i m j n from row 2, 3
494 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
495 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v6); // k o l p from row 2, 3
496 // Transpose 4x4
497 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
498 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
499 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
500 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
501 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
502 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
503 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
504 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
505 _mm_store_ps(packed_w, v0);
506 _mm_store_ps(packed_w + 4, v4);
507 _mm_store_ps(packed_w + 8, v1);
508 _mm_store_ps(packed_w + 12, v5);
509 _mm_store_ps(packed_w + 16, v2);
510 _mm_store_ps(packed_w + 20, v6);
511 _mm_store_ps(packed_w + 24, v3);
512 _mm_store_ps(packed_w + 28, v7);
513 packed_w += 32;
514 }
515 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
516 }
517 weights += nc * kc;
518 } while (--g != 0);
519 }