Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-sse2-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x8s4__sse2_u4_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8);
39 assert(kr == 1);
40 assert(sr == 4);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 const float* b = (const float*) bias;
45 float* packed_w = (float*) packed_weights;
46 do {
47 // NC main loop multiple of 8
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50
51 for (; n >= 8; n -= 8) {
52 if XNN_LIKELY(b != NULL) {
53 const __m128 vb0 = _mm_loadu_ps(b);
54 const __m128 vb4 = _mm_loadu_ps(b + 4);
55 _mm_store_ps(packed_w, vb0);
56 _mm_store_ps(packed_w + 4, vb4);
57 b += 8;
58 } else {
59 const __m128 vzero = _mm_setzero_ps();
60 _mm_store_ps(packed_w, vzero);
61 _mm_store_ps(packed_w + 4, vzero);
62 }
63 packed_w += 8;
64
65 const float* w1 = w0 + kc;
66 const float* w2 = w1 + kc;
67 const float* w3 = w2 + kc;
68 const float* w4 = w3 + kc;
69 const float* w5 = w4 + kc;
70 const float* w6 = w5 + kc;
71 const float* w7 = w6 + kc;
72 xnn_prefetch_to_l1((const int8_t*) w0);
73 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
74 xnn_prefetch_to_l1((const int8_t*) w1);
75 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
76 xnn_prefetch_to_l1((const int8_t*) w2);
77 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
78 xnn_prefetch_to_l1((const int8_t*) w3);
79 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
80 xnn_prefetch_to_l1((const int8_t*) w4);
81 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
82 xnn_prefetch_to_l1((const int8_t*) w5);
83 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
84 xnn_prefetch_to_l1((const int8_t*) w6);
85 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
86 xnn_prefetch_to_l1((const int8_t*) w7);
87 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
88
89 size_t k = kc;
90
91 // KC multiple of 4
92 for (; k >= 4; k -= 4) {
93 // Read blocks of 4x4
94 // a b c d
95 // e f g h
96 // i j k l
97 // m n o p
98 __m128 v0x0123 = _mm_loadu_ps(w0);
99 w0 += 4;
100 __m128 v1x0123 = _mm_loadu_ps(w1);
101 w1 += 4;
102 __m128 v2x0123 = _mm_loadu_ps(w2);
103 w2 += 4;
104 __m128 v3x0123 = _mm_loadu_ps(w3);
105 w3 += 4;
106 __m128 v4x0123 = _mm_loadu_ps(w4);
107 w4 += 4;
108 __m128 v5x0123 = _mm_loadu_ps(w5);
109 w5 += 4;
110 __m128 v6x0123 = _mm_loadu_ps(w6);
111 w6 += 4;
112 __m128 v7x0123 = _mm_loadu_ps(w7);
113 w7 += 4;
114
115 // Apply SR4 shuffle
116 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
117 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
118 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
119 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
120 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
121 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
122
123 // Transpose 2x2
124 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
125 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
126 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
127 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
128 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
129 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
130 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
131 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
132 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
133 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
134 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
135 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
136 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
137 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
138 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
139 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
140 // Transpose 4x4
141 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
142 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
143 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
144 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
145 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
146 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
147 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
148 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
149
150 _mm_store_ps(packed_w, v0x0123);
151 _mm_store_ps(packed_w + 4, v4x0123);
152 _mm_store_ps(packed_w + 8, v1x0123);
153 _mm_store_ps(packed_w + 12, v5x0123);
154 _mm_store_ps(packed_w + 16, v2x0123);
155 _mm_store_ps(packed_w + 20, v6x0123);
156 _mm_store_ps(packed_w + 24, v3x0123);
157 _mm_store_ps(packed_w + 28, v7x0123);
158 packed_w += 32;
159 }
160
161 // KC remainder (1..3)
162 if XNN_UNLIKELY(k != 0) {
163 assert(k >= 1);
164 assert(k <= 3);
165 __m128 v0 = _mm_undefined_ps();
166 __m128 v1 = _mm_undefined_ps();
167 __m128 v2 = _mm_undefined_ps();
168 __m128 v3 = _mm_undefined_ps();
169 __m128 v4 = _mm_undefined_ps();
170 __m128 v5 = _mm_undefined_ps();
171 __m128 v6 = _mm_undefined_ps();
172 __m128 v7 = _mm_undefined_ps();
173
174 switch (k) {
175 case 1:
176 // Read blocks of 4x1
177 // a
178 // e
179 // i
180 // m
181 v0 = _mm_load_ss(w0);
182 w0 += 1;
183 v1 = _mm_load_ss(w1);
184 w1 += 1;
185 v2 = _mm_load_ss(w2);
186 w2 += 1;
187 v3 = _mm_load_ss(w3);
188 w3 += 1;
189 v4 = _mm_load_ss(w4);
190 w4 += 1;
191 v5 = _mm_load_ss(w5);
192 w5 += 1;
193 v6 = _mm_load_ss(w6);
194 w6 += 1;
195 v7 = _mm_load_ss(w7);
196 w7 += 1;
197 break;
198 case 2:
199 // Read blocks of 4x2
200 // a b
201 // e f
202 // i j
203 // m n
204 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
205 w0 += 2;
206 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
207 w1 += 2;
208 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
209 w2 += 2;
210 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
211 w3 += 2;
212 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
213 w4 += 2;
214 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
215 w5 += 2;
216 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
217 w6 += 2;
218 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
219 w7 += 2;
220 break;
221 case 3:
222 {
223 // Read blocks of 4x3
224 // a b c
225 // e f g
226 // i j k
227 // m n o
228 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
229 const __m128 v0hi = _mm_load_ss(w0 + 2);
230 v0 = _mm_movelh_ps(v0lo, v0hi);
231 w0 += 3;
232 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
233 const __m128 v1hi = _mm_load_ss(w1 + 2);
234 v1 = _mm_movelh_ps(v1lo, v1hi);
235 w1 += 3;
236 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
237 const __m128 v2hi = _mm_load_ss(w2 + 2);
238 v2 = _mm_movelh_ps(v2lo, v2hi);
239 w2 += 3;
240 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
241 const __m128 v3hi = _mm_load_ss(w3 + 2);
242 v3 = _mm_movelh_ps(v3lo, v3hi);
243 w3 += 3;
244 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
245 const __m128 v4hi = _mm_load_ss(w4 + 2);
246 v4 = _mm_movelh_ps(v4lo, v4hi);
247 w4 += 3;
248 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
249 const __m128 v5hi = _mm_load_ss(w5 + 2);
250 v5 = _mm_movelh_ps(v5lo, v5hi);
251 w5 += 3;
252 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
253 const __m128 v6hi = _mm_load_ss(w6 + 2);
254 v6 = _mm_movelh_ps(v6lo, v6hi);
255 w6 += 3;
256 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
257 const __m128 v7hi = _mm_load_ss(w7 + 2);
258 v7 = _mm_movelh_ps(v7lo, v7hi);
259 w7 += 3;
260 break;
261 }
262 default:
263 XNN_UNREACHABLE;
264 }
265
266 // Apply SR4 shuffle
267 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
268 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
269 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
270 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
271 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
272 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
273 // Transpose 2x2
274 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
275 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
276 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
277 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
278 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
279 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
280 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
281 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
282 // Transpose 4x4
283 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
284 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
285 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
286 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
287 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
288 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
289 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
290 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
291 _mm_store_ps(packed_w, v0);
292 _mm_store_ps(packed_w + 4, v4);
293 _mm_store_ps(packed_w + 8, v1);
294 _mm_store_ps(packed_w + 12, v5);
295 _mm_store_ps(packed_w + 16, v2);
296 _mm_store_ps(packed_w + 20, v6);
297 _mm_store_ps(packed_w + 24, v3);
298 _mm_store_ps(packed_w + 28, v7);
299 packed_w += 32;
300 }
301 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
302 w0 = w7;
303 }
304
305 // NC remainder (1..7)
306 if XNN_UNLIKELY(n != 0) {
307 assert(n >= 1);
308 assert(n <= 7);
309 if XNN_LIKELY(b != NULL) {
310 size_t nb = n;
311 do {
312 *packed_w++ = *b++;
313 } while (--nb != 0);
314 packed_w += (8 - n);
315 } else {
316 const __m128 vzero = _mm_setzero_ps();
317 _mm_store_ps(packed_w, vzero);
318 _mm_store_ps(packed_w + 4, vzero);
319 packed_w += 8;
320 }
321
322 // NR remainder has less than 8 rows so last row is not loaded
323 const float* w1 = w0 + kc;
324 if XNN_UNPREDICTABLE(n < 2) {
325 w1 = w0;
326 }
327 const float* w2 = w1 + kc;
328 if XNN_UNPREDICTABLE(n <= 2) {
329 w2 = w1;
330 }
331 const float* w3 = w2 + kc;
332 if XNN_UNPREDICTABLE(n < 4) {
333 w3 = w2;
334 }
335 const float* w4 = w3 + kc;
336 if XNN_UNPREDICTABLE(n <= 4) {
337 w4 = w3;
338 }
339 const float* w5 = w4 + kc;
340 if XNN_UNPREDICTABLE(n < 6) {
341 w5 = w4;
342 }
343 const float* w6 = w5 + kc;
344 if XNN_UNPREDICTABLE(n <= 6) {
345 w6 = w5;
346 }
347
348 size_t k = kc;
349
350 // KC multiple of 4
351 for (; k >= 4; k -= 4) {
352 // Read blocks of 4x4
353 // a b c d
354 // e f g h
355 // i j k l
356 // m n o p
357 __m128 v0x0123 = _mm_loadu_ps(w0);
358 w0 += 4;
359 __m128 v1x0123 = _mm_loadu_ps(w1);
360 w1 += 4;
361 __m128 v2x0123 = _mm_loadu_ps(w2);
362 w2 += 4;
363 __m128 v3x0123 = _mm_loadu_ps(w3);
364 w3 += 4;
365 __m128 v4x0123 = _mm_loadu_ps(w4);
366 w4 += 4;
367 __m128 v5x0123 = _mm_loadu_ps(w5);
368 w5 += 4;
369 __m128 v6x0123 = _mm_loadu_ps(w6);
370 w6 += 4;
371 __m128 v7x0123 = _mm_undefined_ps();
372
373 // Apply SR4 shuffle
374 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
375 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
376 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
377 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
378 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
379
380 // Transpose 2x2
381 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
382 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
383 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
384 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
385 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
386 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v6x0123); // i m j n from row 2, 3
387 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
388 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v6x0123); // k o l p from row 2, 3
389 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
390 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
391 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
392 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
393 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
394 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
395 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
396 // Transpose 4x4
397 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
398 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
399 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
400 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
401 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
402 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
403 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
404 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
405
406 _mm_store_ps(packed_w, v0x0123);
407 _mm_store_ps(packed_w + 4, v4x0123);
408 _mm_store_ps(packed_w + 8, v1x0123);
409 _mm_store_ps(packed_w + 12, v5x0123);
410 _mm_store_ps(packed_w + 16, v2x0123);
411 _mm_store_ps(packed_w + 20, v6x0123);
412 _mm_store_ps(packed_w + 24, v3x0123);
413 _mm_store_ps(packed_w + 28, v7x0123);
414 packed_w += 32;
415 }
416
417 // KC remainder (1..3)
418 if XNN_UNLIKELY(k != 0) {
419 assert(k >= 1);
420 assert(k <= 3);
421 __m128 v0 = _mm_undefined_ps();
422 __m128 v1 = _mm_undefined_ps();
423 __m128 v2 = _mm_undefined_ps();
424 __m128 v3 = _mm_undefined_ps();
425 __m128 v4 = _mm_undefined_ps();
426 __m128 v5 = _mm_undefined_ps();
427 __m128 v6 = _mm_undefined_ps();
428 __m128 v7 = _mm_undefined_ps();
429
430 switch (k) {
431 case 1:
432 // Read blocks of 4x1
433 // a
434 // e
435 // i
436 // m
437 v0 = _mm_load_ss(w0);
438 w0 += 1;
439 v1 = _mm_load_ss(w1);
440 w1 += 1;
441 v2 = _mm_load_ss(w2);
442 w2 += 1;
443 v3 = _mm_load_ss(w3);
444 w3 += 1;
445 v4 = _mm_load_ss(w4);
446 w4 += 1;
447 v5 = _mm_load_ss(w5);
448 w5 += 1;
449 v6 = _mm_load_ss(w6);
450 w6 += 1;
451 break;
452 case 2:
453 // Read blocks of 4x2
454 // a b
455 // e f
456 // i j
457 // m n
458 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
459 w0 += 2;
460 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
461 w1 += 2;
462 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
463 w2 += 2;
464 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
465 w3 += 2;
466 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
467 w4 += 2;
468 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
469 w5 += 2;
470 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
471 w6 += 2;
472 break;
473 case 3:
474 {
475 // Read blocks of 4x3
476 // a b c
477 // e f g
478 // i j k
479 // m n o
480 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
481 const __m128 v0hi = _mm_load_ss(w0 + 2);
482 v0 = _mm_movelh_ps(v0lo, v0hi);
483 w0 += 3;
484 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
485 const __m128 v1hi = _mm_load_ss(w1 + 2);
486 v1 = _mm_movelh_ps(v1lo, v1hi);
487 w1 += 3;
488 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
489 const __m128 v2hi = _mm_load_ss(w2 + 2);
490 v2 = _mm_movelh_ps(v2lo, v2hi);
491 w2 += 3;
492 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
493 const __m128 v3hi = _mm_load_ss(w3 + 2);
494 v3 = _mm_movelh_ps(v3lo, v3hi);
495 w3 += 3;
496 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
497 const __m128 v4hi = _mm_load_ss(w4 + 2);
498 v4 = _mm_movelh_ps(v4lo, v4hi);
499 w4 += 3;
500 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
501 const __m128 v5hi = _mm_load_ss(w5 + 2);
502 v5 = _mm_movelh_ps(v5lo, v5hi);
503 w5 += 3;
504 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
505 const __m128 v6hi = _mm_load_ss(w6 + 2);
506 v6 = _mm_movelh_ps(v6lo, v6hi);
507 w6 += 3;
508 break;
509 }
510 default:
511 XNN_UNREACHABLE;
512 }
513
514 // Apply SR4 shuffle
515 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
516 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
517 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
518 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
519 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
520 // Transpose 2x2
521 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
522 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
523 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
524 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
525 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
526 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v6); // i m j n from row 2, 3
527 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
528 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v6); // k o l p from row 2, 3
529 // Transpose 4x4
530 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
531 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
532 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
533 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
534 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
535 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
536 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
537 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
538 _mm_store_ps(packed_w, v0);
539 _mm_store_ps(packed_w + 4, v4);
540 _mm_store_ps(packed_w + 8, v1);
541 _mm_store_ps(packed_w + 12, v5);
542 _mm_store_ps(packed_w + 16, v2);
543 _mm_store_ps(packed_w + 20, v6);
544 _mm_store_ps(packed_w + 24, v3);
545 _mm_store_ps(packed_w + 28, v7);
546 packed_w += 32;
547 }
548 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
549 }
550 weights += nc * kc;
551 } while (--g != 0);
552 }