Debian Codegraph @

src/x32-packw/gen/x32-packw-x16s4-gemm-goi-sse2-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x16s4__sse2_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 16);
38 assert(kr == 1);
39 assert(sr == 4);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 const float* b = (const float*) bias;
44 float* packed_w = (float*) packed_weights;
45 do {
46 // NC main loop multiple of 16
47 const float* w0 = (const float*) weights;
48 size_t n = nc;
49
50 for (; n >= 16; n -= 16) {
51 if XNN_LIKELY(b != NULL) {
52 const __m128 vb0 = _mm_loadu_ps(b);
53 const __m128 vb4 = _mm_loadu_ps(b + 4);
54 const __m128 vb8 = _mm_loadu_ps(b + 8);
55 const __m128 vb12 = _mm_loadu_ps(b + 12);
56 _mm_store_ps(packed_w, vb0);
57 _mm_store_ps(packed_w + 4, vb4);
58 _mm_store_ps(packed_w + 8, vb8);
59 _mm_store_ps(packed_w + 12, vb12);
60 b += 16;
61 } else {
62 const __m128 vzero = _mm_setzero_ps();
63 _mm_store_ps(packed_w, vzero);
64 _mm_store_ps(packed_w + 4, vzero);
65 _mm_store_ps(packed_w + 8, vzero);
66 _mm_store_ps(packed_w + 12, vzero);
67 }
68 packed_w += 16;
69
70 const float* w1 = w0 + kc;
71 const float* w2 = w1 + kc;
72 const float* w3 = w2 + kc;
73 const float* w4 = w3 + kc;
74 const float* w5 = w4 + kc;
75 const float* w6 = w5 + kc;
76 const float* w7 = w6 + kc;
77 const float* w8 = w7 + kc;
78 const float* w9 = w8 + kc;
79 const float* w10 = w9 + kc;
80 const float* w11 = w10 + kc;
81 const float* w12 = w11 + kc;
82 const float* w13 = w12 + kc;
83 const float* w14 = w13 + kc;
84 const float* w15 = w14 + kc;
85
86 size_t k = kc;
87
88 // KC multiple of 4
89 for (; k >= 4; k -= 4) {
90 // Read blocks of 4x4
91 // a b c d
92 // e f g h
93 // i j k l
94 // m n o p
95 __m128 v0x0123 = _mm_loadu_ps(w0);
96 w0 += 4;
97 __m128 v1x0123 = _mm_loadu_ps(w1);
98 w1 += 4;
99 __m128 v2x0123 = _mm_loadu_ps(w2);
100 w2 += 4;
101 __m128 v3x0123 = _mm_loadu_ps(w3);
102 w3 += 4;
103 __m128 v4x0123 = _mm_loadu_ps(w4);
104 w4 += 4;
105 __m128 v5x0123 = _mm_loadu_ps(w5);
106 w5 += 4;
107 __m128 v6x0123 = _mm_loadu_ps(w6);
108 w6 += 4;
109 __m128 v7x0123 = _mm_loadu_ps(w7);
110 w7 += 4;
111 __m128 v8x0123 = _mm_loadu_ps(w8);
112 w8 += 4;
113 __m128 v9x0123 = _mm_loadu_ps(w9);
114 w9 += 4;
115 __m128 v10x0123 = _mm_loadu_ps(w10);
116 w10 += 4;
117 __m128 v11x0123 = _mm_loadu_ps(w11);
118 w11 += 4;
119 __m128 v12x0123 = _mm_loadu_ps(w12);
120 w12 += 4;
121 __m128 v13x0123 = _mm_loadu_ps(w13);
122 w13 += 4;
123 __m128 v14x0123 = _mm_loadu_ps(w14);
124 w14 += 4;
125 __m128 v15x0123 = _mm_loadu_ps(w15);
126 w15 += 4;
127
128 // Apply SR4 shuffle
129 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
130 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
131 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
132 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
133 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
134 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
135 v9x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x0123), _MM_SHUFFLE(0, 3, 2, 1)));
136 v10x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x0123), _MM_SHUFFLE(1, 0, 3, 2)));
137 v11x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x0123), _MM_SHUFFLE(2, 1, 0, 3)));
138 v13x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x0123), _MM_SHUFFLE(0, 3, 2, 1)));
139 v14x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x0123), _MM_SHUFFLE(1, 0, 3, 2)));
140 v15x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15x0123), _MM_SHUFFLE(2, 1, 0, 3)));
141
142 // Transpose 2x2
143 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
144 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
145 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
146 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
147 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
148 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
149 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
150 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
151 const __m128 vtmp8x0123 = _mm_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
152 const __m128 vtmp9x0123 = _mm_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
153 const __m128 vtmp10x0123 = _mm_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
154 const __m128 vtmp11x0123 = _mm_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
155 const __m128 vtmp12x0123 = _mm_unpacklo_ps(v12x0123, v13x0123); // a e b f from row 0, 1
156 const __m128 vtmp13x0123 = _mm_unpacklo_ps(v14x0123, v15x0123); // i m j n from row 2, 3
157 const __m128 vtmp14x0123 = _mm_unpackhi_ps(v12x0123, v13x0123); // c g d h from row 0, 1
158 const __m128 vtmp15x0123 = _mm_unpackhi_ps(v14x0123, v15x0123); // k o l p from row 2, 3
159 // Transpose 4x4
160 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
161 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
162 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
163 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
164 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
165 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
166 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
167 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
168 v8x0123 = _mm_movelh_ps(vtmp8x0123, vtmp9x0123); // a e i m from row 0, 1
169 v9x0123 = _mm_movehl_ps(vtmp9x0123, vtmp8x0123); // b f j n from row 0, 1
170 v10x0123 = _mm_movelh_ps(vtmp10x0123, vtmp11x0123); // c g k o from row 2, 3
171 v11x0123 = _mm_movehl_ps(vtmp11x0123, vtmp10x0123); // d h l p from row 2, 3
172 v12x0123 = _mm_movelh_ps(vtmp12x0123, vtmp13x0123); // a e i m from row 0, 1
173 v13x0123 = _mm_movehl_ps(vtmp13x0123, vtmp12x0123); // b f j n from row 0, 1
174 v14x0123 = _mm_movelh_ps(vtmp14x0123, vtmp15x0123); // c g k o from row 2, 3
175 v15x0123 = _mm_movehl_ps(vtmp15x0123, vtmp14x0123); // d h l p from row 2, 3
176
177 _mm_store_ps(packed_w, v0x0123);
178 _mm_store_ps(packed_w + 4, v4x0123);
179 _mm_store_ps(packed_w + 8, v8x0123);
180 _mm_store_ps(packed_w + 12, v12x0123);
181 _mm_store_ps(packed_w + 16, v1x0123);
182 _mm_store_ps(packed_w + 20, v5x0123);
183 _mm_store_ps(packed_w + 24, v9x0123);
184 _mm_store_ps(packed_w + 28, v13x0123);
185 _mm_store_ps(packed_w + 32, v2x0123);
186 _mm_store_ps(packed_w + 36, v6x0123);
187 _mm_store_ps(packed_w + 40, v10x0123);
188 _mm_store_ps(packed_w + 44, v14x0123);
189 _mm_store_ps(packed_w + 48, v3x0123);
190 _mm_store_ps(packed_w + 52, v7x0123);
191 _mm_store_ps(packed_w + 56, v11x0123);
192 _mm_store_ps(packed_w + 60, v15x0123);
193 packed_w += 64;
194 }
195
196 // KC remainder (1..3)
197 if XNN_UNLIKELY(k != 0) {
198 assert(k >= 1);
199 assert(k <= 3);
200 __m128 v0 = _mm_undefined_ps();
201 __m128 v1 = _mm_undefined_ps();
202 __m128 v2 = _mm_undefined_ps();
203 __m128 v3 = _mm_undefined_ps();
204 __m128 v4 = _mm_undefined_ps();
205 __m128 v5 = _mm_undefined_ps();
206 __m128 v6 = _mm_undefined_ps();
207 __m128 v7 = _mm_undefined_ps();
208 __m128 v8 = _mm_undefined_ps();
209 __m128 v9 = _mm_undefined_ps();
210 __m128 v10 = _mm_undefined_ps();
211 __m128 v11 = _mm_undefined_ps();
212 __m128 v12 = _mm_undefined_ps();
213 __m128 v13 = _mm_undefined_ps();
214 __m128 v14 = _mm_undefined_ps();
215 __m128 v15 = _mm_undefined_ps();
216
217 switch (k) {
218 case 1:
219 // Read blocks of 4x1
220 // a
221 // e
222 // i
223 // m
224 v0 = _mm_load_ss(w0);
225 w0 += 1;
226 v1 = _mm_load_ss(w1);
227 w1 += 1;
228 v2 = _mm_load_ss(w2);
229 w2 += 1;
230 v3 = _mm_load_ss(w3);
231 w3 += 1;
232 v4 = _mm_load_ss(w4);
233 w4 += 1;
234 v5 = _mm_load_ss(w5);
235 w5 += 1;
236 v6 = _mm_load_ss(w6);
237 w6 += 1;
238 v7 = _mm_load_ss(w7);
239 w7 += 1;
240 v8 = _mm_load_ss(w8);
241 w8 += 1;
242 v9 = _mm_load_ss(w9);
243 w9 += 1;
244 v10 = _mm_load_ss(w10);
245 w10 += 1;
246 v11 = _mm_load_ss(w11);
247 w11 += 1;
248 v12 = _mm_load_ss(w12);
249 w12 += 1;
250 v13 = _mm_load_ss(w13);
251 w13 += 1;
252 v14 = _mm_load_ss(w14);
253 w14 += 1;
254 v15 = _mm_load_ss(w15);
255 w15 += 1;
256 break;
257 case 2:
258 // Read blocks of 4x2
259 // a b
260 // e f
261 // i j
262 // m n
263 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
264 w0 += 2;
265 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
266 w1 += 2;
267 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
268 w2 += 2;
269 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
270 w3 += 2;
271 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
272 w4 += 2;
273 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
274 w5 += 2;
275 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
276 w6 += 2;
277 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
278 w7 += 2;
279 v8 = _mm_castpd_ps(_mm_load_sd((const double*) w8));
280 w8 += 2;
281 v9 = _mm_castpd_ps(_mm_load_sd((const double*) w9));
282 w9 += 2;
283 v10 = _mm_castpd_ps(_mm_load_sd((const double*) w10));
284 w10 += 2;
285 v11 = _mm_castpd_ps(_mm_load_sd((const double*) w11));
286 w11 += 2;
287 v12 = _mm_castpd_ps(_mm_load_sd((const double*) w12));
288 w12 += 2;
289 v13 = _mm_castpd_ps(_mm_load_sd((const double*) w13));
290 w13 += 2;
291 v14 = _mm_castpd_ps(_mm_load_sd((const double*) w14));
292 w14 += 2;
293 v15 = _mm_castpd_ps(_mm_load_sd((const double*) w15));
294 w15 += 2;
295 break;
296 case 3:
297 {
298 // Read blocks of 4x3
299 // a b c
300 // e f g
301 // i j k
302 // m n o
303 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
304 const __m128 v0hi = _mm_load_ss(w0 + 2);
305 v0 = _mm_movelh_ps(v0lo, v0hi);
306 w0 += 3;
307 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
308 const __m128 v1hi = _mm_load_ss(w1 + 2);
309 v1 = _mm_movelh_ps(v1lo, v1hi);
310 w1 += 3;
311 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
312 const __m128 v2hi = _mm_load_ss(w2 + 2);
313 v2 = _mm_movelh_ps(v2lo, v2hi);
314 w2 += 3;
315 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
316 const __m128 v3hi = _mm_load_ss(w3 + 2);
317 v3 = _mm_movelh_ps(v3lo, v3hi);
318 w3 += 3;
319 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
320 const __m128 v4hi = _mm_load_ss(w4 + 2);
321 v4 = _mm_movelh_ps(v4lo, v4hi);
322 w4 += 3;
323 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
324 const __m128 v5hi = _mm_load_ss(w5 + 2);
325 v5 = _mm_movelh_ps(v5lo, v5hi);
326 w5 += 3;
327 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
328 const __m128 v6hi = _mm_load_ss(w6 + 2);
329 v6 = _mm_movelh_ps(v6lo, v6hi);
330 w6 += 3;
331 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
332 const __m128 v7hi = _mm_load_ss(w7 + 2);
333 v7 = _mm_movelh_ps(v7lo, v7hi);
334 w7 += 3;
335 const __m128 v8lo = _mm_castpd_ps(_mm_load_sd((const double*) w8));
336 const __m128 v8hi = _mm_load_ss(w8 + 2);
337 v8 = _mm_movelh_ps(v8lo, v8hi);
338 w8 += 3;
339 const __m128 v9lo = _mm_castpd_ps(_mm_load_sd((const double*) w9));
340 const __m128 v9hi = _mm_load_ss(w9 + 2);
341 v9 = _mm_movelh_ps(v9lo, v9hi);
342 w9 += 3;
343 const __m128 v10lo = _mm_castpd_ps(_mm_load_sd((const double*) w10));
344 const __m128 v10hi = _mm_load_ss(w10 + 2);
345 v10 = _mm_movelh_ps(v10lo, v10hi);
346 w10 += 3;
347 const __m128 v11lo = _mm_castpd_ps(_mm_load_sd((const double*) w11));
348 const __m128 v11hi = _mm_load_ss(w11 + 2);
349 v11 = _mm_movelh_ps(v11lo, v11hi);
350 w11 += 3;
351 const __m128 v12lo = _mm_castpd_ps(_mm_load_sd((const double*) w12));
352 const __m128 v12hi = _mm_load_ss(w12 + 2);
353 v12 = _mm_movelh_ps(v12lo, v12hi);
354 w12 += 3;
355 const __m128 v13lo = _mm_castpd_ps(_mm_load_sd((const double*) w13));
356 const __m128 v13hi = _mm_load_ss(w13 + 2);
357 v13 = _mm_movelh_ps(v13lo, v13hi);
358 w13 += 3;
359 const __m128 v14lo = _mm_castpd_ps(_mm_load_sd((const double*) w14));
360 const __m128 v14hi = _mm_load_ss(w14 + 2);
361 v14 = _mm_movelh_ps(v14lo, v14hi);
362 w14 += 3;
363 const __m128 v15lo = _mm_castpd_ps(_mm_load_sd((const double*) w15));
364 const __m128 v15hi = _mm_load_ss(w15 + 2);
365 v15 = _mm_movelh_ps(v15lo, v15hi);
366 w15 += 3;
367 break;
368 }
369 default:
370 XNN_UNREACHABLE;
371 }
372
373 // Apply SR4 shuffle
374 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
375 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
376 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
377 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
378 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
379 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
380 v9 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9), _MM_SHUFFLE(0, 3, 2, 1)));
381 v10 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10), _MM_SHUFFLE(1, 0, 3, 2)));
382 v11 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11), _MM_SHUFFLE(2, 1, 0, 3)));
383 v13 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13), _MM_SHUFFLE(0, 3, 2, 1)));
384 v14 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14), _MM_SHUFFLE(1, 0, 3, 2)));
385 v15 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15), _MM_SHUFFLE(2, 1, 0, 3)));
386 // Transpose 2x2
387 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
388 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
389 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
390 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
391 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
392 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
393 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
394 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
395 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a e b f from row 0, 1
396 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // i m j n from row 2, 3
397 const __m128 vtmp10 = _mm_unpackhi_ps(v8, v9); // c g d h from row 0, 1
398 const __m128 vtmp11 = _mm_unpackhi_ps(v10, v11); // k o l p from row 2, 3
399 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a e b f from row 0, 1
400 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // i m j n from row 2, 3
401 const __m128 vtmp14 = _mm_unpackhi_ps(v12, v13); // c g d h from row 0, 1
402 const __m128 vtmp15 = _mm_unpackhi_ps(v14, v15); // k o l p from row 2, 3
403 // Transpose 4x4
404 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
405 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
406 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
407 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
408 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
409 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
410 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
411 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
412 v8 = _mm_movelh_ps(vtmp8, vtmp9); // a e i m from row 0, 1
413 v9 = _mm_movehl_ps(vtmp9, vtmp8); // b f j n from row 0, 1
414 v10 = _mm_movelh_ps(vtmp10, vtmp11); // c g k o from row 2, 3
415 v11 = _mm_movehl_ps(vtmp11, vtmp10); // d h l p from row 2, 3
416 v12 = _mm_movelh_ps(vtmp12, vtmp13); // a e i m from row 0, 1
417 v13 = _mm_movehl_ps(vtmp13, vtmp12); // b f j n from row 0, 1
418 v14 = _mm_movelh_ps(vtmp14, vtmp15); // c g k o from row 2, 3
419 v15 = _mm_movehl_ps(vtmp15, vtmp14); // d h l p from row 2, 3
420 _mm_store_ps(packed_w, v0);
421 _mm_store_ps(packed_w + 4, v4);
422 _mm_store_ps(packed_w + 8, v8);
423 _mm_store_ps(packed_w + 12, v12);
424 _mm_store_ps(packed_w + 16, v1);
425 _mm_store_ps(packed_w + 20, v5);
426 _mm_store_ps(packed_w + 24, v9);
427 _mm_store_ps(packed_w + 28, v13);
428 _mm_store_ps(packed_w + 32, v2);
429 _mm_store_ps(packed_w + 36, v6);
430 _mm_store_ps(packed_w + 40, v10);
431 _mm_store_ps(packed_w + 44, v14);
432 _mm_store_ps(packed_w + 48, v3);
433 _mm_store_ps(packed_w + 52, v7);
434 _mm_store_ps(packed_w + 56, v11);
435 _mm_store_ps(packed_w + 60, v15);
436 packed_w += 64;
437 }
438 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
439 w0 = w15;
440 }
441
442 // NC remainder (1..15)
443 if XNN_UNLIKELY(n != 0) {
444 assert(n >= 1);
445 assert(n <= 15);
446 if XNN_LIKELY(b != NULL) {
447 size_t nb = n;
448 do {
449 *packed_w++ = *b++;
450 } while (--nb != 0);
451 packed_w += (16 - n);
452 } else {
453 const __m128 vzero = _mm_setzero_ps();
454 _mm_store_ps(packed_w, vzero);
455 _mm_store_ps(packed_w + 4, vzero);
456 _mm_store_ps(packed_w + 8, vzero);
457 _mm_store_ps(packed_w + 12, vzero);
458 packed_w += 16;
459 }
460
461 // NR remainder has less than 16 rows so last row is not loaded
462 const float* w1 = w0 + kc;
463 if XNN_UNPREDICTABLE(n < 2) {
464 w1 = w0;
465 }
466 const float* w2 = w1 + kc;
467 if XNN_UNPREDICTABLE(n <= 2) {
468 w2 = w1;
469 }
470 const float* w3 = w2 + kc;
471 if XNN_UNPREDICTABLE(n < 4) {
472 w3 = w2;
473 }
474 const float* w4 = w3 + kc;
475 if XNN_UNPREDICTABLE(n <= 4) {
476 w4 = w3;
477 }
478 const float* w5 = w4 + kc;
479 if XNN_UNPREDICTABLE(n < 6) {
480 w5 = w4;
481 }
482 const float* w6 = w5 + kc;
483 if XNN_UNPREDICTABLE(n <= 6) {
484 w6 = w5;
485 }
486 const float* w7 = w6 + kc;
487 if XNN_UNPREDICTABLE(n < 8) {
488 w7 = w6;
489 }
490 const float* w8 = w7 + kc;
491 if XNN_UNPREDICTABLE(n <= 8) {
492 w8 = w7;
493 }
494 const float* w9 = w8 + kc;
495 if XNN_UNPREDICTABLE(n < 10) {
496 w9 = w8;
497 }
498 const float* w10 = w9 + kc;
499 if XNN_UNPREDICTABLE(n <= 10) {
500 w10 = w9;
501 }
502 const float* w11 = w10 + kc;
503 if XNN_UNPREDICTABLE(n < 12) {
504 w11 = w10;
505 }
506 const float* w12 = w11 + kc;
507 if XNN_UNPREDICTABLE(n <= 12) {
508 w12 = w11;
509 }
510 const float* w13 = w12 + kc;
511 if XNN_UNPREDICTABLE(n < 14) {
512 w13 = w12;
513 }
514 const float* w14 = w13 + kc;
515 if XNN_UNPREDICTABLE(n <= 14) {
516 w14 = w13;
517 }
518
519 size_t k = kc;
520
521 // KC multiple of 4
522 for (; k >= 4; k -= 4) {
523 // Read blocks of 4x4
524 // a b c d
525 // e f g h
526 // i j k l
527 // m n o p
528 __m128 v0x0123 = _mm_loadu_ps(w0);
529 w0 += 4;
530 __m128 v1x0123 = _mm_loadu_ps(w1);
531 w1 += 4;
532 __m128 v2x0123 = _mm_loadu_ps(w2);
533 w2 += 4;
534 __m128 v3x0123 = _mm_loadu_ps(w3);
535 w3 += 4;
536 __m128 v4x0123 = _mm_loadu_ps(w4);
537 w4 += 4;
538 __m128 v5x0123 = _mm_loadu_ps(w5);
539 w5 += 4;
540 __m128 v6x0123 = _mm_loadu_ps(w6);
541 w6 += 4;
542 __m128 v7x0123 = _mm_loadu_ps(w7);
543 w7 += 4;
544 __m128 v8x0123 = _mm_loadu_ps(w8);
545 w8 += 4;
546 __m128 v9x0123 = _mm_loadu_ps(w9);
547 w9 += 4;
548 __m128 v10x0123 = _mm_loadu_ps(w10);
549 w10 += 4;
550 __m128 v11x0123 = _mm_loadu_ps(w11);
551 w11 += 4;
552 __m128 v12x0123 = _mm_loadu_ps(w12);
553 w12 += 4;
554 __m128 v13x0123 = _mm_loadu_ps(w13);
555 w13 += 4;
556 __m128 v14x0123 = _mm_loadu_ps(w14);
557 w14 += 4;
558 __m128 v15x0123 = _mm_undefined_ps();
559
560 // Apply SR4 shuffle
561 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
562 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
563 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
564 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
565 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
566 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
567 v9x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9x0123), _MM_SHUFFLE(0, 3, 2, 1)));
568 v10x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10x0123), _MM_SHUFFLE(1, 0, 3, 2)));
569 v11x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11x0123), _MM_SHUFFLE(2, 1, 0, 3)));
570 v13x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13x0123), _MM_SHUFFLE(0, 3, 2, 1)));
571 v14x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14x0123), _MM_SHUFFLE(1, 0, 3, 2)));
572
573 // Transpose 2x2
574 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
575 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
576 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
577 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
578 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
579 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
580 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
581 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
582 const __m128 vtmp8x0123 = _mm_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
583 const __m128 vtmp9x0123 = _mm_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
584 const __m128 vtmp10x0123 = _mm_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
585 const __m128 vtmp11x0123 = _mm_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
586 const __m128 vtmp12x0123 = _mm_unpacklo_ps(v12x0123, v13x0123); // a e b f from row 0, 1
587 const __m128 vtmp13x0123 = _mm_unpacklo_ps(v14x0123, v14x0123); // i m j n from row 2, 3
588 const __m128 vtmp14x0123 = _mm_unpackhi_ps(v12x0123, v13x0123); // c g d h from row 0, 1
589 const __m128 vtmp15x0123 = _mm_unpackhi_ps(v14x0123, v14x0123); // k o l p from row 2, 3
590 // Transpose 4x4
591 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
592 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
593 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
594 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
595 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
596 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
597 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
598 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
599 v8x0123 = _mm_movelh_ps(vtmp8x0123, vtmp9x0123); // a e i m from row 0, 1
600 v9x0123 = _mm_movehl_ps(vtmp9x0123, vtmp8x0123); // b f j n from row 0, 1
601 v10x0123 = _mm_movelh_ps(vtmp10x0123, vtmp11x0123); // c g k o from row 2, 3
602 v11x0123 = _mm_movehl_ps(vtmp11x0123, vtmp10x0123); // d h l p from row 2, 3
603 v12x0123 = _mm_movelh_ps(vtmp12x0123, vtmp13x0123); // a e i m from row 0, 1
604 v13x0123 = _mm_movehl_ps(vtmp13x0123, vtmp12x0123); // b f j n from row 0, 1
605 v14x0123 = _mm_movelh_ps(vtmp14x0123, vtmp15x0123); // c g k o from row 2, 3
606 v15x0123 = _mm_movehl_ps(vtmp15x0123, vtmp14x0123); // d h l p from row 2, 3
607
608 _mm_store_ps(packed_w, v0x0123);
609 _mm_store_ps(packed_w + 4, v4x0123);
610 _mm_store_ps(packed_w + 8, v8x0123);
611 _mm_store_ps(packed_w + 12, v12x0123);
612 _mm_store_ps(packed_w + 16, v1x0123);
613 _mm_store_ps(packed_w + 20, v5x0123);
614 _mm_store_ps(packed_w + 24, v9x0123);
615 _mm_store_ps(packed_w + 28, v13x0123);
616 _mm_store_ps(packed_w + 32, v2x0123);
617 _mm_store_ps(packed_w + 36, v6x0123);
618 _mm_store_ps(packed_w + 40, v10x0123);
619 _mm_store_ps(packed_w + 44, v14x0123);
620 _mm_store_ps(packed_w + 48, v3x0123);
621 _mm_store_ps(packed_w + 52, v7x0123);
622 _mm_store_ps(packed_w + 56, v11x0123);
623 _mm_store_ps(packed_w + 60, v15x0123);
624 packed_w += 64;
625 }
626
627 // KC remainder (1..3)
628 if XNN_UNLIKELY(k != 0) {
629 assert(k >= 1);
630 assert(k <= 3);
631 __m128 v0 = _mm_undefined_ps();
632 __m128 v1 = _mm_undefined_ps();
633 __m128 v2 = _mm_undefined_ps();
634 __m128 v3 = _mm_undefined_ps();
635 __m128 v4 = _mm_undefined_ps();
636 __m128 v5 = _mm_undefined_ps();
637 __m128 v6 = _mm_undefined_ps();
638 __m128 v7 = _mm_undefined_ps();
639 __m128 v8 = _mm_undefined_ps();
640 __m128 v9 = _mm_undefined_ps();
641 __m128 v10 = _mm_undefined_ps();
642 __m128 v11 = _mm_undefined_ps();
643 __m128 v12 = _mm_undefined_ps();
644 __m128 v13 = _mm_undefined_ps();
645 __m128 v14 = _mm_undefined_ps();
646 __m128 v15 = _mm_undefined_ps();
647
648 switch (k) {
649 case 1:
650 // Read blocks of 4x1
651 // a
652 // e
653 // i
654 // m
655 v0 = _mm_load_ss(w0);
656 w0 += 1;
657 v1 = _mm_load_ss(w1);
658 w1 += 1;
659 v2 = _mm_load_ss(w2);
660 w2 += 1;
661 v3 = _mm_load_ss(w3);
662 w3 += 1;
663 v4 = _mm_load_ss(w4);
664 w4 += 1;
665 v5 = _mm_load_ss(w5);
666 w5 += 1;
667 v6 = _mm_load_ss(w6);
668 w6 += 1;
669 v7 = _mm_load_ss(w7);
670 w7 += 1;
671 v8 = _mm_load_ss(w8);
672 w8 += 1;
673 v9 = _mm_load_ss(w9);
674 w9 += 1;
675 v10 = _mm_load_ss(w10);
676 w10 += 1;
677 v11 = _mm_load_ss(w11);
678 w11 += 1;
679 v12 = _mm_load_ss(w12);
680 w12 += 1;
681 v13 = _mm_load_ss(w13);
682 w13 += 1;
683 v14 = _mm_load_ss(w14);
684 w14 += 1;
685 break;
686 case 2:
687 // Read blocks of 4x2
688 // a b
689 // e f
690 // i j
691 // m n
692 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
693 w0 += 2;
694 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
695 w1 += 2;
696 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
697 w2 += 2;
698 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
699 w3 += 2;
700 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
701 w4 += 2;
702 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
703 w5 += 2;
704 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
705 w6 += 2;
706 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
707 w7 += 2;
708 v8 = _mm_castpd_ps(_mm_load_sd((const double*) w8));
709 w8 += 2;
710 v9 = _mm_castpd_ps(_mm_load_sd((const double*) w9));
711 w9 += 2;
712 v10 = _mm_castpd_ps(_mm_load_sd((const double*) w10));
713 w10 += 2;
714 v11 = _mm_castpd_ps(_mm_load_sd((const double*) w11));
715 w11 += 2;
716 v12 = _mm_castpd_ps(_mm_load_sd((const double*) w12));
717 w12 += 2;
718 v13 = _mm_castpd_ps(_mm_load_sd((const double*) w13));
719 w13 += 2;
720 v14 = _mm_castpd_ps(_mm_load_sd((const double*) w14));
721 w14 += 2;
722 break;
723 case 3:
724 {
725 // Read blocks of 4x3
726 // a b c
727 // e f g
728 // i j k
729 // m n o
730 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
731 const __m128 v0hi = _mm_load_ss(w0 + 2);
732 v0 = _mm_movelh_ps(v0lo, v0hi);
733 w0 += 3;
734 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
735 const __m128 v1hi = _mm_load_ss(w1 + 2);
736 v1 = _mm_movelh_ps(v1lo, v1hi);
737 w1 += 3;
738 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
739 const __m128 v2hi = _mm_load_ss(w2 + 2);
740 v2 = _mm_movelh_ps(v2lo, v2hi);
741 w2 += 3;
742 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
743 const __m128 v3hi = _mm_load_ss(w3 + 2);
744 v3 = _mm_movelh_ps(v3lo, v3hi);
745 w3 += 3;
746 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
747 const __m128 v4hi = _mm_load_ss(w4 + 2);
748 v4 = _mm_movelh_ps(v4lo, v4hi);
749 w4 += 3;
750 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
751 const __m128 v5hi = _mm_load_ss(w5 + 2);
752 v5 = _mm_movelh_ps(v5lo, v5hi);
753 w5 += 3;
754 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
755 const __m128 v6hi = _mm_load_ss(w6 + 2);
756 v6 = _mm_movelh_ps(v6lo, v6hi);
757 w6 += 3;
758 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
759 const __m128 v7hi = _mm_load_ss(w7 + 2);
760 v7 = _mm_movelh_ps(v7lo, v7hi);
761 w7 += 3;
762 const __m128 v8lo = _mm_castpd_ps(_mm_load_sd((const double*) w8));
763 const __m128 v8hi = _mm_load_ss(w8 + 2);
764 v8 = _mm_movelh_ps(v8lo, v8hi);
765 w8 += 3;
766 const __m128 v9lo = _mm_castpd_ps(_mm_load_sd((const double*) w9));
767 const __m128 v9hi = _mm_load_ss(w9 + 2);
768 v9 = _mm_movelh_ps(v9lo, v9hi);
769 w9 += 3;
770 const __m128 v10lo = _mm_castpd_ps(_mm_load_sd((const double*) w10));
771 const __m128 v10hi = _mm_load_ss(w10 + 2);
772 v10 = _mm_movelh_ps(v10lo, v10hi);
773 w10 += 3;
774 const __m128 v11lo = _mm_castpd_ps(_mm_load_sd((const double*) w11));
775 const __m128 v11hi = _mm_load_ss(w11 + 2);
776 v11 = _mm_movelh_ps(v11lo, v11hi);
777 w11 += 3;
778 const __m128 v12lo = _mm_castpd_ps(_mm_load_sd((const double*) w12));
779 const __m128 v12hi = _mm_load_ss(w12 + 2);
780 v12 = _mm_movelh_ps(v12lo, v12hi);
781 w12 += 3;
782 const __m128 v13lo = _mm_castpd_ps(_mm_load_sd((const double*) w13));
783 const __m128 v13hi = _mm_load_ss(w13 + 2);
784 v13 = _mm_movelh_ps(v13lo, v13hi);
785 w13 += 3;
786 const __m128 v14lo = _mm_castpd_ps(_mm_load_sd((const double*) w14));
787 const __m128 v14hi = _mm_load_ss(w14 + 2);
788 v14 = _mm_movelh_ps(v14lo, v14hi);
789 w14 += 3;
790 break;
791 }
792 default:
793 XNN_UNREACHABLE;
794 }
795
796 // Apply SR4 shuffle
797 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
798 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
799 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
800 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
801 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
802 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
803 v9 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9), _MM_SHUFFLE(0, 3, 2, 1)));
804 v10 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10), _MM_SHUFFLE(1, 0, 3, 2)));
805 v11 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11), _MM_SHUFFLE(2, 1, 0, 3)));
806 v13 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13), _MM_SHUFFLE(0, 3, 2, 1)));
807 v14 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14), _MM_SHUFFLE(1, 0, 3, 2)));
808 // Transpose 2x2
809 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
810 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
811 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
812 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
813 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
814 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
815 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
816 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
817 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a e b f from row 0, 1
818 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // i m j n from row 2, 3
819 const __m128 vtmp10 = _mm_unpackhi_ps(v8, v9); // c g d h from row 0, 1
820 const __m128 vtmp11 = _mm_unpackhi_ps(v10, v11); // k o l p from row 2, 3
821 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a e b f from row 0, 1
822 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // i m j n from row 2, 3
823 const __m128 vtmp14 = _mm_unpackhi_ps(v12, v13); // c g d h from row 0, 1
824 const __m128 vtmp15 = _mm_unpackhi_ps(v14, v14); // k o l p from row 2, 3
825 // Transpose 4x4
826 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
827 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
828 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
829 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
830 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
831 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
832 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
833 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
834 v8 = _mm_movelh_ps(vtmp8, vtmp9); // a e i m from row 0, 1
835 v9 = _mm_movehl_ps(vtmp9, vtmp8); // b f j n from row 0, 1
836 v10 = _mm_movelh_ps(vtmp10, vtmp11); // c g k o from row 2, 3
837 v11 = _mm_movehl_ps(vtmp11, vtmp10); // d h l p from row 2, 3
838 v12 = _mm_movelh_ps(vtmp12, vtmp13); // a e i m from row 0, 1
839 v13 = _mm_movehl_ps(vtmp13, vtmp12); // b f j n from row 0, 1
840 v14 = _mm_movelh_ps(vtmp14, vtmp15); // c g k o from row 2, 3
841 v15 = _mm_movehl_ps(vtmp15, vtmp14); // d h l p from row 2, 3
842 _mm_store_ps(packed_w, v0);
843 _mm_store_ps(packed_w + 4, v4);
844 _mm_store_ps(packed_w + 8, v8);
845 _mm_store_ps(packed_w + 12, v12);
846 _mm_store_ps(packed_w + 16, v1);
847 _mm_store_ps(packed_w + 20, v5);
848 _mm_store_ps(packed_w + 24, v9);
849 _mm_store_ps(packed_w + 28, v13);
850 _mm_store_ps(packed_w + 32, v2);
851 _mm_store_ps(packed_w + 36, v6);
852 _mm_store_ps(packed_w + 40, v10);
853 _mm_store_ps(packed_w + 44, v14);
854 _mm_store_ps(packed_w + 48, v3);
855 _mm_store_ps(packed_w + 52, v7);
856 _mm_store_ps(packed_w + 56, v11);
857 _mm_store_ps(packed_w + 60, v15);
858 packed_w += 64;
859 }
860 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
861 }
862 weights += nc * kc;
863 } while (--g != 0);
864 }