Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-avx-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x8s4__avx_u4(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 8);
38 assert(kr == 1);
39 assert(sr == 4);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 const float* b = (const float*) bias;
44 float* packed_w = (float*) packed_weights;
45 do {
46 // NC main loop multiple of 8
47 const float* w0 = (const float*) weights;
48 size_t n = nc;
49
50 for (; n >= 8; n -= 8) {
51 if XNN_LIKELY(b != NULL) {
52 const __m256 vb0 = _mm256_loadu_ps(b);
53 _mm256_store_ps(packed_w, vb0);
54 b += 8;
55 } else {
56 const __m256 vzero = _mm256_setzero_ps();
57 _mm256_store_ps(packed_w, vzero);
58 }
59 packed_w += 8;
60
61 const float* w1 = w0 + kc;
62 const float* w2 = w1 + kc;
63 const float* w3 = w2 + kc;
64 const float* w4 = w3 + kc;
65 const float* w5 = w4 + kc;
66 const float* w6 = w5 + kc;
67 const float* w7 = w6 + kc;
68
69 // KC main loop multiple of 4
70 size_t k = kc;
71 for (; k >= 4; k -= 4) {
72 // Read blocks of 4x4
73 // a b c d
74 // e f g h
75 // i j k l
76 // m n o p
77 // Load first 4 rows of N into low part of each register
78 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
79 w0 += 4;
80 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
81 w1 += 4;
82 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
83 w2 += 4;
84 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
85 w3 += 4;
86 // Load next 4 rows of N into the high part of each register
87 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
88 w4 += 4;
89 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
90 w5 += 4;
91 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
92 w6 += 4;
93 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
94 w7 += 4;
95
96 // Apply SR4 shuffle
97 v1x0123 = _mm256_permute_ps(v1x0123, _MM_SHUFFLE(0, 3, 2, 1));
98 v2x0123 = _mm256_permute_ps(v2x0123, _MM_SHUFFLE(1, 0, 3, 2));
99 v3x0123 = _mm256_permute_ps(v3x0123, _MM_SHUFFLE(2, 1, 0, 3));
100
101 // Transpose 2x2
102 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
103 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
104 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
105 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
106 // Transpose 4x4
107 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
108 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
109 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
110 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
111
112 _mm256_store_ps(packed_w, v0x0123);
113 _mm256_store_ps(packed_w + 8, v1x0123);
114 _mm256_store_ps(packed_w + 16, v2x0123);
115 _mm256_store_ps(packed_w + 24, v3x0123);
116 packed_w += 32;
117 }
118
119 // KC remainder (1..3)
120 if XNN_UNLIKELY(k != 0) {
121 assert(k >= 1);
122 assert(k <= 3);
123 __m128 v0 = _mm_undefined_ps();
124 __m128 v1 = _mm_undefined_ps();
125 __m128 v2 = _mm_undefined_ps();
126 __m128 v3 = _mm_undefined_ps();
127 __m128 v4 = _mm_undefined_ps();
128 __m128 v5 = _mm_undefined_ps();
129 __m128 v6 = _mm_undefined_ps();
130 __m128 v7 = _mm_undefined_ps();
131
132 switch (k) {
133 case 1:
134 // Read blocks of 4x1
135 // a
136 // e
137 // i
138 // m
139 v0 = _mm_load_ss(w0);
140 w0 += 1;
141 v1 = _mm_load_ss(w1);
142 w1 += 1;
143 v2 = _mm_load_ss(w2);
144 w2 += 1;
145 v3 = _mm_load_ss(w3);
146 w3 += 1;
147 v4 = _mm_load_ss(w4);
148 w4 += 1;
149 v5 = _mm_load_ss(w5);
150 w5 += 1;
151 v6 = _mm_load_ss(w6);
152 w6 += 1;
153 v7 = _mm_load_ss(w7);
154 w7 += 1;
155 break;
156 case 2:
157 // Read blocks of 4x2
158 // a b
159 // e f
160 // i j
161 // m n
162 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
163 w0 += 2;
164 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
165 w1 += 2;
166 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
167 w2 += 2;
168 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
169 w3 += 2;
170 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
171 w4 += 2;
172 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
173 w5 += 2;
174 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
175 w6 += 2;
176 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
177 w7 += 2;
178 break;
179 case 3:
180 {
181 // Read blocks of 4x3
182 // a b c
183 // e f g
184 // i j k
185 // m n o
186 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
187 const __m128 v0hi = _mm_load_ss(w0 + 2);
188 v0 = _mm_movelh_ps(v0lo, v0hi);
189 w0 += 3;
190 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
191 const __m128 v1hi = _mm_load_ss(w1 + 2);
192 v1 = _mm_movelh_ps(v1lo, v1hi);
193 w1 += 3;
194 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
195 const __m128 v2hi = _mm_load_ss(w2 + 2);
196 v2 = _mm_movelh_ps(v2lo, v2hi);
197 w2 += 3;
198 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
199 const __m128 v3hi = _mm_load_ss(w3 + 2);
200 v3 = _mm_movelh_ps(v3lo, v3hi);
201 w3 += 3;
202 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
203 const __m128 v4hi = _mm_load_ss(w4 + 2);
204 v4 = _mm_movelh_ps(v4lo, v4hi);
205 w4 += 3;
206 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
207 const __m128 v5hi = _mm_load_ss(w5 + 2);
208 v5 = _mm_movelh_ps(v5lo, v5hi);
209 w5 += 3;
210 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
211 const __m128 v6hi = _mm_load_ss(w6 + 2);
212 v6 = _mm_movelh_ps(v6lo, v6hi);
213 w6 += 3;
214 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
215 const __m128 v7hi = _mm_load_ss(w7 + 2);
216 v7 = _mm_movelh_ps(v7lo, v7hi);
217 w7 += 3;
218 break;
219 }
220 default:
221 XNN_UNREACHABLE;
222 }
223
224 // Apply SR4 shuffle
225 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
226 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
227 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
228 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
229 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
230 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
231 // Transpose 2x2
232 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
233 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
234 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
235 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
236 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
237 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
238 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
239 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
240 // Transpose 4x4
241 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
242 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
243 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
244 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
245 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
246 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
247 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
248 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
249 _mm_store_ps(packed_w, v0);
250 _mm_store_ps(packed_w + 4, v4);
251 _mm_store_ps(packed_w + 8, v1);
252 _mm_store_ps(packed_w + 12, v5);
253 _mm_store_ps(packed_w + 16, v2);
254 _mm_store_ps(packed_w + 20, v6);
255 _mm_store_ps(packed_w + 24, v3);
256 _mm_store_ps(packed_w + 28, v7);
257 packed_w += 32;
258 }
259 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
260 w0 = w7;
261 }
262
263 // NC remainder (1..7)
264 if XNN_UNLIKELY(n != 0) {
265 assert(n >= 1);
266 assert(n <= 7);
267 if XNN_LIKELY(b != NULL) {
268 size_t nb = n;
269 do {
270 *packed_w++ = *b++;
271 } while (--nb != 0);
272 packed_w += (8 - n);
273 } else {
274 const __m128 vzero = _mm_setzero_ps();
275 _mm_store_ps(packed_w, vzero);
276 _mm_store_ps(packed_w + 4, vzero);
277 packed_w += 8;
278 }
279
280 // NR remainder has less than 8 rows so last row is not loaded
281 const float* w1 = w0 + kc;
282 if XNN_UNPREDICTABLE(n < 2) {
283 w1 = w0;
284 }
285 const float* w2 = w1 + kc;
286 if XNN_UNPREDICTABLE(n <= 2) {
287 w2 = w1;
288 }
289 const float* w3 = w2 + kc;
290 if XNN_UNPREDICTABLE(n < 4) {
291 w3 = w2;
292 }
293 const float* w4 = w3 + kc;
294 if XNN_UNPREDICTABLE(n <= 4) {
295 w4 = w3;
296 }
297 const float* w5 = w4 + kc;
298 if XNN_UNPREDICTABLE(n < 6) {
299 w5 = w4;
300 }
301 const float* w6 = w5 + kc;
302 if XNN_UNPREDICTABLE(n <= 6) {
303 w6 = w5;
304 }
305
306 // KC main loop multiple of 4
307 size_t k = kc;
308 for (; k >= 4; k -= 4) {
309 // Read blocks of 4x4
310 // a b c d
311 // e f g h
312 // i j k l
313 // m n o p
314 // Load first 4 rows of N into low part of each register
315 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
316 w0 += 4;
317 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
318 w1 += 4;
319 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
320 w2 += 4;
321 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
322 w3 += 4;
323 // Load next 4 rows of N into the high part of each register
324 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
325 w4 += 4;
326 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
327 w5 += 4;
328 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
329 w6 += 4;
330 // Apply SR4 shuffle
331 v1x0123 = _mm256_permute_ps(v1x0123, _MM_SHUFFLE(0, 3, 2, 1));
332 v2x0123 = _mm256_permute_ps(v2x0123, _MM_SHUFFLE(1, 0, 3, 2));
333 v3x0123 = _mm256_permute_ps(v3x0123, _MM_SHUFFLE(2, 1, 0, 3));
334 // Transpose 2x2
335 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
336 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
337 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
338 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
339 // Transpose 4x4
340 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
341 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
342 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
343 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
344
345 _mm256_store_ps(packed_w, v0x0123);
346 _mm256_store_ps(packed_w + 8, v1x0123);
347 _mm256_store_ps(packed_w + 16, v2x0123);
348 _mm256_store_ps(packed_w + 24, v3x0123);
349 packed_w += 32;
350 }
351
352 // KC remainder (1..3)
353 if XNN_UNLIKELY(k != 0) {
354 assert(k >= 1);
355 assert(k <= 3);
356 __m128 v0 = _mm_undefined_ps();
357 __m128 v1 = _mm_undefined_ps();
358 __m128 v2 = _mm_undefined_ps();
359 __m128 v3 = _mm_undefined_ps();
360 __m128 v4 = _mm_undefined_ps();
361 __m128 v5 = _mm_undefined_ps();
362 __m128 v6 = _mm_undefined_ps();
363 __m128 v7 = _mm_undefined_ps();
364
365 switch (k) {
366 case 1:
367 // Read blocks of 4x1
368 // a
369 // e
370 // i
371 // m
372 v0 = _mm_load_ss(w0);
373 w0 += 1;
374 v1 = _mm_load_ss(w1);
375 w1 += 1;
376 v2 = _mm_load_ss(w2);
377 w2 += 1;
378 v3 = _mm_load_ss(w3);
379 w3 += 1;
380 v4 = _mm_load_ss(w4);
381 w4 += 1;
382 v5 = _mm_load_ss(w5);
383 w5 += 1;
384 v6 = _mm_load_ss(w6);
385 w6 += 1;
386 break;
387 case 2:
388 // Read blocks of 4x2
389 // a b
390 // e f
391 // i j
392 // m n
393 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
394 w0 += 2;
395 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
396 w1 += 2;
397 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
398 w2 += 2;
399 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
400 w3 += 2;
401 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
402 w4 += 2;
403 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
404 w5 += 2;
405 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
406 w6 += 2;
407 break;
408 case 3:
409 {
410 // Read blocks of 4x3
411 // a b c
412 // e f g
413 // i j k
414 // m n o
415 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
416 const __m128 v0hi = _mm_load_ss(w0 + 2);
417 v0 = _mm_movelh_ps(v0lo, v0hi);
418 w0 += 3;
419 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
420 const __m128 v1hi = _mm_load_ss(w1 + 2);
421 v1 = _mm_movelh_ps(v1lo, v1hi);
422 w1 += 3;
423 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
424 const __m128 v2hi = _mm_load_ss(w2 + 2);
425 v2 = _mm_movelh_ps(v2lo, v2hi);
426 w2 += 3;
427 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
428 const __m128 v3hi = _mm_load_ss(w3 + 2);
429 v3 = _mm_movelh_ps(v3lo, v3hi);
430 w3 += 3;
431 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
432 const __m128 v4hi = _mm_load_ss(w4 + 2);
433 v4 = _mm_movelh_ps(v4lo, v4hi);
434 w4 += 3;
435 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
436 const __m128 v5hi = _mm_load_ss(w5 + 2);
437 v5 = _mm_movelh_ps(v5lo, v5hi);
438 w5 += 3;
439 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
440 const __m128 v6hi = _mm_load_ss(w6 + 2);
441 v6 = _mm_movelh_ps(v6lo, v6hi);
442 w6 += 3;
443 break;
444 }
445 default:
446 XNN_UNREACHABLE;
447 }
448
449 // Apply SR4 shuffle
450 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
451 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
452 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
453 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
454 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
455 // Transpose 2x2
456 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
457 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
458 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
459 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
460 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
461 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v6); // i m j n from row 2, 3
462 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
463 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v6); // k o l p from row 2, 3
464 // Transpose 4x4
465 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
466 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
467 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
468 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
469 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
470 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
471 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
472 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
473 _mm_store_ps(packed_w, v0);
474 _mm_store_ps(packed_w + 4, v4);
475 _mm_store_ps(packed_w + 8, v1);
476 _mm_store_ps(packed_w + 12, v5);
477 _mm_store_ps(packed_w + 16, v2);
478 _mm_store_ps(packed_w + 20, v6);
479 _mm_store_ps(packed_w + 24, v3);
480 _mm_store_ps(packed_w + 28, v7);
481 packed_w += 32;
482 }
483 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
484 }
485 weights += nc * kc;
486 } while (--g != 0);
487 }