Debian Codegraph @

src/x32-packw/gen/x32-packw-x16s4-gemm-goi-avx-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x16s4__avx_u4_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 16);
39 assert(kr == 1);
40 assert(sr == 4);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 const float* b = (const float*) bias;
45 float* packed_w = (float*) packed_weights;
46 do {
47 // NC main loop multiple of 16
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50
51 for (; n >= 16; n -= 16) {
52 if XNN_LIKELY(b != NULL) {
53 const __m256 vb0 = _mm256_loadu_ps(b);
54 const __m256 vb8 = _mm256_loadu_ps(b + 8);
55 _mm256_store_ps(packed_w, vb0);
56 _mm256_store_ps(packed_w + 8, vb8);
57 b += 16;
58 } else {
59 const __m256 vzero = _mm256_setzero_ps();
60 _mm256_store_ps(packed_w, vzero);
61 _mm256_store_ps(packed_w + 8, vzero);
62 }
63 packed_w += 16;
64
65 const float* w1 = w0 + kc;
66 const float* w2 = w1 + kc;
67 const float* w3 = w2 + kc;
68 const float* w4 = w3 + kc;
69 const float* w5 = w4 + kc;
70 const float* w6 = w5 + kc;
71 const float* w7 = w6 + kc;
72 const float* w8 = w7 + kc;
73 const float* w9 = w8 + kc;
74 const float* w10 = w9 + kc;
75 const float* w11 = w10 + kc;
76 const float* w12 = w11 + kc;
77 const float* w13 = w12 + kc;
78 const float* w14 = w13 + kc;
79 const float* w15 = w14 + kc;
80 xnn_prefetch_to_l1((const int8_t*) w0);
81 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
82 xnn_prefetch_to_l1((const int8_t*) w1);
83 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
84 xnn_prefetch_to_l1((const int8_t*) w2);
85 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
86 xnn_prefetch_to_l1((const int8_t*) w3);
87 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
88 xnn_prefetch_to_l1((const int8_t*) w4);
89 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
90 xnn_prefetch_to_l1((const int8_t*) w5);
91 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
92 xnn_prefetch_to_l1((const int8_t*) w6);
93 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
94 xnn_prefetch_to_l1((const int8_t*) w7);
95 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
96 xnn_prefetch_to_l1((const int8_t*) w8);
97 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
98 xnn_prefetch_to_l1((const int8_t*) w9);
99 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
100 xnn_prefetch_to_l1((const int8_t*) w10);
101 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
102 xnn_prefetch_to_l1((const int8_t*) w11);
103 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
104 xnn_prefetch_to_l1((const int8_t*) w12);
105 xnn_prefetch_to_l1((const int8_t*) w12 + 64);
106 xnn_prefetch_to_l1((const int8_t*) w13);
107 xnn_prefetch_to_l1((const int8_t*) w13 + 64);
108 xnn_prefetch_to_l1((const int8_t*) w14);
109 xnn_prefetch_to_l1((const int8_t*) w14 + 64);
110 xnn_prefetch_to_l1((const int8_t*) w15);
111 xnn_prefetch_to_l1((const int8_t*) w15 + 64);
112
113 // KC main loop multiple of 4
114 size_t k = kc;
115 for (; k >= 4; k -= 4) {
116 // Read blocks of 4x4
117 // a b c d
118 // e f g h
119 // i j k l
120 // m n o p
121 // Load first 4 rows of N into low part of each register
122 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
123 w0 += 4;
124 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
125 w1 += 4;
126 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
127 w2 += 4;
128 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
129 w3 += 4;
130 __m256 v8x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w8));
131 w8 += 4;
132 __m256 v9x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w9));
133 w9 += 4;
134 __m256 v10x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w10));
135 w10 += 4;
136 __m256 v11x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w11));
137 w11 += 4;
138 // Load next 4 rows of N into the high part of each register
139 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
140 w4 += 4;
141 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
142 w5 += 4;
143 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
144 w6 += 4;
145 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
146 w7 += 4;
147 v8x0123 = _mm256_insertf128_ps(v8x0123, _mm_loadu_ps(w12), 1);
148 w12 += 4;
149 v9x0123 = _mm256_insertf128_ps(v9x0123, _mm_loadu_ps(w13), 1);
150 w13 += 4;
151 v10x0123 = _mm256_insertf128_ps(v10x0123, _mm_loadu_ps(w14), 1);
152 w14 += 4;
153 v11x0123 = _mm256_insertf128_ps(v11x0123, _mm_loadu_ps(w15), 1);
154 w15 += 4;
155
156 // Apply SR4 shuffle
157 v1x0123 = _mm256_permute_ps(v1x0123, _MM_SHUFFLE(0, 3, 2, 1));
158 v2x0123 = _mm256_permute_ps(v2x0123, _MM_SHUFFLE(1, 0, 3, 2));
159 v3x0123 = _mm256_permute_ps(v3x0123, _MM_SHUFFLE(2, 1, 0, 3));
160 v9x0123 = _mm256_permute_ps(v9x0123, _MM_SHUFFLE(0, 3, 2, 1));
161 v10x0123 = _mm256_permute_ps(v10x0123, _MM_SHUFFLE(1, 0, 3, 2));
162 v11x0123 = _mm256_permute_ps(v11x0123, _MM_SHUFFLE(2, 1, 0, 3));
163
164 // Transpose 2x2
165 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
166 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
167 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
168 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
169 const __m256 vtmp8x0123 = _mm256_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
170 const __m256 vtmp9x0123 = _mm256_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
171 const __m256 vtmp10x0123 = _mm256_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
172 const __m256 vtmp11x0123 = _mm256_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
173 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
174 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
175 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
176 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
177 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
178 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
179 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
180 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
181 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
182 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
183 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
184 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
185 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
186 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
187 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
188 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
189 // Transpose 4x4
190 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
191 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
192 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
193 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
194 v8x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // a e i m from row 0, 1
195 v9x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // b f j n from row 0, 1
196 v10x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // c g k o from row 2, 3
197 v11x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // d h l p from row 2, 3
198
199 _mm256_store_ps(packed_w, v0x0123);
200 _mm256_store_ps(packed_w + 8, v8x0123);
201 _mm256_store_ps(packed_w + 16, v1x0123);
202 _mm256_store_ps(packed_w + 24, v9x0123);
203 _mm256_store_ps(packed_w + 32, v2x0123);
204 _mm256_store_ps(packed_w + 40, v10x0123);
205 _mm256_store_ps(packed_w + 48, v3x0123);
206 _mm256_store_ps(packed_w + 56, v11x0123);
207 packed_w += 64;
208 }
209
210 // KC remainder (1..3)
211 if XNN_UNLIKELY(k != 0) {
212 assert(k >= 1);
213 assert(k <= 3);
214 __m128 v0 = _mm_undefined_ps();
215 __m128 v1 = _mm_undefined_ps();
216 __m128 v2 = _mm_undefined_ps();
217 __m128 v3 = _mm_undefined_ps();
218 __m128 v4 = _mm_undefined_ps();
219 __m128 v5 = _mm_undefined_ps();
220 __m128 v6 = _mm_undefined_ps();
221 __m128 v7 = _mm_undefined_ps();
222 __m128 v8 = _mm_undefined_ps();
223 __m128 v9 = _mm_undefined_ps();
224 __m128 v10 = _mm_undefined_ps();
225 __m128 v11 = _mm_undefined_ps();
226 __m128 v12 = _mm_undefined_ps();
227 __m128 v13 = _mm_undefined_ps();
228 __m128 v14 = _mm_undefined_ps();
229 __m128 v15 = _mm_undefined_ps();
230
231 switch (k) {
232 case 1:
233 // Read blocks of 4x1
234 // a
235 // e
236 // i
237 // m
238 v0 = _mm_load_ss(w0);
239 w0 += 1;
240 v1 = _mm_load_ss(w1);
241 w1 += 1;
242 v2 = _mm_load_ss(w2);
243 w2 += 1;
244 v3 = _mm_load_ss(w3);
245 w3 += 1;
246 v4 = _mm_load_ss(w4);
247 w4 += 1;
248 v5 = _mm_load_ss(w5);
249 w5 += 1;
250 v6 = _mm_load_ss(w6);
251 w6 += 1;
252 v7 = _mm_load_ss(w7);
253 w7 += 1;
254 v8 = _mm_load_ss(w8);
255 w8 += 1;
256 v9 = _mm_load_ss(w9);
257 w9 += 1;
258 v10 = _mm_load_ss(w10);
259 w10 += 1;
260 v11 = _mm_load_ss(w11);
261 w11 += 1;
262 v12 = _mm_load_ss(w12);
263 w12 += 1;
264 v13 = _mm_load_ss(w13);
265 w13 += 1;
266 v14 = _mm_load_ss(w14);
267 w14 += 1;
268 v15 = _mm_load_ss(w15);
269 w15 += 1;
270 break;
271 case 2:
272 // Read blocks of 4x2
273 // a b
274 // e f
275 // i j
276 // m n
277 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
278 w0 += 2;
279 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
280 w1 += 2;
281 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
282 w2 += 2;
283 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
284 w3 += 2;
285 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
286 w4 += 2;
287 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
288 w5 += 2;
289 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
290 w6 += 2;
291 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
292 w7 += 2;
293 v8 = _mm_castpd_ps(_mm_load_sd((const double*) w8));
294 w8 += 2;
295 v9 = _mm_castpd_ps(_mm_load_sd((const double*) w9));
296 w9 += 2;
297 v10 = _mm_castpd_ps(_mm_load_sd((const double*) w10));
298 w10 += 2;
299 v11 = _mm_castpd_ps(_mm_load_sd((const double*) w11));
300 w11 += 2;
301 v12 = _mm_castpd_ps(_mm_load_sd((const double*) w12));
302 w12 += 2;
303 v13 = _mm_castpd_ps(_mm_load_sd((const double*) w13));
304 w13 += 2;
305 v14 = _mm_castpd_ps(_mm_load_sd((const double*) w14));
306 w14 += 2;
307 v15 = _mm_castpd_ps(_mm_load_sd((const double*) w15));
308 w15 += 2;
309 break;
310 case 3:
311 {
312 // Read blocks of 4x3
313 // a b c
314 // e f g
315 // i j k
316 // m n o
317 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
318 const __m128 v0hi = _mm_load_ss(w0 + 2);
319 v0 = _mm_movelh_ps(v0lo, v0hi);
320 w0 += 3;
321 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
322 const __m128 v1hi = _mm_load_ss(w1 + 2);
323 v1 = _mm_movelh_ps(v1lo, v1hi);
324 w1 += 3;
325 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
326 const __m128 v2hi = _mm_load_ss(w2 + 2);
327 v2 = _mm_movelh_ps(v2lo, v2hi);
328 w2 += 3;
329 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
330 const __m128 v3hi = _mm_load_ss(w3 + 2);
331 v3 = _mm_movelh_ps(v3lo, v3hi);
332 w3 += 3;
333 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
334 const __m128 v4hi = _mm_load_ss(w4 + 2);
335 v4 = _mm_movelh_ps(v4lo, v4hi);
336 w4 += 3;
337 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
338 const __m128 v5hi = _mm_load_ss(w5 + 2);
339 v5 = _mm_movelh_ps(v5lo, v5hi);
340 w5 += 3;
341 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
342 const __m128 v6hi = _mm_load_ss(w6 + 2);
343 v6 = _mm_movelh_ps(v6lo, v6hi);
344 w6 += 3;
345 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
346 const __m128 v7hi = _mm_load_ss(w7 + 2);
347 v7 = _mm_movelh_ps(v7lo, v7hi);
348 w7 += 3;
349 const __m128 v8lo = _mm_castpd_ps(_mm_load_sd((const double*) w8));
350 const __m128 v8hi = _mm_load_ss(w8 + 2);
351 v8 = _mm_movelh_ps(v8lo, v8hi);
352 w8 += 3;
353 const __m128 v9lo = _mm_castpd_ps(_mm_load_sd((const double*) w9));
354 const __m128 v9hi = _mm_load_ss(w9 + 2);
355 v9 = _mm_movelh_ps(v9lo, v9hi);
356 w9 += 3;
357 const __m128 v10lo = _mm_castpd_ps(_mm_load_sd((const double*) w10));
358 const __m128 v10hi = _mm_load_ss(w10 + 2);
359 v10 = _mm_movelh_ps(v10lo, v10hi);
360 w10 += 3;
361 const __m128 v11lo = _mm_castpd_ps(_mm_load_sd((const double*) w11));
362 const __m128 v11hi = _mm_load_ss(w11 + 2);
363 v11 = _mm_movelh_ps(v11lo, v11hi);
364 w11 += 3;
365 const __m128 v12lo = _mm_castpd_ps(_mm_load_sd((const double*) w12));
366 const __m128 v12hi = _mm_load_ss(w12 + 2);
367 v12 = _mm_movelh_ps(v12lo, v12hi);
368 w12 += 3;
369 const __m128 v13lo = _mm_castpd_ps(_mm_load_sd((const double*) w13));
370 const __m128 v13hi = _mm_load_ss(w13 + 2);
371 v13 = _mm_movelh_ps(v13lo, v13hi);
372 w13 += 3;
373 const __m128 v14lo = _mm_castpd_ps(_mm_load_sd((const double*) w14));
374 const __m128 v14hi = _mm_load_ss(w14 + 2);
375 v14 = _mm_movelh_ps(v14lo, v14hi);
376 w14 += 3;
377 const __m128 v15lo = _mm_castpd_ps(_mm_load_sd((const double*) w15));
378 const __m128 v15hi = _mm_load_ss(w15 + 2);
379 v15 = _mm_movelh_ps(v15lo, v15hi);
380 w15 += 3;
381 break;
382 }
383 default:
384 XNN_UNREACHABLE;
385 }
386
387 // Apply SR4 shuffle
388 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
389 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
390 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
391 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
392 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
393 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
394 v9 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9), _MM_SHUFFLE(0, 3, 2, 1)));
395 v10 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10), _MM_SHUFFLE(1, 0, 3, 2)));
396 v11 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11), _MM_SHUFFLE(2, 1, 0, 3)));
397 v13 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13), _MM_SHUFFLE(0, 3, 2, 1)));
398 v14 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14), _MM_SHUFFLE(1, 0, 3, 2)));
399 v15 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v15), _MM_SHUFFLE(2, 1, 0, 3)));
400 // Transpose 2x2
401 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
402 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
403 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
404 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
405 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
406 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
407 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
408 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
409 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a e b f from row 0, 1
410 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // i m j n from row 2, 3
411 const __m128 vtmp10 = _mm_unpackhi_ps(v8, v9); // c g d h from row 0, 1
412 const __m128 vtmp11 = _mm_unpackhi_ps(v10, v11); // k o l p from row 2, 3
413 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a e b f from row 0, 1
414 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // i m j n from row 2, 3
415 const __m128 vtmp14 = _mm_unpackhi_ps(v12, v13); // c g d h from row 0, 1
416 const __m128 vtmp15 = _mm_unpackhi_ps(v14, v15); // k o l p from row 2, 3
417 // Transpose 4x4
418 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
419 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
420 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
421 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
422 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
423 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
424 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
425 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
426 v8 = _mm_movelh_ps(vtmp8, vtmp9); // a e i m from row 0, 1
427 v9 = _mm_movehl_ps(vtmp9, vtmp8); // b f j n from row 0, 1
428 v10 = _mm_movelh_ps(vtmp10, vtmp11); // c g k o from row 2, 3
429 v11 = _mm_movehl_ps(vtmp11, vtmp10); // d h l p from row 2, 3
430 v12 = _mm_movelh_ps(vtmp12, vtmp13); // a e i m from row 0, 1
431 v13 = _mm_movehl_ps(vtmp13, vtmp12); // b f j n from row 0, 1
432 v14 = _mm_movelh_ps(vtmp14, vtmp15); // c g k o from row 2, 3
433 v15 = _mm_movehl_ps(vtmp15, vtmp14); // d h l p from row 2, 3
434 _mm_store_ps(packed_w, v0);
435 _mm_store_ps(packed_w + 4, v4);
436 _mm_store_ps(packed_w + 8, v8);
437 _mm_store_ps(packed_w + 12, v12);
438 _mm_store_ps(packed_w + 16, v1);
439 _mm_store_ps(packed_w + 20, v5);
440 _mm_store_ps(packed_w + 24, v9);
441 _mm_store_ps(packed_w + 28, v13);
442 _mm_store_ps(packed_w + 32, v2);
443 _mm_store_ps(packed_w + 36, v6);
444 _mm_store_ps(packed_w + 40, v10);
445 _mm_store_ps(packed_w + 44, v14);
446 _mm_store_ps(packed_w + 48, v3);
447 _mm_store_ps(packed_w + 52, v7);
448 _mm_store_ps(packed_w + 56, v11);
449 _mm_store_ps(packed_w + 60, v15);
450 packed_w += 64;
451 }
452 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
453 w0 = w15;
454 }
455
456 // NC remainder (1..15)
457 if XNN_UNLIKELY(n != 0) {
458 assert(n >= 1);
459 assert(n <= 15);
460 if XNN_LIKELY(b != NULL) {
461 size_t nb = n;
462 do {
463 *packed_w++ = *b++;
464 } while (--nb != 0);
465 packed_w += (16 - n);
466 } else {
467 const __m128 vzero = _mm_setzero_ps();
468 _mm_store_ps(packed_w, vzero);
469 _mm_store_ps(packed_w + 4, vzero);
470 _mm_store_ps(packed_w + 8, vzero);
471 _mm_store_ps(packed_w + 12, vzero);
472 packed_w += 16;
473 }
474
475 // NR remainder has less than 16 rows so last row is not loaded
476 const float* w1 = w0 + kc;
477 if XNN_UNPREDICTABLE(n < 2) {
478 w1 = w0;
479 }
480 const float* w2 = w1 + kc;
481 if XNN_UNPREDICTABLE(n <= 2) {
482 w2 = w1;
483 }
484 const float* w3 = w2 + kc;
485 if XNN_UNPREDICTABLE(n < 4) {
486 w3 = w2;
487 }
488 const float* w4 = w3 + kc;
489 if XNN_UNPREDICTABLE(n <= 4) {
490 w4 = w3;
491 }
492 const float* w5 = w4 + kc;
493 if XNN_UNPREDICTABLE(n < 6) {
494 w5 = w4;
495 }
496 const float* w6 = w5 + kc;
497 if XNN_UNPREDICTABLE(n <= 6) {
498 w6 = w5;
499 }
500 const float* w7 = w6 + kc;
501 if XNN_UNPREDICTABLE(n < 8) {
502 w7 = w6;
503 }
504 const float* w8 = w7 + kc;
505 if XNN_UNPREDICTABLE(n <= 8) {
506 w8 = w7;
507 }
508 const float* w9 = w8 + kc;
509 if XNN_UNPREDICTABLE(n < 10) {
510 w9 = w8;
511 }
512 const float* w10 = w9 + kc;
513 if XNN_UNPREDICTABLE(n <= 10) {
514 w10 = w9;
515 }
516 const float* w11 = w10 + kc;
517 if XNN_UNPREDICTABLE(n < 12) {
518 w11 = w10;
519 }
520 const float* w12 = w11 + kc;
521 if XNN_UNPREDICTABLE(n <= 12) {
522 w12 = w11;
523 }
524 const float* w13 = w12 + kc;
525 if XNN_UNPREDICTABLE(n < 14) {
526 w13 = w12;
527 }
528 const float* w14 = w13 + kc;
529 if XNN_UNPREDICTABLE(n <= 14) {
530 w14 = w13;
531 }
532
533 // KC main loop multiple of 4
534 size_t k = kc;
535 for (; k >= 4; k -= 4) {
536 // Read blocks of 4x4
537 // a b c d
538 // e f g h
539 // i j k l
540 // m n o p
541 // Load first 4 rows of N into low part of each register
542 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
543 w0 += 4;
544 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
545 w1 += 4;
546 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
547 w2 += 4;
548 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
549 w3 += 4;
550 __m256 v8x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w8));
551 w8 += 4;
552 __m256 v9x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w9));
553 w9 += 4;
554 __m256 v10x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w10));
555 w10 += 4;
556 __m256 v11x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w11));
557 w11 += 4;
558 // Load next 4 rows of N into the high part of each register
559 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
560 w4 += 4;
561 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
562 w5 += 4;
563 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
564 w6 += 4;
565 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
566 w7 += 4;
567 v8x0123 = _mm256_insertf128_ps(v8x0123, _mm_loadu_ps(w12), 1);
568 w12 += 4;
569 v9x0123 = _mm256_insertf128_ps(v9x0123, _mm_loadu_ps(w13), 1);
570 w13 += 4;
571 v10x0123 = _mm256_insertf128_ps(v10x0123, _mm_loadu_ps(w14), 1);
572 w14 += 4;
573 // Apply SR4 shuffle
574 v1x0123 = _mm256_permute_ps(v1x0123, _MM_SHUFFLE(0, 3, 2, 1));
575 v2x0123 = _mm256_permute_ps(v2x0123, _MM_SHUFFLE(1, 0, 3, 2));
576 v3x0123 = _mm256_permute_ps(v3x0123, _MM_SHUFFLE(2, 1, 0, 3));
577 v9x0123 = _mm256_permute_ps(v9x0123, _MM_SHUFFLE(0, 3, 2, 1));
578 v10x0123 = _mm256_permute_ps(v10x0123, _MM_SHUFFLE(1, 0, 3, 2));
579 v11x0123 = _mm256_permute_ps(v11x0123, _MM_SHUFFLE(2, 1, 0, 3));
580 // Transpose 2x2
581 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
582 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
583 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
584 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
585 const __m256 vtmp8x0123 = _mm256_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
586 const __m256 vtmp9x0123 = _mm256_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
587 const __m256 vtmp10x0123 = _mm256_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
588 const __m256 vtmp11x0123 = _mm256_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
589 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
590 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
591 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
592 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
593 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
594 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
595 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
596 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
597 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
598 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
599 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
600 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
601 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
602 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
603 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
604 // Transpose 4x4
605 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
606 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
607 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
608 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
609 v8x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // a e i m from row 0, 1
610 v9x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // b f j n from row 0, 1
611 v10x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // c g k o from row 2, 3
612 v11x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // d h l p from row 2, 3
613
614 _mm256_store_ps(packed_w, v0x0123);
615 _mm256_store_ps(packed_w + 8, v8x0123);
616 _mm256_store_ps(packed_w + 16, v1x0123);
617 _mm256_store_ps(packed_w + 24, v9x0123);
618 _mm256_store_ps(packed_w + 32, v2x0123);
619 _mm256_store_ps(packed_w + 40, v10x0123);
620 _mm256_store_ps(packed_w + 48, v3x0123);
621 _mm256_store_ps(packed_w + 56, v11x0123);
622 packed_w += 64;
623 }
624
625 // KC remainder (1..3)
626 if XNN_UNLIKELY(k != 0) {
627 assert(k >= 1);
628 assert(k <= 3);
629 __m128 v0 = _mm_undefined_ps();
630 __m128 v1 = _mm_undefined_ps();
631 __m128 v2 = _mm_undefined_ps();
632 __m128 v3 = _mm_undefined_ps();
633 __m128 v4 = _mm_undefined_ps();
634 __m128 v5 = _mm_undefined_ps();
635 __m128 v6 = _mm_undefined_ps();
636 __m128 v7 = _mm_undefined_ps();
637 __m128 v8 = _mm_undefined_ps();
638 __m128 v9 = _mm_undefined_ps();
639 __m128 v10 = _mm_undefined_ps();
640 __m128 v11 = _mm_undefined_ps();
641 __m128 v12 = _mm_undefined_ps();
642 __m128 v13 = _mm_undefined_ps();
643 __m128 v14 = _mm_undefined_ps();
644 __m128 v15 = _mm_undefined_ps();
645
646 switch (k) {
647 case 1:
648 // Read blocks of 4x1
649 // a
650 // e
651 // i
652 // m
653 v0 = _mm_load_ss(w0);
654 w0 += 1;
655 v1 = _mm_load_ss(w1);
656 w1 += 1;
657 v2 = _mm_load_ss(w2);
658 w2 += 1;
659 v3 = _mm_load_ss(w3);
660 w3 += 1;
661 v4 = _mm_load_ss(w4);
662 w4 += 1;
663 v5 = _mm_load_ss(w5);
664 w5 += 1;
665 v6 = _mm_load_ss(w6);
666 w6 += 1;
667 v7 = _mm_load_ss(w7);
668 w7 += 1;
669 v8 = _mm_load_ss(w8);
670 w8 += 1;
671 v9 = _mm_load_ss(w9);
672 w9 += 1;
673 v10 = _mm_load_ss(w10);
674 w10 += 1;
675 v11 = _mm_load_ss(w11);
676 w11 += 1;
677 v12 = _mm_load_ss(w12);
678 w12 += 1;
679 v13 = _mm_load_ss(w13);
680 w13 += 1;
681 v14 = _mm_load_ss(w14);
682 w14 += 1;
683 break;
684 case 2:
685 // Read blocks of 4x2
686 // a b
687 // e f
688 // i j
689 // m n
690 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
691 w0 += 2;
692 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
693 w1 += 2;
694 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
695 w2 += 2;
696 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
697 w3 += 2;
698 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
699 w4 += 2;
700 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
701 w5 += 2;
702 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
703 w6 += 2;
704 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
705 w7 += 2;
706 v8 = _mm_castpd_ps(_mm_load_sd((const double*) w8));
707 w8 += 2;
708 v9 = _mm_castpd_ps(_mm_load_sd((const double*) w9));
709 w9 += 2;
710 v10 = _mm_castpd_ps(_mm_load_sd((const double*) w10));
711 w10 += 2;
712 v11 = _mm_castpd_ps(_mm_load_sd((const double*) w11));
713 w11 += 2;
714 v12 = _mm_castpd_ps(_mm_load_sd((const double*) w12));
715 w12 += 2;
716 v13 = _mm_castpd_ps(_mm_load_sd((const double*) w13));
717 w13 += 2;
718 v14 = _mm_castpd_ps(_mm_load_sd((const double*) w14));
719 w14 += 2;
720 break;
721 case 3:
722 {
723 // Read blocks of 4x3
724 // a b c
725 // e f g
726 // i j k
727 // m n o
728 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
729 const __m128 v0hi = _mm_load_ss(w0 + 2);
730 v0 = _mm_movelh_ps(v0lo, v0hi);
731 w0 += 3;
732 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
733 const __m128 v1hi = _mm_load_ss(w1 + 2);
734 v1 = _mm_movelh_ps(v1lo, v1hi);
735 w1 += 3;
736 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
737 const __m128 v2hi = _mm_load_ss(w2 + 2);
738 v2 = _mm_movelh_ps(v2lo, v2hi);
739 w2 += 3;
740 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
741 const __m128 v3hi = _mm_load_ss(w3 + 2);
742 v3 = _mm_movelh_ps(v3lo, v3hi);
743 w3 += 3;
744 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
745 const __m128 v4hi = _mm_load_ss(w4 + 2);
746 v4 = _mm_movelh_ps(v4lo, v4hi);
747 w4 += 3;
748 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
749 const __m128 v5hi = _mm_load_ss(w5 + 2);
750 v5 = _mm_movelh_ps(v5lo, v5hi);
751 w5 += 3;
752 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
753 const __m128 v6hi = _mm_load_ss(w6 + 2);
754 v6 = _mm_movelh_ps(v6lo, v6hi);
755 w6 += 3;
756 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
757 const __m128 v7hi = _mm_load_ss(w7 + 2);
758 v7 = _mm_movelh_ps(v7lo, v7hi);
759 w7 += 3;
760 const __m128 v8lo = _mm_castpd_ps(_mm_load_sd((const double*) w8));
761 const __m128 v8hi = _mm_load_ss(w8 + 2);
762 v8 = _mm_movelh_ps(v8lo, v8hi);
763 w8 += 3;
764 const __m128 v9lo = _mm_castpd_ps(_mm_load_sd((const double*) w9));
765 const __m128 v9hi = _mm_load_ss(w9 + 2);
766 v9 = _mm_movelh_ps(v9lo, v9hi);
767 w9 += 3;
768 const __m128 v10lo = _mm_castpd_ps(_mm_load_sd((const double*) w10));
769 const __m128 v10hi = _mm_load_ss(w10 + 2);
770 v10 = _mm_movelh_ps(v10lo, v10hi);
771 w10 += 3;
772 const __m128 v11lo = _mm_castpd_ps(_mm_load_sd((const double*) w11));
773 const __m128 v11hi = _mm_load_ss(w11 + 2);
774 v11 = _mm_movelh_ps(v11lo, v11hi);
775 w11 += 3;
776 const __m128 v12lo = _mm_castpd_ps(_mm_load_sd((const double*) w12));
777 const __m128 v12hi = _mm_load_ss(w12 + 2);
778 v12 = _mm_movelh_ps(v12lo, v12hi);
779 w12 += 3;
780 const __m128 v13lo = _mm_castpd_ps(_mm_load_sd((const double*) w13));
781 const __m128 v13hi = _mm_load_ss(w13 + 2);
782 v13 = _mm_movelh_ps(v13lo, v13hi);
783 w13 += 3;
784 const __m128 v14lo = _mm_castpd_ps(_mm_load_sd((const double*) w14));
785 const __m128 v14hi = _mm_load_ss(w14 + 2);
786 v14 = _mm_movelh_ps(v14lo, v14hi);
787 w14 += 3;
788 break;
789 }
790 default:
791 XNN_UNREACHABLE;
792 }
793
794 // Apply SR4 shuffle
795 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
796 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
797 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
798 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
799 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
800 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
801 v9 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v9), _MM_SHUFFLE(0, 3, 2, 1)));
802 v10 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v10), _MM_SHUFFLE(1, 0, 3, 2)));
803 v11 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v11), _MM_SHUFFLE(2, 1, 0, 3)));
804 v13 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v13), _MM_SHUFFLE(0, 3, 2, 1)));
805 v14 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v14), _MM_SHUFFLE(1, 0, 3, 2)));
806 // Transpose 2x2
807 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
808 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
809 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
810 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
811 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
812 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
813 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
814 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
815 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a e b f from row 0, 1
816 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // i m j n from row 2, 3
817 const __m128 vtmp10 = _mm_unpackhi_ps(v8, v9); // c g d h from row 0, 1
818 const __m128 vtmp11 = _mm_unpackhi_ps(v10, v11); // k o l p from row 2, 3
819 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a e b f from row 0, 1
820 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // i m j n from row 2, 3
821 const __m128 vtmp14 = _mm_unpackhi_ps(v12, v13); // c g d h from row 0, 1
822 const __m128 vtmp15 = _mm_unpackhi_ps(v14, v14); // k o l p from row 2, 3
823 // Transpose 4x4
824 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
825 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
826 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
827 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
828 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
829 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
830 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
831 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
832 v8 = _mm_movelh_ps(vtmp8, vtmp9); // a e i m from row 0, 1
833 v9 = _mm_movehl_ps(vtmp9, vtmp8); // b f j n from row 0, 1
834 v10 = _mm_movelh_ps(vtmp10, vtmp11); // c g k o from row 2, 3
835 v11 = _mm_movehl_ps(vtmp11, vtmp10); // d h l p from row 2, 3
836 v12 = _mm_movelh_ps(vtmp12, vtmp13); // a e i m from row 0, 1
837 v13 = _mm_movehl_ps(vtmp13, vtmp12); // b f j n from row 0, 1
838 v14 = _mm_movelh_ps(vtmp14, vtmp15); // c g k o from row 2, 3
839 v15 = _mm_movehl_ps(vtmp15, vtmp14); // d h l p from row 2, 3
840 _mm_store_ps(packed_w, v0);
841 _mm_store_ps(packed_w + 4, v4);
842 _mm_store_ps(packed_w + 8, v8);
843 _mm_store_ps(packed_w + 12, v12);
844 _mm_store_ps(packed_w + 16, v1);
845 _mm_store_ps(packed_w + 20, v5);
846 _mm_store_ps(packed_w + 24, v9);
847 _mm_store_ps(packed_w + 28, v13);
848 _mm_store_ps(packed_w + 32, v2);
849 _mm_store_ps(packed_w + 36, v6);
850 _mm_store_ps(packed_w + 40, v10);
851 _mm_store_ps(packed_w + 44, v14);
852 _mm_store_ps(packed_w + 48, v3);
853 _mm_store_ps(packed_w + 52, v7);
854 _mm_store_ps(packed_w + 56, v11);
855 _mm_store_ps(packed_w + 60, v15);
856 packed_w += 64;
857 }
858 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
859 }
860 weights += nc * kc;
861 } while (--g != 0);
862 }