Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-sse2-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <emmintrin.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20 void xnn_x32_packw_gemm_goi_ukernel_x8__sse2_u4_prfm(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 8);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 const float* b = (const float*) bias;
44 float* packed_w = (float*) packed_weights;
45 do {
46 // NC main loop multiple of 8
47 const float* w0 = (const float*) weights;
48 size_t n = nc;
49
50 for (; n >= 8; n -= 8) {
51 if XNN_LIKELY(b != NULL) {
52 const __m128 vb0123 = _mm_loadu_ps(b);
53 const __m128 vb4567 = _mm_loadu_ps(b + 4);
54 b += 8;
55
56 _mm_store_ps(packed_w, vb0123);
57 _mm_store_ps(packed_w + 4, vb4567);
58 } else {
59 const __m128 vzero = _mm_setzero_ps();
60 _mm_store_ps(packed_w, vzero);
61 _mm_store_ps(packed_w + 4, vzero);
62 }
63 packed_w += 8;
64
65 const float* w1 = w0 + kc;
66 const float* w2 = w1 + kc;
67 const float* w3 = w2 + kc;
68 const float* w4 = w3 + kc;
69 const float* w5 = w4 + kc;
70 const float* w6 = w5 + kc;
71 const float* w7 = w6 + kc;
72 xnn_prefetch_to_l1((const int8_t*) w0);
73 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
74 xnn_prefetch_to_l1((const int8_t*) w1);
75 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
76 xnn_prefetch_to_l1((const int8_t*) w2);
77 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
78 xnn_prefetch_to_l1((const int8_t*) w3);
79 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
80 xnn_prefetch_to_l1((const int8_t*) w4);
81 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
82 xnn_prefetch_to_l1((const int8_t*) w5);
83 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
84 xnn_prefetch_to_l1((const int8_t*) w6);
85 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
86 xnn_prefetch_to_l1((const int8_t*) w7);
87 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
88
89 size_t k = kc;
90
91 // KC multiple of 4
92 for (; k >= 4; k -= 4) {
93 const __m128 v0x0123 = _mm_loadu_ps(w0);
94 w0 += 4;
95 const __m128 v1x0123 = _mm_loadu_ps(w1);
96 w1 += 4;
97 const __m128 v2x0123 = _mm_loadu_ps(w2);
98 w2 += 4;
99 const __m128 v3x0123 = _mm_loadu_ps(w3);
100 w3 += 4;
101 const __m128 v4x0123 = _mm_loadu_ps(w4);
102 w4 += 4;
103 const __m128 v5x0123 = _mm_loadu_ps(w5);
104 w5 += 4;
105 const __m128 v6x0123 = _mm_loadu_ps(w6);
106 w6 += 4;
107 const __m128 v7x0123 = _mm_loadu_ps(w7);
108 w7 += 4;
109
110 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
111 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
112 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
113 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
114 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
115 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
116 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
117 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
118 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
119 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
120 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
121 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
122 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
123 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
124 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
125 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
126 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
127 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
128 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
129 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
130 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
131 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
132 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
133 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
134
135 _mm_store_ps(packed_w, v0123x0);
136 _mm_store_ps(packed_w + 4, v4567x0);
137 _mm_store_ps(packed_w + 8, v0123x1);
138 _mm_store_ps(packed_w + 12, v4567x1);
139 _mm_store_ps(packed_w + 16, v0123x2);
140 _mm_store_ps(packed_w + 20, v4567x2);
141 _mm_store_ps(packed_w + 24, v0123x3);
142 _mm_store_ps(packed_w + 28, v4567x3);
143 packed_w += 32;
144 }
145
146 // KC remainder (1..3)
147 if XNN_UNLIKELY(k != 0) {
148 assert(k >= 1);
149 assert(k <= 3);
150 switch (k) {
151 case 1:
152 {
153 const __m128 v0x0 = _mm_load_ss(w0);
154 w0 += 1;
155 const __m128 v1x0 = _mm_load_ss(w1);
156 w1 += 1;
157 const __m128 v2x0 = _mm_load_ss(w2);
158 w2 += 1;
159 const __m128 v3x0 = _mm_load_ss(w3);
160 w3 += 1;
161 const __m128 v4x0 = _mm_load_ss(w4);
162 w4 += 1;
163 const __m128 v5x0 = _mm_load_ss(w5);
164 w5 += 1;
165 const __m128 v6x0 = _mm_load_ss(w6);
166 w6 += 1;
167 const __m128 v7x0 = _mm_load_ss(w7);
168 w7 += 1;
169
170 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
171 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
172 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
173 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
174
175 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
176 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
177
178 _mm_store_ps(packed_w, v0123x0);
179 _mm_store_ps(packed_w + 4, v4567x0);
180 packed_w += 8;
181 break;
182 }
183 case 2:
184 {
185 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
186 w0 += 2;
187 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
188 w1 += 2;
189 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
190 w2 += 2;
191 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
192 w3 += 2;
193 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
194 w4 += 2;
195 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
196 w5 += 2;
197 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
198 w6 += 2;
199 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
200 w7 += 2;
201
202 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
203 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
204 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
205 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
206
207 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
208 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
209 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
210 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
211
212 _mm_store_ps(packed_w, v0123x0);
213 _mm_store_ps(packed_w + 4, v4567x0);
214 _mm_store_ps(packed_w + 8, v0123x1);
215 _mm_store_ps(packed_w + 12, v4567x1);
216 packed_w += 16;
217 break;
218 }
219 case 3:
220 {
221 __m128 v0x012 = _mm_load_ss(w0 + 2);
222 __m128 v1x012 = _mm_load_ss(w1 + 2);
223 __m128 v2x012 = _mm_load_ss(w2 + 2);
224 __m128 v3x012 = _mm_load_ss(w3 + 2);
225 __m128 v4x012 = _mm_load_ss(w4 + 2);
226 __m128 v5x012 = _mm_load_ss(w5 + 2);
227 __m128 v6x012 = _mm_load_ss(w6 + 2);
228 __m128 v7x012 = _mm_load_ss(w7 + 2);
229
230 v0x012 = _mm_movelh_ps(v0x012, v0x012);
231 v1x012 = _mm_movelh_ps(v1x012, v1x012);
232 v2x012 = _mm_movelh_ps(v2x012, v2x012);
233 v3x012 = _mm_movelh_ps(v3x012, v3x012);
234 v4x012 = _mm_movelh_ps(v4x012, v4x012);
235 v5x012 = _mm_movelh_ps(v5x012, v5x012);
236 v6x012 = _mm_movelh_ps(v6x012, v6x012);
237 v7x012 = _mm_movelh_ps(v7x012, v7x012);
238
239 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
240 w0 += 3;
241 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
242 w1 += 3;
243 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
244 w2 += 3;
245 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
246 w3 += 3;
247 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
248 w4 += 3;
249 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
250 w5 += 3;
251 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
252 w6 += 3;
253 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
254 w7 += 3;
255
256 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
257 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
258 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
259 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
260 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
261 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
262 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
263 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
264
265 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
266 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
267 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
268 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
269 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
270 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
271
272 _mm_store_ps(packed_w, v0123x0);
273 _mm_store_ps(packed_w + 4, v4567x0);
274 _mm_store_ps(packed_w + 8, v0123x1);
275 _mm_store_ps(packed_w + 12, v4567x1);
276 _mm_store_ps(packed_w + 16, v0123x2);
277 _mm_store_ps(packed_w + 20, v4567x2);
278 packed_w += 24;
279 break;
280 }
281 default:
282 XNN_UNREACHABLE;
283 }
284 }
285 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
286 w0 = w7;
287 }
288
289 // NC remainder (1..7)
290 if XNN_UNLIKELY(n != 0) {
291 assert(n >= 1);
292 assert(n <= 7);
293 if XNN_LIKELY(b != NULL) {
294 size_t nb = n;
295 do {
296 *packed_w++ = *b++;
297 } while (--nb != 0);
298 packed_w += (8 - n);
299 } else {
300 const __m128 vzero = _mm_setzero_ps();
301 _mm_store_ps(packed_w, vzero);
302 _mm_store_ps(packed_w + 4, vzero);
303 packed_w += 8;
304 }
305
306 const float* w1 = w0 + kc;
307 if XNN_UNPREDICTABLE(n < 2) {
308 w1 = w0;
309 }
310 const float* w2 = w1 + kc;
311 if XNN_UNPREDICTABLE(n <= 2) {
312 w2 = w1;
313 }
314 const float* w3 = w2 + kc;
315 if XNN_UNPREDICTABLE(n < 4) {
316 w3 = w2;
317 }
318 const float* w4 = w3 + kc;
319 if XNN_UNPREDICTABLE(n <= 4) {
320 w4 = w3;
321 }
322 const float* w5 = w4 + kc;
323 if XNN_UNPREDICTABLE(n < 6) {
324 w5 = w4;
325 }
326 const float* w6 = w5 + kc;
327 if XNN_UNPREDICTABLE(n <= 6) {
328 w6 = w5;
329 }
330
331 size_t k = kc;
332
333 // KC multiple of 4
334 for (; k >= 4; k -= 4) {
335 const __m128 v0x0123 = _mm_loadu_ps(w0);
336 w0 += 4;
337 const __m128 v1x0123 = _mm_loadu_ps(w1);
338 w1 += 4;
339 const __m128 v2x0123 = _mm_loadu_ps(w2);
340 w2 += 4;
341 const __m128 v3x0123 = _mm_loadu_ps(w3);
342 w3 += 4;
343 const __m128 v4x0123 = _mm_loadu_ps(w4);
344 w4 += 4;
345 const __m128 v5x0123 = _mm_loadu_ps(w5);
346 w5 += 4;
347 const __m128 v6x0123 = _mm_loadu_ps(w6);
348 w6 += 4;
349
350 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
351 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
352 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
353 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
354 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
355 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v6x0123);
356 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
357 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v6x0123);
358 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
359 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
360 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
361 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
362 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
363 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
364 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
365 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
366 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
367 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
368 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
369 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
370 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
371 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
372 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
373
374 _mm_store_ps(packed_w, v0123x0);
375 _mm_store_ps(packed_w + 4, v4567x0);
376 _mm_store_ps(packed_w + 8, v0123x1);
377 _mm_store_ps(packed_w + 12, v4567x1);
378 _mm_store_ps(packed_w + 16, v0123x2);
379 _mm_store_ps(packed_w + 20, v4567x2);
380 _mm_store_ps(packed_w + 24, v0123x3);
381 _mm_store_ps(packed_w + 28, v4567x3);
382 packed_w += 32;
383 }
384
385 // KC remainder (1..3)
386 if XNN_UNLIKELY(k != 0) {
387 assert(k >= 1);
388 assert(k <= 3);
389 switch (k) {
390 case 1:
391 {
392 const __m128 v0x0 = _mm_load_ss(w0);
393 const __m128 v1x0 = _mm_load_ss(w1);
394 const __m128 v2x0 = _mm_load_ss(w2);
395 const __m128 v3x0 = _mm_load_ss(w3);
396 const __m128 v4x0 = _mm_load_ss(w4);
397 const __m128 v5x0 = _mm_load_ss(w5);
398 const __m128 v6x0 = _mm_load_ss(w6);
399
400 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
401 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
402 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
403 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v6x0);
404
405 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
406 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
407
408 _mm_store_ps(packed_w, v0123x0);
409 _mm_store_ps(packed_w + 4, v4567x0);
410 packed_w += 8;
411 break;
412 }
413 case 2:
414 {
415 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
416 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
417 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
418 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
419 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
420 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
421 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
422
423 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
424 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
425 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
426 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v6x01);
427
428 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
429 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
430 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
431 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
432
433 _mm_store_ps(packed_w, v0123x0);
434 _mm_store_ps(packed_w + 4, v4567x0);
435 _mm_store_ps(packed_w + 8, v0123x1);
436 _mm_store_ps(packed_w + 12, v4567x1);
437 packed_w += 16;
438 break;
439 }
440 case 3:
441 {
442 __m128 v0x012 = _mm_load_ss(w0 + 2);
443 __m128 v1x012 = _mm_load_ss(w1 + 2);
444 __m128 v2x012 = _mm_load_ss(w2 + 2);
445 __m128 v3x012 = _mm_load_ss(w3 + 2);
446 __m128 v4x012 = _mm_load_ss(w4 + 2);
447 __m128 v5x012 = _mm_load_ss(w5 + 2);
448 __m128 v6x012 = _mm_load_ss(w6 + 2);
449
450 v0x012 = _mm_movelh_ps(v0x012, v0x012);
451 v1x012 = _mm_movelh_ps(v1x012, v1x012);
452 v2x012 = _mm_movelh_ps(v2x012, v2x012);
453 v3x012 = _mm_movelh_ps(v3x012, v3x012);
454 v4x012 = _mm_movelh_ps(v4x012, v4x012);
455 v5x012 = _mm_movelh_ps(v5x012, v5x012);
456 v6x012 = _mm_movelh_ps(v6x012, v6x012);
457
458 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
459 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
460 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
461 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
462 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
463 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
464 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
465
466 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
467 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
468 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
469 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
470 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
471 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v6x012);
472 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
473 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v6x012);
474
475 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
476 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
477 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
478 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
479 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
480 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
481
482 _mm_store_ps(packed_w, v0123x0);
483 _mm_store_ps(packed_w + 4, v4567x0);
484 _mm_store_ps(packed_w + 8, v0123x1);
485 _mm_store_ps(packed_w + 12, v4567x1);
486 _mm_store_ps(packed_w + 16, v0123x2);
487 _mm_store_ps(packed_w + 20, v4567x2);
488 packed_w += 24;
489 break;
490 }
491 default:
492 XNN_UNREACHABLE;
493 }
494 }
495 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
496 }
497 weights += nc * kc;
498 } while (--g != 0);
499 }