Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-sse2-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <emmintrin.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20 void xnn_x32_packw_gemm_goi_ukernel_x16__sse2_u4_prfm(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 16);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 const float* b = (const float*) bias;
44 float* packed_w = (float*) packed_weights;
45 do {
46 // NC main loop multiple of 16
47 const float* w0 = (const float*) weights;
48 size_t n = nc;
49
50 for (; n >= 16; n -= 16) {
51 if XNN_LIKELY(b != NULL) {
52 const __m128 vb0123 = _mm_loadu_ps(b);
53 const __m128 vb4567 = _mm_loadu_ps(b + 4);
54 const __m128 vb89AB = _mm_loadu_ps(b + 8);
55 const __m128 vbCDEF = _mm_loadu_ps(b + 12);
56 b += 16;
57
58 _mm_store_ps(packed_w, vb0123);
59 _mm_store_ps(packed_w + 4, vb4567);
60 _mm_store_ps(packed_w + 8, vb89AB);
61 _mm_store_ps(packed_w + 12, vbCDEF);
62 } else {
63 const __m128 vzero = _mm_setzero_ps();
64 _mm_store_ps(packed_w, vzero);
65 _mm_store_ps(packed_w + 4, vzero);
66 _mm_store_ps(packed_w + 8, vzero);
67 _mm_store_ps(packed_w + 12, vzero);
68 }
69 packed_w += 16;
70
71 const float* w1 = w0 + kc;
72 const float* w2 = w1 + kc;
73 const float* w3 = w2 + kc;
74 const float* w4 = w3 + kc;
75 const float* w5 = w4 + kc;
76 const float* w6 = w5 + kc;
77 const float* w7 = w6 + kc;
78 const float* w8 = w7 + kc;
79 const float* w9 = w8 + kc;
80 const float* w10 = w9 + kc;
81 const float* w11 = w10 + kc;
82 const float* w12 = w11 + kc;
83 const float* w13 = w12 + kc;
84 const float* w14 = w13 + kc;
85 const float* w15 = w14 + kc;
86 xnn_prefetch_to_l1((const int8_t*) w0);
87 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
88 xnn_prefetch_to_l1((const int8_t*) w1);
89 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
90 xnn_prefetch_to_l1((const int8_t*) w2);
91 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
92 xnn_prefetch_to_l1((const int8_t*) w3);
93 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
94 xnn_prefetch_to_l1((const int8_t*) w4);
95 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
96 xnn_prefetch_to_l1((const int8_t*) w5);
97 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
98 xnn_prefetch_to_l1((const int8_t*) w6);
99 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
100 xnn_prefetch_to_l1((const int8_t*) w7);
101 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
102 xnn_prefetch_to_l1((const int8_t*) w8);
103 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
104 xnn_prefetch_to_l1((const int8_t*) w9);
105 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
106 xnn_prefetch_to_l1((const int8_t*) w10);
107 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
108 xnn_prefetch_to_l1((const int8_t*) w11);
109 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
110 xnn_prefetch_to_l1((const int8_t*) w12);
111 xnn_prefetch_to_l1((const int8_t*) w12 + 64);
112 xnn_prefetch_to_l1((const int8_t*) w13);
113 xnn_prefetch_to_l1((const int8_t*) w13 + 64);
114 xnn_prefetch_to_l1((const int8_t*) w14);
115 xnn_prefetch_to_l1((const int8_t*) w14 + 64);
116 xnn_prefetch_to_l1((const int8_t*) w15);
117 xnn_prefetch_to_l1((const int8_t*) w15 + 64);
118
119 size_t k = kc;
120
121 // KC multiple of 4
122 for (; k >= 4; k -= 4) {
123 const __m128 v0x0123 = _mm_loadu_ps(w0);
124 w0 += 4;
125 const __m128 v1x0123 = _mm_loadu_ps(w1);
126 w1 += 4;
127 const __m128 v2x0123 = _mm_loadu_ps(w2);
128 w2 += 4;
129 const __m128 v3x0123 = _mm_loadu_ps(w3);
130 w3 += 4;
131 const __m128 v4x0123 = _mm_loadu_ps(w4);
132 w4 += 4;
133 const __m128 v5x0123 = _mm_loadu_ps(w5);
134 w5 += 4;
135 const __m128 v6x0123 = _mm_loadu_ps(w6);
136 w6 += 4;
137 const __m128 v7x0123 = _mm_loadu_ps(w7);
138 w7 += 4;
139 const __m128 v8x0123 = _mm_loadu_ps(w8);
140 w8 += 4;
141 const __m128 v9x0123 = _mm_loadu_ps(w9);
142 w9 += 4;
143 const __m128 vAx0123 = _mm_loadu_ps(w10);
144 w10 += 4;
145 const __m128 vBx0123 = _mm_loadu_ps(w11);
146 w11 += 4;
147 const __m128 vCx0123 = _mm_loadu_ps(w12);
148 w12 += 4;
149 const __m128 vDx0123 = _mm_loadu_ps(w13);
150 w13 += 4;
151 const __m128 vEx0123 = _mm_loadu_ps(w14);
152 w14 += 4;
153 const __m128 vFx0123 = _mm_loadu_ps(w15);
154 w15 += 4;
155
156 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
157 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
158 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
159 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
160 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
161 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
162 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
163 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
164 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
165 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
166 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
167 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
168 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
169 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vFx0123);
170 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
171 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vFx0123);
172 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
173 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
174 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
175 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
176 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
177 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
178 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
179 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
180 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
181 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
182 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
183 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
184 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
185 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
186 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
187 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
188 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
189 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
190 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
191 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
192 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
193 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
194 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
195 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
196 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
197 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
198 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
199 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
200 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
201 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
202 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
203 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
204
205 _mm_store_ps(packed_w, v0123x0);
206 _mm_store_ps(packed_w + 4, v4567x0);
207 _mm_store_ps(packed_w + 8, v89ABx0);
208 _mm_store_ps(packed_w + 12, vCDEFx0);
209 _mm_store_ps(packed_w + 16, v0123x1);
210 _mm_store_ps(packed_w + 20, v4567x1);
211 _mm_store_ps(packed_w + 24, v89ABx1);
212 _mm_store_ps(packed_w + 28, vCDEFx1);
213 _mm_store_ps(packed_w + 32, v0123x2);
214 _mm_store_ps(packed_w + 36, v4567x2);
215 _mm_store_ps(packed_w + 40, v89ABx2);
216 _mm_store_ps(packed_w + 44, vCDEFx2);
217 _mm_store_ps(packed_w + 48, v0123x3);
218 _mm_store_ps(packed_w + 52, v4567x3);
219 _mm_store_ps(packed_w + 56, v89ABx3);
220 _mm_store_ps(packed_w + 60, vCDEFx3);
221 packed_w += 64;
222 }
223
224 // KC remainder (1..3)
225 if XNN_UNLIKELY(k != 0) {
226 assert(k >= 1);
227 assert(k <= 3);
228 switch (k) {
229 case 1:
230 {
231 const __m128 v0x0 = _mm_load_ss(w0);
232 w0 += 1;
233 const __m128 v1x0 = _mm_load_ss(w1);
234 w1 += 1;
235 const __m128 v2x0 = _mm_load_ss(w2);
236 w2 += 1;
237 const __m128 v3x0 = _mm_load_ss(w3);
238 w3 += 1;
239 const __m128 v4x0 = _mm_load_ss(w4);
240 w4 += 1;
241 const __m128 v5x0 = _mm_load_ss(w5);
242 w5 += 1;
243 const __m128 v6x0 = _mm_load_ss(w6);
244 w6 += 1;
245 const __m128 v7x0 = _mm_load_ss(w7);
246 w7 += 1;
247 const __m128 v8x0 = _mm_load_ss(w8);
248 w8 += 1;
249 const __m128 v9x0 = _mm_load_ss(w9);
250 w9 += 1;
251 const __m128 vAx0 = _mm_load_ss(w10);
252 w10 += 1;
253 const __m128 vBx0 = _mm_load_ss(w11);
254 w11 += 1;
255 const __m128 vCx0 = _mm_load_ss(w12);
256 w12 += 1;
257 const __m128 vDx0 = _mm_load_ss(w13);
258 w13 += 1;
259 const __m128 vEx0 = _mm_load_ss(w14);
260 w14 += 1;
261 const __m128 vFx0 = _mm_load_ss(w15);
262 w15 += 1;
263
264 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
265 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
266 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
267 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
268 const __m128 v89x0 = _mm_unpacklo_ps(v8x0, v9x0);
269 const __m128 vABx0 = _mm_unpacklo_ps(vAx0, vBx0);
270 const __m128 vCDx0 = _mm_unpacklo_ps(vCx0, vDx0);
271 const __m128 vEFx0 = _mm_unpacklo_ps(vEx0, vFx0);
272
273 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
274 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
275 const __m128 v89ABx0 = _mm_movelh_ps(v89x0, vABx0);
276 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0, vEFx0);
277
278 _mm_store_ps(packed_w, v0123x0);
279 _mm_store_ps(packed_w + 4, v4567x0);
280 _mm_store_ps(packed_w + 8, v89ABx0);
281 _mm_store_ps(packed_w + 12, vCDEFx0);
282 packed_w += 16;
283 break;
284 }
285 case 2:
286 {
287 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
288 w0 += 2;
289 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
290 w1 += 2;
291 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
292 w2 += 2;
293 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
294 w3 += 2;
295 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
296 w4 += 2;
297 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
298 w5 += 2;
299 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
300 w6 += 2;
301 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
302 w7 += 2;
303 const __m128 v8x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
304 w8 += 2;
305 const __m128 v9x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
306 w9 += 2;
307 const __m128 vAx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
308 w10 += 2;
309 const __m128 vBx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
310 w11 += 2;
311 const __m128 vCx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
312 w12 += 2;
313 const __m128 vDx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
314 w13 += 2;
315 const __m128 vEx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
316 w14 += 2;
317 const __m128 vFx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
318 w15 += 2;
319
320 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
321 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
322 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
323 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
324 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x01, v9x01);
325 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx01, vBx01);
326 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx01, vDx01);
327 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx01, vFx01);
328
329 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
330 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
331 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
332 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
333 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
334 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
335 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
336 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
337
338 _mm_store_ps(packed_w, v0123x0);
339 _mm_store_ps(packed_w + 4, v4567x0);
340 _mm_store_ps(packed_w + 8, v89ABx0);
341 _mm_store_ps(packed_w + 12, vCDEFx0);
342 _mm_store_ps(packed_w + 16, v0123x1);
343 _mm_store_ps(packed_w + 20, v4567x1);
344 _mm_store_ps(packed_w + 24, v89ABx1);
345 _mm_store_ps(packed_w + 28, vCDEFx1);
346 packed_w += 32;
347 break;
348 }
349 case 3:
350 {
351 __m128 v0x012 = _mm_load_ss(w0 + 2);
352 __m128 v1x012 = _mm_load_ss(w1 + 2);
353 __m128 v2x012 = _mm_load_ss(w2 + 2);
354 __m128 v3x012 = _mm_load_ss(w3 + 2);
355 __m128 v4x012 = _mm_load_ss(w4 + 2);
356 __m128 v5x012 = _mm_load_ss(w5 + 2);
357 __m128 v6x012 = _mm_load_ss(w6 + 2);
358 __m128 v7x012 = _mm_load_ss(w7 + 2);
359 __m128 v8x012 = _mm_load_ss(w8 + 2);
360 __m128 v9x012 = _mm_load_ss(w9 + 2);
361 __m128 vAx012 = _mm_load_ss(w10 + 2);
362 __m128 vBx012 = _mm_load_ss(w11 + 2);
363 __m128 vCx012 = _mm_load_ss(w12 + 2);
364 __m128 vDx012 = _mm_load_ss(w13 + 2);
365 __m128 vEx012 = _mm_load_ss(w14 + 2);
366 __m128 vFx012 = _mm_load_ss(w15 + 2);
367
368 v0x012 = _mm_movelh_ps(v0x012, v0x012);
369 v1x012 = _mm_movelh_ps(v1x012, v1x012);
370 v2x012 = _mm_movelh_ps(v2x012, v2x012);
371 v3x012 = _mm_movelh_ps(v3x012, v3x012);
372 v4x012 = _mm_movelh_ps(v4x012, v4x012);
373 v5x012 = _mm_movelh_ps(v5x012, v5x012);
374 v6x012 = _mm_movelh_ps(v6x012, v6x012);
375 v7x012 = _mm_movelh_ps(v7x012, v7x012);
376 v8x012 = _mm_movelh_ps(v8x012, v8x012);
377 v9x012 = _mm_movelh_ps(v9x012, v9x012);
378 vAx012 = _mm_movelh_ps(vAx012, vAx012);
379 vBx012 = _mm_movelh_ps(vBx012, vBx012);
380 vCx012 = _mm_movelh_ps(vCx012, vCx012);
381 vDx012 = _mm_movelh_ps(vDx012, vDx012);
382 vEx012 = _mm_movelh_ps(vEx012, vEx012);
383 vFx012 = _mm_movelh_ps(vFx012, vFx012);
384
385 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
386 w0 += 3;
387 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
388 w1 += 3;
389 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
390 w2 += 3;
391 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
392 w3 += 3;
393 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
394 w4 += 3;
395 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
396 w5 += 3;
397 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
398 w6 += 3;
399 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
400 w7 += 3;
401 v8x012 = _mm_loadl_pi(v8x012, (const __m64*) w8);
402 w8 += 3;
403 v9x012 = _mm_loadl_pi(v9x012, (const __m64*) w9);
404 w9 += 3;
405 vAx012 = _mm_loadl_pi(vAx012, (const __m64*) w10);
406 w10 += 3;
407 vBx012 = _mm_loadl_pi(vBx012, (const __m64*) w11);
408 w11 += 3;
409 vCx012 = _mm_loadl_pi(vCx012, (const __m64*) w12);
410 w12 += 3;
411 vDx012 = _mm_loadl_pi(vDx012, (const __m64*) w13);
412 w13 += 3;
413 vEx012 = _mm_loadl_pi(vEx012, (const __m64*) w14);
414 w14 += 3;
415 vFx012 = _mm_loadl_pi(vFx012, (const __m64*) w15);
416 w15 += 3;
417
418 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
419 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
420 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
421 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
422 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
423 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
424 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
425 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
426 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x012, v9x012);
427 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx012, vBx012);
428 const __m128 v89x2 = _mm_unpackhi_ps(v8x012, v9x012);
429 const __m128 vABx2 = _mm_unpackhi_ps(vAx012, vBx012);
430 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx012, vDx012);
431 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx012, vFx012);
432 const __m128 vCDx2 = _mm_unpackhi_ps(vCx012, vDx012);
433 const __m128 vEFx2 = _mm_unpackhi_ps(vEx012, vFx012);
434
435 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
436 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
437 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
438 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
439 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
440 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
441 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
442 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
443 const __m128 v89ABx2 = _mm_movelh_ps(v89x2, vABx2);
444 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
445 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
446 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2, vEFx2);
447
448 _mm_store_ps(packed_w, v0123x0);
449 _mm_store_ps(packed_w + 4, v4567x0);
450 _mm_store_ps(packed_w + 8, v89ABx0);
451 _mm_store_ps(packed_w + 12, vCDEFx0);
452 _mm_store_ps(packed_w + 16, v0123x1);
453 _mm_store_ps(packed_w + 20, v4567x1);
454 _mm_store_ps(packed_w + 24, v89ABx1);
455 _mm_store_ps(packed_w + 28, vCDEFx1);
456 _mm_store_ps(packed_w + 32, v0123x2);
457 _mm_store_ps(packed_w + 36, v4567x2);
458 _mm_store_ps(packed_w + 40, v89ABx2);
459 _mm_store_ps(packed_w + 44, vCDEFx2);
460 packed_w += 48;
461 break;
462 }
463 default:
464 XNN_UNREACHABLE;
465 }
466 }
467 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
468 w0 = w15;
469 }
470
471 // NC remainder (1..15)
472 if XNN_UNLIKELY(n != 0) {
473 assert(n >= 1);
474 assert(n <= 15);
475 if XNN_LIKELY(b != NULL) {
476 size_t nb = n;
477 do {
478 *packed_w++ = *b++;
479 } while (--nb != 0);
480 packed_w += (16 - n);
481 } else {
482 const __m128 vzero = _mm_setzero_ps();
483 _mm_store_ps(packed_w, vzero);
484 _mm_store_ps(packed_w + 4, vzero);
485 _mm_store_ps(packed_w + 8, vzero);
486 _mm_store_ps(packed_w + 12, vzero);
487 packed_w += 16;
488 }
489
490 const float* w1 = w0 + kc;
491 if XNN_UNPREDICTABLE(n < 2) {
492 w1 = w0;
493 }
494 const float* w2 = w1 + kc;
495 if XNN_UNPREDICTABLE(n <= 2) {
496 w2 = w1;
497 }
498 const float* w3 = w2 + kc;
499 if XNN_UNPREDICTABLE(n < 4) {
500 w3 = w2;
501 }
502 const float* w4 = w3 + kc;
503 if XNN_UNPREDICTABLE(n <= 4) {
504 w4 = w3;
505 }
506 const float* w5 = w4 + kc;
507 if XNN_UNPREDICTABLE(n < 6) {
508 w5 = w4;
509 }
510 const float* w6 = w5 + kc;
511 if XNN_UNPREDICTABLE(n <= 6) {
512 w6 = w5;
513 }
514 const float* w7 = w6 + kc;
515 if XNN_UNPREDICTABLE(n < 8) {
516 w7 = w6;
517 }
518 const float* w8 = w7 + kc;
519 if XNN_UNPREDICTABLE(n <= 8) {
520 w8 = w7;
521 }
522 const float* w9 = w8 + kc;
523 if XNN_UNPREDICTABLE(n < 10) {
524 w9 = w8;
525 }
526 const float* w10 = w9 + kc;
527 if XNN_UNPREDICTABLE(n <= 10) {
528 w10 = w9;
529 }
530 const float* w11 = w10 + kc;
531 if XNN_UNPREDICTABLE(n < 12) {
532 w11 = w10;
533 }
534 const float* w12 = w11 + kc;
535 if XNN_UNPREDICTABLE(n <= 12) {
536 w12 = w11;
537 }
538 const float* w13 = w12 + kc;
539 if XNN_UNPREDICTABLE(n < 14) {
540 w13 = w12;
541 }
542 const float* w14 = w13 + kc;
543 if XNN_UNPREDICTABLE(n <= 14) {
544 w14 = w13;
545 }
546
547 size_t k = kc;
548
549 // KC multiple of 4
550 for (; k >= 4; k -= 4) {
551 const __m128 v0x0123 = _mm_loadu_ps(w0);
552 w0 += 4;
553 const __m128 v1x0123 = _mm_loadu_ps(w1);
554 w1 += 4;
555 const __m128 v2x0123 = _mm_loadu_ps(w2);
556 w2 += 4;
557 const __m128 v3x0123 = _mm_loadu_ps(w3);
558 w3 += 4;
559 const __m128 v4x0123 = _mm_loadu_ps(w4);
560 w4 += 4;
561 const __m128 v5x0123 = _mm_loadu_ps(w5);
562 w5 += 4;
563 const __m128 v6x0123 = _mm_loadu_ps(w6);
564 w6 += 4;
565 const __m128 v7x0123 = _mm_loadu_ps(w7);
566 w7 += 4;
567 const __m128 v8x0123 = _mm_loadu_ps(w8);
568 w8 += 4;
569 const __m128 v9x0123 = _mm_loadu_ps(w9);
570 w9 += 4;
571 const __m128 vAx0123 = _mm_loadu_ps(w10);
572 w10 += 4;
573 const __m128 vBx0123 = _mm_loadu_ps(w11);
574 w11 += 4;
575 const __m128 vCx0123 = _mm_loadu_ps(w12);
576 w12 += 4;
577 const __m128 vDx0123 = _mm_loadu_ps(w13);
578 w13 += 4;
579 const __m128 vEx0123 = _mm_loadu_ps(w14);
580 w14 += 4;
581
582 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
583 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
584 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
585 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
586 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
587 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
588 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
589 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
590 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
591 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
592 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
593 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
594 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
595 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vEx0123);
596 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
597 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vEx0123);
598 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
599 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
600 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
601 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
602 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
603 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
604 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
605 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
606 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
607 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
608 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
609 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
610 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
611 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
612 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
613 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
614 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
615 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
616 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
617 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
618 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
619 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
620 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
621 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
622 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
623 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
624 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
625 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
626 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
627 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
628 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
629
630 _mm_store_ps(packed_w, v0123x0);
631 _mm_store_ps(packed_w + 4, v4567x0);
632 _mm_store_ps(packed_w + 8, v89ABx0);
633 _mm_store_ps(packed_w + 12, vCDEFx0);
634 _mm_store_ps(packed_w + 16, v0123x1);
635 _mm_store_ps(packed_w + 20, v4567x1);
636 _mm_store_ps(packed_w + 24, v89ABx1);
637 _mm_store_ps(packed_w + 28, vCDEFx1);
638 _mm_store_ps(packed_w + 32, v0123x2);
639 _mm_store_ps(packed_w + 36, v4567x2);
640 _mm_store_ps(packed_w + 40, v89ABx2);
641 _mm_store_ps(packed_w + 44, vCDEFx2);
642 _mm_store_ps(packed_w + 48, v0123x3);
643 _mm_store_ps(packed_w + 52, v4567x3);
644 _mm_store_ps(packed_w + 56, v89ABx3);
645 _mm_store_ps(packed_w + 60, vCDEFx3);
646 packed_w += 64;
647 }
648
649 // KC remainder (1..3)
650 if XNN_UNLIKELY(k != 0) {
651 assert(k >= 1);
652 assert(k <= 3);
653 switch (k) {
654 case 1:
655 {
656 const __m128 v0x0 = _mm_load_ss(w0);
657 const __m128 v1x0 = _mm_load_ss(w1);
658 const __m128 v2x0 = _mm_load_ss(w2);
659 const __m128 v3x0 = _mm_load_ss(w3);
660 const __m128 v4x0 = _mm_load_ss(w4);
661 const __m128 v5x0 = _mm_load_ss(w5);
662 const __m128 v6x0 = _mm_load_ss(w6);
663 const __m128 v7x0 = _mm_load_ss(w7);
664 const __m128 v8x0 = _mm_load_ss(w8);
665 const __m128 v9x0 = _mm_load_ss(w9);
666 const __m128 vAx0 = _mm_load_ss(w10);
667 const __m128 vBx0 = _mm_load_ss(w11);
668 const __m128 vCx0 = _mm_load_ss(w12);
669 const __m128 vDx0 = _mm_load_ss(w13);
670 const __m128 vEx0 = _mm_load_ss(w14);
671
672 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
673 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
674 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
675 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
676 const __m128 v89x0 = _mm_unpacklo_ps(v8x0, v9x0);
677 const __m128 vABx0 = _mm_unpacklo_ps(vAx0, vBx0);
678 const __m128 vCDx0 = _mm_unpacklo_ps(vCx0, vDx0);
679 const __m128 vEFx0 = _mm_unpacklo_ps(vEx0, vEx0);
680
681 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
682 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
683 const __m128 v89ABx0 = _mm_movelh_ps(v89x0, vABx0);
684 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0, vEFx0);
685
686 _mm_store_ps(packed_w, v0123x0);
687 _mm_store_ps(packed_w + 4, v4567x0);
688 _mm_store_ps(packed_w + 8, v89ABx0);
689 _mm_store_ps(packed_w + 12, vCDEFx0);
690 packed_w += 16;
691 break;
692 }
693 case 2:
694 {
695 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
696 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
697 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
698 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
699 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
700 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
701 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
702 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
703 const __m128 v8x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
704 const __m128 v9x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
705 const __m128 vAx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
706 const __m128 vBx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
707 const __m128 vCx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
708 const __m128 vDx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
709 const __m128 vEx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
710
711 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
712 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
713 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
714 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
715 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x01, v9x01);
716 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx01, vBx01);
717 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx01, vDx01);
718 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx01, vEx01);
719
720 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
721 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
722 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
723 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
724 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
725 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
726 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
727 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
728
729 _mm_store_ps(packed_w, v0123x0);
730 _mm_store_ps(packed_w + 4, v4567x0);
731 _mm_store_ps(packed_w + 8, v89ABx0);
732 _mm_store_ps(packed_w + 12, vCDEFx0);
733 _mm_store_ps(packed_w + 16, v0123x1);
734 _mm_store_ps(packed_w + 20, v4567x1);
735 _mm_store_ps(packed_w + 24, v89ABx1);
736 _mm_store_ps(packed_w + 28, vCDEFx1);
737 packed_w += 32;
738 break;
739 }
740 case 3:
741 {
742 __m128 v0x012 = _mm_load_ss(w0 + 2);
743 __m128 v1x012 = _mm_load_ss(w1 + 2);
744 __m128 v2x012 = _mm_load_ss(w2 + 2);
745 __m128 v3x012 = _mm_load_ss(w3 + 2);
746 __m128 v4x012 = _mm_load_ss(w4 + 2);
747 __m128 v5x012 = _mm_load_ss(w5 + 2);
748 __m128 v6x012 = _mm_load_ss(w6 + 2);
749 __m128 v7x012 = _mm_load_ss(w7 + 2);
750 __m128 v8x012 = _mm_load_ss(w8 + 2);
751 __m128 v9x012 = _mm_load_ss(w9 + 2);
752 __m128 vAx012 = _mm_load_ss(w10 + 2);
753 __m128 vBx012 = _mm_load_ss(w11 + 2);
754 __m128 vCx012 = _mm_load_ss(w12 + 2);
755 __m128 vDx012 = _mm_load_ss(w13 + 2);
756 __m128 vEx012 = _mm_load_ss(w14 + 2);
757
758 v0x012 = _mm_movelh_ps(v0x012, v0x012);
759 v1x012 = _mm_movelh_ps(v1x012, v1x012);
760 v2x012 = _mm_movelh_ps(v2x012, v2x012);
761 v3x012 = _mm_movelh_ps(v3x012, v3x012);
762 v4x012 = _mm_movelh_ps(v4x012, v4x012);
763 v5x012 = _mm_movelh_ps(v5x012, v5x012);
764 v6x012 = _mm_movelh_ps(v6x012, v6x012);
765 v7x012 = _mm_movelh_ps(v7x012, v7x012);
766 v8x012 = _mm_movelh_ps(v8x012, v8x012);
767 v9x012 = _mm_movelh_ps(v9x012, v9x012);
768 vAx012 = _mm_movelh_ps(vAx012, vAx012);
769 vBx012 = _mm_movelh_ps(vBx012, vBx012);
770 vCx012 = _mm_movelh_ps(vCx012, vCx012);
771 vDx012 = _mm_movelh_ps(vDx012, vDx012);
772 vEx012 = _mm_movelh_ps(vEx012, vEx012);
773
774 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
775 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
776 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
777 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
778 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
779 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
780 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
781 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
782 v8x012 = _mm_loadl_pi(v8x012, (const __m64*) w8);
783 v9x012 = _mm_loadl_pi(v9x012, (const __m64*) w9);
784 vAx012 = _mm_loadl_pi(vAx012, (const __m64*) w10);
785 vBx012 = _mm_loadl_pi(vBx012, (const __m64*) w11);
786 vCx012 = _mm_loadl_pi(vCx012, (const __m64*) w12);
787 vDx012 = _mm_loadl_pi(vDx012, (const __m64*) w13);
788 vEx012 = _mm_loadl_pi(vEx012, (const __m64*) w14);
789
790 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
791 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
792 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
793 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
794 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
795 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
796 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
797 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
798 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x012, v9x012);
799 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx012, vBx012);
800 const __m128 v89x2 = _mm_unpackhi_ps(v8x012, v9x012);
801 const __m128 vABx2 = _mm_unpackhi_ps(vAx012, vBx012);
802 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx012, vDx012);
803 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx012, vEx012);
804 const __m128 vCDx2 = _mm_unpackhi_ps(vCx012, vDx012);
805 const __m128 vEFx2 = _mm_unpackhi_ps(vEx012, vEx012);
806
807 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
808 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
809 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
810 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
811 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
812 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
813 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
814 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
815 const __m128 v89ABx2 = _mm_movelh_ps(v89x2, vABx2);
816 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
817 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
818 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2, vEFx2);
819
820 _mm_store_ps(packed_w, v0123x0);
821 _mm_store_ps(packed_w + 4, v4567x0);
822 _mm_store_ps(packed_w + 8, v89ABx0);
823 _mm_store_ps(packed_w + 12, vCDEFx0);
824 _mm_store_ps(packed_w + 16, v0123x1);
825 _mm_store_ps(packed_w + 20, v4567x1);
826 _mm_store_ps(packed_w + 24, v89ABx1);
827 _mm_store_ps(packed_w + 28, vCDEFx1);
828 _mm_store_ps(packed_w + 32, v0123x2);
829 _mm_store_ps(packed_w + 36, v4567x2);
830 _mm_store_ps(packed_w + 40, v89ABx2);
831 _mm_store_ps(packed_w + 44, vCDEFx2);
832 packed_w += 48;
833 break;
834 }
835 default:
836 XNN_UNREACHABLE;
837 }
838 }
839 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
840 }
841 weights += nc * kc;
842 } while (--g != 0);
843 }