Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-sse2-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <emmintrin.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20 void xnn_x32_packw_gemm_goi_ukernel_x16__sse2_u8_prfm(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 16);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 const float* b = (const float*) bias;
44 float* packed_w = (float*) packed_weights;
45 do {
46 // NC main loop multiple of 16
47 const float* w0 = (const float*) weights;
48 size_t n = nc;
49
50 for (; n >= 16; n -= 16) {
51 if XNN_LIKELY(b != NULL) {
52 const __m128 vb0123 = _mm_loadu_ps(b);
53 const __m128 vb4567 = _mm_loadu_ps(b + 4);
54 const __m128 vb89AB = _mm_loadu_ps(b + 8);
55 const __m128 vbCDEF = _mm_loadu_ps(b + 12);
56 b += 16;
57
58 _mm_store_ps(packed_w, vb0123);
59 _mm_store_ps(packed_w + 4, vb4567);
60 _mm_store_ps(packed_w + 8, vb89AB);
61 _mm_store_ps(packed_w + 12, vbCDEF);
62 } else {
63 const __m128 vzero = _mm_setzero_ps();
64 _mm_store_ps(packed_w, vzero);
65 _mm_store_ps(packed_w + 4, vzero);
66 _mm_store_ps(packed_w + 8, vzero);
67 _mm_store_ps(packed_w + 12, vzero);
68 }
69 packed_w += 16;
70
71 const float* w1 = w0 + kc;
72 const float* w2 = w1 + kc;
73 const float* w3 = w2 + kc;
74 const float* w4 = w3 + kc;
75 const float* w5 = w4 + kc;
76 const float* w6 = w5 + kc;
77 const float* w7 = w6 + kc;
78 const float* w8 = w7 + kc;
79 const float* w9 = w8 + kc;
80 const float* w10 = w9 + kc;
81 const float* w11 = w10 + kc;
82 const float* w12 = w11 + kc;
83 const float* w13 = w12 + kc;
84 const float* w14 = w13 + kc;
85 const float* w15 = w14 + kc;
86 xnn_prefetch_to_l1((const int8_t*) w0);
87 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
88 xnn_prefetch_to_l1((const int8_t*) w1);
89 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
90 xnn_prefetch_to_l1((const int8_t*) w2);
91 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
92 xnn_prefetch_to_l1((const int8_t*) w3);
93 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
94 xnn_prefetch_to_l1((const int8_t*) w4);
95 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
96 xnn_prefetch_to_l1((const int8_t*) w5);
97 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
98 xnn_prefetch_to_l1((const int8_t*) w6);
99 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
100 xnn_prefetch_to_l1((const int8_t*) w7);
101 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
102 xnn_prefetch_to_l1((const int8_t*) w8);
103 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
104 xnn_prefetch_to_l1((const int8_t*) w9);
105 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
106 xnn_prefetch_to_l1((const int8_t*) w10);
107 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
108 xnn_prefetch_to_l1((const int8_t*) w11);
109 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
110 xnn_prefetch_to_l1((const int8_t*) w12);
111 xnn_prefetch_to_l1((const int8_t*) w12 + 64);
112 xnn_prefetch_to_l1((const int8_t*) w13);
113 xnn_prefetch_to_l1((const int8_t*) w13 + 64);
114 xnn_prefetch_to_l1((const int8_t*) w14);
115 xnn_prefetch_to_l1((const int8_t*) w14 + 64);
116 xnn_prefetch_to_l1((const int8_t*) w15);
117 xnn_prefetch_to_l1((const int8_t*) w15 + 64);
118
119 size_t k = kc;
120 // KC main loop multiple of 8
121 for (; k >= 8; k -= 8) {
122 const __m128 v0x0123 = _mm_loadu_ps(w0);
123 const __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
124 w0 += 8;
125 const __m128 v1x0123 = _mm_loadu_ps(w1);
126 const __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
127 w1 += 8;
128 const __m128 v2x0123 = _mm_loadu_ps(w2);
129 const __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
130 w2 += 8;
131 const __m128 v3x0123 = _mm_loadu_ps(w3);
132 const __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
133 w3 += 8;
134 const __m128 v4x0123 = _mm_loadu_ps(w4);
135 const __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
136 w4 += 8;
137 const __m128 v5x0123 = _mm_loadu_ps(w5);
138 const __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
139 w5 += 8;
140 const __m128 v6x0123 = _mm_loadu_ps(w6);
141 const __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
142 w6 += 8;
143 const __m128 v7x0123 = _mm_loadu_ps(w7);
144 const __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
145 w7 += 8;
146 const __m128 v8x0123 = _mm_loadu_ps(w8);
147 const __m128 v8x4567 = _mm_loadu_ps(w8 + 4);
148 w8 += 8;
149 const __m128 v9x0123 = _mm_loadu_ps(w9);
150 const __m128 v9x4567 = _mm_loadu_ps(w9 + 4);
151 w9 += 8;
152 const __m128 vAx0123 = _mm_loadu_ps(w10);
153 const __m128 vAx4567 = _mm_loadu_ps(w10 + 4);
154 w10 += 8;
155 const __m128 vBx0123 = _mm_loadu_ps(w11);
156 const __m128 vBx4567 = _mm_loadu_ps(w11 + 4);
157 w11 += 8;
158 const __m128 vCx0123 = _mm_loadu_ps(w12);
159 const __m128 vCx4567 = _mm_loadu_ps(w12 + 4);
160 w12 += 8;
161 const __m128 vDx0123 = _mm_loadu_ps(w13);
162 const __m128 vDx4567 = _mm_loadu_ps(w13 + 4);
163 w13 += 8;
164 const __m128 vEx0123 = _mm_loadu_ps(w14);
165 const __m128 vEx4567 = _mm_loadu_ps(w14 + 4);
166 w14 += 8;
167 const __m128 vFx0123 = _mm_loadu_ps(w15);
168 const __m128 vFx4567 = _mm_loadu_ps(w15 + 4);
169 w15 += 8;
170
171 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
172 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
173 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
174 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
175 const __m128 v01x4_01x5 = _mm_unpacklo_ps(v0x4567, v1x4567);
176 const __m128 v23x4_23x5 = _mm_unpacklo_ps(v2x4567, v3x4567);
177 const __m128 v01x6_01x7 = _mm_unpackhi_ps(v0x4567, v1x4567);
178 const __m128 v23x6_23x7 = _mm_unpackhi_ps(v2x4567, v3x4567);
179 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
180 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
181 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
182 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
183 const __m128 v45x4_45x5 = _mm_unpacklo_ps(v4x4567, v5x4567);
184 const __m128 v67x4_67x5 = _mm_unpacklo_ps(v6x4567, v7x4567);
185 const __m128 v45x6_45x7 = _mm_unpackhi_ps(v4x4567, v5x4567);
186 const __m128 v67x6_67x7 = _mm_unpackhi_ps(v6x4567, v7x4567);
187 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
188 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
189 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
190 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
191 const __m128 v89x4_89x5 = _mm_unpacklo_ps(v8x4567, v9x4567);
192 const __m128 vABx4_ABx5 = _mm_unpacklo_ps(vAx4567, vBx4567);
193 const __m128 v89x6_89x7 = _mm_unpackhi_ps(v8x4567, v9x4567);
194 const __m128 vABx6_ABx7 = _mm_unpackhi_ps(vAx4567, vBx4567);
195 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
196 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vFx0123);
197 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
198 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vFx0123);
199 const __m128 vCDx4_CDx5 = _mm_unpacklo_ps(vCx4567, vDx4567);
200 const __m128 vEFx4_EFx5 = _mm_unpacklo_ps(vEx4567, vFx4567);
201 const __m128 vCDx6_CDx7 = _mm_unpackhi_ps(vCx4567, vDx4567);
202 const __m128 vEFx6_EFx7 = _mm_unpackhi_ps(vEx4567, vFx4567);
203 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
204 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
205 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
206 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
207 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
208 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
209 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
210 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
211 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
212 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
213 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
214 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
215 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
216 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
217 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
218 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
219 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
220 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
221 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
222 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
223 const __m128 v0123x4 = _mm_movelh_ps(v01x4_01x5, v23x4_23x5);
224 const __m128 v0123x5 = _mm_movehl_ps(v23x4_23x5, v01x4_01x5);
225 const __m128 v0123x6 = _mm_movelh_ps(v01x6_01x7, v23x6_23x7);
226 const __m128 v0123x7 = _mm_movehl_ps(v23x6_23x7, v01x6_01x7);
227 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
228 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
229 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
230 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
231 const __m128 v4567x4 = _mm_movelh_ps(v45x4_45x5, v67x4_67x5);
232 const __m128 v4567x5 = _mm_movehl_ps(v67x4_67x5, v45x4_45x5);
233 const __m128 v4567x6 = _mm_movelh_ps(v45x6_45x7, v67x6_67x7);
234 const __m128 v4567x7 = _mm_movehl_ps(v67x6_67x7, v45x6_45x7);
235 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
236 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
237 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
238 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
239 const __m128 v89ABx4 = _mm_movelh_ps(v89x4_89x5, vABx4_ABx5);
240 const __m128 v89ABx5 = _mm_movehl_ps(vABx4_ABx5, v89x4_89x5);
241 const __m128 v89ABx6 = _mm_movelh_ps(v89x6_89x7, vABx6_ABx7);
242 const __m128 v89ABx7 = _mm_movehl_ps(vABx6_ABx7, v89x6_89x7);
243 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
244 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
245 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
246 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
247 const __m128 vCDEFx4 = _mm_movelh_ps(vCDx4_CDx5, vEFx4_EFx5);
248 const __m128 vCDEFx5 = _mm_movehl_ps(vEFx4_EFx5, vCDx4_CDx5);
249 const __m128 vCDEFx6 = _mm_movelh_ps(vCDx6_CDx7, vEFx6_EFx7);
250 const __m128 vCDEFx7 = _mm_movehl_ps(vEFx6_EFx7, vCDx6_CDx7);
251
252 _mm_store_ps(packed_w, v0123x0);
253 _mm_store_ps(packed_w + 4, v4567x0);
254 _mm_store_ps(packed_w + 8, v89ABx0);
255 _mm_store_ps(packed_w + 12, vCDEFx0);
256 _mm_store_ps(packed_w + 16, v0123x1);
257 _mm_store_ps(packed_w + 20, v4567x1);
258 _mm_store_ps(packed_w + 24, v89ABx1);
259 _mm_store_ps(packed_w + 28, vCDEFx1);
260 _mm_store_ps(packed_w + 32, v0123x2);
261 _mm_store_ps(packed_w + 36, v4567x2);
262 _mm_store_ps(packed_w + 40, v89ABx2);
263 _mm_store_ps(packed_w + 44, vCDEFx2);
264 _mm_store_ps(packed_w + 48, v0123x3);
265 _mm_store_ps(packed_w + 52, v4567x3);
266 _mm_store_ps(packed_w + 56, v89ABx3);
267 _mm_store_ps(packed_w + 60, vCDEFx3);
268 _mm_store_ps(packed_w + 64, v0123x4);
269 _mm_store_ps(packed_w + 68, v4567x4);
270 _mm_store_ps(packed_w + 72, v89ABx4);
271 _mm_store_ps(packed_w + 76, vCDEFx4);
272 _mm_store_ps(packed_w + 80, v0123x5);
273 _mm_store_ps(packed_w + 84, v4567x5);
274 _mm_store_ps(packed_w + 88, v89ABx5);
275 _mm_store_ps(packed_w + 92, vCDEFx5);
276 _mm_store_ps(packed_w + 96, v0123x6);
277 _mm_store_ps(packed_w + 100, v4567x6);
278 _mm_store_ps(packed_w + 104, v89ABx6);
279 _mm_store_ps(packed_w + 108, vCDEFx6);
280 _mm_store_ps(packed_w + 112, v0123x7);
281 _mm_store_ps(packed_w + 116, v4567x7);
282 _mm_store_ps(packed_w + 120, v89ABx7);
283 _mm_store_ps(packed_w + 124, vCDEFx7);
284 packed_w += 128;
285 }
286
287 // KC multiple of 4
288 for (; k >= 4; k -= 4) {
289 const __m128 v0x0123 = _mm_loadu_ps(w0);
290 w0 += 4;
291 const __m128 v1x0123 = _mm_loadu_ps(w1);
292 w1 += 4;
293 const __m128 v2x0123 = _mm_loadu_ps(w2);
294 w2 += 4;
295 const __m128 v3x0123 = _mm_loadu_ps(w3);
296 w3 += 4;
297 const __m128 v4x0123 = _mm_loadu_ps(w4);
298 w4 += 4;
299 const __m128 v5x0123 = _mm_loadu_ps(w5);
300 w5 += 4;
301 const __m128 v6x0123 = _mm_loadu_ps(w6);
302 w6 += 4;
303 const __m128 v7x0123 = _mm_loadu_ps(w7);
304 w7 += 4;
305 const __m128 v8x0123 = _mm_loadu_ps(w8);
306 w8 += 4;
307 const __m128 v9x0123 = _mm_loadu_ps(w9);
308 w9 += 4;
309 const __m128 vAx0123 = _mm_loadu_ps(w10);
310 w10 += 4;
311 const __m128 vBx0123 = _mm_loadu_ps(w11);
312 w11 += 4;
313 const __m128 vCx0123 = _mm_loadu_ps(w12);
314 w12 += 4;
315 const __m128 vDx0123 = _mm_loadu_ps(w13);
316 w13 += 4;
317 const __m128 vEx0123 = _mm_loadu_ps(w14);
318 w14 += 4;
319 const __m128 vFx0123 = _mm_loadu_ps(w15);
320 w15 += 4;
321
322 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
323 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
324 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
325 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
326 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
327 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
328 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
329 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
330 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
331 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
332 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
333 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
334 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
335 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vFx0123);
336 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
337 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vFx0123);
338 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
339 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
340 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
341 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
342 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
343 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
344 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
345 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
346 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
347 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
348 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
349 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
350 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
351 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
352 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
353 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
354 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
355 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
356 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
357 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
358 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
359 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
360 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
361 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
362 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
363 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
364 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
365 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
366 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
367 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
368 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
369 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
370
371 _mm_store_ps(packed_w, v0123x0);
372 _mm_store_ps(packed_w + 4, v4567x0);
373 _mm_store_ps(packed_w + 8, v89ABx0);
374 _mm_store_ps(packed_w + 12, vCDEFx0);
375 _mm_store_ps(packed_w + 16, v0123x1);
376 _mm_store_ps(packed_w + 20, v4567x1);
377 _mm_store_ps(packed_w + 24, v89ABx1);
378 _mm_store_ps(packed_w + 28, vCDEFx1);
379 _mm_store_ps(packed_w + 32, v0123x2);
380 _mm_store_ps(packed_w + 36, v4567x2);
381 _mm_store_ps(packed_w + 40, v89ABx2);
382 _mm_store_ps(packed_w + 44, vCDEFx2);
383 _mm_store_ps(packed_w + 48, v0123x3);
384 _mm_store_ps(packed_w + 52, v4567x3);
385 _mm_store_ps(packed_w + 56, v89ABx3);
386 _mm_store_ps(packed_w + 60, vCDEFx3);
387 packed_w += 64;
388 }
389
390 // KC remainder (1..3)
391 if XNN_UNLIKELY(k != 0) {
392 assert(k >= 1);
393 assert(k <= 3);
394 switch (k) {
395 case 1:
396 {
397 const __m128 v0x0 = _mm_load_ss(w0);
398 w0 += 1;
399 const __m128 v1x0 = _mm_load_ss(w1);
400 w1 += 1;
401 const __m128 v2x0 = _mm_load_ss(w2);
402 w2 += 1;
403 const __m128 v3x0 = _mm_load_ss(w3);
404 w3 += 1;
405 const __m128 v4x0 = _mm_load_ss(w4);
406 w4 += 1;
407 const __m128 v5x0 = _mm_load_ss(w5);
408 w5 += 1;
409 const __m128 v6x0 = _mm_load_ss(w6);
410 w6 += 1;
411 const __m128 v7x0 = _mm_load_ss(w7);
412 w7 += 1;
413 const __m128 v8x0 = _mm_load_ss(w8);
414 w8 += 1;
415 const __m128 v9x0 = _mm_load_ss(w9);
416 w9 += 1;
417 const __m128 vAx0 = _mm_load_ss(w10);
418 w10 += 1;
419 const __m128 vBx0 = _mm_load_ss(w11);
420 w11 += 1;
421 const __m128 vCx0 = _mm_load_ss(w12);
422 w12 += 1;
423 const __m128 vDx0 = _mm_load_ss(w13);
424 w13 += 1;
425 const __m128 vEx0 = _mm_load_ss(w14);
426 w14 += 1;
427 const __m128 vFx0 = _mm_load_ss(w15);
428 w15 += 1;
429
430 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
431 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
432 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
433 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
434 const __m128 v89x0 = _mm_unpacklo_ps(v8x0, v9x0);
435 const __m128 vABx0 = _mm_unpacklo_ps(vAx0, vBx0);
436 const __m128 vCDx0 = _mm_unpacklo_ps(vCx0, vDx0);
437 const __m128 vEFx0 = _mm_unpacklo_ps(vEx0, vFx0);
438
439 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
440 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
441 const __m128 v89ABx0 = _mm_movelh_ps(v89x0, vABx0);
442 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0, vEFx0);
443
444 _mm_store_ps(packed_w, v0123x0);
445 _mm_store_ps(packed_w + 4, v4567x0);
446 _mm_store_ps(packed_w + 8, v89ABx0);
447 _mm_store_ps(packed_w + 12, vCDEFx0);
448 packed_w += 16;
449 break;
450 }
451 case 2:
452 {
453 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
454 w0 += 2;
455 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
456 w1 += 2;
457 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
458 w2 += 2;
459 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
460 w3 += 2;
461 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
462 w4 += 2;
463 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
464 w5 += 2;
465 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
466 w6 += 2;
467 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
468 w7 += 2;
469 const __m128 v8x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
470 w8 += 2;
471 const __m128 v9x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
472 w9 += 2;
473 const __m128 vAx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
474 w10 += 2;
475 const __m128 vBx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
476 w11 += 2;
477 const __m128 vCx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
478 w12 += 2;
479 const __m128 vDx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
480 w13 += 2;
481 const __m128 vEx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
482 w14 += 2;
483 const __m128 vFx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
484 w15 += 2;
485
486 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
487 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
488 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
489 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
490 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x01, v9x01);
491 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx01, vBx01);
492 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx01, vDx01);
493 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx01, vFx01);
494
495 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
496 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
497 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
498 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
499 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
500 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
501 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
502 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
503
504 _mm_store_ps(packed_w, v0123x0);
505 _mm_store_ps(packed_w + 4, v4567x0);
506 _mm_store_ps(packed_w + 8, v89ABx0);
507 _mm_store_ps(packed_w + 12, vCDEFx0);
508 _mm_store_ps(packed_w + 16, v0123x1);
509 _mm_store_ps(packed_w + 20, v4567x1);
510 _mm_store_ps(packed_w + 24, v89ABx1);
511 _mm_store_ps(packed_w + 28, vCDEFx1);
512 packed_w += 32;
513 break;
514 }
515 case 3:
516 {
517 __m128 v0x012 = _mm_load_ss(w0 + 2);
518 __m128 v1x012 = _mm_load_ss(w1 + 2);
519 __m128 v2x012 = _mm_load_ss(w2 + 2);
520 __m128 v3x012 = _mm_load_ss(w3 + 2);
521 __m128 v4x012 = _mm_load_ss(w4 + 2);
522 __m128 v5x012 = _mm_load_ss(w5 + 2);
523 __m128 v6x012 = _mm_load_ss(w6 + 2);
524 __m128 v7x012 = _mm_load_ss(w7 + 2);
525 __m128 v8x012 = _mm_load_ss(w8 + 2);
526 __m128 v9x012 = _mm_load_ss(w9 + 2);
527 __m128 vAx012 = _mm_load_ss(w10 + 2);
528 __m128 vBx012 = _mm_load_ss(w11 + 2);
529 __m128 vCx012 = _mm_load_ss(w12 + 2);
530 __m128 vDx012 = _mm_load_ss(w13 + 2);
531 __m128 vEx012 = _mm_load_ss(w14 + 2);
532 __m128 vFx012 = _mm_load_ss(w15 + 2);
533
534 v0x012 = _mm_movelh_ps(v0x012, v0x012);
535 v1x012 = _mm_movelh_ps(v1x012, v1x012);
536 v2x012 = _mm_movelh_ps(v2x012, v2x012);
537 v3x012 = _mm_movelh_ps(v3x012, v3x012);
538 v4x012 = _mm_movelh_ps(v4x012, v4x012);
539 v5x012 = _mm_movelh_ps(v5x012, v5x012);
540 v6x012 = _mm_movelh_ps(v6x012, v6x012);
541 v7x012 = _mm_movelh_ps(v7x012, v7x012);
542 v8x012 = _mm_movelh_ps(v8x012, v8x012);
543 v9x012 = _mm_movelh_ps(v9x012, v9x012);
544 vAx012 = _mm_movelh_ps(vAx012, vAx012);
545 vBx012 = _mm_movelh_ps(vBx012, vBx012);
546 vCx012 = _mm_movelh_ps(vCx012, vCx012);
547 vDx012 = _mm_movelh_ps(vDx012, vDx012);
548 vEx012 = _mm_movelh_ps(vEx012, vEx012);
549 vFx012 = _mm_movelh_ps(vFx012, vFx012);
550
551 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
552 w0 += 3;
553 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
554 w1 += 3;
555 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
556 w2 += 3;
557 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
558 w3 += 3;
559 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
560 w4 += 3;
561 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
562 w5 += 3;
563 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
564 w6 += 3;
565 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
566 w7 += 3;
567 v8x012 = _mm_loadl_pi(v8x012, (const __m64*) w8);
568 w8 += 3;
569 v9x012 = _mm_loadl_pi(v9x012, (const __m64*) w9);
570 w9 += 3;
571 vAx012 = _mm_loadl_pi(vAx012, (const __m64*) w10);
572 w10 += 3;
573 vBx012 = _mm_loadl_pi(vBx012, (const __m64*) w11);
574 w11 += 3;
575 vCx012 = _mm_loadl_pi(vCx012, (const __m64*) w12);
576 w12 += 3;
577 vDx012 = _mm_loadl_pi(vDx012, (const __m64*) w13);
578 w13 += 3;
579 vEx012 = _mm_loadl_pi(vEx012, (const __m64*) w14);
580 w14 += 3;
581 vFx012 = _mm_loadl_pi(vFx012, (const __m64*) w15);
582 w15 += 3;
583
584 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
585 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
586 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
587 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
588 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
589 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
590 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
591 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
592 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x012, v9x012);
593 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx012, vBx012);
594 const __m128 v89x2 = _mm_unpackhi_ps(v8x012, v9x012);
595 const __m128 vABx2 = _mm_unpackhi_ps(vAx012, vBx012);
596 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx012, vDx012);
597 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx012, vFx012);
598 const __m128 vCDx2 = _mm_unpackhi_ps(vCx012, vDx012);
599 const __m128 vEFx2 = _mm_unpackhi_ps(vEx012, vFx012);
600
601 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
602 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
603 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
604 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
605 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
606 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
607 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
608 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
609 const __m128 v89ABx2 = _mm_movelh_ps(v89x2, vABx2);
610 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
611 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
612 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2, vEFx2);
613
614 _mm_store_ps(packed_w, v0123x0);
615 _mm_store_ps(packed_w + 4, v4567x0);
616 _mm_store_ps(packed_w + 8, v89ABx0);
617 _mm_store_ps(packed_w + 12, vCDEFx0);
618 _mm_store_ps(packed_w + 16, v0123x1);
619 _mm_store_ps(packed_w + 20, v4567x1);
620 _mm_store_ps(packed_w + 24, v89ABx1);
621 _mm_store_ps(packed_w + 28, vCDEFx1);
622 _mm_store_ps(packed_w + 32, v0123x2);
623 _mm_store_ps(packed_w + 36, v4567x2);
624 _mm_store_ps(packed_w + 40, v89ABx2);
625 _mm_store_ps(packed_w + 44, vCDEFx2);
626 packed_w += 48;
627 break;
628 }
629 default:
630 XNN_UNREACHABLE;
631 }
632 }
633 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
634 w0 = w15;
635 }
636
637 // NC remainder (1..15)
638 if XNN_UNLIKELY(n != 0) {
639 assert(n >= 1);
640 assert(n <= 15);
641 if XNN_LIKELY(b != NULL) {
642 size_t nb = n;
643 do {
644 *packed_w++ = *b++;
645 } while (--nb != 0);
646 packed_w += (16 - n);
647 } else {
648 const __m128 vzero = _mm_setzero_ps();
649 _mm_store_ps(packed_w, vzero);
650 _mm_store_ps(packed_w + 4, vzero);
651 _mm_store_ps(packed_w + 8, vzero);
652 _mm_store_ps(packed_w + 12, vzero);
653 packed_w += 16;
654 }
655
656 const float* w1 = w0 + kc;
657 if XNN_UNPREDICTABLE(n < 2) {
658 w1 = w0;
659 }
660 const float* w2 = w1 + kc;
661 if XNN_UNPREDICTABLE(n <= 2) {
662 w2 = w1;
663 }
664 const float* w3 = w2 + kc;
665 if XNN_UNPREDICTABLE(n < 4) {
666 w3 = w2;
667 }
668 const float* w4 = w3 + kc;
669 if XNN_UNPREDICTABLE(n <= 4) {
670 w4 = w3;
671 }
672 const float* w5 = w4 + kc;
673 if XNN_UNPREDICTABLE(n < 6) {
674 w5 = w4;
675 }
676 const float* w6 = w5 + kc;
677 if XNN_UNPREDICTABLE(n <= 6) {
678 w6 = w5;
679 }
680 const float* w7 = w6 + kc;
681 if XNN_UNPREDICTABLE(n < 8) {
682 w7 = w6;
683 }
684 const float* w8 = w7 + kc;
685 if XNN_UNPREDICTABLE(n <= 8) {
686 w8 = w7;
687 }
688 const float* w9 = w8 + kc;
689 if XNN_UNPREDICTABLE(n < 10) {
690 w9 = w8;
691 }
692 const float* w10 = w9 + kc;
693 if XNN_UNPREDICTABLE(n <= 10) {
694 w10 = w9;
695 }
696 const float* w11 = w10 + kc;
697 if XNN_UNPREDICTABLE(n < 12) {
698 w11 = w10;
699 }
700 const float* w12 = w11 + kc;
701 if XNN_UNPREDICTABLE(n <= 12) {
702 w12 = w11;
703 }
704 const float* w13 = w12 + kc;
705 if XNN_UNPREDICTABLE(n < 14) {
706 w13 = w12;
707 }
708 const float* w14 = w13 + kc;
709 if XNN_UNPREDICTABLE(n <= 14) {
710 w14 = w13;
711 }
712
713 size_t k = kc;
714 // KC main loop multiple of 8
715 for (; k >= 8; k -= 8) {
716 const __m128 v0x0123 = _mm_loadu_ps(w0);
717 const __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
718 w0 += 8;
719 const __m128 v1x0123 = _mm_loadu_ps(w1);
720 const __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
721 w1 += 8;
722 const __m128 v2x0123 = _mm_loadu_ps(w2);
723 const __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
724 w2 += 8;
725 const __m128 v3x0123 = _mm_loadu_ps(w3);
726 const __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
727 w3 += 8;
728 const __m128 v4x0123 = _mm_loadu_ps(w4);
729 const __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
730 w4 += 8;
731 const __m128 v5x0123 = _mm_loadu_ps(w5);
732 const __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
733 w5 += 8;
734 const __m128 v6x0123 = _mm_loadu_ps(w6);
735 const __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
736 w6 += 8;
737 const __m128 v7x0123 = _mm_loadu_ps(w7);
738 const __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
739 w7 += 8;
740 const __m128 v8x0123 = _mm_loadu_ps(w8);
741 const __m128 v8x4567 = _mm_loadu_ps(w8 + 4);
742 w8 += 8;
743 const __m128 v9x0123 = _mm_loadu_ps(w9);
744 const __m128 v9x4567 = _mm_loadu_ps(w9 + 4);
745 w9 += 8;
746 const __m128 vAx0123 = _mm_loadu_ps(w10);
747 const __m128 vAx4567 = _mm_loadu_ps(w10 + 4);
748 w10 += 8;
749 const __m128 vBx0123 = _mm_loadu_ps(w11);
750 const __m128 vBx4567 = _mm_loadu_ps(w11 + 4);
751 w11 += 8;
752 const __m128 vCx0123 = _mm_loadu_ps(w12);
753 const __m128 vCx4567 = _mm_loadu_ps(w12 + 4);
754 w12 += 8;
755 const __m128 vDx0123 = _mm_loadu_ps(w13);
756 const __m128 vDx4567 = _mm_loadu_ps(w13 + 4);
757 w13 += 8;
758 const __m128 vEx0123 = _mm_loadu_ps(w14);
759 const __m128 vEx4567 = _mm_loadu_ps(w14 + 4);
760 w14 += 8;
761
762 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
763 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
764 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
765 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
766 const __m128 v01x4_01x5 = _mm_unpacklo_ps(v0x4567, v1x4567);
767 const __m128 v23x4_23x5 = _mm_unpacklo_ps(v2x4567, v3x4567);
768 const __m128 v01x6_01x7 = _mm_unpackhi_ps(v0x4567, v1x4567);
769 const __m128 v23x6_23x7 = _mm_unpackhi_ps(v2x4567, v3x4567);
770 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
771 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
772 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
773 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
774 const __m128 v45x4_45x5 = _mm_unpacklo_ps(v4x4567, v5x4567);
775 const __m128 v67x4_67x5 = _mm_unpacklo_ps(v6x4567, v7x4567);
776 const __m128 v45x6_45x7 = _mm_unpackhi_ps(v4x4567, v5x4567);
777 const __m128 v67x6_67x7 = _mm_unpackhi_ps(v6x4567, v7x4567);
778 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
779 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
780 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
781 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
782 const __m128 v89x4_89x5 = _mm_unpacklo_ps(v8x4567, v9x4567);
783 const __m128 vABx4_ABx5 = _mm_unpacklo_ps(vAx4567, vBx4567);
784 const __m128 v89x6_89x7 = _mm_unpackhi_ps(v8x4567, v9x4567);
785 const __m128 vABx6_ABx7 = _mm_unpackhi_ps(vAx4567, vBx4567);
786 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
787 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vEx0123);
788 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
789 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vEx0123);
790 const __m128 vCDx4_CDx5 = _mm_unpacklo_ps(vCx4567, vDx4567);
791 const __m128 vEFx4_EFx5 = _mm_unpacklo_ps(vEx4567, vEx4567);
792 const __m128 vCDx6_CDx7 = _mm_unpackhi_ps(vCx4567, vDx4567);
793 const __m128 vEFx6_EFx7 = _mm_unpackhi_ps(vEx4567, vEx4567);
794 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
795 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
796 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
797 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
798 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
799 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
800 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
801 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
802 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
803 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
804 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
805 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
806 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
807 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
808 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
809 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
810 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
811 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
812 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
813 const __m128 v0123x4 = _mm_movelh_ps(v01x4_01x5, v23x4_23x5);
814 const __m128 v0123x5 = _mm_movehl_ps(v23x4_23x5, v01x4_01x5);
815 const __m128 v0123x6 = _mm_movelh_ps(v01x6_01x7, v23x6_23x7);
816 const __m128 v0123x7 = _mm_movehl_ps(v23x6_23x7, v01x6_01x7);
817 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
818 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
819 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
820 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
821 const __m128 v4567x4 = _mm_movelh_ps(v45x4_45x5, v67x4_67x5);
822 const __m128 v4567x5 = _mm_movehl_ps(v67x4_67x5, v45x4_45x5);
823 const __m128 v4567x6 = _mm_movelh_ps(v45x6_45x7, v67x6_67x7);
824 const __m128 v4567x7 = _mm_movehl_ps(v67x6_67x7, v45x6_45x7);
825 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
826 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
827 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
828 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
829 const __m128 v89ABx4 = _mm_movelh_ps(v89x4_89x5, vABx4_ABx5);
830 const __m128 v89ABx5 = _mm_movehl_ps(vABx4_ABx5, v89x4_89x5);
831 const __m128 v89ABx6 = _mm_movelh_ps(v89x6_89x7, vABx6_ABx7);
832 const __m128 v89ABx7 = _mm_movehl_ps(vABx6_ABx7, v89x6_89x7);
833 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
834 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
835 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
836 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
837 const __m128 vCDEFx4 = _mm_movelh_ps(vCDx4_CDx5, vEFx4_EFx5);
838 const __m128 vCDEFx5 = _mm_movehl_ps(vEFx4_EFx5, vCDx4_CDx5);
839 const __m128 vCDEFx6 = _mm_movelh_ps(vCDx6_CDx7, vEFx6_EFx7);
840 const __m128 vCDEFx7 = _mm_movehl_ps(vEFx6_EFx7, vCDx6_CDx7);
841
842 _mm_store_ps(packed_w, v0123x0);
843 _mm_store_ps(packed_w + 4, v4567x0);
844 _mm_store_ps(packed_w + 8, v89ABx0);
845 _mm_store_ps(packed_w + 12, vCDEFx0);
846 _mm_store_ps(packed_w + 16, v0123x1);
847 _mm_store_ps(packed_w + 20, v4567x1);
848 _mm_store_ps(packed_w + 24, v89ABx1);
849 _mm_store_ps(packed_w + 28, vCDEFx1);
850 _mm_store_ps(packed_w + 32, v0123x2);
851 _mm_store_ps(packed_w + 36, v4567x2);
852 _mm_store_ps(packed_w + 40, v89ABx2);
853 _mm_store_ps(packed_w + 44, vCDEFx2);
854 _mm_store_ps(packed_w + 48, v0123x3);
855 _mm_store_ps(packed_w + 52, v4567x3);
856 _mm_store_ps(packed_w + 56, v89ABx3);
857 _mm_store_ps(packed_w + 60, vCDEFx3);
858 _mm_store_ps(packed_w + 64, v0123x4);
859 _mm_store_ps(packed_w + 68, v4567x4);
860 _mm_store_ps(packed_w + 72, v89ABx4);
861 _mm_store_ps(packed_w + 76, vCDEFx4);
862 _mm_store_ps(packed_w + 80, v0123x5);
863 _mm_store_ps(packed_w + 84, v4567x5);
864 _mm_store_ps(packed_w + 88, v89ABx5);
865 _mm_store_ps(packed_w + 92, vCDEFx5);
866 _mm_store_ps(packed_w + 96, v0123x6);
867 _mm_store_ps(packed_w + 100, v4567x6);
868 _mm_store_ps(packed_w + 104, v89ABx6);
869 _mm_store_ps(packed_w + 108, vCDEFx6);
870 _mm_store_ps(packed_w + 112, v0123x7);
871 _mm_store_ps(packed_w + 116, v4567x7);
872 _mm_store_ps(packed_w + 120, v89ABx7);
873 _mm_store_ps(packed_w + 124, vCDEFx7);
874 packed_w += 128;
875 }
876
877 // KC multiple of 4
878 for (; k >= 4; k -= 4) {
879 const __m128 v0x0123 = _mm_loadu_ps(w0);
880 w0 += 4;
881 const __m128 v1x0123 = _mm_loadu_ps(w1);
882 w1 += 4;
883 const __m128 v2x0123 = _mm_loadu_ps(w2);
884 w2 += 4;
885 const __m128 v3x0123 = _mm_loadu_ps(w3);
886 w3 += 4;
887 const __m128 v4x0123 = _mm_loadu_ps(w4);
888 w4 += 4;
889 const __m128 v5x0123 = _mm_loadu_ps(w5);
890 w5 += 4;
891 const __m128 v6x0123 = _mm_loadu_ps(w6);
892 w6 += 4;
893 const __m128 v7x0123 = _mm_loadu_ps(w7);
894 w7 += 4;
895 const __m128 v8x0123 = _mm_loadu_ps(w8);
896 w8 += 4;
897 const __m128 v9x0123 = _mm_loadu_ps(w9);
898 w9 += 4;
899 const __m128 vAx0123 = _mm_loadu_ps(w10);
900 w10 += 4;
901 const __m128 vBx0123 = _mm_loadu_ps(w11);
902 w11 += 4;
903 const __m128 vCx0123 = _mm_loadu_ps(w12);
904 w12 += 4;
905 const __m128 vDx0123 = _mm_loadu_ps(w13);
906 w13 += 4;
907 const __m128 vEx0123 = _mm_loadu_ps(w14);
908 w14 += 4;
909
910 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
911 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
912 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
913 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
914 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
915 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
916 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
917 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
918 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
919 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
920 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
921 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
922 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
923 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vEx0123);
924 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
925 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vEx0123);
926 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
927 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
928 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
929 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
930 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
931 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
932 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
933 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
934 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
935 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
936 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
937 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
938 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
939 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
940 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
941 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
942 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
943 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
944 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
945 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
946 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
947 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
948 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
949 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
950 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
951 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
952 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
953 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
954 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
955 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
956 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
957
958 _mm_store_ps(packed_w, v0123x0);
959 _mm_store_ps(packed_w + 4, v4567x0);
960 _mm_store_ps(packed_w + 8, v89ABx0);
961 _mm_store_ps(packed_w + 12, vCDEFx0);
962 _mm_store_ps(packed_w + 16, v0123x1);
963 _mm_store_ps(packed_w + 20, v4567x1);
964 _mm_store_ps(packed_w + 24, v89ABx1);
965 _mm_store_ps(packed_w + 28, vCDEFx1);
966 _mm_store_ps(packed_w + 32, v0123x2);
967 _mm_store_ps(packed_w + 36, v4567x2);
968 _mm_store_ps(packed_w + 40, v89ABx2);
969 _mm_store_ps(packed_w + 44, vCDEFx2);
970 _mm_store_ps(packed_w + 48, v0123x3);
971 _mm_store_ps(packed_w + 52, v4567x3);
972 _mm_store_ps(packed_w + 56, v89ABx3);
973 _mm_store_ps(packed_w + 60, vCDEFx3);
974 packed_w += 64;
975 }
976
977 // KC remainder (1..3)
978 if XNN_UNLIKELY(k != 0) {
979 assert(k >= 1);
980 assert(k <= 3);
981 switch (k) {
982 case 1:
983 {
984 const __m128 v0x0 = _mm_load_ss(w0);
985 const __m128 v1x0 = _mm_load_ss(w1);
986 const __m128 v2x0 = _mm_load_ss(w2);
987 const __m128 v3x0 = _mm_load_ss(w3);
988 const __m128 v4x0 = _mm_load_ss(w4);
989 const __m128 v5x0 = _mm_load_ss(w5);
990 const __m128 v6x0 = _mm_load_ss(w6);
991 const __m128 v7x0 = _mm_load_ss(w7);
992 const __m128 v8x0 = _mm_load_ss(w8);
993 const __m128 v9x0 = _mm_load_ss(w9);
994 const __m128 vAx0 = _mm_load_ss(w10);
995 const __m128 vBx0 = _mm_load_ss(w11);
996 const __m128 vCx0 = _mm_load_ss(w12);
997 const __m128 vDx0 = _mm_load_ss(w13);
998 const __m128 vEx0 = _mm_load_ss(w14);
999
1000 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
1001 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
1002 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
1003 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
1004 const __m128 v89x0 = _mm_unpacklo_ps(v8x0, v9x0);
1005 const __m128 vABx0 = _mm_unpacklo_ps(vAx0, vBx0);
1006 const __m128 vCDx0 = _mm_unpacklo_ps(vCx0, vDx0);
1007 const __m128 vEFx0 = _mm_unpacklo_ps(vEx0, vEx0);
1008
1009 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
1010 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
1011 const __m128 v89ABx0 = _mm_movelh_ps(v89x0, vABx0);
1012 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0, vEFx0);
1013
1014 _mm_store_ps(packed_w, v0123x0);
1015 _mm_store_ps(packed_w + 4, v4567x0);
1016 _mm_store_ps(packed_w + 8, v89ABx0);
1017 _mm_store_ps(packed_w + 12, vCDEFx0);
1018 packed_w += 16;
1019 break;
1020 }
1021 case 2:
1022 {
1023 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
1024 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
1025 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
1026 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
1027 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
1028 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
1029 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
1030 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
1031 const __m128 v8x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
1032 const __m128 v9x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
1033 const __m128 vAx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
1034 const __m128 vBx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
1035 const __m128 vCx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
1036 const __m128 vDx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
1037 const __m128 vEx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
1038
1039 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
1040 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
1041 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
1042 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
1043 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x01, v9x01);
1044 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx01, vBx01);
1045 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx01, vDx01);
1046 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx01, vEx01);
1047
1048 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
1049 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
1050 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
1051 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
1052 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
1053 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
1054 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
1055 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
1056
1057 _mm_store_ps(packed_w, v0123x0);
1058 _mm_store_ps(packed_w + 4, v4567x0);
1059 _mm_store_ps(packed_w + 8, v89ABx0);
1060 _mm_store_ps(packed_w + 12, vCDEFx0);
1061 _mm_store_ps(packed_w + 16, v0123x1);
1062 _mm_store_ps(packed_w + 20, v4567x1);
1063 _mm_store_ps(packed_w + 24, v89ABx1);
1064 _mm_store_ps(packed_w + 28, vCDEFx1);
1065 packed_w += 32;
1066 break;
1067 }
1068 case 3:
1069 {
1070 __m128 v0x012 = _mm_load_ss(w0 + 2);
1071 __m128 v1x012 = _mm_load_ss(w1 + 2);
1072 __m128 v2x012 = _mm_load_ss(w2 + 2);
1073 __m128 v3x012 = _mm_load_ss(w3 + 2);
1074 __m128 v4x012 = _mm_load_ss(w4 + 2);
1075 __m128 v5x012 = _mm_load_ss(w5 + 2);
1076 __m128 v6x012 = _mm_load_ss(w6 + 2);
1077 __m128 v7x012 = _mm_load_ss(w7 + 2);
1078 __m128 v8x012 = _mm_load_ss(w8 + 2);
1079 __m128 v9x012 = _mm_load_ss(w9 + 2);
1080 __m128 vAx012 = _mm_load_ss(w10 + 2);
1081 __m128 vBx012 = _mm_load_ss(w11 + 2);
1082 __m128 vCx012 = _mm_load_ss(w12 + 2);
1083 __m128 vDx012 = _mm_load_ss(w13 + 2);
1084 __m128 vEx012 = _mm_load_ss(w14 + 2);
1085
1086 v0x012 = _mm_movelh_ps(v0x012, v0x012);
1087 v1x012 = _mm_movelh_ps(v1x012, v1x012);
1088 v2x012 = _mm_movelh_ps(v2x012, v2x012);
1089 v3x012 = _mm_movelh_ps(v3x012, v3x012);
1090 v4x012 = _mm_movelh_ps(v4x012, v4x012);
1091 v5x012 = _mm_movelh_ps(v5x012, v5x012);
1092 v6x012 = _mm_movelh_ps(v6x012, v6x012);
1093 v7x012 = _mm_movelh_ps(v7x012, v7x012);
1094 v8x012 = _mm_movelh_ps(v8x012, v8x012);
1095 v9x012 = _mm_movelh_ps(v9x012, v9x012);
1096 vAx012 = _mm_movelh_ps(vAx012, vAx012);
1097 vBx012 = _mm_movelh_ps(vBx012, vBx012);
1098 vCx012 = _mm_movelh_ps(vCx012, vCx012);
1099 vDx012 = _mm_movelh_ps(vDx012, vDx012);
1100 vEx012 = _mm_movelh_ps(vEx012, vEx012);
1101
1102 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
1103 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
1104 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
1105 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
1106 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
1107 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
1108 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
1109 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
1110 v8x012 = _mm_loadl_pi(v8x012, (const __m64*) w8);
1111 v9x012 = _mm_loadl_pi(v9x012, (const __m64*) w9);
1112 vAx012 = _mm_loadl_pi(vAx012, (const __m64*) w10);
1113 vBx012 = _mm_loadl_pi(vBx012, (const __m64*) w11);
1114 vCx012 = _mm_loadl_pi(vCx012, (const __m64*) w12);
1115 vDx012 = _mm_loadl_pi(vDx012, (const __m64*) w13);
1116 vEx012 = _mm_loadl_pi(vEx012, (const __m64*) w14);
1117
1118 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
1119 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
1120 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
1121 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
1122 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
1123 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
1124 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
1125 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
1126 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x012, v9x012);
1127 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx012, vBx012);
1128 const __m128 v89x2 = _mm_unpackhi_ps(v8x012, v9x012);
1129 const __m128 vABx2 = _mm_unpackhi_ps(vAx012, vBx012);
1130 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx012, vDx012);
1131 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx012, vEx012);
1132 const __m128 vCDx2 = _mm_unpackhi_ps(vCx012, vDx012);
1133 const __m128 vEFx2 = _mm_unpackhi_ps(vEx012, vEx012);
1134
1135 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
1136 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
1137 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
1138 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
1139 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
1140 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
1141 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
1142 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
1143 const __m128 v89ABx2 = _mm_movelh_ps(v89x2, vABx2);
1144 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
1145 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
1146 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2, vEFx2);
1147
1148 _mm_store_ps(packed_w, v0123x0);
1149 _mm_store_ps(packed_w + 4, v4567x0);
1150 _mm_store_ps(packed_w + 8, v89ABx0);
1151 _mm_store_ps(packed_w + 12, vCDEFx0);
1152 _mm_store_ps(packed_w + 16, v0123x1);
1153 _mm_store_ps(packed_w + 20, v4567x1);
1154 _mm_store_ps(packed_w + 24, v89ABx1);
1155 _mm_store_ps(packed_w + 28, vCDEFx1);
1156 _mm_store_ps(packed_w + 32, v0123x2);
1157 _mm_store_ps(packed_w + 36, v4567x2);
1158 _mm_store_ps(packed_w + 40, v89ABx2);
1159 _mm_store_ps(packed_w + 44, vCDEFx2);
1160 packed_w += 48;
1161 break;
1162 }
1163 default:
1164 XNN_UNREACHABLE;
1165 }
1166 }
1167 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
1168 }
1169 weights += nc * kc;
1170 } while (--g != 0);
1171 }