Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-sse2-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <emmintrin.h>
16
17 #include "xnnpack/packw.h"
18
19 void xnn_x32_packw_gemm_goi_ukernel_x16__sse2_u8(
20 size_t g,
21 size_t nc,
22 size_t kc,
23 size_t nr,
24 size_t kr,
25 size_t sr,
26 const uint32_t* weights,
27 const uint32_t* bias,
28 const void* scale,
29 uint32_t* packed_weights,
30 size_t extra_bytes,
31 const void* params)
32 {
33 assert(g != 0);
34 assert(nc != 0);
35 assert(kc != 0);
36 assert(nr == 16);
37 assert(kr == 1);
38 assert(sr == 1);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 const float* b = (const float*) bias;
43 float* packed_w = (float*) packed_weights;
44 do {
45 // NC main loop multiple of 16
46 const float* w0 = (const float*) weights;
47 size_t n = nc;
48
49 for (; n >= 16; n -= 16) {
50 if XNN_LIKELY(b != NULL) {
51 const __m128 vb0123 = _mm_loadu_ps(b);
52 const __m128 vb4567 = _mm_loadu_ps(b + 4);
53 const __m128 vb89AB = _mm_loadu_ps(b + 8);
54 const __m128 vbCDEF = _mm_loadu_ps(b + 12);
55 b += 16;
56
57 _mm_store_ps(packed_w, vb0123);
58 _mm_store_ps(packed_w + 4, vb4567);
59 _mm_store_ps(packed_w + 8, vb89AB);
60 _mm_store_ps(packed_w + 12, vbCDEF);
61 } else {
62 const __m128 vzero = _mm_setzero_ps();
63 _mm_store_ps(packed_w, vzero);
64 _mm_store_ps(packed_w + 4, vzero);
65 _mm_store_ps(packed_w + 8, vzero);
66 _mm_store_ps(packed_w + 12, vzero);
67 }
68 packed_w += 16;
69
70 const float* w1 = w0 + kc;
71 const float* w2 = w1 + kc;
72 const float* w3 = w2 + kc;
73 const float* w4 = w3 + kc;
74 const float* w5 = w4 + kc;
75 const float* w6 = w5 + kc;
76 const float* w7 = w6 + kc;
77 const float* w8 = w7 + kc;
78 const float* w9 = w8 + kc;
79 const float* w10 = w9 + kc;
80 const float* w11 = w10 + kc;
81 const float* w12 = w11 + kc;
82 const float* w13 = w12 + kc;
83 const float* w14 = w13 + kc;
84 const float* w15 = w14 + kc;
85
86 size_t k = kc;
87 // KC main loop multiple of 8
88 for (; k >= 8; k -= 8) {
89 const __m128 v0x0123 = _mm_loadu_ps(w0);
90 const __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
91 w0 += 8;
92 const __m128 v1x0123 = _mm_loadu_ps(w1);
93 const __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
94 w1 += 8;
95 const __m128 v2x0123 = _mm_loadu_ps(w2);
96 const __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
97 w2 += 8;
98 const __m128 v3x0123 = _mm_loadu_ps(w3);
99 const __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
100 w3 += 8;
101 const __m128 v4x0123 = _mm_loadu_ps(w4);
102 const __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
103 w4 += 8;
104 const __m128 v5x0123 = _mm_loadu_ps(w5);
105 const __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
106 w5 += 8;
107 const __m128 v6x0123 = _mm_loadu_ps(w6);
108 const __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
109 w6 += 8;
110 const __m128 v7x0123 = _mm_loadu_ps(w7);
111 const __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
112 w7 += 8;
113 const __m128 v8x0123 = _mm_loadu_ps(w8);
114 const __m128 v8x4567 = _mm_loadu_ps(w8 + 4);
115 w8 += 8;
116 const __m128 v9x0123 = _mm_loadu_ps(w9);
117 const __m128 v9x4567 = _mm_loadu_ps(w9 + 4);
118 w9 += 8;
119 const __m128 vAx0123 = _mm_loadu_ps(w10);
120 const __m128 vAx4567 = _mm_loadu_ps(w10 + 4);
121 w10 += 8;
122 const __m128 vBx0123 = _mm_loadu_ps(w11);
123 const __m128 vBx4567 = _mm_loadu_ps(w11 + 4);
124 w11 += 8;
125 const __m128 vCx0123 = _mm_loadu_ps(w12);
126 const __m128 vCx4567 = _mm_loadu_ps(w12 + 4);
127 w12 += 8;
128 const __m128 vDx0123 = _mm_loadu_ps(w13);
129 const __m128 vDx4567 = _mm_loadu_ps(w13 + 4);
130 w13 += 8;
131 const __m128 vEx0123 = _mm_loadu_ps(w14);
132 const __m128 vEx4567 = _mm_loadu_ps(w14 + 4);
133 w14 += 8;
134 const __m128 vFx0123 = _mm_loadu_ps(w15);
135 const __m128 vFx4567 = _mm_loadu_ps(w15 + 4);
136 w15 += 8;
137
138 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
139 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
140 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
141 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
142 const __m128 v01x4_01x5 = _mm_unpacklo_ps(v0x4567, v1x4567);
143 const __m128 v23x4_23x5 = _mm_unpacklo_ps(v2x4567, v3x4567);
144 const __m128 v01x6_01x7 = _mm_unpackhi_ps(v0x4567, v1x4567);
145 const __m128 v23x6_23x7 = _mm_unpackhi_ps(v2x4567, v3x4567);
146 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
147 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
148 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
149 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
150 const __m128 v45x4_45x5 = _mm_unpacklo_ps(v4x4567, v5x4567);
151 const __m128 v67x4_67x5 = _mm_unpacklo_ps(v6x4567, v7x4567);
152 const __m128 v45x6_45x7 = _mm_unpackhi_ps(v4x4567, v5x4567);
153 const __m128 v67x6_67x7 = _mm_unpackhi_ps(v6x4567, v7x4567);
154 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
155 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
156 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
157 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
158 const __m128 v89x4_89x5 = _mm_unpacklo_ps(v8x4567, v9x4567);
159 const __m128 vABx4_ABx5 = _mm_unpacklo_ps(vAx4567, vBx4567);
160 const __m128 v89x6_89x7 = _mm_unpackhi_ps(v8x4567, v9x4567);
161 const __m128 vABx6_ABx7 = _mm_unpackhi_ps(vAx4567, vBx4567);
162 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
163 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vFx0123);
164 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
165 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vFx0123);
166 const __m128 vCDx4_CDx5 = _mm_unpacklo_ps(vCx4567, vDx4567);
167 const __m128 vEFx4_EFx5 = _mm_unpacklo_ps(vEx4567, vFx4567);
168 const __m128 vCDx6_CDx7 = _mm_unpackhi_ps(vCx4567, vDx4567);
169 const __m128 vEFx6_EFx7 = _mm_unpackhi_ps(vEx4567, vFx4567);
170 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
171 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
172 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
173 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
174 const __m128 v0123x4 = _mm_movelh_ps(v01x4_01x5, v23x4_23x5);
175 const __m128 v0123x5 = _mm_movehl_ps(v23x4_23x5, v01x4_01x5);
176 const __m128 v0123x6 = _mm_movelh_ps(v01x6_01x7, v23x6_23x7);
177 const __m128 v0123x7 = _mm_movehl_ps(v23x6_23x7, v01x6_01x7);
178 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
179 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
180 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
181 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
182 const __m128 v4567x4 = _mm_movelh_ps(v45x4_45x5, v67x4_67x5);
183 const __m128 v4567x5 = _mm_movehl_ps(v67x4_67x5, v45x4_45x5);
184 const __m128 v4567x6 = _mm_movelh_ps(v45x6_45x7, v67x6_67x7);
185 const __m128 v4567x7 = _mm_movehl_ps(v67x6_67x7, v45x6_45x7);
186 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
187 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
188 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
189 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
190 const __m128 v89ABx4 = _mm_movelh_ps(v89x4_89x5, vABx4_ABx5);
191 const __m128 v89ABx5 = _mm_movehl_ps(vABx4_ABx5, v89x4_89x5);
192 const __m128 v89ABx6 = _mm_movelh_ps(v89x6_89x7, vABx6_ABx7);
193 const __m128 v89ABx7 = _mm_movehl_ps(vABx6_ABx7, v89x6_89x7);
194 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
195 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
196 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
197 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
198 const __m128 vCDEFx4 = _mm_movelh_ps(vCDx4_CDx5, vEFx4_EFx5);
199 const __m128 vCDEFx5 = _mm_movehl_ps(vEFx4_EFx5, vCDx4_CDx5);
200 const __m128 vCDEFx6 = _mm_movelh_ps(vCDx6_CDx7, vEFx6_EFx7);
201 const __m128 vCDEFx7 = _mm_movehl_ps(vEFx6_EFx7, vCDx6_CDx7);
202
203 _mm_store_ps(packed_w, v0123x0);
204 _mm_store_ps(packed_w + 4, v4567x0);
205 _mm_store_ps(packed_w + 8, v89ABx0);
206 _mm_store_ps(packed_w + 12, vCDEFx0);
207 _mm_store_ps(packed_w + 16, v0123x1);
208 _mm_store_ps(packed_w + 20, v4567x1);
209 _mm_store_ps(packed_w + 24, v89ABx1);
210 _mm_store_ps(packed_w + 28, vCDEFx1);
211 _mm_store_ps(packed_w + 32, v0123x2);
212 _mm_store_ps(packed_w + 36, v4567x2);
213 _mm_store_ps(packed_w + 40, v89ABx2);
214 _mm_store_ps(packed_w + 44, vCDEFx2);
215 _mm_store_ps(packed_w + 48, v0123x3);
216 _mm_store_ps(packed_w + 52, v4567x3);
217 _mm_store_ps(packed_w + 56, v89ABx3);
218 _mm_store_ps(packed_w + 60, vCDEFx3);
219 _mm_store_ps(packed_w + 64, v0123x4);
220 _mm_store_ps(packed_w + 68, v4567x4);
221 _mm_store_ps(packed_w + 72, v89ABx4);
222 _mm_store_ps(packed_w + 76, vCDEFx4);
223 _mm_store_ps(packed_w + 80, v0123x5);
224 _mm_store_ps(packed_w + 84, v4567x5);
225 _mm_store_ps(packed_w + 88, v89ABx5);
226 _mm_store_ps(packed_w + 92, vCDEFx5);
227 _mm_store_ps(packed_w + 96, v0123x6);
228 _mm_store_ps(packed_w + 100, v4567x6);
229 _mm_store_ps(packed_w + 104, v89ABx6);
230 _mm_store_ps(packed_w + 108, vCDEFx6);
231 _mm_store_ps(packed_w + 112, v0123x7);
232 _mm_store_ps(packed_w + 116, v4567x7);
233 _mm_store_ps(packed_w + 120, v89ABx7);
234 _mm_store_ps(packed_w + 124, vCDEFx7);
235 packed_w += 128;
236 }
237
238 // KC multiple of 4
239 for (; k >= 4; k -= 4) {
240 const __m128 v0x0123 = _mm_loadu_ps(w0);
241 w0 += 4;
242 const __m128 v1x0123 = _mm_loadu_ps(w1);
243 w1 += 4;
244 const __m128 v2x0123 = _mm_loadu_ps(w2);
245 w2 += 4;
246 const __m128 v3x0123 = _mm_loadu_ps(w3);
247 w3 += 4;
248 const __m128 v4x0123 = _mm_loadu_ps(w4);
249 w4 += 4;
250 const __m128 v5x0123 = _mm_loadu_ps(w5);
251 w5 += 4;
252 const __m128 v6x0123 = _mm_loadu_ps(w6);
253 w6 += 4;
254 const __m128 v7x0123 = _mm_loadu_ps(w7);
255 w7 += 4;
256 const __m128 v8x0123 = _mm_loadu_ps(w8);
257 w8 += 4;
258 const __m128 v9x0123 = _mm_loadu_ps(w9);
259 w9 += 4;
260 const __m128 vAx0123 = _mm_loadu_ps(w10);
261 w10 += 4;
262 const __m128 vBx0123 = _mm_loadu_ps(w11);
263 w11 += 4;
264 const __m128 vCx0123 = _mm_loadu_ps(w12);
265 w12 += 4;
266 const __m128 vDx0123 = _mm_loadu_ps(w13);
267 w13 += 4;
268 const __m128 vEx0123 = _mm_loadu_ps(w14);
269 w14 += 4;
270 const __m128 vFx0123 = _mm_loadu_ps(w15);
271 w15 += 4;
272
273 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
274 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
275 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
276 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
277 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
278 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
279 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
280 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
281 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
282 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
283 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
284 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
285 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
286 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vFx0123);
287 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
288 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vFx0123);
289 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
290 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
291 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
292 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
293 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
294 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
295 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
296 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
297 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
298 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
299 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
300 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
301 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
302 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
303 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
304 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
305
306 _mm_store_ps(packed_w, v0123x0);
307 _mm_store_ps(packed_w + 4, v4567x0);
308 _mm_store_ps(packed_w + 8, v89ABx0);
309 _mm_store_ps(packed_w + 12, vCDEFx0);
310 _mm_store_ps(packed_w + 16, v0123x1);
311 _mm_store_ps(packed_w + 20, v4567x1);
312 _mm_store_ps(packed_w + 24, v89ABx1);
313 _mm_store_ps(packed_w + 28, vCDEFx1);
314 _mm_store_ps(packed_w + 32, v0123x2);
315 _mm_store_ps(packed_w + 36, v4567x2);
316 _mm_store_ps(packed_w + 40, v89ABx2);
317 _mm_store_ps(packed_w + 44, vCDEFx2);
318 _mm_store_ps(packed_w + 48, v0123x3);
319 _mm_store_ps(packed_w + 52, v4567x3);
320 _mm_store_ps(packed_w + 56, v89ABx3);
321 _mm_store_ps(packed_w + 60, vCDEFx3);
322 packed_w += 64;
323 }
324
325 // KC remainder (1..3)
326 if XNN_UNLIKELY(k != 0) {
327 assert(k >= 1);
328 assert(k <= 3);
329 switch (k) {
330 case 1:
331 {
332 const __m128 v0x0 = _mm_load_ss(w0);
333 w0 += 1;
334 const __m128 v1x0 = _mm_load_ss(w1);
335 w1 += 1;
336 const __m128 v2x0 = _mm_load_ss(w2);
337 w2 += 1;
338 const __m128 v3x0 = _mm_load_ss(w3);
339 w3 += 1;
340 const __m128 v4x0 = _mm_load_ss(w4);
341 w4 += 1;
342 const __m128 v5x0 = _mm_load_ss(w5);
343 w5 += 1;
344 const __m128 v6x0 = _mm_load_ss(w6);
345 w6 += 1;
346 const __m128 v7x0 = _mm_load_ss(w7);
347 w7 += 1;
348 const __m128 v8x0 = _mm_load_ss(w8);
349 w8 += 1;
350 const __m128 v9x0 = _mm_load_ss(w9);
351 w9 += 1;
352 const __m128 vAx0 = _mm_load_ss(w10);
353 w10 += 1;
354 const __m128 vBx0 = _mm_load_ss(w11);
355 w11 += 1;
356 const __m128 vCx0 = _mm_load_ss(w12);
357 w12 += 1;
358 const __m128 vDx0 = _mm_load_ss(w13);
359 w13 += 1;
360 const __m128 vEx0 = _mm_load_ss(w14);
361 w14 += 1;
362 const __m128 vFx0 = _mm_load_ss(w15);
363 w15 += 1;
364
365 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
366 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
367 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
368 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
369 const __m128 v89x0 = _mm_unpacklo_ps(v8x0, v9x0);
370 const __m128 vABx0 = _mm_unpacklo_ps(vAx0, vBx0);
371 const __m128 vCDx0 = _mm_unpacklo_ps(vCx0, vDx0);
372 const __m128 vEFx0 = _mm_unpacklo_ps(vEx0, vFx0);
373
374 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
375 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
376 const __m128 v89ABx0 = _mm_movelh_ps(v89x0, vABx0);
377 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0, vEFx0);
378
379 _mm_store_ps(packed_w, v0123x0);
380 _mm_store_ps(packed_w + 4, v4567x0);
381 _mm_store_ps(packed_w + 8, v89ABx0);
382 _mm_store_ps(packed_w + 12, vCDEFx0);
383 packed_w += 16;
384 break;
385 }
386 case 2:
387 {
388 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
389 w0 += 2;
390 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
391 w1 += 2;
392 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
393 w2 += 2;
394 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
395 w3 += 2;
396 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
397 w4 += 2;
398 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
399 w5 += 2;
400 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
401 w6 += 2;
402 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
403 w7 += 2;
404 const __m128 v8x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
405 w8 += 2;
406 const __m128 v9x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
407 w9 += 2;
408 const __m128 vAx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
409 w10 += 2;
410 const __m128 vBx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
411 w11 += 2;
412 const __m128 vCx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
413 w12 += 2;
414 const __m128 vDx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
415 w13 += 2;
416 const __m128 vEx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
417 w14 += 2;
418 const __m128 vFx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
419 w15 += 2;
420
421 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
422 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
423 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
424 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
425 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x01, v9x01);
426 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx01, vBx01);
427 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx01, vDx01);
428 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx01, vFx01);
429
430 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
431 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
432 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
433 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
434 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
435 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
436 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
437 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
438
439 _mm_store_ps(packed_w, v0123x0);
440 _mm_store_ps(packed_w + 4, v4567x0);
441 _mm_store_ps(packed_w + 8, v89ABx0);
442 _mm_store_ps(packed_w + 12, vCDEFx0);
443 _mm_store_ps(packed_w + 16, v0123x1);
444 _mm_store_ps(packed_w + 20, v4567x1);
445 _mm_store_ps(packed_w + 24, v89ABx1);
446 _mm_store_ps(packed_w + 28, vCDEFx1);
447 packed_w += 32;
448 break;
449 }
450 case 3:
451 {
452 __m128 v0x012 = _mm_load_ss(w0 + 2);
453 __m128 v1x012 = _mm_load_ss(w1 + 2);
454 __m128 v2x012 = _mm_load_ss(w2 + 2);
455 __m128 v3x012 = _mm_load_ss(w3 + 2);
456 __m128 v4x012 = _mm_load_ss(w4 + 2);
457 __m128 v5x012 = _mm_load_ss(w5 + 2);
458 __m128 v6x012 = _mm_load_ss(w6 + 2);
459 __m128 v7x012 = _mm_load_ss(w7 + 2);
460 __m128 v8x012 = _mm_load_ss(w8 + 2);
461 __m128 v9x012 = _mm_load_ss(w9 + 2);
462 __m128 vAx012 = _mm_load_ss(w10 + 2);
463 __m128 vBx012 = _mm_load_ss(w11 + 2);
464 __m128 vCx012 = _mm_load_ss(w12 + 2);
465 __m128 vDx012 = _mm_load_ss(w13 + 2);
466 __m128 vEx012 = _mm_load_ss(w14 + 2);
467 __m128 vFx012 = _mm_load_ss(w15 + 2);
468
469 v0x012 = _mm_movelh_ps(v0x012, v0x012);
470 v1x012 = _mm_movelh_ps(v1x012, v1x012);
471 v2x012 = _mm_movelh_ps(v2x012, v2x012);
472 v3x012 = _mm_movelh_ps(v3x012, v3x012);
473 v4x012 = _mm_movelh_ps(v4x012, v4x012);
474 v5x012 = _mm_movelh_ps(v5x012, v5x012);
475 v6x012 = _mm_movelh_ps(v6x012, v6x012);
476 v7x012 = _mm_movelh_ps(v7x012, v7x012);
477 v8x012 = _mm_movelh_ps(v8x012, v8x012);
478 v9x012 = _mm_movelh_ps(v9x012, v9x012);
479 vAx012 = _mm_movelh_ps(vAx012, vAx012);
480 vBx012 = _mm_movelh_ps(vBx012, vBx012);
481 vCx012 = _mm_movelh_ps(vCx012, vCx012);
482 vDx012 = _mm_movelh_ps(vDx012, vDx012);
483 vEx012 = _mm_movelh_ps(vEx012, vEx012);
484 vFx012 = _mm_movelh_ps(vFx012, vFx012);
485
486 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
487 w0 += 3;
488 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
489 w1 += 3;
490 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
491 w2 += 3;
492 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
493 w3 += 3;
494 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
495 w4 += 3;
496 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
497 w5 += 3;
498 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
499 w6 += 3;
500 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
501 w7 += 3;
502 v8x012 = _mm_loadl_pi(v8x012, (const __m64*) w8);
503 w8 += 3;
504 v9x012 = _mm_loadl_pi(v9x012, (const __m64*) w9);
505 w9 += 3;
506 vAx012 = _mm_loadl_pi(vAx012, (const __m64*) w10);
507 w10 += 3;
508 vBx012 = _mm_loadl_pi(vBx012, (const __m64*) w11);
509 w11 += 3;
510 vCx012 = _mm_loadl_pi(vCx012, (const __m64*) w12);
511 w12 += 3;
512 vDx012 = _mm_loadl_pi(vDx012, (const __m64*) w13);
513 w13 += 3;
514 vEx012 = _mm_loadl_pi(vEx012, (const __m64*) w14);
515 w14 += 3;
516 vFx012 = _mm_loadl_pi(vFx012, (const __m64*) w15);
517 w15 += 3;
518
519 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
520 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
521 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
522 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
523 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
524 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
525 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
526 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
527 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x012, v9x012);
528 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx012, vBx012);
529 const __m128 v89x2 = _mm_unpackhi_ps(v8x012, v9x012);
530 const __m128 vABx2 = _mm_unpackhi_ps(vAx012, vBx012);
531 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx012, vDx012);
532 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx012, vFx012);
533 const __m128 vCDx2 = _mm_unpackhi_ps(vCx012, vDx012);
534 const __m128 vEFx2 = _mm_unpackhi_ps(vEx012, vFx012);
535
536 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
537 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
538 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
539 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
540 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
541 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
542 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
543 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
544 const __m128 v89ABx2 = _mm_movelh_ps(v89x2, vABx2);
545 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
546 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
547 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2, vEFx2);
548
549 _mm_store_ps(packed_w, v0123x0);
550 _mm_store_ps(packed_w + 4, v4567x0);
551 _mm_store_ps(packed_w + 8, v89ABx0);
552 _mm_store_ps(packed_w + 12, vCDEFx0);
553 _mm_store_ps(packed_w + 16, v0123x1);
554 _mm_store_ps(packed_w + 20, v4567x1);
555 _mm_store_ps(packed_w + 24, v89ABx1);
556 _mm_store_ps(packed_w + 28, vCDEFx1);
557 _mm_store_ps(packed_w + 32, v0123x2);
558 _mm_store_ps(packed_w + 36, v4567x2);
559 _mm_store_ps(packed_w + 40, v89ABx2);
560 _mm_store_ps(packed_w + 44, vCDEFx2);
561 packed_w += 48;
562 break;
563 }
564 default:
565 XNN_UNREACHABLE;
566 }
567 }
568 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
569 w0 = w15;
570 }
571
572 // NC remainder (1..15)
573 if XNN_UNLIKELY(n != 0) {
574 assert(n >= 1);
575 assert(n <= 15);
576 if XNN_LIKELY(b != NULL) {
577 size_t nb = n;
578 do {
579 *packed_w++ = *b++;
580 } while (--nb != 0);
581 packed_w += (16 - n);
582 } else {
583 const __m128 vzero = _mm_setzero_ps();
584 _mm_store_ps(packed_w, vzero);
585 _mm_store_ps(packed_w + 4, vzero);
586 _mm_store_ps(packed_w + 8, vzero);
587 _mm_store_ps(packed_w + 12, vzero);
588 packed_w += 16;
589 }
590
591 const float* w1 = w0 + kc;
592 if XNN_UNPREDICTABLE(n < 2) {
593 w1 = w0;
594 }
595 const float* w2 = w1 + kc;
596 if XNN_UNPREDICTABLE(n <= 2) {
597 w2 = w1;
598 }
599 const float* w3 = w2 + kc;
600 if XNN_UNPREDICTABLE(n < 4) {
601 w3 = w2;
602 }
603 const float* w4 = w3 + kc;
604 if XNN_UNPREDICTABLE(n <= 4) {
605 w4 = w3;
606 }
607 const float* w5 = w4 + kc;
608 if XNN_UNPREDICTABLE(n < 6) {
609 w5 = w4;
610 }
611 const float* w6 = w5 + kc;
612 if XNN_UNPREDICTABLE(n <= 6) {
613 w6 = w5;
614 }
615 const float* w7 = w6 + kc;
616 if XNN_UNPREDICTABLE(n < 8) {
617 w7 = w6;
618 }
619 const float* w8 = w7 + kc;
620 if XNN_UNPREDICTABLE(n <= 8) {
621 w8 = w7;
622 }
623 const float* w9 = w8 + kc;
624 if XNN_UNPREDICTABLE(n < 10) {
625 w9 = w8;
626 }
627 const float* w10 = w9 + kc;
628 if XNN_UNPREDICTABLE(n <= 10) {
629 w10 = w9;
630 }
631 const float* w11 = w10 + kc;
632 if XNN_UNPREDICTABLE(n < 12) {
633 w11 = w10;
634 }
635 const float* w12 = w11 + kc;
636 if XNN_UNPREDICTABLE(n <= 12) {
637 w12 = w11;
638 }
639 const float* w13 = w12 + kc;
640 if XNN_UNPREDICTABLE(n < 14) {
641 w13 = w12;
642 }
643 const float* w14 = w13 + kc;
644 if XNN_UNPREDICTABLE(n <= 14) {
645 w14 = w13;
646 }
647
648 size_t k = kc;
649 // KC main loop multiple of 8
650 for (; k >= 8; k -= 8) {
651 const __m128 v0x0123 = _mm_loadu_ps(w0);
652 const __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
653 w0 += 8;
654 const __m128 v1x0123 = _mm_loadu_ps(w1);
655 const __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
656 w1 += 8;
657 const __m128 v2x0123 = _mm_loadu_ps(w2);
658 const __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
659 w2 += 8;
660 const __m128 v3x0123 = _mm_loadu_ps(w3);
661 const __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
662 w3 += 8;
663 const __m128 v4x0123 = _mm_loadu_ps(w4);
664 const __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
665 w4 += 8;
666 const __m128 v5x0123 = _mm_loadu_ps(w5);
667 const __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
668 w5 += 8;
669 const __m128 v6x0123 = _mm_loadu_ps(w6);
670 const __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
671 w6 += 8;
672 const __m128 v7x0123 = _mm_loadu_ps(w7);
673 const __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
674 w7 += 8;
675 const __m128 v8x0123 = _mm_loadu_ps(w8);
676 const __m128 v8x4567 = _mm_loadu_ps(w8 + 4);
677 w8 += 8;
678 const __m128 v9x0123 = _mm_loadu_ps(w9);
679 const __m128 v9x4567 = _mm_loadu_ps(w9 + 4);
680 w9 += 8;
681 const __m128 vAx0123 = _mm_loadu_ps(w10);
682 const __m128 vAx4567 = _mm_loadu_ps(w10 + 4);
683 w10 += 8;
684 const __m128 vBx0123 = _mm_loadu_ps(w11);
685 const __m128 vBx4567 = _mm_loadu_ps(w11 + 4);
686 w11 += 8;
687 const __m128 vCx0123 = _mm_loadu_ps(w12);
688 const __m128 vCx4567 = _mm_loadu_ps(w12 + 4);
689 w12 += 8;
690 const __m128 vDx0123 = _mm_loadu_ps(w13);
691 const __m128 vDx4567 = _mm_loadu_ps(w13 + 4);
692 w13 += 8;
693 const __m128 vEx0123 = _mm_loadu_ps(w14);
694 const __m128 vEx4567 = _mm_loadu_ps(w14 + 4);
695 w14 += 8;
696
697 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
698 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
699 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
700 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
701 const __m128 v01x4_01x5 = _mm_unpacklo_ps(v0x4567, v1x4567);
702 const __m128 v23x4_23x5 = _mm_unpacklo_ps(v2x4567, v3x4567);
703 const __m128 v01x6_01x7 = _mm_unpackhi_ps(v0x4567, v1x4567);
704 const __m128 v23x6_23x7 = _mm_unpackhi_ps(v2x4567, v3x4567);
705 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
706 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
707 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
708 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
709 const __m128 v45x4_45x5 = _mm_unpacklo_ps(v4x4567, v5x4567);
710 const __m128 v67x4_67x5 = _mm_unpacklo_ps(v6x4567, v7x4567);
711 const __m128 v45x6_45x7 = _mm_unpackhi_ps(v4x4567, v5x4567);
712 const __m128 v67x6_67x7 = _mm_unpackhi_ps(v6x4567, v7x4567);
713 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
714 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
715 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
716 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
717 const __m128 v89x4_89x5 = _mm_unpacklo_ps(v8x4567, v9x4567);
718 const __m128 vABx4_ABx5 = _mm_unpacklo_ps(vAx4567, vBx4567);
719 const __m128 v89x6_89x7 = _mm_unpackhi_ps(v8x4567, v9x4567);
720 const __m128 vABx6_ABx7 = _mm_unpackhi_ps(vAx4567, vBx4567);
721 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
722 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vEx0123);
723 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
724 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vEx0123);
725 const __m128 vCDx4_CDx5 = _mm_unpacklo_ps(vCx4567, vDx4567);
726 const __m128 vEFx4_EFx5 = _mm_unpacklo_ps(vEx4567, vEx4567);
727 const __m128 vCDx6_CDx7 = _mm_unpackhi_ps(vCx4567, vDx4567);
728 const __m128 vEFx6_EFx7 = _mm_unpackhi_ps(vEx4567, vEx4567);
729 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
730 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
731 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
732 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
733 const __m128 v0123x4 = _mm_movelh_ps(v01x4_01x5, v23x4_23x5);
734 const __m128 v0123x5 = _mm_movehl_ps(v23x4_23x5, v01x4_01x5);
735 const __m128 v0123x6 = _mm_movelh_ps(v01x6_01x7, v23x6_23x7);
736 const __m128 v0123x7 = _mm_movehl_ps(v23x6_23x7, v01x6_01x7);
737 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
738 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
739 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
740 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
741 const __m128 v4567x4 = _mm_movelh_ps(v45x4_45x5, v67x4_67x5);
742 const __m128 v4567x5 = _mm_movehl_ps(v67x4_67x5, v45x4_45x5);
743 const __m128 v4567x6 = _mm_movelh_ps(v45x6_45x7, v67x6_67x7);
744 const __m128 v4567x7 = _mm_movehl_ps(v67x6_67x7, v45x6_45x7);
745 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
746 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
747 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
748 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
749 const __m128 v89ABx4 = _mm_movelh_ps(v89x4_89x5, vABx4_ABx5);
750 const __m128 v89ABx5 = _mm_movehl_ps(vABx4_ABx5, v89x4_89x5);
751 const __m128 v89ABx6 = _mm_movelh_ps(v89x6_89x7, vABx6_ABx7);
752 const __m128 v89ABx7 = _mm_movehl_ps(vABx6_ABx7, v89x6_89x7);
753 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
754 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
755 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
756 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
757 const __m128 vCDEFx4 = _mm_movelh_ps(vCDx4_CDx5, vEFx4_EFx5);
758 const __m128 vCDEFx5 = _mm_movehl_ps(vEFx4_EFx5, vCDx4_CDx5);
759 const __m128 vCDEFx6 = _mm_movelh_ps(vCDx6_CDx7, vEFx6_EFx7);
760 const __m128 vCDEFx7 = _mm_movehl_ps(vEFx6_EFx7, vCDx6_CDx7);
761
762 _mm_store_ps(packed_w, v0123x0);
763 _mm_store_ps(packed_w + 4, v4567x0);
764 _mm_store_ps(packed_w + 8, v89ABx0);
765 _mm_store_ps(packed_w + 12, vCDEFx0);
766 _mm_store_ps(packed_w + 16, v0123x1);
767 _mm_store_ps(packed_w + 20, v4567x1);
768 _mm_store_ps(packed_w + 24, v89ABx1);
769 _mm_store_ps(packed_w + 28, vCDEFx1);
770 _mm_store_ps(packed_w + 32, v0123x2);
771 _mm_store_ps(packed_w + 36, v4567x2);
772 _mm_store_ps(packed_w + 40, v89ABx2);
773 _mm_store_ps(packed_w + 44, vCDEFx2);
774 _mm_store_ps(packed_w + 48, v0123x3);
775 _mm_store_ps(packed_w + 52, v4567x3);
776 _mm_store_ps(packed_w + 56, v89ABx3);
777 _mm_store_ps(packed_w + 60, vCDEFx3);
778 _mm_store_ps(packed_w + 64, v0123x4);
779 _mm_store_ps(packed_w + 68, v4567x4);
780 _mm_store_ps(packed_w + 72, v89ABx4);
781 _mm_store_ps(packed_w + 76, vCDEFx4);
782 _mm_store_ps(packed_w + 80, v0123x5);
783 _mm_store_ps(packed_w + 84, v4567x5);
784 _mm_store_ps(packed_w + 88, v89ABx5);
785 _mm_store_ps(packed_w + 92, vCDEFx5);
786 _mm_store_ps(packed_w + 96, v0123x6);
787 _mm_store_ps(packed_w + 100, v4567x6);
788 _mm_store_ps(packed_w + 104, v89ABx6);
789 _mm_store_ps(packed_w + 108, vCDEFx6);
790 _mm_store_ps(packed_w + 112, v0123x7);
791 _mm_store_ps(packed_w + 116, v4567x7);
792 _mm_store_ps(packed_w + 120, v89ABx7);
793 _mm_store_ps(packed_w + 124, vCDEFx7);
794 packed_w += 128;
795 }
796
797 // KC multiple of 4
798 for (; k >= 4; k -= 4) {
799 const __m128 v0x0123 = _mm_loadu_ps(w0);
800 w0 += 4;
801 const __m128 v1x0123 = _mm_loadu_ps(w1);
802 w1 += 4;
803 const __m128 v2x0123 = _mm_loadu_ps(w2);
804 w2 += 4;
805 const __m128 v3x0123 = _mm_loadu_ps(w3);
806 w3 += 4;
807 const __m128 v4x0123 = _mm_loadu_ps(w4);
808 w4 += 4;
809 const __m128 v5x0123 = _mm_loadu_ps(w5);
810 w5 += 4;
811 const __m128 v6x0123 = _mm_loadu_ps(w6);
812 w6 += 4;
813 const __m128 v7x0123 = _mm_loadu_ps(w7);
814 w7 += 4;
815 const __m128 v8x0123 = _mm_loadu_ps(w8);
816 w8 += 4;
817 const __m128 v9x0123 = _mm_loadu_ps(w9);
818 w9 += 4;
819 const __m128 vAx0123 = _mm_loadu_ps(w10);
820 w10 += 4;
821 const __m128 vBx0123 = _mm_loadu_ps(w11);
822 w11 += 4;
823 const __m128 vCx0123 = _mm_loadu_ps(w12);
824 w12 += 4;
825 const __m128 vDx0123 = _mm_loadu_ps(w13);
826 w13 += 4;
827 const __m128 vEx0123 = _mm_loadu_ps(w14);
828 w14 += 4;
829
830 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
831 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
832 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
833 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
834 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
835 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
836 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
837 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
838 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
839 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
840 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
841 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
842 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
843 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vEx0123);
844 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
845 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vEx0123);
846 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
847 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
848 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
849 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
850 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
851 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
852 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
853 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
854 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
855 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
856 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
857 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
858 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
859 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
860 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
861 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
862
863 _mm_store_ps(packed_w, v0123x0);
864 _mm_store_ps(packed_w + 4, v4567x0);
865 _mm_store_ps(packed_w + 8, v89ABx0);
866 _mm_store_ps(packed_w + 12, vCDEFx0);
867 _mm_store_ps(packed_w + 16, v0123x1);
868 _mm_store_ps(packed_w + 20, v4567x1);
869 _mm_store_ps(packed_w + 24, v89ABx1);
870 _mm_store_ps(packed_w + 28, vCDEFx1);
871 _mm_store_ps(packed_w + 32, v0123x2);
872 _mm_store_ps(packed_w + 36, v4567x2);
873 _mm_store_ps(packed_w + 40, v89ABx2);
874 _mm_store_ps(packed_w + 44, vCDEFx2);
875 _mm_store_ps(packed_w + 48, v0123x3);
876 _mm_store_ps(packed_w + 52, v4567x3);
877 _mm_store_ps(packed_w + 56, v89ABx3);
878 _mm_store_ps(packed_w + 60, vCDEFx3);
879 packed_w += 64;
880 }
881
882 // KC remainder (1..3)
883 if XNN_UNLIKELY(k != 0) {
884 assert(k >= 1);
885 assert(k <= 3);
886 switch (k) {
887 case 1:
888 {
889 const __m128 v0x0 = _mm_load_ss(w0);
890 const __m128 v1x0 = _mm_load_ss(w1);
891 const __m128 v2x0 = _mm_load_ss(w2);
892 const __m128 v3x0 = _mm_load_ss(w3);
893 const __m128 v4x0 = _mm_load_ss(w4);
894 const __m128 v5x0 = _mm_load_ss(w5);
895 const __m128 v6x0 = _mm_load_ss(w6);
896 const __m128 v7x0 = _mm_load_ss(w7);
897 const __m128 v8x0 = _mm_load_ss(w8);
898 const __m128 v9x0 = _mm_load_ss(w9);
899 const __m128 vAx0 = _mm_load_ss(w10);
900 const __m128 vBx0 = _mm_load_ss(w11);
901 const __m128 vCx0 = _mm_load_ss(w12);
902 const __m128 vDx0 = _mm_load_ss(w13);
903 const __m128 vEx0 = _mm_load_ss(w14);
904
905 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
906 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
907 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
908 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
909 const __m128 v89x0 = _mm_unpacklo_ps(v8x0, v9x0);
910 const __m128 vABx0 = _mm_unpacklo_ps(vAx0, vBx0);
911 const __m128 vCDx0 = _mm_unpacklo_ps(vCx0, vDx0);
912 const __m128 vEFx0 = _mm_unpacklo_ps(vEx0, vEx0);
913
914 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
915 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
916 const __m128 v89ABx0 = _mm_movelh_ps(v89x0, vABx0);
917 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0, vEFx0);
918
919 _mm_store_ps(packed_w, v0123x0);
920 _mm_store_ps(packed_w + 4, v4567x0);
921 _mm_store_ps(packed_w + 8, v89ABx0);
922 _mm_store_ps(packed_w + 12, vCDEFx0);
923 packed_w += 16;
924 break;
925 }
926 case 2:
927 {
928 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
929 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
930 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
931 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
932 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
933 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
934 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
935 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
936 const __m128 v8x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
937 const __m128 v9x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
938 const __m128 vAx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
939 const __m128 vBx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
940 const __m128 vCx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
941 const __m128 vDx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
942 const __m128 vEx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
943
944 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
945 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
946 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
947 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
948 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x01, v9x01);
949 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx01, vBx01);
950 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx01, vDx01);
951 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx01, vEx01);
952
953 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
954 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
955 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
956 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
957 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
958 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
959 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
960 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
961
962 _mm_store_ps(packed_w, v0123x0);
963 _mm_store_ps(packed_w + 4, v4567x0);
964 _mm_store_ps(packed_w + 8, v89ABx0);
965 _mm_store_ps(packed_w + 12, vCDEFx0);
966 _mm_store_ps(packed_w + 16, v0123x1);
967 _mm_store_ps(packed_w + 20, v4567x1);
968 _mm_store_ps(packed_w + 24, v89ABx1);
969 _mm_store_ps(packed_w + 28, vCDEFx1);
970 packed_w += 32;
971 break;
972 }
973 case 3:
974 {
975 __m128 v0x012 = _mm_load_ss(w0 + 2);
976 __m128 v1x012 = _mm_load_ss(w1 + 2);
977 __m128 v2x012 = _mm_load_ss(w2 + 2);
978 __m128 v3x012 = _mm_load_ss(w3 + 2);
979 __m128 v4x012 = _mm_load_ss(w4 + 2);
980 __m128 v5x012 = _mm_load_ss(w5 + 2);
981 __m128 v6x012 = _mm_load_ss(w6 + 2);
982 __m128 v7x012 = _mm_load_ss(w7 + 2);
983 __m128 v8x012 = _mm_load_ss(w8 + 2);
984 __m128 v9x012 = _mm_load_ss(w9 + 2);
985 __m128 vAx012 = _mm_load_ss(w10 + 2);
986 __m128 vBx012 = _mm_load_ss(w11 + 2);
987 __m128 vCx012 = _mm_load_ss(w12 + 2);
988 __m128 vDx012 = _mm_load_ss(w13 + 2);
989 __m128 vEx012 = _mm_load_ss(w14 + 2);
990
991 v0x012 = _mm_movelh_ps(v0x012, v0x012);
992 v1x012 = _mm_movelh_ps(v1x012, v1x012);
993 v2x012 = _mm_movelh_ps(v2x012, v2x012);
994 v3x012 = _mm_movelh_ps(v3x012, v3x012);
995 v4x012 = _mm_movelh_ps(v4x012, v4x012);
996 v5x012 = _mm_movelh_ps(v5x012, v5x012);
997 v6x012 = _mm_movelh_ps(v6x012, v6x012);
998 v7x012 = _mm_movelh_ps(v7x012, v7x012);
999 v8x012 = _mm_movelh_ps(v8x012, v8x012);
1000 v9x012 = _mm_movelh_ps(v9x012, v9x012);
1001 vAx012 = _mm_movelh_ps(vAx012, vAx012);
1002 vBx012 = _mm_movelh_ps(vBx012, vBx012);
1003 vCx012 = _mm_movelh_ps(vCx012, vCx012);
1004 vDx012 = _mm_movelh_ps(vDx012, vDx012);
1005 vEx012 = _mm_movelh_ps(vEx012, vEx012);
1006
1007 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
1008 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
1009 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
1010 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
1011 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
1012 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
1013 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
1014 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
1015 v8x012 = _mm_loadl_pi(v8x012, (const __m64*) w8);
1016 v9x012 = _mm_loadl_pi(v9x012, (const __m64*) w9);
1017 vAx012 = _mm_loadl_pi(vAx012, (const __m64*) w10);
1018 vBx012 = _mm_loadl_pi(vBx012, (const __m64*) w11);
1019 vCx012 = _mm_loadl_pi(vCx012, (const __m64*) w12);
1020 vDx012 = _mm_loadl_pi(vDx012, (const __m64*) w13);
1021 vEx012 = _mm_loadl_pi(vEx012, (const __m64*) w14);
1022
1023 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
1024 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
1025 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
1026 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
1027 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
1028 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
1029 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
1030 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
1031 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x012, v9x012);
1032 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx012, vBx012);
1033 const __m128 v89x2 = _mm_unpackhi_ps(v8x012, v9x012);
1034 const __m128 vABx2 = _mm_unpackhi_ps(vAx012, vBx012);
1035 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx012, vDx012);
1036 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx012, vEx012);
1037 const __m128 vCDx2 = _mm_unpackhi_ps(vCx012, vDx012);
1038 const __m128 vEFx2 = _mm_unpackhi_ps(vEx012, vEx012);
1039
1040 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
1041 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
1042 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
1043 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
1044 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
1045 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
1046 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
1047 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
1048 const __m128 v89ABx2 = _mm_movelh_ps(v89x2, vABx2);
1049 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
1050 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
1051 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2, vEFx2);
1052
1053 _mm_store_ps(packed_w, v0123x0);
1054 _mm_store_ps(packed_w + 4, v4567x0);
1055 _mm_store_ps(packed_w + 8, v89ABx0);
1056 _mm_store_ps(packed_w + 12, vCDEFx0);
1057 _mm_store_ps(packed_w + 16, v0123x1);
1058 _mm_store_ps(packed_w + 20, v4567x1);
1059 _mm_store_ps(packed_w + 24, v89ABx1);
1060 _mm_store_ps(packed_w + 28, vCDEFx1);
1061 _mm_store_ps(packed_w + 32, v0123x2);
1062 _mm_store_ps(packed_w + 36, v4567x2);
1063 _mm_store_ps(packed_w + 40, v89ABx2);
1064 _mm_store_ps(packed_w + 44, vCDEFx2);
1065 packed_w += 48;
1066 break;
1067 }
1068 default:
1069 XNN_UNREACHABLE;
1070 }
1071 }
1072 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
1073 }
1074 weights += nc * kc;
1075 } while (--g != 0);
1076 }