Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-sse2-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <emmintrin.h>
16
17 #include "xnnpack/packw.h"
18
19 void xnn_x32_packw_gemm_goi_ukernel_x16__sse2_u4(
20 size_t g,
21 size_t nc,
22 size_t kc,
23 size_t nr,
24 size_t kr,
25 size_t sr,
26 const uint32_t* weights,
27 const uint32_t* bias,
28 const void* scale,
29 uint32_t* packed_weights,
30 size_t extra_bytes,
31 const void* params)
32 {
33 assert(g != 0);
34 assert(nc != 0);
35 assert(kc != 0);
36 assert(nr == 16);
37 assert(kr == 1);
38 assert(sr == 1);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 const float* b = (const float*) bias;
43 float* packed_w = (float*) packed_weights;
44 do {
45 // NC main loop multiple of 16
46 const float* w0 = (const float*) weights;
47 size_t n = nc;
48
49 for (; n >= 16; n -= 16) {
50 if XNN_LIKELY(b != NULL) {
51 const __m128 vb0123 = _mm_loadu_ps(b);
52 const __m128 vb4567 = _mm_loadu_ps(b + 4);
53 const __m128 vb89AB = _mm_loadu_ps(b + 8);
54 const __m128 vbCDEF = _mm_loadu_ps(b + 12);
55 b += 16;
56
57 _mm_store_ps(packed_w, vb0123);
58 _mm_store_ps(packed_w + 4, vb4567);
59 _mm_store_ps(packed_w + 8, vb89AB);
60 _mm_store_ps(packed_w + 12, vbCDEF);
61 } else {
62 const __m128 vzero = _mm_setzero_ps();
63 _mm_store_ps(packed_w, vzero);
64 _mm_store_ps(packed_w + 4, vzero);
65 _mm_store_ps(packed_w + 8, vzero);
66 _mm_store_ps(packed_w + 12, vzero);
67 }
68 packed_w += 16;
69
70 const float* w1 = w0 + kc;
71 const float* w2 = w1 + kc;
72 const float* w3 = w2 + kc;
73 const float* w4 = w3 + kc;
74 const float* w5 = w4 + kc;
75 const float* w6 = w5 + kc;
76 const float* w7 = w6 + kc;
77 const float* w8 = w7 + kc;
78 const float* w9 = w8 + kc;
79 const float* w10 = w9 + kc;
80 const float* w11 = w10 + kc;
81 const float* w12 = w11 + kc;
82 const float* w13 = w12 + kc;
83 const float* w14 = w13 + kc;
84 const float* w15 = w14 + kc;
85
86 size_t k = kc;
87
88 // KC multiple of 4
89 for (; k >= 4; k -= 4) {
90 const __m128 v0x0123 = _mm_loadu_ps(w0);
91 w0 += 4;
92 const __m128 v1x0123 = _mm_loadu_ps(w1);
93 w1 += 4;
94 const __m128 v2x0123 = _mm_loadu_ps(w2);
95 w2 += 4;
96 const __m128 v3x0123 = _mm_loadu_ps(w3);
97 w3 += 4;
98 const __m128 v4x0123 = _mm_loadu_ps(w4);
99 w4 += 4;
100 const __m128 v5x0123 = _mm_loadu_ps(w5);
101 w5 += 4;
102 const __m128 v6x0123 = _mm_loadu_ps(w6);
103 w6 += 4;
104 const __m128 v7x0123 = _mm_loadu_ps(w7);
105 w7 += 4;
106 const __m128 v8x0123 = _mm_loadu_ps(w8);
107 w8 += 4;
108 const __m128 v9x0123 = _mm_loadu_ps(w9);
109 w9 += 4;
110 const __m128 vAx0123 = _mm_loadu_ps(w10);
111 w10 += 4;
112 const __m128 vBx0123 = _mm_loadu_ps(w11);
113 w11 += 4;
114 const __m128 vCx0123 = _mm_loadu_ps(w12);
115 w12 += 4;
116 const __m128 vDx0123 = _mm_loadu_ps(w13);
117 w13 += 4;
118 const __m128 vEx0123 = _mm_loadu_ps(w14);
119 w14 += 4;
120 const __m128 vFx0123 = _mm_loadu_ps(w15);
121 w15 += 4;
122
123 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
124 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
125 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
126 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
127 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
128 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
129 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
130 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
131 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
132 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
133 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
134 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
135 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
136 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vFx0123);
137 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
138 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vFx0123);
139 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
140 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
141 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
142 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
143 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
144 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
145 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
146 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
147 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
148 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
149 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
150 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
151 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
152 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
153 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
154 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
155
156 _mm_store_ps(packed_w, v0123x0);
157 _mm_store_ps(packed_w + 4, v4567x0);
158 _mm_store_ps(packed_w + 8, v89ABx0);
159 _mm_store_ps(packed_w + 12, vCDEFx0);
160 _mm_store_ps(packed_w + 16, v0123x1);
161 _mm_store_ps(packed_w + 20, v4567x1);
162 _mm_store_ps(packed_w + 24, v89ABx1);
163 _mm_store_ps(packed_w + 28, vCDEFx1);
164 _mm_store_ps(packed_w + 32, v0123x2);
165 _mm_store_ps(packed_w + 36, v4567x2);
166 _mm_store_ps(packed_w + 40, v89ABx2);
167 _mm_store_ps(packed_w + 44, vCDEFx2);
168 _mm_store_ps(packed_w + 48, v0123x3);
169 _mm_store_ps(packed_w + 52, v4567x3);
170 _mm_store_ps(packed_w + 56, v89ABx3);
171 _mm_store_ps(packed_w + 60, vCDEFx3);
172 packed_w += 64;
173 }
174
175 // KC remainder (1..3)
176 if XNN_UNLIKELY(k != 0) {
177 assert(k >= 1);
178 assert(k <= 3);
179 switch (k) {
180 case 1:
181 {
182 const __m128 v0x0 = _mm_load_ss(w0);
183 w0 += 1;
184 const __m128 v1x0 = _mm_load_ss(w1);
185 w1 += 1;
186 const __m128 v2x0 = _mm_load_ss(w2);
187 w2 += 1;
188 const __m128 v3x0 = _mm_load_ss(w3);
189 w3 += 1;
190 const __m128 v4x0 = _mm_load_ss(w4);
191 w4 += 1;
192 const __m128 v5x0 = _mm_load_ss(w5);
193 w5 += 1;
194 const __m128 v6x0 = _mm_load_ss(w6);
195 w6 += 1;
196 const __m128 v7x0 = _mm_load_ss(w7);
197 w7 += 1;
198 const __m128 v8x0 = _mm_load_ss(w8);
199 w8 += 1;
200 const __m128 v9x0 = _mm_load_ss(w9);
201 w9 += 1;
202 const __m128 vAx0 = _mm_load_ss(w10);
203 w10 += 1;
204 const __m128 vBx0 = _mm_load_ss(w11);
205 w11 += 1;
206 const __m128 vCx0 = _mm_load_ss(w12);
207 w12 += 1;
208 const __m128 vDx0 = _mm_load_ss(w13);
209 w13 += 1;
210 const __m128 vEx0 = _mm_load_ss(w14);
211 w14 += 1;
212 const __m128 vFx0 = _mm_load_ss(w15);
213 w15 += 1;
214
215 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
216 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
217 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
218 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
219 const __m128 v89x0 = _mm_unpacklo_ps(v8x0, v9x0);
220 const __m128 vABx0 = _mm_unpacklo_ps(vAx0, vBx0);
221 const __m128 vCDx0 = _mm_unpacklo_ps(vCx0, vDx0);
222 const __m128 vEFx0 = _mm_unpacklo_ps(vEx0, vFx0);
223
224 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
225 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
226 const __m128 v89ABx0 = _mm_movelh_ps(v89x0, vABx0);
227 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0, vEFx0);
228
229 _mm_store_ps(packed_w, v0123x0);
230 _mm_store_ps(packed_w + 4, v4567x0);
231 _mm_store_ps(packed_w + 8, v89ABx0);
232 _mm_store_ps(packed_w + 12, vCDEFx0);
233 packed_w += 16;
234 break;
235 }
236 case 2:
237 {
238 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
239 w0 += 2;
240 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
241 w1 += 2;
242 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
243 w2 += 2;
244 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
245 w3 += 2;
246 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
247 w4 += 2;
248 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
249 w5 += 2;
250 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
251 w6 += 2;
252 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
253 w7 += 2;
254 const __m128 v8x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
255 w8 += 2;
256 const __m128 v9x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
257 w9 += 2;
258 const __m128 vAx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
259 w10 += 2;
260 const __m128 vBx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
261 w11 += 2;
262 const __m128 vCx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
263 w12 += 2;
264 const __m128 vDx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
265 w13 += 2;
266 const __m128 vEx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
267 w14 += 2;
268 const __m128 vFx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
269 w15 += 2;
270
271 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
272 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
273 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
274 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
275 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x01, v9x01);
276 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx01, vBx01);
277 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx01, vDx01);
278 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx01, vFx01);
279
280 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
281 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
282 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
283 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
284 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
285 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
286 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
287 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
288
289 _mm_store_ps(packed_w, v0123x0);
290 _mm_store_ps(packed_w + 4, v4567x0);
291 _mm_store_ps(packed_w + 8, v89ABx0);
292 _mm_store_ps(packed_w + 12, vCDEFx0);
293 _mm_store_ps(packed_w + 16, v0123x1);
294 _mm_store_ps(packed_w + 20, v4567x1);
295 _mm_store_ps(packed_w + 24, v89ABx1);
296 _mm_store_ps(packed_w + 28, vCDEFx1);
297 packed_w += 32;
298 break;
299 }
300 case 3:
301 {
302 __m128 v0x012 = _mm_load_ss(w0 + 2);
303 __m128 v1x012 = _mm_load_ss(w1 + 2);
304 __m128 v2x012 = _mm_load_ss(w2 + 2);
305 __m128 v3x012 = _mm_load_ss(w3 + 2);
306 __m128 v4x012 = _mm_load_ss(w4 + 2);
307 __m128 v5x012 = _mm_load_ss(w5 + 2);
308 __m128 v6x012 = _mm_load_ss(w6 + 2);
309 __m128 v7x012 = _mm_load_ss(w7 + 2);
310 __m128 v8x012 = _mm_load_ss(w8 + 2);
311 __m128 v9x012 = _mm_load_ss(w9 + 2);
312 __m128 vAx012 = _mm_load_ss(w10 + 2);
313 __m128 vBx012 = _mm_load_ss(w11 + 2);
314 __m128 vCx012 = _mm_load_ss(w12 + 2);
315 __m128 vDx012 = _mm_load_ss(w13 + 2);
316 __m128 vEx012 = _mm_load_ss(w14 + 2);
317 __m128 vFx012 = _mm_load_ss(w15 + 2);
318
319 v0x012 = _mm_movelh_ps(v0x012, v0x012);
320 v1x012 = _mm_movelh_ps(v1x012, v1x012);
321 v2x012 = _mm_movelh_ps(v2x012, v2x012);
322 v3x012 = _mm_movelh_ps(v3x012, v3x012);
323 v4x012 = _mm_movelh_ps(v4x012, v4x012);
324 v5x012 = _mm_movelh_ps(v5x012, v5x012);
325 v6x012 = _mm_movelh_ps(v6x012, v6x012);
326 v7x012 = _mm_movelh_ps(v7x012, v7x012);
327 v8x012 = _mm_movelh_ps(v8x012, v8x012);
328 v9x012 = _mm_movelh_ps(v9x012, v9x012);
329 vAx012 = _mm_movelh_ps(vAx012, vAx012);
330 vBx012 = _mm_movelh_ps(vBx012, vBx012);
331 vCx012 = _mm_movelh_ps(vCx012, vCx012);
332 vDx012 = _mm_movelh_ps(vDx012, vDx012);
333 vEx012 = _mm_movelh_ps(vEx012, vEx012);
334 vFx012 = _mm_movelh_ps(vFx012, vFx012);
335
336 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
337 w0 += 3;
338 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
339 w1 += 3;
340 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
341 w2 += 3;
342 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
343 w3 += 3;
344 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
345 w4 += 3;
346 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
347 w5 += 3;
348 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
349 w6 += 3;
350 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
351 w7 += 3;
352 v8x012 = _mm_loadl_pi(v8x012, (const __m64*) w8);
353 w8 += 3;
354 v9x012 = _mm_loadl_pi(v9x012, (const __m64*) w9);
355 w9 += 3;
356 vAx012 = _mm_loadl_pi(vAx012, (const __m64*) w10);
357 w10 += 3;
358 vBx012 = _mm_loadl_pi(vBx012, (const __m64*) w11);
359 w11 += 3;
360 vCx012 = _mm_loadl_pi(vCx012, (const __m64*) w12);
361 w12 += 3;
362 vDx012 = _mm_loadl_pi(vDx012, (const __m64*) w13);
363 w13 += 3;
364 vEx012 = _mm_loadl_pi(vEx012, (const __m64*) w14);
365 w14 += 3;
366 vFx012 = _mm_loadl_pi(vFx012, (const __m64*) w15);
367 w15 += 3;
368
369 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
370 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
371 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
372 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
373 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
374 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
375 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
376 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
377 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x012, v9x012);
378 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx012, vBx012);
379 const __m128 v89x2 = _mm_unpackhi_ps(v8x012, v9x012);
380 const __m128 vABx2 = _mm_unpackhi_ps(vAx012, vBx012);
381 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx012, vDx012);
382 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx012, vFx012);
383 const __m128 vCDx2 = _mm_unpackhi_ps(vCx012, vDx012);
384 const __m128 vEFx2 = _mm_unpackhi_ps(vEx012, vFx012);
385
386 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
387 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
388 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
389 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
390 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
391 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
392 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
393 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
394 const __m128 v89ABx2 = _mm_movelh_ps(v89x2, vABx2);
395 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
396 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
397 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2, vEFx2);
398
399 _mm_store_ps(packed_w, v0123x0);
400 _mm_store_ps(packed_w + 4, v4567x0);
401 _mm_store_ps(packed_w + 8, v89ABx0);
402 _mm_store_ps(packed_w + 12, vCDEFx0);
403 _mm_store_ps(packed_w + 16, v0123x1);
404 _mm_store_ps(packed_w + 20, v4567x1);
405 _mm_store_ps(packed_w + 24, v89ABx1);
406 _mm_store_ps(packed_w + 28, vCDEFx1);
407 _mm_store_ps(packed_w + 32, v0123x2);
408 _mm_store_ps(packed_w + 36, v4567x2);
409 _mm_store_ps(packed_w + 40, v89ABx2);
410 _mm_store_ps(packed_w + 44, vCDEFx2);
411 packed_w += 48;
412 break;
413 }
414 default:
415 XNN_UNREACHABLE;
416 }
417 }
418 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
419 w0 = w15;
420 }
421
422 // NC remainder (1..15)
423 if XNN_UNLIKELY(n != 0) {
424 assert(n >= 1);
425 assert(n <= 15);
426 if XNN_LIKELY(b != NULL) {
427 size_t nb = n;
428 do {
429 *packed_w++ = *b++;
430 } while (--nb != 0);
431 packed_w += (16 - n);
432 } else {
433 const __m128 vzero = _mm_setzero_ps();
434 _mm_store_ps(packed_w, vzero);
435 _mm_store_ps(packed_w + 4, vzero);
436 _mm_store_ps(packed_w + 8, vzero);
437 _mm_store_ps(packed_w + 12, vzero);
438 packed_w += 16;
439 }
440
441 const float* w1 = w0 + kc;
442 if XNN_UNPREDICTABLE(n < 2) {
443 w1 = w0;
444 }
445 const float* w2 = w1 + kc;
446 if XNN_UNPREDICTABLE(n <= 2) {
447 w2 = w1;
448 }
449 const float* w3 = w2 + kc;
450 if XNN_UNPREDICTABLE(n < 4) {
451 w3 = w2;
452 }
453 const float* w4 = w3 + kc;
454 if XNN_UNPREDICTABLE(n <= 4) {
455 w4 = w3;
456 }
457 const float* w5 = w4 + kc;
458 if XNN_UNPREDICTABLE(n < 6) {
459 w5 = w4;
460 }
461 const float* w6 = w5 + kc;
462 if XNN_UNPREDICTABLE(n <= 6) {
463 w6 = w5;
464 }
465 const float* w7 = w6 + kc;
466 if XNN_UNPREDICTABLE(n < 8) {
467 w7 = w6;
468 }
469 const float* w8 = w7 + kc;
470 if XNN_UNPREDICTABLE(n <= 8) {
471 w8 = w7;
472 }
473 const float* w9 = w8 + kc;
474 if XNN_UNPREDICTABLE(n < 10) {
475 w9 = w8;
476 }
477 const float* w10 = w9 + kc;
478 if XNN_UNPREDICTABLE(n <= 10) {
479 w10 = w9;
480 }
481 const float* w11 = w10 + kc;
482 if XNN_UNPREDICTABLE(n < 12) {
483 w11 = w10;
484 }
485 const float* w12 = w11 + kc;
486 if XNN_UNPREDICTABLE(n <= 12) {
487 w12 = w11;
488 }
489 const float* w13 = w12 + kc;
490 if XNN_UNPREDICTABLE(n < 14) {
491 w13 = w12;
492 }
493 const float* w14 = w13 + kc;
494 if XNN_UNPREDICTABLE(n <= 14) {
495 w14 = w13;
496 }
497
498 size_t k = kc;
499
500 // KC multiple of 4
501 for (; k >= 4; k -= 4) {
502 const __m128 v0x0123 = _mm_loadu_ps(w0);
503 w0 += 4;
504 const __m128 v1x0123 = _mm_loadu_ps(w1);
505 w1 += 4;
506 const __m128 v2x0123 = _mm_loadu_ps(w2);
507 w2 += 4;
508 const __m128 v3x0123 = _mm_loadu_ps(w3);
509 w3 += 4;
510 const __m128 v4x0123 = _mm_loadu_ps(w4);
511 w4 += 4;
512 const __m128 v5x0123 = _mm_loadu_ps(w5);
513 w5 += 4;
514 const __m128 v6x0123 = _mm_loadu_ps(w6);
515 w6 += 4;
516 const __m128 v7x0123 = _mm_loadu_ps(w7);
517 w7 += 4;
518 const __m128 v8x0123 = _mm_loadu_ps(w8);
519 w8 += 4;
520 const __m128 v9x0123 = _mm_loadu_ps(w9);
521 w9 += 4;
522 const __m128 vAx0123 = _mm_loadu_ps(w10);
523 w10 += 4;
524 const __m128 vBx0123 = _mm_loadu_ps(w11);
525 w11 += 4;
526 const __m128 vCx0123 = _mm_loadu_ps(w12);
527 w12 += 4;
528 const __m128 vDx0123 = _mm_loadu_ps(w13);
529 w13 += 4;
530 const __m128 vEx0123 = _mm_loadu_ps(w14);
531 w14 += 4;
532
533 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
534 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
535 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
536 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
537 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
538 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
539 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
540 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
541 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x0123, v9x0123);
542 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx0123, vBx0123);
543 const __m128 v89x2_89x3 = _mm_unpackhi_ps(v8x0123, v9x0123);
544 const __m128 vABx2_ABx3 = _mm_unpackhi_ps(vAx0123, vBx0123);
545 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx0123, vDx0123);
546 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx0123, vEx0123);
547 const __m128 vCDx2_CDx3 = _mm_unpackhi_ps(vCx0123, vDx0123);
548 const __m128 vEFx2_EFx3 = _mm_unpackhi_ps(vEx0123, vEx0123);
549 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
550 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
551 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
552 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
553 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
554 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
555 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
556 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
557 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
558 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
559 const __m128 v89ABx2 = _mm_movelh_ps(v89x2_89x3, vABx2_ABx3);
560 const __m128 v89ABx3 = _mm_movehl_ps(vABx2_ABx3, v89x2_89x3);
561 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
562 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
563 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2_CDx3, vEFx2_EFx3);
564 const __m128 vCDEFx3 = _mm_movehl_ps(vEFx2_EFx3, vCDx2_CDx3);
565
566 _mm_store_ps(packed_w, v0123x0);
567 _mm_store_ps(packed_w + 4, v4567x0);
568 _mm_store_ps(packed_w + 8, v89ABx0);
569 _mm_store_ps(packed_w + 12, vCDEFx0);
570 _mm_store_ps(packed_w + 16, v0123x1);
571 _mm_store_ps(packed_w + 20, v4567x1);
572 _mm_store_ps(packed_w + 24, v89ABx1);
573 _mm_store_ps(packed_w + 28, vCDEFx1);
574 _mm_store_ps(packed_w + 32, v0123x2);
575 _mm_store_ps(packed_w + 36, v4567x2);
576 _mm_store_ps(packed_w + 40, v89ABx2);
577 _mm_store_ps(packed_w + 44, vCDEFx2);
578 _mm_store_ps(packed_w + 48, v0123x3);
579 _mm_store_ps(packed_w + 52, v4567x3);
580 _mm_store_ps(packed_w + 56, v89ABx3);
581 _mm_store_ps(packed_w + 60, vCDEFx3);
582 packed_w += 64;
583 }
584
585 // KC remainder (1..3)
586 if XNN_UNLIKELY(k != 0) {
587 assert(k >= 1);
588 assert(k <= 3);
589 switch (k) {
590 case 1:
591 {
592 const __m128 v0x0 = _mm_load_ss(w0);
593 const __m128 v1x0 = _mm_load_ss(w1);
594 const __m128 v2x0 = _mm_load_ss(w2);
595 const __m128 v3x0 = _mm_load_ss(w3);
596 const __m128 v4x0 = _mm_load_ss(w4);
597 const __m128 v5x0 = _mm_load_ss(w5);
598 const __m128 v6x0 = _mm_load_ss(w6);
599 const __m128 v7x0 = _mm_load_ss(w7);
600 const __m128 v8x0 = _mm_load_ss(w8);
601 const __m128 v9x0 = _mm_load_ss(w9);
602 const __m128 vAx0 = _mm_load_ss(w10);
603 const __m128 vBx0 = _mm_load_ss(w11);
604 const __m128 vCx0 = _mm_load_ss(w12);
605 const __m128 vDx0 = _mm_load_ss(w13);
606 const __m128 vEx0 = _mm_load_ss(w14);
607
608 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
609 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
610 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
611 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
612 const __m128 v89x0 = _mm_unpacklo_ps(v8x0, v9x0);
613 const __m128 vABx0 = _mm_unpacklo_ps(vAx0, vBx0);
614 const __m128 vCDx0 = _mm_unpacklo_ps(vCx0, vDx0);
615 const __m128 vEFx0 = _mm_unpacklo_ps(vEx0, vEx0);
616
617 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
618 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
619 const __m128 v89ABx0 = _mm_movelh_ps(v89x0, vABx0);
620 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0, vEFx0);
621
622 _mm_store_ps(packed_w, v0123x0);
623 _mm_store_ps(packed_w + 4, v4567x0);
624 _mm_store_ps(packed_w + 8, v89ABx0);
625 _mm_store_ps(packed_w + 12, vCDEFx0);
626 packed_w += 16;
627 break;
628 }
629 case 2:
630 {
631 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
632 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
633 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
634 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
635 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
636 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
637 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
638 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
639 const __m128 v8x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
640 const __m128 v9x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
641 const __m128 vAx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
642 const __m128 vBx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
643 const __m128 vCx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
644 const __m128 vDx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
645 const __m128 vEx01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
646
647 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
648 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
649 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
650 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
651 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x01, v9x01);
652 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx01, vBx01);
653 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx01, vDx01);
654 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx01, vEx01);
655
656 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
657 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
658 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
659 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
660 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
661 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
662 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
663 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
664
665 _mm_store_ps(packed_w, v0123x0);
666 _mm_store_ps(packed_w + 4, v4567x0);
667 _mm_store_ps(packed_w + 8, v89ABx0);
668 _mm_store_ps(packed_w + 12, vCDEFx0);
669 _mm_store_ps(packed_w + 16, v0123x1);
670 _mm_store_ps(packed_w + 20, v4567x1);
671 _mm_store_ps(packed_w + 24, v89ABx1);
672 _mm_store_ps(packed_w + 28, vCDEFx1);
673 packed_w += 32;
674 break;
675 }
676 case 3:
677 {
678 __m128 v0x012 = _mm_load_ss(w0 + 2);
679 __m128 v1x012 = _mm_load_ss(w1 + 2);
680 __m128 v2x012 = _mm_load_ss(w2 + 2);
681 __m128 v3x012 = _mm_load_ss(w3 + 2);
682 __m128 v4x012 = _mm_load_ss(w4 + 2);
683 __m128 v5x012 = _mm_load_ss(w5 + 2);
684 __m128 v6x012 = _mm_load_ss(w6 + 2);
685 __m128 v7x012 = _mm_load_ss(w7 + 2);
686 __m128 v8x012 = _mm_load_ss(w8 + 2);
687 __m128 v9x012 = _mm_load_ss(w9 + 2);
688 __m128 vAx012 = _mm_load_ss(w10 + 2);
689 __m128 vBx012 = _mm_load_ss(w11 + 2);
690 __m128 vCx012 = _mm_load_ss(w12 + 2);
691 __m128 vDx012 = _mm_load_ss(w13 + 2);
692 __m128 vEx012 = _mm_load_ss(w14 + 2);
693
694 v0x012 = _mm_movelh_ps(v0x012, v0x012);
695 v1x012 = _mm_movelh_ps(v1x012, v1x012);
696 v2x012 = _mm_movelh_ps(v2x012, v2x012);
697 v3x012 = _mm_movelh_ps(v3x012, v3x012);
698 v4x012 = _mm_movelh_ps(v4x012, v4x012);
699 v5x012 = _mm_movelh_ps(v5x012, v5x012);
700 v6x012 = _mm_movelh_ps(v6x012, v6x012);
701 v7x012 = _mm_movelh_ps(v7x012, v7x012);
702 v8x012 = _mm_movelh_ps(v8x012, v8x012);
703 v9x012 = _mm_movelh_ps(v9x012, v9x012);
704 vAx012 = _mm_movelh_ps(vAx012, vAx012);
705 vBx012 = _mm_movelh_ps(vBx012, vBx012);
706 vCx012 = _mm_movelh_ps(vCx012, vCx012);
707 vDx012 = _mm_movelh_ps(vDx012, vDx012);
708 vEx012 = _mm_movelh_ps(vEx012, vEx012);
709
710 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
711 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
712 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
713 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
714 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
715 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
716 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
717 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
718 v8x012 = _mm_loadl_pi(v8x012, (const __m64*) w8);
719 v9x012 = _mm_loadl_pi(v9x012, (const __m64*) w9);
720 vAx012 = _mm_loadl_pi(vAx012, (const __m64*) w10);
721 vBx012 = _mm_loadl_pi(vBx012, (const __m64*) w11);
722 vCx012 = _mm_loadl_pi(vCx012, (const __m64*) w12);
723 vDx012 = _mm_loadl_pi(vDx012, (const __m64*) w13);
724 vEx012 = _mm_loadl_pi(vEx012, (const __m64*) w14);
725
726 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
727 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
728 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
729 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
730 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
731 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
732 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
733 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
734 const __m128 v89x0_89x1 = _mm_unpacklo_ps(v8x012, v9x012);
735 const __m128 vABx0_ABx1 = _mm_unpacklo_ps(vAx012, vBx012);
736 const __m128 v89x2 = _mm_unpackhi_ps(v8x012, v9x012);
737 const __m128 vABx2 = _mm_unpackhi_ps(vAx012, vBx012);
738 const __m128 vCDx0_CDx1 = _mm_unpacklo_ps(vCx012, vDx012);
739 const __m128 vEFx0_EFx1 = _mm_unpacklo_ps(vEx012, vEx012);
740 const __m128 vCDx2 = _mm_unpackhi_ps(vCx012, vDx012);
741 const __m128 vEFx2 = _mm_unpackhi_ps(vEx012, vEx012);
742
743 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
744 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
745 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
746 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
747 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
748 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
749 const __m128 v89ABx0 = _mm_movelh_ps(v89x0_89x1, vABx0_ABx1);
750 const __m128 v89ABx1 = _mm_movehl_ps(vABx0_ABx1, v89x0_89x1);
751 const __m128 v89ABx2 = _mm_movelh_ps(v89x2, vABx2);
752 const __m128 vCDEFx0 = _mm_movelh_ps(vCDx0_CDx1, vEFx0_EFx1);
753 const __m128 vCDEFx1 = _mm_movehl_ps(vEFx0_EFx1, vCDx0_CDx1);
754 const __m128 vCDEFx2 = _mm_movelh_ps(vCDx2, vEFx2);
755
756 _mm_store_ps(packed_w, v0123x0);
757 _mm_store_ps(packed_w + 4, v4567x0);
758 _mm_store_ps(packed_w + 8, v89ABx0);
759 _mm_store_ps(packed_w + 12, vCDEFx0);
760 _mm_store_ps(packed_w + 16, v0123x1);
761 _mm_store_ps(packed_w + 20, v4567x1);
762 _mm_store_ps(packed_w + 24, v89ABx1);
763 _mm_store_ps(packed_w + 28, vCDEFx1);
764 _mm_store_ps(packed_w + 32, v0123x2);
765 _mm_store_ps(packed_w + 36, v4567x2);
766 _mm_store_ps(packed_w + 40, v89ABx2);
767 _mm_store_ps(packed_w + 44, vCDEFx2);
768 packed_w += 48;
769 break;
770 }
771 default:
772 XNN_UNREACHABLE;
773 }
774 }
775 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
776 }
777 weights += nc * kc;
778 } while (--g != 0);
779 }