Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-sse2-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <emmintrin.h>
16
17 #include "xnnpack/packw.h"
18
19 void xnn_x32_packw_gemm_goi_ukernel_x8__sse2_u8(
20 size_t g,
21 size_t nc,
22 size_t kc,
23 size_t nr,
24 size_t kr,
25 size_t sr,
26 const uint32_t* weights,
27 const uint32_t* bias,
28 const void* scale,
29 uint32_t* packed_weights,
30 size_t extra_bytes,
31 const void* params)
32 {
33 assert(g != 0);
34 assert(nc != 0);
35 assert(kc != 0);
36 assert(nr == 8);
37 assert(kr == 1);
38 assert(sr == 1);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 const float* b = (const float*) bias;
43 float* packed_w = (float*) packed_weights;
44 do {
45 // NC main loop multiple of 8
46 const float* w0 = (const float*) weights;
47 size_t n = nc;
48
49 for (; n >= 8; n -= 8) {
50 if XNN_LIKELY(b != NULL) {
51 const __m128 vb0123 = _mm_loadu_ps(b);
52 const __m128 vb4567 = _mm_loadu_ps(b + 4);
53 b += 8;
54
55 _mm_store_ps(packed_w, vb0123);
56 _mm_store_ps(packed_w + 4, vb4567);
57 } else {
58 const __m128 vzero = _mm_setzero_ps();
59 _mm_store_ps(packed_w, vzero);
60 _mm_store_ps(packed_w + 4, vzero);
61 }
62 packed_w += 8;
63
64 const float* w1 = w0 + kc;
65 const float* w2 = w1 + kc;
66 const float* w3 = w2 + kc;
67 const float* w4 = w3 + kc;
68 const float* w5 = w4 + kc;
69 const float* w6 = w5 + kc;
70 const float* w7 = w6 + kc;
71
72 size_t k = kc;
73 // KC main loop multiple of 8
74 for (; k >= 8; k -= 8) {
75 const __m128 v0x0123 = _mm_loadu_ps(w0);
76 const __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
77 w0 += 8;
78 const __m128 v1x0123 = _mm_loadu_ps(w1);
79 const __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
80 w1 += 8;
81 const __m128 v2x0123 = _mm_loadu_ps(w2);
82 const __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
83 w2 += 8;
84 const __m128 v3x0123 = _mm_loadu_ps(w3);
85 const __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
86 w3 += 8;
87 const __m128 v4x0123 = _mm_loadu_ps(w4);
88 const __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
89 w4 += 8;
90 const __m128 v5x0123 = _mm_loadu_ps(w5);
91 const __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
92 w5 += 8;
93 const __m128 v6x0123 = _mm_loadu_ps(w6);
94 const __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
95 w6 += 8;
96 const __m128 v7x0123 = _mm_loadu_ps(w7);
97 const __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
98 w7 += 8;
99
100 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
101 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
102 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
103 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
104 const __m128 v01x4_01x5 = _mm_unpacklo_ps(v0x4567, v1x4567);
105 const __m128 v23x4_23x5 = _mm_unpacklo_ps(v2x4567, v3x4567);
106 const __m128 v01x6_01x7 = _mm_unpackhi_ps(v0x4567, v1x4567);
107 const __m128 v23x6_23x7 = _mm_unpackhi_ps(v2x4567, v3x4567);
108 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
109 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
110 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
111 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
112 const __m128 v45x4_45x5 = _mm_unpacklo_ps(v4x4567, v5x4567);
113 const __m128 v67x4_67x5 = _mm_unpacklo_ps(v6x4567, v7x4567);
114 const __m128 v45x6_45x7 = _mm_unpackhi_ps(v4x4567, v5x4567);
115 const __m128 v67x6_67x7 = _mm_unpackhi_ps(v6x4567, v7x4567);
116 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
117 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
118 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
119 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
120 const __m128 v0123x4 = _mm_movelh_ps(v01x4_01x5, v23x4_23x5);
121 const __m128 v0123x5 = _mm_movehl_ps(v23x4_23x5, v01x4_01x5);
122 const __m128 v0123x6 = _mm_movelh_ps(v01x6_01x7, v23x6_23x7);
123 const __m128 v0123x7 = _mm_movehl_ps(v23x6_23x7, v01x6_01x7);
124 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
125 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
126 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
127 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
128 const __m128 v4567x4 = _mm_movelh_ps(v45x4_45x5, v67x4_67x5);
129 const __m128 v4567x5 = _mm_movehl_ps(v67x4_67x5, v45x4_45x5);
130 const __m128 v4567x6 = _mm_movelh_ps(v45x6_45x7, v67x6_67x7);
131 const __m128 v4567x7 = _mm_movehl_ps(v67x6_67x7, v45x6_45x7);
132
133 _mm_store_ps(packed_w, v0123x0);
134 _mm_store_ps(packed_w + 4, v4567x0);
135 _mm_store_ps(packed_w + 8, v0123x1);
136 _mm_store_ps(packed_w + 12, v4567x1);
137 _mm_store_ps(packed_w + 16, v0123x2);
138 _mm_store_ps(packed_w + 20, v4567x2);
139 _mm_store_ps(packed_w + 24, v0123x3);
140 _mm_store_ps(packed_w + 28, v4567x3);
141 _mm_store_ps(packed_w + 32, v0123x4);
142 _mm_store_ps(packed_w + 36, v4567x4);
143 _mm_store_ps(packed_w + 40, v0123x5);
144 _mm_store_ps(packed_w + 44, v4567x5);
145 _mm_store_ps(packed_w + 48, v0123x6);
146 _mm_store_ps(packed_w + 52, v4567x6);
147 _mm_store_ps(packed_w + 56, v0123x7);
148 _mm_store_ps(packed_w + 60, v4567x7);
149 packed_w += 64;
150 }
151
152 // KC multiple of 4
153 for (; k >= 4; k -= 4) {
154 const __m128 v0x0123 = _mm_loadu_ps(w0);
155 w0 += 4;
156 const __m128 v1x0123 = _mm_loadu_ps(w1);
157 w1 += 4;
158 const __m128 v2x0123 = _mm_loadu_ps(w2);
159 w2 += 4;
160 const __m128 v3x0123 = _mm_loadu_ps(w3);
161 w3 += 4;
162 const __m128 v4x0123 = _mm_loadu_ps(w4);
163 w4 += 4;
164 const __m128 v5x0123 = _mm_loadu_ps(w5);
165 w5 += 4;
166 const __m128 v6x0123 = _mm_loadu_ps(w6);
167 w6 += 4;
168 const __m128 v7x0123 = _mm_loadu_ps(w7);
169 w7 += 4;
170
171 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
172 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
173 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
174 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
175 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
176 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
177 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
178 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
179 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
180 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
181 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
182 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
183 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
184 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
185 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
186 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
187
188 _mm_store_ps(packed_w, v0123x0);
189 _mm_store_ps(packed_w + 4, v4567x0);
190 _mm_store_ps(packed_w + 8, v0123x1);
191 _mm_store_ps(packed_w + 12, v4567x1);
192 _mm_store_ps(packed_w + 16, v0123x2);
193 _mm_store_ps(packed_w + 20, v4567x2);
194 _mm_store_ps(packed_w + 24, v0123x3);
195 _mm_store_ps(packed_w + 28, v4567x3);
196 packed_w += 32;
197 }
198
199 // KC remainder (1..3)
200 if XNN_UNLIKELY(k != 0) {
201 assert(k >= 1);
202 assert(k <= 3);
203 switch (k) {
204 case 1:
205 {
206 const __m128 v0x0 = _mm_load_ss(w0);
207 w0 += 1;
208 const __m128 v1x0 = _mm_load_ss(w1);
209 w1 += 1;
210 const __m128 v2x0 = _mm_load_ss(w2);
211 w2 += 1;
212 const __m128 v3x0 = _mm_load_ss(w3);
213 w3 += 1;
214 const __m128 v4x0 = _mm_load_ss(w4);
215 w4 += 1;
216 const __m128 v5x0 = _mm_load_ss(w5);
217 w5 += 1;
218 const __m128 v6x0 = _mm_load_ss(w6);
219 w6 += 1;
220 const __m128 v7x0 = _mm_load_ss(w7);
221 w7 += 1;
222
223 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
224 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
225 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
226 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
227
228 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
229 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
230
231 _mm_store_ps(packed_w, v0123x0);
232 _mm_store_ps(packed_w + 4, v4567x0);
233 packed_w += 8;
234 break;
235 }
236 case 2:
237 {
238 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
239 w0 += 2;
240 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
241 w1 += 2;
242 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
243 w2 += 2;
244 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
245 w3 += 2;
246 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
247 w4 += 2;
248 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
249 w5 += 2;
250 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
251 w6 += 2;
252 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
253 w7 += 2;
254
255 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
256 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
257 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
258 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
259
260 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
261 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
262 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
263 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
264
265 _mm_store_ps(packed_w, v0123x0);
266 _mm_store_ps(packed_w + 4, v4567x0);
267 _mm_store_ps(packed_w + 8, v0123x1);
268 _mm_store_ps(packed_w + 12, v4567x1);
269 packed_w += 16;
270 break;
271 }
272 case 3:
273 {
274 __m128 v0x012 = _mm_load_ss(w0 + 2);
275 __m128 v1x012 = _mm_load_ss(w1 + 2);
276 __m128 v2x012 = _mm_load_ss(w2 + 2);
277 __m128 v3x012 = _mm_load_ss(w3 + 2);
278 __m128 v4x012 = _mm_load_ss(w4 + 2);
279 __m128 v5x012 = _mm_load_ss(w5 + 2);
280 __m128 v6x012 = _mm_load_ss(w6 + 2);
281 __m128 v7x012 = _mm_load_ss(w7 + 2);
282
283 v0x012 = _mm_movelh_ps(v0x012, v0x012);
284 v1x012 = _mm_movelh_ps(v1x012, v1x012);
285 v2x012 = _mm_movelh_ps(v2x012, v2x012);
286 v3x012 = _mm_movelh_ps(v3x012, v3x012);
287 v4x012 = _mm_movelh_ps(v4x012, v4x012);
288 v5x012 = _mm_movelh_ps(v5x012, v5x012);
289 v6x012 = _mm_movelh_ps(v6x012, v6x012);
290 v7x012 = _mm_movelh_ps(v7x012, v7x012);
291
292 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
293 w0 += 3;
294 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
295 w1 += 3;
296 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
297 w2 += 3;
298 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
299 w3 += 3;
300 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
301 w4 += 3;
302 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
303 w5 += 3;
304 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
305 w6 += 3;
306 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
307 w7 += 3;
308
309 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
310 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
311 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
312 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
313 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
314 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
315 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
316 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
317
318 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
319 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
320 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
321 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
322 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
323 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
324
325 _mm_store_ps(packed_w, v0123x0);
326 _mm_store_ps(packed_w + 4, v4567x0);
327 _mm_store_ps(packed_w + 8, v0123x1);
328 _mm_store_ps(packed_w + 12, v4567x1);
329 _mm_store_ps(packed_w + 16, v0123x2);
330 _mm_store_ps(packed_w + 20, v4567x2);
331 packed_w += 24;
332 break;
333 }
334 default:
335 XNN_UNREACHABLE;
336 }
337 }
338 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
339 w0 = w7;
340 }
341
342 // NC remainder (1..7)
343 if XNN_UNLIKELY(n != 0) {
344 assert(n >= 1);
345 assert(n <= 7);
346 if XNN_LIKELY(b != NULL) {
347 size_t nb = n;
348 do {
349 *packed_w++ = *b++;
350 } while (--nb != 0);
351 packed_w += (8 - n);
352 } else {
353 const __m128 vzero = _mm_setzero_ps();
354 _mm_store_ps(packed_w, vzero);
355 _mm_store_ps(packed_w + 4, vzero);
356 packed_w += 8;
357 }
358
359 const float* w1 = w0 + kc;
360 if XNN_UNPREDICTABLE(n < 2) {
361 w1 = w0;
362 }
363 const float* w2 = w1 + kc;
364 if XNN_UNPREDICTABLE(n <= 2) {
365 w2 = w1;
366 }
367 const float* w3 = w2 + kc;
368 if XNN_UNPREDICTABLE(n < 4) {
369 w3 = w2;
370 }
371 const float* w4 = w3 + kc;
372 if XNN_UNPREDICTABLE(n <= 4) {
373 w4 = w3;
374 }
375 const float* w5 = w4 + kc;
376 if XNN_UNPREDICTABLE(n < 6) {
377 w5 = w4;
378 }
379 const float* w6 = w5 + kc;
380 if XNN_UNPREDICTABLE(n <= 6) {
381 w6 = w5;
382 }
383
384 size_t k = kc;
385 // KC main loop multiple of 8
386 for (; k >= 8; k -= 8) {
387 const __m128 v0x0123 = _mm_loadu_ps(w0);
388 const __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
389 w0 += 8;
390 const __m128 v1x0123 = _mm_loadu_ps(w1);
391 const __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
392 w1 += 8;
393 const __m128 v2x0123 = _mm_loadu_ps(w2);
394 const __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
395 w2 += 8;
396 const __m128 v3x0123 = _mm_loadu_ps(w3);
397 const __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
398 w3 += 8;
399 const __m128 v4x0123 = _mm_loadu_ps(w4);
400 const __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
401 w4 += 8;
402 const __m128 v5x0123 = _mm_loadu_ps(w5);
403 const __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
404 w5 += 8;
405 const __m128 v6x0123 = _mm_loadu_ps(w6);
406 const __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
407 w6 += 8;
408
409 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
410 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
411 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
412 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
413 const __m128 v01x4_01x5 = _mm_unpacklo_ps(v0x4567, v1x4567);
414 const __m128 v23x4_23x5 = _mm_unpacklo_ps(v2x4567, v3x4567);
415 const __m128 v01x6_01x7 = _mm_unpackhi_ps(v0x4567, v1x4567);
416 const __m128 v23x6_23x7 = _mm_unpackhi_ps(v2x4567, v3x4567);
417 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
418 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v6x0123);
419 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
420 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v6x0123);
421 const __m128 v45x4_45x5 = _mm_unpacklo_ps(v4x4567, v5x4567);
422 const __m128 v67x4_67x5 = _mm_unpacklo_ps(v6x4567, v6x4567);
423 const __m128 v45x6_45x7 = _mm_unpackhi_ps(v4x4567, v5x4567);
424 const __m128 v67x6_67x7 = _mm_unpackhi_ps(v6x4567, v6x4567);
425 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
426 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
427 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
428 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
429 const __m128 v0123x4 = _mm_movelh_ps(v01x4_01x5, v23x4_23x5);
430 const __m128 v0123x5 = _mm_movehl_ps(v23x4_23x5, v01x4_01x5);
431 const __m128 v0123x6 = _mm_movelh_ps(v01x6_01x7, v23x6_23x7);
432 const __m128 v0123x7 = _mm_movehl_ps(v23x6_23x7, v01x6_01x7);
433 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
434 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
435 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
436 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
437 const __m128 v4567x4 = _mm_movelh_ps(v45x4_45x5, v67x4_67x5);
438 const __m128 v4567x5 = _mm_movehl_ps(v67x4_67x5, v45x4_45x5);
439 const __m128 v4567x6 = _mm_movelh_ps(v45x6_45x7, v67x6_67x7);
440 const __m128 v4567x7 = _mm_movehl_ps(v67x6_67x7, v45x6_45x7);
441
442 _mm_store_ps(packed_w, v0123x0);
443 _mm_store_ps(packed_w + 4, v4567x0);
444 _mm_store_ps(packed_w + 8, v0123x1);
445 _mm_store_ps(packed_w + 12, v4567x1);
446 _mm_store_ps(packed_w + 16, v0123x2);
447 _mm_store_ps(packed_w + 20, v4567x2);
448 _mm_store_ps(packed_w + 24, v0123x3);
449 _mm_store_ps(packed_w + 28, v4567x3);
450 _mm_store_ps(packed_w + 32, v0123x4);
451 _mm_store_ps(packed_w + 36, v4567x4);
452 _mm_store_ps(packed_w + 40, v0123x5);
453 _mm_store_ps(packed_w + 44, v4567x5);
454 _mm_store_ps(packed_w + 48, v0123x6);
455 _mm_store_ps(packed_w + 52, v4567x6);
456 _mm_store_ps(packed_w + 56, v0123x7);
457 _mm_store_ps(packed_w + 60, v4567x7);
458 packed_w += 64;
459 }
460
461 // KC multiple of 4
462 for (; k >= 4; k -= 4) {
463 const __m128 v0x0123 = _mm_loadu_ps(w0);
464 w0 += 4;
465 const __m128 v1x0123 = _mm_loadu_ps(w1);
466 w1 += 4;
467 const __m128 v2x0123 = _mm_loadu_ps(w2);
468 w2 += 4;
469 const __m128 v3x0123 = _mm_loadu_ps(w3);
470 w3 += 4;
471 const __m128 v4x0123 = _mm_loadu_ps(w4);
472 w4 += 4;
473 const __m128 v5x0123 = _mm_loadu_ps(w5);
474 w5 += 4;
475 const __m128 v6x0123 = _mm_loadu_ps(w6);
476 w6 += 4;
477
478 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
479 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
480 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
481 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
482 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
483 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v6x0123);
484 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
485 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v6x0123);
486 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
487 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
488 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
489 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
490 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
491 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
492 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
493 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
494
495 _mm_store_ps(packed_w, v0123x0);
496 _mm_store_ps(packed_w + 4, v4567x0);
497 _mm_store_ps(packed_w + 8, v0123x1);
498 _mm_store_ps(packed_w + 12, v4567x1);
499 _mm_store_ps(packed_w + 16, v0123x2);
500 _mm_store_ps(packed_w + 20, v4567x2);
501 _mm_store_ps(packed_w + 24, v0123x3);
502 _mm_store_ps(packed_w + 28, v4567x3);
503 packed_w += 32;
504 }
505
506 // KC remainder (1..3)
507 if XNN_UNLIKELY(k != 0) {
508 assert(k >= 1);
509 assert(k <= 3);
510 switch (k) {
511 case 1:
512 {
513 const __m128 v0x0 = _mm_load_ss(w0);
514 const __m128 v1x0 = _mm_load_ss(w1);
515 const __m128 v2x0 = _mm_load_ss(w2);
516 const __m128 v3x0 = _mm_load_ss(w3);
517 const __m128 v4x0 = _mm_load_ss(w4);
518 const __m128 v5x0 = _mm_load_ss(w5);
519 const __m128 v6x0 = _mm_load_ss(w6);
520
521 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
522 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
523 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
524 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v6x0);
525
526 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
527 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
528
529 _mm_store_ps(packed_w, v0123x0);
530 _mm_store_ps(packed_w + 4, v4567x0);
531 packed_w += 8;
532 break;
533 }
534 case 2:
535 {
536 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
537 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
538 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
539 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
540 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
541 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
542 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
543
544 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
545 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
546 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
547 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v6x01);
548
549 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
550 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
551 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
552 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
553
554 _mm_store_ps(packed_w, v0123x0);
555 _mm_store_ps(packed_w + 4, v4567x0);
556 _mm_store_ps(packed_w + 8, v0123x1);
557 _mm_store_ps(packed_w + 12, v4567x1);
558 packed_w += 16;
559 break;
560 }
561 case 3:
562 {
563 __m128 v0x012 = _mm_load_ss(w0 + 2);
564 __m128 v1x012 = _mm_load_ss(w1 + 2);
565 __m128 v2x012 = _mm_load_ss(w2 + 2);
566 __m128 v3x012 = _mm_load_ss(w3 + 2);
567 __m128 v4x012 = _mm_load_ss(w4 + 2);
568 __m128 v5x012 = _mm_load_ss(w5 + 2);
569 __m128 v6x012 = _mm_load_ss(w6 + 2);
570
571 v0x012 = _mm_movelh_ps(v0x012, v0x012);
572 v1x012 = _mm_movelh_ps(v1x012, v1x012);
573 v2x012 = _mm_movelh_ps(v2x012, v2x012);
574 v3x012 = _mm_movelh_ps(v3x012, v3x012);
575 v4x012 = _mm_movelh_ps(v4x012, v4x012);
576 v5x012 = _mm_movelh_ps(v5x012, v5x012);
577 v6x012 = _mm_movelh_ps(v6x012, v6x012);
578
579 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
580 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
581 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
582 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
583 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
584 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
585 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
586
587 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
588 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
589 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
590 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
591 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
592 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v6x012);
593 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
594 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v6x012);
595
596 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
597 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
598 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
599 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
600 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
601 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
602
603 _mm_store_ps(packed_w, v0123x0);
604 _mm_store_ps(packed_w + 4, v4567x0);
605 _mm_store_ps(packed_w + 8, v0123x1);
606 _mm_store_ps(packed_w + 12, v4567x1);
607 _mm_store_ps(packed_w + 16, v0123x2);
608 _mm_store_ps(packed_w + 20, v4567x2);
609 packed_w += 24;
610 break;
611 }
612 default:
613 XNN_UNREACHABLE;
614 }
615 }
616 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
617 }
618 weights += nc * kc;
619 } while (--g != 0);
620 }