Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-sse2-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <emmintrin.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20 void xnn_x32_packw_gemm_goi_ukernel_x8__sse2_u8_prfm(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 8);
38 assert(kr == 1);
39 assert(sr == 1);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 const float* b = (const float*) bias;
44 float* packed_w = (float*) packed_weights;
45 do {
46 // NC main loop multiple of 8
47 const float* w0 = (const float*) weights;
48 size_t n = nc;
49
50 for (; n >= 8; n -= 8) {
51 if XNN_LIKELY(b != NULL) {
52 const __m128 vb0123 = _mm_loadu_ps(b);
53 const __m128 vb4567 = _mm_loadu_ps(b + 4);
54 b += 8;
55
56 _mm_store_ps(packed_w, vb0123);
57 _mm_store_ps(packed_w + 4, vb4567);
58 } else {
59 const __m128 vzero = _mm_setzero_ps();
60 _mm_store_ps(packed_w, vzero);
61 _mm_store_ps(packed_w + 4, vzero);
62 }
63 packed_w += 8;
64
65 const float* w1 = w0 + kc;
66 const float* w2 = w1 + kc;
67 const float* w3 = w2 + kc;
68 const float* w4 = w3 + kc;
69 const float* w5 = w4 + kc;
70 const float* w6 = w5 + kc;
71 const float* w7 = w6 + kc;
72 xnn_prefetch_to_l1((const int8_t*) w0);
73 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
74 xnn_prefetch_to_l1((const int8_t*) w1);
75 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
76 xnn_prefetch_to_l1((const int8_t*) w2);
77 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
78 xnn_prefetch_to_l1((const int8_t*) w3);
79 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
80 xnn_prefetch_to_l1((const int8_t*) w4);
81 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
82 xnn_prefetch_to_l1((const int8_t*) w5);
83 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
84 xnn_prefetch_to_l1((const int8_t*) w6);
85 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
86 xnn_prefetch_to_l1((const int8_t*) w7);
87 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
88
89 size_t k = kc;
90 // KC main loop multiple of 8
91 for (; k >= 8; k -= 8) {
92 const __m128 v0x0123 = _mm_loadu_ps(w0);
93 const __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
94 w0 += 8;
95 const __m128 v1x0123 = _mm_loadu_ps(w1);
96 const __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
97 w1 += 8;
98 const __m128 v2x0123 = _mm_loadu_ps(w2);
99 const __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
100 w2 += 8;
101 const __m128 v3x0123 = _mm_loadu_ps(w3);
102 const __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
103 w3 += 8;
104 const __m128 v4x0123 = _mm_loadu_ps(w4);
105 const __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
106 w4 += 8;
107 const __m128 v5x0123 = _mm_loadu_ps(w5);
108 const __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
109 w5 += 8;
110 const __m128 v6x0123 = _mm_loadu_ps(w6);
111 const __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
112 w6 += 8;
113 const __m128 v7x0123 = _mm_loadu_ps(w7);
114 const __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
115 w7 += 8;
116
117 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
118 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
119 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
120 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
121 const __m128 v01x4_01x5 = _mm_unpacklo_ps(v0x4567, v1x4567);
122 const __m128 v23x4_23x5 = _mm_unpacklo_ps(v2x4567, v3x4567);
123 const __m128 v01x6_01x7 = _mm_unpackhi_ps(v0x4567, v1x4567);
124 const __m128 v23x6_23x7 = _mm_unpackhi_ps(v2x4567, v3x4567);
125 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
126 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
127 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
128 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
129 const __m128 v45x4_45x5 = _mm_unpacklo_ps(v4x4567, v5x4567);
130 const __m128 v67x4_67x5 = _mm_unpacklo_ps(v6x4567, v7x4567);
131 const __m128 v45x6_45x7 = _mm_unpackhi_ps(v4x4567, v5x4567);
132 const __m128 v67x6_67x7 = _mm_unpackhi_ps(v6x4567, v7x4567);
133 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
134 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
135 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
136 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
137 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
138 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
139 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
140 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
141 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
142 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
143 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
144 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
145 const __m128 v0123x4 = _mm_movelh_ps(v01x4_01x5, v23x4_23x5);
146 const __m128 v0123x5 = _mm_movehl_ps(v23x4_23x5, v01x4_01x5);
147 const __m128 v0123x6 = _mm_movelh_ps(v01x6_01x7, v23x6_23x7);
148 const __m128 v0123x7 = _mm_movehl_ps(v23x6_23x7, v01x6_01x7);
149 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
150 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
151 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
152 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
153 const __m128 v4567x4 = _mm_movelh_ps(v45x4_45x5, v67x4_67x5);
154 const __m128 v4567x5 = _mm_movehl_ps(v67x4_67x5, v45x4_45x5);
155 const __m128 v4567x6 = _mm_movelh_ps(v45x6_45x7, v67x6_67x7);
156 const __m128 v4567x7 = _mm_movehl_ps(v67x6_67x7, v45x6_45x7);
157
158 _mm_store_ps(packed_w, v0123x0);
159 _mm_store_ps(packed_w + 4, v4567x0);
160 _mm_store_ps(packed_w + 8, v0123x1);
161 _mm_store_ps(packed_w + 12, v4567x1);
162 _mm_store_ps(packed_w + 16, v0123x2);
163 _mm_store_ps(packed_w + 20, v4567x2);
164 _mm_store_ps(packed_w + 24, v0123x3);
165 _mm_store_ps(packed_w + 28, v4567x3);
166 _mm_store_ps(packed_w + 32, v0123x4);
167 _mm_store_ps(packed_w + 36, v4567x4);
168 _mm_store_ps(packed_w + 40, v0123x5);
169 _mm_store_ps(packed_w + 44, v4567x5);
170 _mm_store_ps(packed_w + 48, v0123x6);
171 _mm_store_ps(packed_w + 52, v4567x6);
172 _mm_store_ps(packed_w + 56, v0123x7);
173 _mm_store_ps(packed_w + 60, v4567x7);
174 packed_w += 64;
175 }
176
177 // KC multiple of 4
178 for (; k >= 4; k -= 4) {
179 const __m128 v0x0123 = _mm_loadu_ps(w0);
180 w0 += 4;
181 const __m128 v1x0123 = _mm_loadu_ps(w1);
182 w1 += 4;
183 const __m128 v2x0123 = _mm_loadu_ps(w2);
184 w2 += 4;
185 const __m128 v3x0123 = _mm_loadu_ps(w3);
186 w3 += 4;
187 const __m128 v4x0123 = _mm_loadu_ps(w4);
188 w4 += 4;
189 const __m128 v5x0123 = _mm_loadu_ps(w5);
190 w5 += 4;
191 const __m128 v6x0123 = _mm_loadu_ps(w6);
192 w6 += 4;
193 const __m128 v7x0123 = _mm_loadu_ps(w7);
194 w7 += 4;
195
196 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
197 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
198 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
199 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
200 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
201 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
202 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
203 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
204 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
205 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
206 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
207 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
208 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
209 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
210 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
211 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
212 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
213 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
214 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
215 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
216 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
217 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
218 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
219 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
220
221 _mm_store_ps(packed_w, v0123x0);
222 _mm_store_ps(packed_w + 4, v4567x0);
223 _mm_store_ps(packed_w + 8, v0123x1);
224 _mm_store_ps(packed_w + 12, v4567x1);
225 _mm_store_ps(packed_w + 16, v0123x2);
226 _mm_store_ps(packed_w + 20, v4567x2);
227 _mm_store_ps(packed_w + 24, v0123x3);
228 _mm_store_ps(packed_w + 28, v4567x3);
229 packed_w += 32;
230 }
231
232 // KC remainder (1..3)
233 if XNN_UNLIKELY(k != 0) {
234 assert(k >= 1);
235 assert(k <= 3);
236 switch (k) {
237 case 1:
238 {
239 const __m128 v0x0 = _mm_load_ss(w0);
240 w0 += 1;
241 const __m128 v1x0 = _mm_load_ss(w1);
242 w1 += 1;
243 const __m128 v2x0 = _mm_load_ss(w2);
244 w2 += 1;
245 const __m128 v3x0 = _mm_load_ss(w3);
246 w3 += 1;
247 const __m128 v4x0 = _mm_load_ss(w4);
248 w4 += 1;
249 const __m128 v5x0 = _mm_load_ss(w5);
250 w5 += 1;
251 const __m128 v6x0 = _mm_load_ss(w6);
252 w6 += 1;
253 const __m128 v7x0 = _mm_load_ss(w7);
254 w7 += 1;
255
256 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
257 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
258 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
259 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
260
261 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
262 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
263
264 _mm_store_ps(packed_w, v0123x0);
265 _mm_store_ps(packed_w + 4, v4567x0);
266 packed_w += 8;
267 break;
268 }
269 case 2:
270 {
271 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
272 w0 += 2;
273 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
274 w1 += 2;
275 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
276 w2 += 2;
277 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
278 w3 += 2;
279 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
280 w4 += 2;
281 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
282 w5 += 2;
283 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
284 w6 += 2;
285 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
286 w7 += 2;
287
288 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
289 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
290 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
291 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
292
293 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
294 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
295 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
296 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
297
298 _mm_store_ps(packed_w, v0123x0);
299 _mm_store_ps(packed_w + 4, v4567x0);
300 _mm_store_ps(packed_w + 8, v0123x1);
301 _mm_store_ps(packed_w + 12, v4567x1);
302 packed_w += 16;
303 break;
304 }
305 case 3:
306 {
307 __m128 v0x012 = _mm_load_ss(w0 + 2);
308 __m128 v1x012 = _mm_load_ss(w1 + 2);
309 __m128 v2x012 = _mm_load_ss(w2 + 2);
310 __m128 v3x012 = _mm_load_ss(w3 + 2);
311 __m128 v4x012 = _mm_load_ss(w4 + 2);
312 __m128 v5x012 = _mm_load_ss(w5 + 2);
313 __m128 v6x012 = _mm_load_ss(w6 + 2);
314 __m128 v7x012 = _mm_load_ss(w7 + 2);
315
316 v0x012 = _mm_movelh_ps(v0x012, v0x012);
317 v1x012 = _mm_movelh_ps(v1x012, v1x012);
318 v2x012 = _mm_movelh_ps(v2x012, v2x012);
319 v3x012 = _mm_movelh_ps(v3x012, v3x012);
320 v4x012 = _mm_movelh_ps(v4x012, v4x012);
321 v5x012 = _mm_movelh_ps(v5x012, v5x012);
322 v6x012 = _mm_movelh_ps(v6x012, v6x012);
323 v7x012 = _mm_movelh_ps(v7x012, v7x012);
324
325 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
326 w0 += 3;
327 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
328 w1 += 3;
329 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
330 w2 += 3;
331 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
332 w3 += 3;
333 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
334 w4 += 3;
335 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
336 w5 += 3;
337 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
338 w6 += 3;
339 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
340 w7 += 3;
341
342 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
343 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
344 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
345 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
346 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
347 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
348 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
349 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
350
351 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
352 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
353 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
354 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
355 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
356 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
357
358 _mm_store_ps(packed_w, v0123x0);
359 _mm_store_ps(packed_w + 4, v4567x0);
360 _mm_store_ps(packed_w + 8, v0123x1);
361 _mm_store_ps(packed_w + 12, v4567x1);
362 _mm_store_ps(packed_w + 16, v0123x2);
363 _mm_store_ps(packed_w + 20, v4567x2);
364 packed_w += 24;
365 break;
366 }
367 default:
368 XNN_UNREACHABLE;
369 }
370 }
371 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
372 w0 = w7;
373 }
374
375 // NC remainder (1..7)
376 if XNN_UNLIKELY(n != 0) {
377 assert(n >= 1);
378 assert(n <= 7);
379 if XNN_LIKELY(b != NULL) {
380 size_t nb = n;
381 do {
382 *packed_w++ = *b++;
383 } while (--nb != 0);
384 packed_w += (8 - n);
385 } else {
386 const __m128 vzero = _mm_setzero_ps();
387 _mm_store_ps(packed_w, vzero);
388 _mm_store_ps(packed_w + 4, vzero);
389 packed_w += 8;
390 }
391
392 const float* w1 = w0 + kc;
393 if XNN_UNPREDICTABLE(n < 2) {
394 w1 = w0;
395 }
396 const float* w2 = w1 + kc;
397 if XNN_UNPREDICTABLE(n <= 2) {
398 w2 = w1;
399 }
400 const float* w3 = w2 + kc;
401 if XNN_UNPREDICTABLE(n < 4) {
402 w3 = w2;
403 }
404 const float* w4 = w3 + kc;
405 if XNN_UNPREDICTABLE(n <= 4) {
406 w4 = w3;
407 }
408 const float* w5 = w4 + kc;
409 if XNN_UNPREDICTABLE(n < 6) {
410 w5 = w4;
411 }
412 const float* w6 = w5 + kc;
413 if XNN_UNPREDICTABLE(n <= 6) {
414 w6 = w5;
415 }
416
417 size_t k = kc;
418 // KC main loop multiple of 8
419 for (; k >= 8; k -= 8) {
420 const __m128 v0x0123 = _mm_loadu_ps(w0);
421 const __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
422 w0 += 8;
423 const __m128 v1x0123 = _mm_loadu_ps(w1);
424 const __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
425 w1 += 8;
426 const __m128 v2x0123 = _mm_loadu_ps(w2);
427 const __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
428 w2 += 8;
429 const __m128 v3x0123 = _mm_loadu_ps(w3);
430 const __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
431 w3 += 8;
432 const __m128 v4x0123 = _mm_loadu_ps(w4);
433 const __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
434 w4 += 8;
435 const __m128 v5x0123 = _mm_loadu_ps(w5);
436 const __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
437 w5 += 8;
438 const __m128 v6x0123 = _mm_loadu_ps(w6);
439 const __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
440 w6 += 8;
441
442 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
443 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
444 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
445 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
446 const __m128 v01x4_01x5 = _mm_unpacklo_ps(v0x4567, v1x4567);
447 const __m128 v23x4_23x5 = _mm_unpacklo_ps(v2x4567, v3x4567);
448 const __m128 v01x6_01x7 = _mm_unpackhi_ps(v0x4567, v1x4567);
449 const __m128 v23x6_23x7 = _mm_unpackhi_ps(v2x4567, v3x4567);
450 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
451 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v6x0123);
452 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
453 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v6x0123);
454 const __m128 v45x4_45x5 = _mm_unpacklo_ps(v4x4567, v5x4567);
455 const __m128 v67x4_67x5 = _mm_unpacklo_ps(v6x4567, v6x4567);
456 const __m128 v45x6_45x7 = _mm_unpackhi_ps(v4x4567, v5x4567);
457 const __m128 v67x6_67x7 = _mm_unpackhi_ps(v6x4567, v6x4567);
458 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
459 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
460 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
461 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
462 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
463 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
464 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
465 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
466 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
467 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
468 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
469 const __m128 v0123x4 = _mm_movelh_ps(v01x4_01x5, v23x4_23x5);
470 const __m128 v0123x5 = _mm_movehl_ps(v23x4_23x5, v01x4_01x5);
471 const __m128 v0123x6 = _mm_movelh_ps(v01x6_01x7, v23x6_23x7);
472 const __m128 v0123x7 = _mm_movehl_ps(v23x6_23x7, v01x6_01x7);
473 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
474 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
475 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
476 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
477 const __m128 v4567x4 = _mm_movelh_ps(v45x4_45x5, v67x4_67x5);
478 const __m128 v4567x5 = _mm_movehl_ps(v67x4_67x5, v45x4_45x5);
479 const __m128 v4567x6 = _mm_movelh_ps(v45x6_45x7, v67x6_67x7);
480 const __m128 v4567x7 = _mm_movehl_ps(v67x6_67x7, v45x6_45x7);
481
482 _mm_store_ps(packed_w, v0123x0);
483 _mm_store_ps(packed_w + 4, v4567x0);
484 _mm_store_ps(packed_w + 8, v0123x1);
485 _mm_store_ps(packed_w + 12, v4567x1);
486 _mm_store_ps(packed_w + 16, v0123x2);
487 _mm_store_ps(packed_w + 20, v4567x2);
488 _mm_store_ps(packed_w + 24, v0123x3);
489 _mm_store_ps(packed_w + 28, v4567x3);
490 _mm_store_ps(packed_w + 32, v0123x4);
491 _mm_store_ps(packed_w + 36, v4567x4);
492 _mm_store_ps(packed_w + 40, v0123x5);
493 _mm_store_ps(packed_w + 44, v4567x5);
494 _mm_store_ps(packed_w + 48, v0123x6);
495 _mm_store_ps(packed_w + 52, v4567x6);
496 _mm_store_ps(packed_w + 56, v0123x7);
497 _mm_store_ps(packed_w + 60, v4567x7);
498 packed_w += 64;
499 }
500
501 // KC multiple of 4
502 for (; k >= 4; k -= 4) {
503 const __m128 v0x0123 = _mm_loadu_ps(w0);
504 w0 += 4;
505 const __m128 v1x0123 = _mm_loadu_ps(w1);
506 w1 += 4;
507 const __m128 v2x0123 = _mm_loadu_ps(w2);
508 w2 += 4;
509 const __m128 v3x0123 = _mm_loadu_ps(w3);
510 w3 += 4;
511 const __m128 v4x0123 = _mm_loadu_ps(w4);
512 w4 += 4;
513 const __m128 v5x0123 = _mm_loadu_ps(w5);
514 w5 += 4;
515 const __m128 v6x0123 = _mm_loadu_ps(w6);
516 w6 += 4;
517
518 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
519 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
520 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
521 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
522 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
523 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v6x0123);
524 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
525 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v6x0123);
526 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
527 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
528 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
529 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
530 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
531 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
532 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
533 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
534 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
535 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
536 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
537 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
538 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
539 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
540 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
541
542 _mm_store_ps(packed_w, v0123x0);
543 _mm_store_ps(packed_w + 4, v4567x0);
544 _mm_store_ps(packed_w + 8, v0123x1);
545 _mm_store_ps(packed_w + 12, v4567x1);
546 _mm_store_ps(packed_w + 16, v0123x2);
547 _mm_store_ps(packed_w + 20, v4567x2);
548 _mm_store_ps(packed_w + 24, v0123x3);
549 _mm_store_ps(packed_w + 28, v4567x3);
550 packed_w += 32;
551 }
552
553 // KC remainder (1..3)
554 if XNN_UNLIKELY(k != 0) {
555 assert(k >= 1);
556 assert(k <= 3);
557 switch (k) {
558 case 1:
559 {
560 const __m128 v0x0 = _mm_load_ss(w0);
561 const __m128 v1x0 = _mm_load_ss(w1);
562 const __m128 v2x0 = _mm_load_ss(w2);
563 const __m128 v3x0 = _mm_load_ss(w3);
564 const __m128 v4x0 = _mm_load_ss(w4);
565 const __m128 v5x0 = _mm_load_ss(w5);
566 const __m128 v6x0 = _mm_load_ss(w6);
567
568 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
569 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
570 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
571 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v6x0);
572
573 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
574 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
575
576 _mm_store_ps(packed_w, v0123x0);
577 _mm_store_ps(packed_w + 4, v4567x0);
578 packed_w += 8;
579 break;
580 }
581 case 2:
582 {
583 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
584 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
585 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
586 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
587 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
588 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
589 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
590
591 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
592 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
593 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
594 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v6x01);
595
596 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
597 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
598 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
599 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
600
601 _mm_store_ps(packed_w, v0123x0);
602 _mm_store_ps(packed_w + 4, v4567x0);
603 _mm_store_ps(packed_w + 8, v0123x1);
604 _mm_store_ps(packed_w + 12, v4567x1);
605 packed_w += 16;
606 break;
607 }
608 case 3:
609 {
610 __m128 v0x012 = _mm_load_ss(w0 + 2);
611 __m128 v1x012 = _mm_load_ss(w1 + 2);
612 __m128 v2x012 = _mm_load_ss(w2 + 2);
613 __m128 v3x012 = _mm_load_ss(w3 + 2);
614 __m128 v4x012 = _mm_load_ss(w4 + 2);
615 __m128 v5x012 = _mm_load_ss(w5 + 2);
616 __m128 v6x012 = _mm_load_ss(w6 + 2);
617
618 v0x012 = _mm_movelh_ps(v0x012, v0x012);
619 v1x012 = _mm_movelh_ps(v1x012, v1x012);
620 v2x012 = _mm_movelh_ps(v2x012, v2x012);
621 v3x012 = _mm_movelh_ps(v3x012, v3x012);
622 v4x012 = _mm_movelh_ps(v4x012, v4x012);
623 v5x012 = _mm_movelh_ps(v5x012, v5x012);
624 v6x012 = _mm_movelh_ps(v6x012, v6x012);
625
626 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
627 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
628 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
629 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
630 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
631 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
632 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
633
634 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
635 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
636 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
637 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
638 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
639 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v6x012);
640 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
641 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v6x012);
642
643 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
644 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
645 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
646 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
647 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
648 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
649
650 _mm_store_ps(packed_w, v0123x0);
651 _mm_store_ps(packed_w + 4, v4567x0);
652 _mm_store_ps(packed_w + 8, v0123x1);
653 _mm_store_ps(packed_w + 12, v4567x1);
654 _mm_store_ps(packed_w + 16, v0123x2);
655 _mm_store_ps(packed_w + 20, v4567x2);
656 packed_w += 24;
657 break;
658 }
659 default:
660 XNN_UNREACHABLE;
661 }
662 }
663 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
664 }
665 weights += nc * kc;
666 } while (--g != 0);
667 }