Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-sse2-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <emmintrin.h>
16
17 #include "xnnpack/packw.h"
18
19 void xnn_x32_packw_gemm_goi_ukernel_x8__sse2_u4(
20 size_t g,
21 size_t nc,
22 size_t kc,
23 size_t nr,
24 size_t kr,
25 size_t sr,
26 const uint32_t* weights,
27 const uint32_t* bias,
28 const void* scale,
29 uint32_t* packed_weights,
30 size_t extra_bytes,
31 const void* params)
32 {
33 assert(g != 0);
34 assert(nc != 0);
35 assert(kc != 0);
36 assert(nr == 8);
37 assert(kr == 1);
38 assert(sr == 1);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 const float* b = (const float*) bias;
43 float* packed_w = (float*) packed_weights;
44 do {
45 // NC main loop multiple of 8
46 const float* w0 = (const float*) weights;
47 size_t n = nc;
48
49 for (; n >= 8; n -= 8) {
50 if XNN_LIKELY(b != NULL) {
51 const __m128 vb0123 = _mm_loadu_ps(b);
52 const __m128 vb4567 = _mm_loadu_ps(b + 4);
53 b += 8;
54
55 _mm_store_ps(packed_w, vb0123);
56 _mm_store_ps(packed_w + 4, vb4567);
57 } else {
58 const __m128 vzero = _mm_setzero_ps();
59 _mm_store_ps(packed_w, vzero);
60 _mm_store_ps(packed_w + 4, vzero);
61 }
62 packed_w += 8;
63
64 const float* w1 = w0 + kc;
65 const float* w2 = w1 + kc;
66 const float* w3 = w2 + kc;
67 const float* w4 = w3 + kc;
68 const float* w5 = w4 + kc;
69 const float* w6 = w5 + kc;
70 const float* w7 = w6 + kc;
71
72 size_t k = kc;
73
74 // KC multiple of 4
75 for (; k >= 4; k -= 4) {
76 const __m128 v0x0123 = _mm_loadu_ps(w0);
77 w0 += 4;
78 const __m128 v1x0123 = _mm_loadu_ps(w1);
79 w1 += 4;
80 const __m128 v2x0123 = _mm_loadu_ps(w2);
81 w2 += 4;
82 const __m128 v3x0123 = _mm_loadu_ps(w3);
83 w3 += 4;
84 const __m128 v4x0123 = _mm_loadu_ps(w4);
85 w4 += 4;
86 const __m128 v5x0123 = _mm_loadu_ps(w5);
87 w5 += 4;
88 const __m128 v6x0123 = _mm_loadu_ps(w6);
89 w6 += 4;
90 const __m128 v7x0123 = _mm_loadu_ps(w7);
91 w7 += 4;
92
93 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
94 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
95 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
96 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
97 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
98 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v7x0123);
99 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
100 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v7x0123);
101 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
102 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
103 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
104 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
105 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
106 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
107 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
108 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
109
110 _mm_store_ps(packed_w, v0123x0);
111 _mm_store_ps(packed_w + 4, v4567x0);
112 _mm_store_ps(packed_w + 8, v0123x1);
113 _mm_store_ps(packed_w + 12, v4567x1);
114 _mm_store_ps(packed_w + 16, v0123x2);
115 _mm_store_ps(packed_w + 20, v4567x2);
116 _mm_store_ps(packed_w + 24, v0123x3);
117 _mm_store_ps(packed_w + 28, v4567x3);
118 packed_w += 32;
119 }
120
121 // KC remainder (1..3)
122 if XNN_UNLIKELY(k != 0) {
123 assert(k >= 1);
124 assert(k <= 3);
125 switch (k) {
126 case 1:
127 {
128 const __m128 v0x0 = _mm_load_ss(w0);
129 w0 += 1;
130 const __m128 v1x0 = _mm_load_ss(w1);
131 w1 += 1;
132 const __m128 v2x0 = _mm_load_ss(w2);
133 w2 += 1;
134 const __m128 v3x0 = _mm_load_ss(w3);
135 w3 += 1;
136 const __m128 v4x0 = _mm_load_ss(w4);
137 w4 += 1;
138 const __m128 v5x0 = _mm_load_ss(w5);
139 w5 += 1;
140 const __m128 v6x0 = _mm_load_ss(w6);
141 w6 += 1;
142 const __m128 v7x0 = _mm_load_ss(w7);
143 w7 += 1;
144
145 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
146 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
147 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
148 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v7x0);
149
150 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
151 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
152
153 _mm_store_ps(packed_w, v0123x0);
154 _mm_store_ps(packed_w + 4, v4567x0);
155 packed_w += 8;
156 break;
157 }
158 case 2:
159 {
160 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
161 w0 += 2;
162 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
163 w1 += 2;
164 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
165 w2 += 2;
166 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
167 w3 += 2;
168 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
169 w4 += 2;
170 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
171 w5 += 2;
172 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
173 w6 += 2;
174 const __m128 v7x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
175 w7 += 2;
176
177 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
178 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
179 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
180 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v7x01);
181
182 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
183 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
184 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
185 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
186
187 _mm_store_ps(packed_w, v0123x0);
188 _mm_store_ps(packed_w + 4, v4567x0);
189 _mm_store_ps(packed_w + 8, v0123x1);
190 _mm_store_ps(packed_w + 12, v4567x1);
191 packed_w += 16;
192 break;
193 }
194 case 3:
195 {
196 __m128 v0x012 = _mm_load_ss(w0 + 2);
197 __m128 v1x012 = _mm_load_ss(w1 + 2);
198 __m128 v2x012 = _mm_load_ss(w2 + 2);
199 __m128 v3x012 = _mm_load_ss(w3 + 2);
200 __m128 v4x012 = _mm_load_ss(w4 + 2);
201 __m128 v5x012 = _mm_load_ss(w5 + 2);
202 __m128 v6x012 = _mm_load_ss(w6 + 2);
203 __m128 v7x012 = _mm_load_ss(w7 + 2);
204
205 v0x012 = _mm_movelh_ps(v0x012, v0x012);
206 v1x012 = _mm_movelh_ps(v1x012, v1x012);
207 v2x012 = _mm_movelh_ps(v2x012, v2x012);
208 v3x012 = _mm_movelh_ps(v3x012, v3x012);
209 v4x012 = _mm_movelh_ps(v4x012, v4x012);
210 v5x012 = _mm_movelh_ps(v5x012, v5x012);
211 v6x012 = _mm_movelh_ps(v6x012, v6x012);
212 v7x012 = _mm_movelh_ps(v7x012, v7x012);
213
214 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
215 w0 += 3;
216 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
217 w1 += 3;
218 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
219 w2 += 3;
220 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
221 w3 += 3;
222 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
223 w4 += 3;
224 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
225 w5 += 3;
226 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
227 w6 += 3;
228 v7x012 = _mm_loadl_pi(v7x012, (const __m64*) w7);
229 w7 += 3;
230
231 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
232 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
233 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
234 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
235 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
236 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v7x012);
237 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
238 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v7x012);
239
240 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
241 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
242 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
243 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
244 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
245 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
246
247 _mm_store_ps(packed_w, v0123x0);
248 _mm_store_ps(packed_w + 4, v4567x0);
249 _mm_store_ps(packed_w + 8, v0123x1);
250 _mm_store_ps(packed_w + 12, v4567x1);
251 _mm_store_ps(packed_w + 16, v0123x2);
252 _mm_store_ps(packed_w + 20, v4567x2);
253 packed_w += 24;
254 break;
255 }
256 default:
257 XNN_UNREACHABLE;
258 }
259 }
260 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
261 w0 = w7;
262 }
263
264 // NC remainder (1..7)
265 if XNN_UNLIKELY(n != 0) {
266 assert(n >= 1);
267 assert(n <= 7);
268 if XNN_LIKELY(b != NULL) {
269 size_t nb = n;
270 do {
271 *packed_w++ = *b++;
272 } while (--nb != 0);
273 packed_w += (8 - n);
274 } else {
275 const __m128 vzero = _mm_setzero_ps();
276 _mm_store_ps(packed_w, vzero);
277 _mm_store_ps(packed_w + 4, vzero);
278 packed_w += 8;
279 }
280
281 const float* w1 = w0 + kc;
282 if XNN_UNPREDICTABLE(n < 2) {
283 w1 = w0;
284 }
285 const float* w2 = w1 + kc;
286 if XNN_UNPREDICTABLE(n <= 2) {
287 w2 = w1;
288 }
289 const float* w3 = w2 + kc;
290 if XNN_UNPREDICTABLE(n < 4) {
291 w3 = w2;
292 }
293 const float* w4 = w3 + kc;
294 if XNN_UNPREDICTABLE(n <= 4) {
295 w4 = w3;
296 }
297 const float* w5 = w4 + kc;
298 if XNN_UNPREDICTABLE(n < 6) {
299 w5 = w4;
300 }
301 const float* w6 = w5 + kc;
302 if XNN_UNPREDICTABLE(n <= 6) {
303 w6 = w5;
304 }
305
306 size_t k = kc;
307
308 // KC multiple of 4
309 for (; k >= 4; k -= 4) {
310 const __m128 v0x0123 = _mm_loadu_ps(w0);
311 w0 += 4;
312 const __m128 v1x0123 = _mm_loadu_ps(w1);
313 w1 += 4;
314 const __m128 v2x0123 = _mm_loadu_ps(w2);
315 w2 += 4;
316 const __m128 v3x0123 = _mm_loadu_ps(w3);
317 w3 += 4;
318 const __m128 v4x0123 = _mm_loadu_ps(w4);
319 w4 += 4;
320 const __m128 v5x0123 = _mm_loadu_ps(w5);
321 w5 += 4;
322 const __m128 v6x0123 = _mm_loadu_ps(w6);
323 w6 += 4;
324
325 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x0123, v1x0123);
326 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x0123, v3x0123);
327 const __m128 v01x2_01x3 = _mm_unpackhi_ps(v0x0123, v1x0123);
328 const __m128 v23x2_23x3 = _mm_unpackhi_ps(v2x0123, v3x0123);
329 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x0123, v5x0123);
330 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x0123, v6x0123);
331 const __m128 v45x2_45x3 = _mm_unpackhi_ps(v4x0123, v5x0123);
332 const __m128 v67x2_67x3 = _mm_unpackhi_ps(v6x0123, v6x0123);
333 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
334 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
335 const __m128 v0123x2 = _mm_movelh_ps(v01x2_01x3, v23x2_23x3);
336 const __m128 v0123x3 = _mm_movehl_ps(v23x2_23x3, v01x2_01x3);
337 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
338 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
339 const __m128 v4567x2 = _mm_movelh_ps(v45x2_45x3, v67x2_67x3);
340 const __m128 v4567x3 = _mm_movehl_ps(v67x2_67x3, v45x2_45x3);
341
342 _mm_store_ps(packed_w, v0123x0);
343 _mm_store_ps(packed_w + 4, v4567x0);
344 _mm_store_ps(packed_w + 8, v0123x1);
345 _mm_store_ps(packed_w + 12, v4567x1);
346 _mm_store_ps(packed_w + 16, v0123x2);
347 _mm_store_ps(packed_w + 20, v4567x2);
348 _mm_store_ps(packed_w + 24, v0123x3);
349 _mm_store_ps(packed_w + 28, v4567x3);
350 packed_w += 32;
351 }
352
353 // KC remainder (1..3)
354 if XNN_UNLIKELY(k != 0) {
355 assert(k >= 1);
356 assert(k <= 3);
357 switch (k) {
358 case 1:
359 {
360 const __m128 v0x0 = _mm_load_ss(w0);
361 const __m128 v1x0 = _mm_load_ss(w1);
362 const __m128 v2x0 = _mm_load_ss(w2);
363 const __m128 v3x0 = _mm_load_ss(w3);
364 const __m128 v4x0 = _mm_load_ss(w4);
365 const __m128 v5x0 = _mm_load_ss(w5);
366 const __m128 v6x0 = _mm_load_ss(w6);
367
368 const __m128 v01x0 = _mm_unpacklo_ps(v0x0, v1x0);
369 const __m128 v23x0 = _mm_unpacklo_ps(v2x0, v3x0);
370 const __m128 v45x0 = _mm_unpacklo_ps(v4x0, v5x0);
371 const __m128 v67x0 = _mm_unpacklo_ps(v6x0, v6x0);
372
373 const __m128 v0123x0 = _mm_movelh_ps(v01x0, v23x0);
374 const __m128 v4567x0 = _mm_movelh_ps(v45x0, v67x0);
375
376 _mm_store_ps(packed_w, v0123x0);
377 _mm_store_ps(packed_w + 4, v4567x0);
378 packed_w += 8;
379 break;
380 }
381 case 2:
382 {
383 const __m128 v0x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
384 const __m128 v1x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
385 const __m128 v2x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
386 const __m128 v3x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
387 const __m128 v4x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
388 const __m128 v5x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
389 const __m128 v6x01 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
390
391 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x01, v1x01);
392 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x01, v3x01);
393 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x01, v5x01);
394 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x01, v6x01);
395
396 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
397 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
398 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
399 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
400
401 _mm_store_ps(packed_w, v0123x0);
402 _mm_store_ps(packed_w + 4, v4567x0);
403 _mm_store_ps(packed_w + 8, v0123x1);
404 _mm_store_ps(packed_w + 12, v4567x1);
405 packed_w += 16;
406 break;
407 }
408 case 3:
409 {
410 __m128 v0x012 = _mm_load_ss(w0 + 2);
411 __m128 v1x012 = _mm_load_ss(w1 + 2);
412 __m128 v2x012 = _mm_load_ss(w2 + 2);
413 __m128 v3x012 = _mm_load_ss(w3 + 2);
414 __m128 v4x012 = _mm_load_ss(w4 + 2);
415 __m128 v5x012 = _mm_load_ss(w5 + 2);
416 __m128 v6x012 = _mm_load_ss(w6 + 2);
417
418 v0x012 = _mm_movelh_ps(v0x012, v0x012);
419 v1x012 = _mm_movelh_ps(v1x012, v1x012);
420 v2x012 = _mm_movelh_ps(v2x012, v2x012);
421 v3x012 = _mm_movelh_ps(v3x012, v3x012);
422 v4x012 = _mm_movelh_ps(v4x012, v4x012);
423 v5x012 = _mm_movelh_ps(v5x012, v5x012);
424 v6x012 = _mm_movelh_ps(v6x012, v6x012);
425
426 v0x012 = _mm_loadl_pi(v0x012, (const __m64*) w0);
427 v1x012 = _mm_loadl_pi(v1x012, (const __m64*) w1);
428 v2x012 = _mm_loadl_pi(v2x012, (const __m64*) w2);
429 v3x012 = _mm_loadl_pi(v3x012, (const __m64*) w3);
430 v4x012 = _mm_loadl_pi(v4x012, (const __m64*) w4);
431 v5x012 = _mm_loadl_pi(v5x012, (const __m64*) w5);
432 v6x012 = _mm_loadl_pi(v6x012, (const __m64*) w6);
433
434 const __m128 v01x0_01x1 = _mm_unpacklo_ps(v0x012, v1x012);
435 const __m128 v23x0_23x1 = _mm_unpacklo_ps(v2x012, v3x012);
436 const __m128 v01x2 = _mm_unpackhi_ps(v0x012, v1x012);
437 const __m128 v23x2 = _mm_unpackhi_ps(v2x012, v3x012);
438 const __m128 v45x0_45x1 = _mm_unpacklo_ps(v4x012, v5x012);
439 const __m128 v67x0_67x1 = _mm_unpacklo_ps(v6x012, v6x012);
440 const __m128 v45x2 = _mm_unpackhi_ps(v4x012, v5x012);
441 const __m128 v67x2 = _mm_unpackhi_ps(v6x012, v6x012);
442
443 const __m128 v0123x0 = _mm_movelh_ps(v01x0_01x1, v23x0_23x1);
444 const __m128 v0123x1 = _mm_movehl_ps(v23x0_23x1, v01x0_01x1);
445 const __m128 v0123x2 = _mm_movelh_ps(v01x2, v23x2);
446 const __m128 v4567x0 = _mm_movelh_ps(v45x0_45x1, v67x0_67x1);
447 const __m128 v4567x1 = _mm_movehl_ps(v67x0_67x1, v45x0_45x1);
448 const __m128 v4567x2 = _mm_movelh_ps(v45x2, v67x2);
449
450 _mm_store_ps(packed_w, v0123x0);
451 _mm_store_ps(packed_w + 4, v4567x0);
452 _mm_store_ps(packed_w + 8, v0123x1);
453 _mm_store_ps(packed_w + 12, v4567x1);
454 _mm_store_ps(packed_w + 16, v0123x2);
455 _mm_store_ps(packed_w + 20, v4567x2);
456 packed_w += 24;
457 break;
458 }
459 default:
460 XNN_UNREACHABLE;
461 }
462 }
463 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
464 }
465 weights += nc * kc;
466 } while (--g != 0);
467 }