Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-sse2-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_goi_ukernel_x8s4__sse2_u8(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 const uint32_t* weights,
28 const uint32_t* bias,
29 const void* scale,
30 uint32_t* packed_weights,
31 size_t extra_bytes,
32 const void* params)
33 {
34 assert(g != 0);
35 assert(nc != 0);
36 assert(kc != 0);
37 assert(nr == 8);
38 assert(kr == 1);
39 assert(sr == 4);
40 assert(weights != NULL);
41 assert(packed_weights != NULL);
42
43 const float* b = (const float*) bias;
44 float* packed_w = (float*) packed_weights;
45 do {
46 // NC main loop multiple of 8
47 const float* w0 = (const float*) weights;
48 size_t n = nc;
49
50 for (; n >= 8; n -= 8) {
51 if XNN_LIKELY(b != NULL) {
52 const __m128 vb0 = _mm_loadu_ps(b);
53 const __m128 vb4 = _mm_loadu_ps(b + 4);
54 _mm_store_ps(packed_w, vb0);
55 _mm_store_ps(packed_w + 4, vb4);
56 b += 8;
57 } else {
58 const __m128 vzero = _mm_setzero_ps();
59 _mm_store_ps(packed_w, vzero);
60 _mm_store_ps(packed_w + 4, vzero);
61 }
62 packed_w += 8;
63
64 const float* w1 = w0 + kc;
65 const float* w2 = w1 + kc;
66 const float* w3 = w2 + kc;
67 const float* w4 = w3 + kc;
68 const float* w5 = w4 + kc;
69 const float* w6 = w5 + kc;
70 const float* w7 = w6 + kc;
71
72 size_t k = kc;
73 // KC main loop multiple of 8
74 for (; k >= 8; k -= 8) {
75 // Read blocks of 4x4
76 // a b c d
77 // e f g h
78 // i j k l
79 // m n o p
80 __m128 v0x0123 = _mm_loadu_ps(w0);
81 __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
82 w0 += 8;
83 __m128 v1x0123 = _mm_loadu_ps(w1);
84 __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
85 w1 += 8;
86 __m128 v2x0123 = _mm_loadu_ps(w2);
87 __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
88 w2 += 8;
89 __m128 v3x0123 = _mm_loadu_ps(w3);
90 __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
91 w3 += 8;
92 __m128 v4x0123 = _mm_loadu_ps(w4);
93 __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
94 w4 += 8;
95 __m128 v5x0123 = _mm_loadu_ps(w5);
96 __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
97 w5 += 8;
98 __m128 v6x0123 = _mm_loadu_ps(w6);
99 __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
100 w6 += 8;
101 __m128 v7x0123 = _mm_loadu_ps(w7);
102 __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
103 w7 += 8;
104
105 // Apply SR4 shuffle
106 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
107 v1x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x4567), _MM_SHUFFLE(0, 3, 2, 1)));
108 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
109 v2x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x4567), _MM_SHUFFLE(1, 0, 3, 2)));
110 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
111 v3x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x4567), _MM_SHUFFLE(2, 1, 0, 3)));
112 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
113 v5x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x4567), _MM_SHUFFLE(0, 3, 2, 1)));
114 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
115 v6x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x4567), _MM_SHUFFLE(1, 0, 3, 2)));
116 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
117 v7x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x4567), _MM_SHUFFLE(2, 1, 0, 3)));
118 // Transpose 2x2
119 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
120 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
121 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
122 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
123 const __m128 vtmp0x4567 = _mm_unpacklo_ps(v0x4567, v1x4567); // a e b f from row 0, 1
124 const __m128 vtmp1x4567 = _mm_unpacklo_ps(v2x4567, v3x4567); // i m j n from row 2, 3
125 const __m128 vtmp2x4567 = _mm_unpackhi_ps(v0x4567, v1x4567); // c g d h from row 0, 1
126 const __m128 vtmp3x4567 = _mm_unpackhi_ps(v2x4567, v3x4567); // k o l p from row 2, 3
127 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
128 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
129 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
130 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
131 const __m128 vtmp4x4567 = _mm_unpacklo_ps(v4x4567, v5x4567); // a e b f from row 0, 1
132 const __m128 vtmp5x4567 = _mm_unpacklo_ps(v6x4567, v7x4567); // i m j n from row 2, 3
133 const __m128 vtmp6x4567 = _mm_unpackhi_ps(v4x4567, v5x4567); // c g d h from row 0, 1
134 const __m128 vtmp7x4567 = _mm_unpackhi_ps(v6x4567, v7x4567); // k o l p from row 2, 3
135 // Transpose 4x4
136 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
137 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
138 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
139 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
140 v0x4567 = _mm_movelh_ps(vtmp0x4567, vtmp1x4567); // a e i m from row 0, 1
141 v1x4567 = _mm_movehl_ps(vtmp1x4567, vtmp0x4567); // b f j n from row 0, 1
142 v2x4567 = _mm_movelh_ps(vtmp2x4567, vtmp3x4567); // c g k o from row 2, 3
143 v3x4567 = _mm_movehl_ps(vtmp3x4567, vtmp2x4567); // d h l p from row 2, 3
144 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
145 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
146 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
147 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
148 v4x4567 = _mm_movelh_ps(vtmp4x4567, vtmp5x4567); // a e i m from row 0, 1
149 v5x4567 = _mm_movehl_ps(vtmp5x4567, vtmp4x4567); // b f j n from row 0, 1
150 v6x4567 = _mm_movelh_ps(vtmp6x4567, vtmp7x4567); // c g k o from row 2, 3
151 v7x4567 = _mm_movehl_ps(vtmp7x4567, vtmp6x4567); // d h l p from row 2, 3
152
153 _mm_store_ps(packed_w, v0x0123);
154 _mm_store_ps(packed_w + 4, v4x0123);
155 _mm_store_ps(packed_w + 8, v1x0123);
156 _mm_store_ps(packed_w + 12, v5x0123);
157 _mm_store_ps(packed_w + 16, v2x0123);
158 _mm_store_ps(packed_w + 20, v6x0123);
159 _mm_store_ps(packed_w + 24, v3x0123);
160 _mm_store_ps(packed_w + 28, v7x0123);
161 _mm_store_ps(packed_w + 32, v0x4567);
162 _mm_store_ps(packed_w + 36, v4x4567);
163 _mm_store_ps(packed_w + 40, v1x4567);
164 _mm_store_ps(packed_w + 44, v5x4567);
165 _mm_store_ps(packed_w + 48, v2x4567);
166 _mm_store_ps(packed_w + 52, v6x4567);
167 _mm_store_ps(packed_w + 56, v3x4567);
168 _mm_store_ps(packed_w + 60, v7x4567);
169 packed_w += 64;
170 }
171
172 // KC multiple of 4
173 for (; k >= 4; k -= 4) {
174 // Read blocks of 4x4
175 // a b c d
176 // e f g h
177 // i j k l
178 // m n o p
179 __m128 v0x0123 = _mm_loadu_ps(w0);
180 w0 += 4;
181 __m128 v1x0123 = _mm_loadu_ps(w1);
182 w1 += 4;
183 __m128 v2x0123 = _mm_loadu_ps(w2);
184 w2 += 4;
185 __m128 v3x0123 = _mm_loadu_ps(w3);
186 w3 += 4;
187 __m128 v4x0123 = _mm_loadu_ps(w4);
188 w4 += 4;
189 __m128 v5x0123 = _mm_loadu_ps(w5);
190 w5 += 4;
191 __m128 v6x0123 = _mm_loadu_ps(w6);
192 w6 += 4;
193 __m128 v7x0123 = _mm_loadu_ps(w7);
194 w7 += 4;
195
196 // Apply SR4 shuffle
197 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
198 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
199 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
200 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
201 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
202 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
203
204 // Transpose 2x2
205 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
206 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
207 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
208 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
209 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
210 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
211 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
212 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
213 // Transpose 4x4
214 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
215 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
216 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
217 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
218 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
219 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
220 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
221 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
222
223 _mm_store_ps(packed_w, v0x0123);
224 _mm_store_ps(packed_w + 4, v4x0123);
225 _mm_store_ps(packed_w + 8, v1x0123);
226 _mm_store_ps(packed_w + 12, v5x0123);
227 _mm_store_ps(packed_w + 16, v2x0123);
228 _mm_store_ps(packed_w + 20, v6x0123);
229 _mm_store_ps(packed_w + 24, v3x0123);
230 _mm_store_ps(packed_w + 28, v7x0123);
231 packed_w += 32;
232 }
233
234 // KC remainder (1..3)
235 if XNN_UNLIKELY(k != 0) {
236 assert(k >= 1);
237 assert(k <= 3);
238 __m128 v0 = _mm_undefined_ps();
239 __m128 v1 = _mm_undefined_ps();
240 __m128 v2 = _mm_undefined_ps();
241 __m128 v3 = _mm_undefined_ps();
242 __m128 v4 = _mm_undefined_ps();
243 __m128 v5 = _mm_undefined_ps();
244 __m128 v6 = _mm_undefined_ps();
245 __m128 v7 = _mm_undefined_ps();
246
247 switch (k) {
248 case 1:
249 // Read blocks of 4x1
250 // a
251 // e
252 // i
253 // m
254 v0 = _mm_load_ss(w0);
255 w0 += 1;
256 v1 = _mm_load_ss(w1);
257 w1 += 1;
258 v2 = _mm_load_ss(w2);
259 w2 += 1;
260 v3 = _mm_load_ss(w3);
261 w3 += 1;
262 v4 = _mm_load_ss(w4);
263 w4 += 1;
264 v5 = _mm_load_ss(w5);
265 w5 += 1;
266 v6 = _mm_load_ss(w6);
267 w6 += 1;
268 v7 = _mm_load_ss(w7);
269 w7 += 1;
270 break;
271 case 2:
272 // Read blocks of 4x2
273 // a b
274 // e f
275 // i j
276 // m n
277 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
278 w0 += 2;
279 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
280 w1 += 2;
281 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
282 w2 += 2;
283 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
284 w3 += 2;
285 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
286 w4 += 2;
287 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
288 w5 += 2;
289 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
290 w6 += 2;
291 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
292 w7 += 2;
293 break;
294 case 3:
295 {
296 // Read blocks of 4x3
297 // a b c
298 // e f g
299 // i j k
300 // m n o
301 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
302 const __m128 v0hi = _mm_load_ss(w0 + 2);
303 v0 = _mm_movelh_ps(v0lo, v0hi);
304 w0 += 3;
305 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
306 const __m128 v1hi = _mm_load_ss(w1 + 2);
307 v1 = _mm_movelh_ps(v1lo, v1hi);
308 w1 += 3;
309 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
310 const __m128 v2hi = _mm_load_ss(w2 + 2);
311 v2 = _mm_movelh_ps(v2lo, v2hi);
312 w2 += 3;
313 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
314 const __m128 v3hi = _mm_load_ss(w3 + 2);
315 v3 = _mm_movelh_ps(v3lo, v3hi);
316 w3 += 3;
317 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
318 const __m128 v4hi = _mm_load_ss(w4 + 2);
319 v4 = _mm_movelh_ps(v4lo, v4hi);
320 w4 += 3;
321 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
322 const __m128 v5hi = _mm_load_ss(w5 + 2);
323 v5 = _mm_movelh_ps(v5lo, v5hi);
324 w5 += 3;
325 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
326 const __m128 v6hi = _mm_load_ss(w6 + 2);
327 v6 = _mm_movelh_ps(v6lo, v6hi);
328 w6 += 3;
329 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
330 const __m128 v7hi = _mm_load_ss(w7 + 2);
331 v7 = _mm_movelh_ps(v7lo, v7hi);
332 w7 += 3;
333 break;
334 }
335 default:
336 XNN_UNREACHABLE;
337 }
338
339 // Apply SR4 shuffle
340 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
341 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
342 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
343 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
344 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
345 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
346 // Transpose 2x2
347 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
348 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
349 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
350 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
351 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
352 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
353 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
354 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
355 // Transpose 4x4
356 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
357 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
358 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
359 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
360 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
361 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
362 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
363 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
364 _mm_store_ps(packed_w, v0);
365 _mm_store_ps(packed_w + 4, v4);
366 _mm_store_ps(packed_w + 8, v1);
367 _mm_store_ps(packed_w + 12, v5);
368 _mm_store_ps(packed_w + 16, v2);
369 _mm_store_ps(packed_w + 20, v6);
370 _mm_store_ps(packed_w + 24, v3);
371 _mm_store_ps(packed_w + 28, v7);
372 packed_w += 32;
373 }
374 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
375 w0 = w7;
376 }
377
378 // NC remainder (1..7)
379 if XNN_UNLIKELY(n != 0) {
380 assert(n >= 1);
381 assert(n <= 7);
382 if XNN_LIKELY(b != NULL) {
383 size_t nb = n;
384 do {
385 *packed_w++ = *b++;
386 } while (--nb != 0);
387 packed_w += (8 - n);
388 } else {
389 const __m128 vzero = _mm_setzero_ps();
390 _mm_store_ps(packed_w, vzero);
391 _mm_store_ps(packed_w + 4, vzero);
392 packed_w += 8;
393 }
394
395 // NR remainder has less than 8 rows so last row is not loaded
396 const float* w1 = w0 + kc;
397 if XNN_UNPREDICTABLE(n < 2) {
398 w1 = w0;
399 }
400 const float* w2 = w1 + kc;
401 if XNN_UNPREDICTABLE(n <= 2) {
402 w2 = w1;
403 }
404 const float* w3 = w2 + kc;
405 if XNN_UNPREDICTABLE(n < 4) {
406 w3 = w2;
407 }
408 const float* w4 = w3 + kc;
409 if XNN_UNPREDICTABLE(n <= 4) {
410 w4 = w3;
411 }
412 const float* w5 = w4 + kc;
413 if XNN_UNPREDICTABLE(n < 6) {
414 w5 = w4;
415 }
416 const float* w6 = w5 + kc;
417 if XNN_UNPREDICTABLE(n <= 6) {
418 w6 = w5;
419 }
420
421 size_t k = kc;
422 // KC main loop multiple of 8
423 for (; k >= 8; k -= 8) {
424 // Read blocks of 4x4
425 // a b c d
426 // e f g h
427 // i j k l
428 // m n o p
429 __m128 v0x0123 = _mm_loadu_ps(w0);
430 __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
431 w0 += 8;
432 __m128 v1x0123 = _mm_loadu_ps(w1);
433 __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
434 w1 += 8;
435 __m128 v2x0123 = _mm_loadu_ps(w2);
436 __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
437 w2 += 8;
438 __m128 v3x0123 = _mm_loadu_ps(w3);
439 __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
440 w3 += 8;
441 __m128 v4x0123 = _mm_loadu_ps(w4);
442 __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
443 w4 += 8;
444 __m128 v5x0123 = _mm_loadu_ps(w5);
445 __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
446 w5 += 8;
447 __m128 v6x0123 = _mm_loadu_ps(w6);
448 __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
449 w6 += 8;
450
451 __m128 v7x0123 = _mm_undefined_ps();
452 __m128 v7x4567 = _mm_undefined_ps();
453
454 // Apply SR4 shuffle
455 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
456 v1x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x4567), _MM_SHUFFLE(0, 3, 2, 1)));
457 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
458 v2x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x4567), _MM_SHUFFLE(1, 0, 3, 2)));
459 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
460 v3x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x4567), _MM_SHUFFLE(2, 1, 0, 3)));
461 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
462 v5x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x4567), _MM_SHUFFLE(0, 3, 2, 1)));
463 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
464 v6x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x4567), _MM_SHUFFLE(1, 0, 3, 2)));
465
466 // Transpose 2x2
467 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
468 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
469 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
470 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
471 const __m128 vtmp0x4567 = _mm_unpacklo_ps(v0x4567, v1x4567); // a e b f from row 0, 1
472 const __m128 vtmp1x4567 = _mm_unpacklo_ps(v2x4567, v3x4567); // i m j n from row 2, 3
473 const __m128 vtmp2x4567 = _mm_unpackhi_ps(v0x4567, v1x4567); // c g d h from row 0, 1
474 const __m128 vtmp3x4567 = _mm_unpackhi_ps(v2x4567, v3x4567); // k o l p from row 2, 3
475 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
476 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v6x0123); // i m j n from row 2, 3
477 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
478 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v6x0123); // k o l p from row 2, 3
479 const __m128 vtmp4x4567 = _mm_unpacklo_ps(v4x4567, v5x4567); // a e b f from row 0, 1
480 const __m128 vtmp5x4567 = _mm_unpacklo_ps(v6x4567, v6x4567); // i m j n from row 2, 3
481 const __m128 vtmp6x4567 = _mm_unpackhi_ps(v4x4567, v5x4567); // c g d h from row 0, 1
482 const __m128 vtmp7x4567 = _mm_unpackhi_ps(v6x4567, v6x4567); // k o l p from row 2, 3
483 // Transpose 4x4
484 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
485 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
486 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
487 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
488 v0x4567 = _mm_movelh_ps(vtmp0x4567, vtmp1x4567); // a e i m from row 0, 1
489 v1x4567 = _mm_movehl_ps(vtmp1x4567, vtmp0x4567); // b f j n from row 0, 1
490 v2x4567 = _mm_movelh_ps(vtmp2x4567, vtmp3x4567); // c g k o from row 2, 3
491 v3x4567 = _mm_movehl_ps(vtmp3x4567, vtmp2x4567); // d h l p from row 2, 3
492 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
493 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
494 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
495 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
496 v4x4567 = _mm_movelh_ps(vtmp4x4567, vtmp5x4567); // a e i m from row 0, 1
497 v5x4567 = _mm_movehl_ps(vtmp5x4567, vtmp4x4567); // b f j n from row 0, 1
498 v6x4567 = _mm_movelh_ps(vtmp6x4567, vtmp7x4567); // c g k o from row 2, 3
499 v7x4567 = _mm_movehl_ps(vtmp7x4567, vtmp6x4567); // d h l p from row 2, 3
500
501 _mm_store_ps(packed_w, v0x0123);
502 _mm_store_ps(packed_w + 4, v4x0123);
503 _mm_store_ps(packed_w + 8, v1x0123);
504 _mm_store_ps(packed_w + 12, v5x0123);
505 _mm_store_ps(packed_w + 16, v2x0123);
506 _mm_store_ps(packed_w + 20, v6x0123);
507 _mm_store_ps(packed_w + 24, v3x0123);
508 _mm_store_ps(packed_w + 28, v7x0123);
509 _mm_store_ps(packed_w + 32, v0x4567);
510 _mm_store_ps(packed_w + 36, v4x4567);
511 _mm_store_ps(packed_w + 40, v1x4567);
512 _mm_store_ps(packed_w + 44, v5x4567);
513 _mm_store_ps(packed_w + 48, v2x4567);
514 _mm_store_ps(packed_w + 52, v6x4567);
515 _mm_store_ps(packed_w + 56, v3x4567);
516 _mm_store_ps(packed_w + 60, v7x4567);
517 packed_w += 64;
518 }
519
520 // KC multiple of 4
521 for (; k >= 4; k -= 4) {
522 // Read blocks of 4x4
523 // a b c d
524 // e f g h
525 // i j k l
526 // m n o p
527 __m128 v0x0123 = _mm_loadu_ps(w0);
528 w0 += 4;
529 __m128 v1x0123 = _mm_loadu_ps(w1);
530 w1 += 4;
531 __m128 v2x0123 = _mm_loadu_ps(w2);
532 w2 += 4;
533 __m128 v3x0123 = _mm_loadu_ps(w3);
534 w3 += 4;
535 __m128 v4x0123 = _mm_loadu_ps(w4);
536 w4 += 4;
537 __m128 v5x0123 = _mm_loadu_ps(w5);
538 w5 += 4;
539 __m128 v6x0123 = _mm_loadu_ps(w6);
540 w6 += 4;
541 __m128 v7x0123 = _mm_undefined_ps();
542
543 // Apply SR4 shuffle
544 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
545 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
546 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
547 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
548 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
549
550 // Transpose 2x2
551 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
552 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
553 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
554 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
555 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
556 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v6x0123); // i m j n from row 2, 3
557 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
558 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v6x0123); // k o l p from row 2, 3
559 // Transpose 4x4
560 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
561 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
562 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
563 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
564 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
565 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
566 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
567 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
568
569 _mm_store_ps(packed_w, v0x0123);
570 _mm_store_ps(packed_w + 4, v4x0123);
571 _mm_store_ps(packed_w + 8, v1x0123);
572 _mm_store_ps(packed_w + 12, v5x0123);
573 _mm_store_ps(packed_w + 16, v2x0123);
574 _mm_store_ps(packed_w + 20, v6x0123);
575 _mm_store_ps(packed_w + 24, v3x0123);
576 _mm_store_ps(packed_w + 28, v7x0123);
577 packed_w += 32;
578 }
579
580 // KC remainder (1..3)
581 if XNN_UNLIKELY(k != 0) {
582 assert(k >= 1);
583 assert(k <= 3);
584 __m128 v0 = _mm_undefined_ps();
585 __m128 v1 = _mm_undefined_ps();
586 __m128 v2 = _mm_undefined_ps();
587 __m128 v3 = _mm_undefined_ps();
588 __m128 v4 = _mm_undefined_ps();
589 __m128 v5 = _mm_undefined_ps();
590 __m128 v6 = _mm_undefined_ps();
591 __m128 v7 = _mm_undefined_ps();
592
593 switch (k) {
594 case 1:
595 // Read blocks of 4x1
596 // a
597 // e
598 // i
599 // m
600 v0 = _mm_load_ss(w0);
601 w0 += 1;
602 v1 = _mm_load_ss(w1);
603 w1 += 1;
604 v2 = _mm_load_ss(w2);
605 w2 += 1;
606 v3 = _mm_load_ss(w3);
607 w3 += 1;
608 v4 = _mm_load_ss(w4);
609 w4 += 1;
610 v5 = _mm_load_ss(w5);
611 w5 += 1;
612 v6 = _mm_load_ss(w6);
613 w6 += 1;
614 break;
615 case 2:
616 // Read blocks of 4x2
617 // a b
618 // e f
619 // i j
620 // m n
621 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
622 w0 += 2;
623 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
624 w1 += 2;
625 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
626 w2 += 2;
627 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
628 w3 += 2;
629 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
630 w4 += 2;
631 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
632 w5 += 2;
633 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
634 w6 += 2;
635 break;
636 case 3:
637 {
638 // Read blocks of 4x3
639 // a b c
640 // e f g
641 // i j k
642 // m n o
643 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
644 const __m128 v0hi = _mm_load_ss(w0 + 2);
645 v0 = _mm_movelh_ps(v0lo, v0hi);
646 w0 += 3;
647 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
648 const __m128 v1hi = _mm_load_ss(w1 + 2);
649 v1 = _mm_movelh_ps(v1lo, v1hi);
650 w1 += 3;
651 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
652 const __m128 v2hi = _mm_load_ss(w2 + 2);
653 v2 = _mm_movelh_ps(v2lo, v2hi);
654 w2 += 3;
655 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
656 const __m128 v3hi = _mm_load_ss(w3 + 2);
657 v3 = _mm_movelh_ps(v3lo, v3hi);
658 w3 += 3;
659 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
660 const __m128 v4hi = _mm_load_ss(w4 + 2);
661 v4 = _mm_movelh_ps(v4lo, v4hi);
662 w4 += 3;
663 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
664 const __m128 v5hi = _mm_load_ss(w5 + 2);
665 v5 = _mm_movelh_ps(v5lo, v5hi);
666 w5 += 3;
667 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
668 const __m128 v6hi = _mm_load_ss(w6 + 2);
669 v6 = _mm_movelh_ps(v6lo, v6hi);
670 w6 += 3;
671 break;
672 }
673 default:
674 XNN_UNREACHABLE;
675 }
676
677 // Apply SR4 shuffle
678 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
679 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
680 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
681 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
682 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
683 // Transpose 2x2
684 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
685 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
686 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
687 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
688 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
689 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v6); // i m j n from row 2, 3
690 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
691 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v6); // k o l p from row 2, 3
692 // Transpose 4x4
693 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
694 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
695 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
696 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
697 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
698 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
699 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
700 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
701 _mm_store_ps(packed_w, v0);
702 _mm_store_ps(packed_w + 4, v4);
703 _mm_store_ps(packed_w + 8, v1);
704 _mm_store_ps(packed_w + 12, v5);
705 _mm_store_ps(packed_w + 16, v2);
706 _mm_store_ps(packed_w + 20, v6);
707 _mm_store_ps(packed_w + 24, v3);
708 _mm_store_ps(packed_w + 28, v7);
709 packed_w += 32;
710 }
711 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
712 }
713 weights += nc * kc;
714 } while (--g != 0);
715 }