Debian Codegraph @

src/x32-packw/gen/x32-packw-x8s4-gemm-goi-sse2-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/s4-sse2.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x8s4__sse2_u8_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8);
39 assert(kr == 1);
40 assert(sr == 4);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 const float* b = (const float*) bias;
45 float* packed_w = (float*) packed_weights;
46 do {
47 // NC main loop multiple of 8
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50
51 for (; n >= 8; n -= 8) {
52 if XNN_LIKELY(b != NULL) {
53 const __m128 vb0 = _mm_loadu_ps(b);
54 const __m128 vb4 = _mm_loadu_ps(b + 4);
55 _mm_store_ps(packed_w, vb0);
56 _mm_store_ps(packed_w + 4, vb4);
57 b += 8;
58 } else {
59 const __m128 vzero = _mm_setzero_ps();
60 _mm_store_ps(packed_w, vzero);
61 _mm_store_ps(packed_w + 4, vzero);
62 }
63 packed_w += 8;
64
65 const float* w1 = w0 + kc;
66 const float* w2 = w1 + kc;
67 const float* w3 = w2 + kc;
68 const float* w4 = w3 + kc;
69 const float* w5 = w4 + kc;
70 const float* w6 = w5 + kc;
71 const float* w7 = w6 + kc;
72 xnn_prefetch_to_l1((const int8_t*) w0);
73 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
74 xnn_prefetch_to_l1((const int8_t*) w1);
75 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
76 xnn_prefetch_to_l1((const int8_t*) w2);
77 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
78 xnn_prefetch_to_l1((const int8_t*) w3);
79 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
80 xnn_prefetch_to_l1((const int8_t*) w4);
81 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
82 xnn_prefetch_to_l1((const int8_t*) w5);
83 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
84 xnn_prefetch_to_l1((const int8_t*) w6);
85 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
86 xnn_prefetch_to_l1((const int8_t*) w7);
87 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
88
89 size_t k = kc;
90 // KC main loop multiple of 8
91 for (; k >= 8; k -= 8) {
92 // Read blocks of 4x4
93 // a b c d
94 // e f g h
95 // i j k l
96 // m n o p
97 __m128 v0x0123 = _mm_loadu_ps(w0);
98 __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
99 w0 += 8;
100 __m128 v1x0123 = _mm_loadu_ps(w1);
101 __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
102 w1 += 8;
103 __m128 v2x0123 = _mm_loadu_ps(w2);
104 __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
105 w2 += 8;
106 __m128 v3x0123 = _mm_loadu_ps(w3);
107 __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
108 w3 += 8;
109 __m128 v4x0123 = _mm_loadu_ps(w4);
110 __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
111 w4 += 8;
112 __m128 v5x0123 = _mm_loadu_ps(w5);
113 __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
114 w5 += 8;
115 __m128 v6x0123 = _mm_loadu_ps(w6);
116 __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
117 w6 += 8;
118 __m128 v7x0123 = _mm_loadu_ps(w7);
119 __m128 v7x4567 = _mm_loadu_ps(w7 + 4);
120 w7 += 8;
121
122 // Apply SR4 shuffle
123 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
124 v1x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x4567), _MM_SHUFFLE(0, 3, 2, 1)));
125 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
126 v2x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x4567), _MM_SHUFFLE(1, 0, 3, 2)));
127 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
128 v3x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x4567), _MM_SHUFFLE(2, 1, 0, 3)));
129 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
130 v5x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x4567), _MM_SHUFFLE(0, 3, 2, 1)));
131 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
132 v6x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x4567), _MM_SHUFFLE(1, 0, 3, 2)));
133 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
134 v7x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x4567), _MM_SHUFFLE(2, 1, 0, 3)));
135 // Transpose 2x2
136 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
137 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
138 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
139 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
140 const __m128 vtmp0x4567 = _mm_unpacklo_ps(v0x4567, v1x4567); // a e b f from row 0, 1
141 const __m128 vtmp1x4567 = _mm_unpacklo_ps(v2x4567, v3x4567); // i m j n from row 2, 3
142 const __m128 vtmp2x4567 = _mm_unpackhi_ps(v0x4567, v1x4567); // c g d h from row 0, 1
143 const __m128 vtmp3x4567 = _mm_unpackhi_ps(v2x4567, v3x4567); // k o l p from row 2, 3
144 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
145 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
146 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
147 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
148 const __m128 vtmp4x4567 = _mm_unpacklo_ps(v4x4567, v5x4567); // a e b f from row 0, 1
149 const __m128 vtmp5x4567 = _mm_unpacklo_ps(v6x4567, v7x4567); // i m j n from row 2, 3
150 const __m128 vtmp6x4567 = _mm_unpackhi_ps(v4x4567, v5x4567); // c g d h from row 0, 1
151 const __m128 vtmp7x4567 = _mm_unpackhi_ps(v6x4567, v7x4567); // k o l p from row 2, 3
152 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
153 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
154 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
155 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
156 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
157 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
158 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
159 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
160 // Transpose 4x4
161 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
162 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
163 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
164 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
165 v0x4567 = _mm_movelh_ps(vtmp0x4567, vtmp1x4567); // a e i m from row 0, 1
166 v1x4567 = _mm_movehl_ps(vtmp1x4567, vtmp0x4567); // b f j n from row 0, 1
167 v2x4567 = _mm_movelh_ps(vtmp2x4567, vtmp3x4567); // c g k o from row 2, 3
168 v3x4567 = _mm_movehl_ps(vtmp3x4567, vtmp2x4567); // d h l p from row 2, 3
169 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
170 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
171 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
172 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
173 v4x4567 = _mm_movelh_ps(vtmp4x4567, vtmp5x4567); // a e i m from row 0, 1
174 v5x4567 = _mm_movehl_ps(vtmp5x4567, vtmp4x4567); // b f j n from row 0, 1
175 v6x4567 = _mm_movelh_ps(vtmp6x4567, vtmp7x4567); // c g k o from row 2, 3
176 v7x4567 = _mm_movehl_ps(vtmp7x4567, vtmp6x4567); // d h l p from row 2, 3
177
178 _mm_store_ps(packed_w, v0x0123);
179 _mm_store_ps(packed_w + 4, v4x0123);
180 _mm_store_ps(packed_w + 8, v1x0123);
181 _mm_store_ps(packed_w + 12, v5x0123);
182 _mm_store_ps(packed_w + 16, v2x0123);
183 _mm_store_ps(packed_w + 20, v6x0123);
184 _mm_store_ps(packed_w + 24, v3x0123);
185 _mm_store_ps(packed_w + 28, v7x0123);
186 _mm_store_ps(packed_w + 32, v0x4567);
187 _mm_store_ps(packed_w + 36, v4x4567);
188 _mm_store_ps(packed_w + 40, v1x4567);
189 _mm_store_ps(packed_w + 44, v5x4567);
190 _mm_store_ps(packed_w + 48, v2x4567);
191 _mm_store_ps(packed_w + 52, v6x4567);
192 _mm_store_ps(packed_w + 56, v3x4567);
193 _mm_store_ps(packed_w + 60, v7x4567);
194 packed_w += 64;
195 }
196
197 // KC multiple of 4
198 for (; k >= 4; k -= 4) {
199 // Read blocks of 4x4
200 // a b c d
201 // e f g h
202 // i j k l
203 // m n o p
204 __m128 v0x0123 = _mm_loadu_ps(w0);
205 w0 += 4;
206 __m128 v1x0123 = _mm_loadu_ps(w1);
207 w1 += 4;
208 __m128 v2x0123 = _mm_loadu_ps(w2);
209 w2 += 4;
210 __m128 v3x0123 = _mm_loadu_ps(w3);
211 w3 += 4;
212 __m128 v4x0123 = _mm_loadu_ps(w4);
213 w4 += 4;
214 __m128 v5x0123 = _mm_loadu_ps(w5);
215 w5 += 4;
216 __m128 v6x0123 = _mm_loadu_ps(w6);
217 w6 += 4;
218 __m128 v7x0123 = _mm_loadu_ps(w7);
219 w7 += 4;
220
221 // Apply SR4 shuffle
222 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
223 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
224 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
225 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
226 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
227 v7x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7x0123), _MM_SHUFFLE(2, 1, 0, 3)));
228
229 // Transpose 2x2
230 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
231 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
232 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
233 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
234 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
235 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v7x0123); // i m j n from row 2, 3
236 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
237 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v7x0123); // k o l p from row 2, 3
238 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
239 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
240 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
241 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
242 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
243 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
244 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
245 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
246 // Transpose 4x4
247 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
248 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
249 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
250 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
251 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
252 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
253 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
254 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
255
256 _mm_store_ps(packed_w, v0x0123);
257 _mm_store_ps(packed_w + 4, v4x0123);
258 _mm_store_ps(packed_w + 8, v1x0123);
259 _mm_store_ps(packed_w + 12, v5x0123);
260 _mm_store_ps(packed_w + 16, v2x0123);
261 _mm_store_ps(packed_w + 20, v6x0123);
262 _mm_store_ps(packed_w + 24, v3x0123);
263 _mm_store_ps(packed_w + 28, v7x0123);
264 packed_w += 32;
265 }
266
267 // KC remainder (1..3)
268 if XNN_UNLIKELY(k != 0) {
269 assert(k >= 1);
270 assert(k <= 3);
271 __m128 v0 = _mm_undefined_ps();
272 __m128 v1 = _mm_undefined_ps();
273 __m128 v2 = _mm_undefined_ps();
274 __m128 v3 = _mm_undefined_ps();
275 __m128 v4 = _mm_undefined_ps();
276 __m128 v5 = _mm_undefined_ps();
277 __m128 v6 = _mm_undefined_ps();
278 __m128 v7 = _mm_undefined_ps();
279
280 switch (k) {
281 case 1:
282 // Read blocks of 4x1
283 // a
284 // e
285 // i
286 // m
287 v0 = _mm_load_ss(w0);
288 w0 += 1;
289 v1 = _mm_load_ss(w1);
290 w1 += 1;
291 v2 = _mm_load_ss(w2);
292 w2 += 1;
293 v3 = _mm_load_ss(w3);
294 w3 += 1;
295 v4 = _mm_load_ss(w4);
296 w4 += 1;
297 v5 = _mm_load_ss(w5);
298 w5 += 1;
299 v6 = _mm_load_ss(w6);
300 w6 += 1;
301 v7 = _mm_load_ss(w7);
302 w7 += 1;
303 break;
304 case 2:
305 // Read blocks of 4x2
306 // a b
307 // e f
308 // i j
309 // m n
310 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
311 w0 += 2;
312 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
313 w1 += 2;
314 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
315 w2 += 2;
316 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
317 w3 += 2;
318 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
319 w4 += 2;
320 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
321 w5 += 2;
322 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
323 w6 += 2;
324 v7 = _mm_castpd_ps(_mm_load_sd((const double*) w7));
325 w7 += 2;
326 break;
327 case 3:
328 {
329 // Read blocks of 4x3
330 // a b c
331 // e f g
332 // i j k
333 // m n o
334 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
335 const __m128 v0hi = _mm_load_ss(w0 + 2);
336 v0 = _mm_movelh_ps(v0lo, v0hi);
337 w0 += 3;
338 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
339 const __m128 v1hi = _mm_load_ss(w1 + 2);
340 v1 = _mm_movelh_ps(v1lo, v1hi);
341 w1 += 3;
342 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
343 const __m128 v2hi = _mm_load_ss(w2 + 2);
344 v2 = _mm_movelh_ps(v2lo, v2hi);
345 w2 += 3;
346 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
347 const __m128 v3hi = _mm_load_ss(w3 + 2);
348 v3 = _mm_movelh_ps(v3lo, v3hi);
349 w3 += 3;
350 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
351 const __m128 v4hi = _mm_load_ss(w4 + 2);
352 v4 = _mm_movelh_ps(v4lo, v4hi);
353 w4 += 3;
354 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
355 const __m128 v5hi = _mm_load_ss(w5 + 2);
356 v5 = _mm_movelh_ps(v5lo, v5hi);
357 w5 += 3;
358 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
359 const __m128 v6hi = _mm_load_ss(w6 + 2);
360 v6 = _mm_movelh_ps(v6lo, v6hi);
361 w6 += 3;
362 const __m128 v7lo = _mm_castpd_ps(_mm_load_sd((const double*) w7));
363 const __m128 v7hi = _mm_load_ss(w7 + 2);
364 v7 = _mm_movelh_ps(v7lo, v7hi);
365 w7 += 3;
366 break;
367 }
368 default:
369 XNN_UNREACHABLE;
370 }
371
372 // Apply SR4 shuffle
373 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
374 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
375 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
376 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
377 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
378 v7 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v7), _MM_SHUFFLE(2, 1, 0, 3)));
379 // Transpose 2x2
380 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
381 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
382 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
383 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
384 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
385 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // i m j n from row 2, 3
386 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
387 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v7); // k o l p from row 2, 3
388 // Transpose 4x4
389 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
390 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
391 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
392 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
393 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
394 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
395 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
396 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
397 _mm_store_ps(packed_w, v0);
398 _mm_store_ps(packed_w + 4, v4);
399 _mm_store_ps(packed_w + 8, v1);
400 _mm_store_ps(packed_w + 12, v5);
401 _mm_store_ps(packed_w + 16, v2);
402 _mm_store_ps(packed_w + 20, v6);
403 _mm_store_ps(packed_w + 24, v3);
404 _mm_store_ps(packed_w + 28, v7);
405 packed_w += 32;
406 }
407 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
408 w0 = w7;
409 }
410
411 // NC remainder (1..7)
412 if XNN_UNLIKELY(n != 0) {
413 assert(n >= 1);
414 assert(n <= 7);
415 if XNN_LIKELY(b != NULL) {
416 size_t nb = n;
417 do {
418 *packed_w++ = *b++;
419 } while (--nb != 0);
420 packed_w += (8 - n);
421 } else {
422 const __m128 vzero = _mm_setzero_ps();
423 _mm_store_ps(packed_w, vzero);
424 _mm_store_ps(packed_w + 4, vzero);
425 packed_w += 8;
426 }
427
428 // NR remainder has less than 8 rows so last row is not loaded
429 const float* w1 = w0 + kc;
430 if XNN_UNPREDICTABLE(n < 2) {
431 w1 = w0;
432 }
433 const float* w2 = w1 + kc;
434 if XNN_UNPREDICTABLE(n <= 2) {
435 w2 = w1;
436 }
437 const float* w3 = w2 + kc;
438 if XNN_UNPREDICTABLE(n < 4) {
439 w3 = w2;
440 }
441 const float* w4 = w3 + kc;
442 if XNN_UNPREDICTABLE(n <= 4) {
443 w4 = w3;
444 }
445 const float* w5 = w4 + kc;
446 if XNN_UNPREDICTABLE(n < 6) {
447 w5 = w4;
448 }
449 const float* w6 = w5 + kc;
450 if XNN_UNPREDICTABLE(n <= 6) {
451 w6 = w5;
452 }
453
454 size_t k = kc;
455 // KC main loop multiple of 8
456 for (; k >= 8; k -= 8) {
457 // Read blocks of 4x4
458 // a b c d
459 // e f g h
460 // i j k l
461 // m n o p
462 __m128 v0x0123 = _mm_loadu_ps(w0);
463 __m128 v0x4567 = _mm_loadu_ps(w0 + 4);
464 w0 += 8;
465 __m128 v1x0123 = _mm_loadu_ps(w1);
466 __m128 v1x4567 = _mm_loadu_ps(w1 + 4);
467 w1 += 8;
468 __m128 v2x0123 = _mm_loadu_ps(w2);
469 __m128 v2x4567 = _mm_loadu_ps(w2 + 4);
470 w2 += 8;
471 __m128 v3x0123 = _mm_loadu_ps(w3);
472 __m128 v3x4567 = _mm_loadu_ps(w3 + 4);
473 w3 += 8;
474 __m128 v4x0123 = _mm_loadu_ps(w4);
475 __m128 v4x4567 = _mm_loadu_ps(w4 + 4);
476 w4 += 8;
477 __m128 v5x0123 = _mm_loadu_ps(w5);
478 __m128 v5x4567 = _mm_loadu_ps(w5 + 4);
479 w5 += 8;
480 __m128 v6x0123 = _mm_loadu_ps(w6);
481 __m128 v6x4567 = _mm_loadu_ps(w6 + 4);
482 w6 += 8;
483
484 __m128 v7x0123 = _mm_undefined_ps();
485 __m128 v7x4567 = _mm_undefined_ps();
486
487 // Apply SR4 shuffle
488 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
489 v1x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x4567), _MM_SHUFFLE(0, 3, 2, 1)));
490 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
491 v2x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x4567), _MM_SHUFFLE(1, 0, 3, 2)));
492 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
493 v3x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x4567), _MM_SHUFFLE(2, 1, 0, 3)));
494 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
495 v5x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x4567), _MM_SHUFFLE(0, 3, 2, 1)));
496 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
497 v6x4567 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x4567), _MM_SHUFFLE(1, 0, 3, 2)));
498
499 // Transpose 2x2
500 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
501 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
502 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
503 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
504 const __m128 vtmp0x4567 = _mm_unpacklo_ps(v0x4567, v1x4567); // a e b f from row 0, 1
505 const __m128 vtmp1x4567 = _mm_unpacklo_ps(v2x4567, v3x4567); // i m j n from row 2, 3
506 const __m128 vtmp2x4567 = _mm_unpackhi_ps(v0x4567, v1x4567); // c g d h from row 0, 1
507 const __m128 vtmp3x4567 = _mm_unpackhi_ps(v2x4567, v3x4567); // k o l p from row 2, 3
508 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
509 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v6x0123); // i m j n from row 2, 3
510 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
511 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v6x0123); // k o l p from row 2, 3
512 const __m128 vtmp4x4567 = _mm_unpacklo_ps(v4x4567, v5x4567); // a e b f from row 0, 1
513 const __m128 vtmp5x4567 = _mm_unpacklo_ps(v6x4567, v6x4567); // i m j n from row 2, 3
514 const __m128 vtmp6x4567 = _mm_unpackhi_ps(v4x4567, v5x4567); // c g d h from row 0, 1
515 const __m128 vtmp7x4567 = _mm_unpackhi_ps(v6x4567, v6x4567); // k o l p from row 2, 3
516 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
517 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
518 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
519 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
520 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
521 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
522 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
523 // Transpose 4x4
524 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
525 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
526 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
527 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
528 v0x4567 = _mm_movelh_ps(vtmp0x4567, vtmp1x4567); // a e i m from row 0, 1
529 v1x4567 = _mm_movehl_ps(vtmp1x4567, vtmp0x4567); // b f j n from row 0, 1
530 v2x4567 = _mm_movelh_ps(vtmp2x4567, vtmp3x4567); // c g k o from row 2, 3
531 v3x4567 = _mm_movehl_ps(vtmp3x4567, vtmp2x4567); // d h l p from row 2, 3
532 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
533 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
534 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
535 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
536 v4x4567 = _mm_movelh_ps(vtmp4x4567, vtmp5x4567); // a e i m from row 0, 1
537 v5x4567 = _mm_movehl_ps(vtmp5x4567, vtmp4x4567); // b f j n from row 0, 1
538 v6x4567 = _mm_movelh_ps(vtmp6x4567, vtmp7x4567); // c g k o from row 2, 3
539 v7x4567 = _mm_movehl_ps(vtmp7x4567, vtmp6x4567); // d h l p from row 2, 3
540
541 _mm_store_ps(packed_w, v0x0123);
542 _mm_store_ps(packed_w + 4, v4x0123);
543 _mm_store_ps(packed_w + 8, v1x0123);
544 _mm_store_ps(packed_w + 12, v5x0123);
545 _mm_store_ps(packed_w + 16, v2x0123);
546 _mm_store_ps(packed_w + 20, v6x0123);
547 _mm_store_ps(packed_w + 24, v3x0123);
548 _mm_store_ps(packed_w + 28, v7x0123);
549 _mm_store_ps(packed_w + 32, v0x4567);
550 _mm_store_ps(packed_w + 36, v4x4567);
551 _mm_store_ps(packed_w + 40, v1x4567);
552 _mm_store_ps(packed_w + 44, v5x4567);
553 _mm_store_ps(packed_w + 48, v2x4567);
554 _mm_store_ps(packed_w + 52, v6x4567);
555 _mm_store_ps(packed_w + 56, v3x4567);
556 _mm_store_ps(packed_w + 60, v7x4567);
557 packed_w += 64;
558 }
559
560 // KC multiple of 4
561 for (; k >= 4; k -= 4) {
562 // Read blocks of 4x4
563 // a b c d
564 // e f g h
565 // i j k l
566 // m n o p
567 __m128 v0x0123 = _mm_loadu_ps(w0);
568 w0 += 4;
569 __m128 v1x0123 = _mm_loadu_ps(w1);
570 w1 += 4;
571 __m128 v2x0123 = _mm_loadu_ps(w2);
572 w2 += 4;
573 __m128 v3x0123 = _mm_loadu_ps(w3);
574 w3 += 4;
575 __m128 v4x0123 = _mm_loadu_ps(w4);
576 w4 += 4;
577 __m128 v5x0123 = _mm_loadu_ps(w5);
578 w5 += 4;
579 __m128 v6x0123 = _mm_loadu_ps(w6);
580 w6 += 4;
581 __m128 v7x0123 = _mm_undefined_ps();
582
583 // Apply SR4 shuffle
584 v1x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1x0123), _MM_SHUFFLE(0, 3, 2, 1)));
585 v2x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2x0123), _MM_SHUFFLE(1, 0, 3, 2)));
586 v3x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3x0123), _MM_SHUFFLE(2, 1, 0, 3)));
587 v5x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5x0123), _MM_SHUFFLE(0, 3, 2, 1)));
588 v6x0123 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6x0123), _MM_SHUFFLE(1, 0, 3, 2)));
589
590 // Transpose 2x2
591 const __m128 vtmp0x0123 = _mm_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
592 const __m128 vtmp1x0123 = _mm_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
593 const __m128 vtmp2x0123 = _mm_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
594 const __m128 vtmp3x0123 = _mm_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
595 const __m128 vtmp4x0123 = _mm_unpacklo_ps(v4x0123, v5x0123); // a e b f from row 0, 1
596 const __m128 vtmp5x0123 = _mm_unpacklo_ps(v6x0123, v6x0123); // i m j n from row 2, 3
597 const __m128 vtmp6x0123 = _mm_unpackhi_ps(v4x0123, v5x0123); // c g d h from row 0, 1
598 const __m128 vtmp7x0123 = _mm_unpackhi_ps(v6x0123, v6x0123); // k o l p from row 2, 3
599 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
600 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
601 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
602 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
603 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
604 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
605 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
606 // Transpose 4x4
607 v0x0123 = _mm_movelh_ps(vtmp0x0123, vtmp1x0123); // a e i m from row 0, 1
608 v1x0123 = _mm_movehl_ps(vtmp1x0123, vtmp0x0123); // b f j n from row 0, 1
609 v2x0123 = _mm_movelh_ps(vtmp2x0123, vtmp3x0123); // c g k o from row 2, 3
610 v3x0123 = _mm_movehl_ps(vtmp3x0123, vtmp2x0123); // d h l p from row 2, 3
611 v4x0123 = _mm_movelh_ps(vtmp4x0123, vtmp5x0123); // a e i m from row 0, 1
612 v5x0123 = _mm_movehl_ps(vtmp5x0123, vtmp4x0123); // b f j n from row 0, 1
613 v6x0123 = _mm_movelh_ps(vtmp6x0123, vtmp7x0123); // c g k o from row 2, 3
614 v7x0123 = _mm_movehl_ps(vtmp7x0123, vtmp6x0123); // d h l p from row 2, 3
615
616 _mm_store_ps(packed_w, v0x0123);
617 _mm_store_ps(packed_w + 4, v4x0123);
618 _mm_store_ps(packed_w + 8, v1x0123);
619 _mm_store_ps(packed_w + 12, v5x0123);
620 _mm_store_ps(packed_w + 16, v2x0123);
621 _mm_store_ps(packed_w + 20, v6x0123);
622 _mm_store_ps(packed_w + 24, v3x0123);
623 _mm_store_ps(packed_w + 28, v7x0123);
624 packed_w += 32;
625 }
626
627 // KC remainder (1..3)
628 if XNN_UNLIKELY(k != 0) {
629 assert(k >= 1);
630 assert(k <= 3);
631 __m128 v0 = _mm_undefined_ps();
632 __m128 v1 = _mm_undefined_ps();
633 __m128 v2 = _mm_undefined_ps();
634 __m128 v3 = _mm_undefined_ps();
635 __m128 v4 = _mm_undefined_ps();
636 __m128 v5 = _mm_undefined_ps();
637 __m128 v6 = _mm_undefined_ps();
638 __m128 v7 = _mm_undefined_ps();
639
640 switch (k) {
641 case 1:
642 // Read blocks of 4x1
643 // a
644 // e
645 // i
646 // m
647 v0 = _mm_load_ss(w0);
648 w0 += 1;
649 v1 = _mm_load_ss(w1);
650 w1 += 1;
651 v2 = _mm_load_ss(w2);
652 w2 += 1;
653 v3 = _mm_load_ss(w3);
654 w3 += 1;
655 v4 = _mm_load_ss(w4);
656 w4 += 1;
657 v5 = _mm_load_ss(w5);
658 w5 += 1;
659 v6 = _mm_load_ss(w6);
660 w6 += 1;
661 break;
662 case 2:
663 // Read blocks of 4x2
664 // a b
665 // e f
666 // i j
667 // m n
668 v0 = _mm_castpd_ps(_mm_load_sd((const double*) w0));
669 w0 += 2;
670 v1 = _mm_castpd_ps(_mm_load_sd((const double*) w1));
671 w1 += 2;
672 v2 = _mm_castpd_ps(_mm_load_sd((const double*) w2));
673 w2 += 2;
674 v3 = _mm_castpd_ps(_mm_load_sd((const double*) w3));
675 w3 += 2;
676 v4 = _mm_castpd_ps(_mm_load_sd((const double*) w4));
677 w4 += 2;
678 v5 = _mm_castpd_ps(_mm_load_sd((const double*) w5));
679 w5 += 2;
680 v6 = _mm_castpd_ps(_mm_load_sd((const double*) w6));
681 w6 += 2;
682 break;
683 case 3:
684 {
685 // Read blocks of 4x3
686 // a b c
687 // e f g
688 // i j k
689 // m n o
690 const __m128 v0lo = _mm_castpd_ps(_mm_load_sd((const double*) w0));
691 const __m128 v0hi = _mm_load_ss(w0 + 2);
692 v0 = _mm_movelh_ps(v0lo, v0hi);
693 w0 += 3;
694 const __m128 v1lo = _mm_castpd_ps(_mm_load_sd((const double*) w1));
695 const __m128 v1hi = _mm_load_ss(w1 + 2);
696 v1 = _mm_movelh_ps(v1lo, v1hi);
697 w1 += 3;
698 const __m128 v2lo = _mm_castpd_ps(_mm_load_sd((const double*) w2));
699 const __m128 v2hi = _mm_load_ss(w2 + 2);
700 v2 = _mm_movelh_ps(v2lo, v2hi);
701 w2 += 3;
702 const __m128 v3lo = _mm_castpd_ps(_mm_load_sd((const double*) w3));
703 const __m128 v3hi = _mm_load_ss(w3 + 2);
704 v3 = _mm_movelh_ps(v3lo, v3hi);
705 w3 += 3;
706 const __m128 v4lo = _mm_castpd_ps(_mm_load_sd((const double*) w4));
707 const __m128 v4hi = _mm_load_ss(w4 + 2);
708 v4 = _mm_movelh_ps(v4lo, v4hi);
709 w4 += 3;
710 const __m128 v5lo = _mm_castpd_ps(_mm_load_sd((const double*) w5));
711 const __m128 v5hi = _mm_load_ss(w5 + 2);
712 v5 = _mm_movelh_ps(v5lo, v5hi);
713 w5 += 3;
714 const __m128 v6lo = _mm_castpd_ps(_mm_load_sd((const double*) w6));
715 const __m128 v6hi = _mm_load_ss(w6 + 2);
716 v6 = _mm_movelh_ps(v6lo, v6hi);
717 w6 += 3;
718 break;
719 }
720 default:
721 XNN_UNREACHABLE;
722 }
723
724 // Apply SR4 shuffle
725 v1 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v1), _MM_SHUFFLE(0, 3, 2, 1)));
726 v2 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v2), _MM_SHUFFLE(1, 0, 3, 2)));
727 v3 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v3), _MM_SHUFFLE(2, 1, 0, 3)));
728 v5 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v5), _MM_SHUFFLE(0, 3, 2, 1)));
729 v6 = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v6), _MM_SHUFFLE(1, 0, 3, 2)));
730 // Transpose 2x2
731 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a e b f from row 0, 1
732 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // i m j n from row 2, 3
733 const __m128 vtmp2 = _mm_unpackhi_ps(v0, v1); // c g d h from row 0, 1
734 const __m128 vtmp3 = _mm_unpackhi_ps(v2, v3); // k o l p from row 2, 3
735 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a e b f from row 0, 1
736 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v6); // i m j n from row 2, 3
737 const __m128 vtmp6 = _mm_unpackhi_ps(v4, v5); // c g d h from row 0, 1
738 const __m128 vtmp7 = _mm_unpackhi_ps(v6, v6); // k o l p from row 2, 3
739 // Transpose 4x4
740 v0 = _mm_movelh_ps(vtmp0, vtmp1); // a e i m from row 0, 1
741 v1 = _mm_movehl_ps(vtmp1, vtmp0); // b f j n from row 0, 1
742 v2 = _mm_movelh_ps(vtmp2, vtmp3); // c g k o from row 2, 3
743 v3 = _mm_movehl_ps(vtmp3, vtmp2); // d h l p from row 2, 3
744 v4 = _mm_movelh_ps(vtmp4, vtmp5); // a e i m from row 0, 1
745 v5 = _mm_movehl_ps(vtmp5, vtmp4); // b f j n from row 0, 1
746 v6 = _mm_movelh_ps(vtmp6, vtmp7); // c g k o from row 2, 3
747 v7 = _mm_movehl_ps(vtmp7, vtmp6); // d h l p from row 2, 3
748 _mm_store_ps(packed_w, v0);
749 _mm_store_ps(packed_w + 4, v4);
750 _mm_store_ps(packed_w + 8, v1);
751 _mm_store_ps(packed_w + 12, v5);
752 _mm_store_ps(packed_w + 16, v2);
753 _mm_store_ps(packed_w + 20, v6);
754 _mm_store_ps(packed_w + 24, v3);
755 _mm_store_ps(packed_w + 28, v7);
756 packed_w += 32;
757 }
758 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
759 }
760 weights += nc * kc;
761 } while (--g != 0);
762 }