Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-avx-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/intrinsics-polyfill.h"
18 #include "xnnpack/packw.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x16__avx_u4(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 16); // This kernel is for NR=16
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 const float* b = (const float*) bias;
45 float* packed_w = (float*) packed_weights;
46 do {
47 // NC main loop multiple of 16
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50
51 for (; n >= 16; n -= 16) {
52 if XNN_LIKELY(b != NULL) {
53 const __m256 vb0 = _mm256_loadu_ps(b);
54 const __m256 vb8 = _mm256_loadu_ps(b + 8);
55 _mm256_store_ps(packed_w, vb0);
56 _mm256_store_ps(packed_w + 8, vb8);
57 b += 16;
58 } else {
59 const __m256 vzero = _mm256_setzero_ps();
60 _mm256_store_ps(packed_w, vzero);
61 _mm256_store_ps(packed_w + 8, vzero);
62 }
63 packed_w += 16;
64
65 const float* w1 = w0 + kc;
66 const float* w2 = w1 + kc;
67 const float* w3 = w2 + kc;
68 const float* w4 = w3 + kc;
69 const float* w5 = w4 + kc;
70 const float* w6 = w5 + kc;
71 const float* w7 = w6 + kc;
72 const float* w8 = w7 + kc;
73 const float* w9 = w8 + kc;
74 const float* w10 = w9 + kc;
75 const float* w11 = w10 + kc;
76 const float* w12 = w11 + kc;
77 const float* w13 = w12 + kc;
78 const float* w14 = w13 + kc;
79 const float* w15 = w14 + kc;
80
81 // KC main loop multiple of 4
82 size_t k = kc;
83 for (; k >= 4; k -= 4) {
84 // Read blocks of 4x4
85 // a b c d
86 // e f g h
87 // i j k l
88 // m n o p
89 // Load first 4 rows of N into low part of each register
90 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
91 w0 += 4;
92 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
93 w1 += 4;
94 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
95 w2 += 4;
96 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
97 w3 += 4;
98 __m256 v8x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w8));
99 w8 += 4;
100 __m256 v9x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w9));
101 w9 += 4;
102 __m256 v10x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w10));
103 w10 += 4;
104 __m256 v11x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w11));
105 w11 += 4;
106 // Load next 4 rows of N into the high part of each register
107 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
108 w4 += 4;
109 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
110 w5 += 4;
111 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
112 w6 += 4;
113 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
114 w7 += 4;
115 v8x0123 = _mm256_insertf128_ps(v8x0123, _mm_loadu_ps(w12), 1);
116 w12 += 4;
117 v9x0123 = _mm256_insertf128_ps(v9x0123, _mm_loadu_ps(w13), 1);
118 w13 += 4;
119 v10x0123 = _mm256_insertf128_ps(v10x0123, _mm_loadu_ps(w14), 1);
120 w14 += 4;
121 v11x0123 = _mm256_insertf128_ps(v11x0123, _mm_loadu_ps(w15), 1);
122 w15 += 4;
123
124 // Transpose 2x2
125 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
126 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
127 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
128 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
129 const __m256 vtmp8x0123 = _mm256_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
130 const __m256 vtmp9x0123 = _mm256_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
131 const __m256 vtmp10x0123 = _mm256_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
132 const __m256 vtmp11x0123 = _mm256_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
133 // Transpose 4x4
134 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
135 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
136 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
137 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
138 v8x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // a e i m from row 0, 1
139 v9x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // b f j n from row 0, 1
140 v10x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // c g k o from row 2, 3
141 v11x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // d h l p from row 2, 3
142
143 _mm256_store_ps(packed_w, v0x0123);
144 _mm256_store_ps(packed_w + 8, v8x0123);
145 _mm256_store_ps(packed_w + 16, v1x0123);
146 _mm256_store_ps(packed_w + 24, v9x0123);
147 _mm256_store_ps(packed_w + 32, v2x0123);
148 _mm256_store_ps(packed_w + 40, v10x0123);
149 _mm256_store_ps(packed_w + 48, v3x0123);
150 _mm256_store_ps(packed_w + 56, v11x0123);
151 packed_w += 64;
152 }
153
154 // KC remainder (1..3)
155 if XNN_UNLIKELY(k != 0) {
156 assert(k >= 1);
157 assert(k <= 3);
158 if (k & 2) {
159 // Read blocks of 4x2
160 // a b
161 // c d
162 // e f
163 // g h
164 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
165 w0 += 2;
166 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
167 w1 += 2;
168 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
169 w2 += 2;
170 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
171 w3 += 2;
172 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
173 w4 += 2;
174 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
175 w5 += 2;
176 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
177 w6 += 2;
178 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
179 w7 += 2;
180 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
181 w8 += 2;
182 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
183 w9 += 2;
184 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
185 w10 += 2;
186 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
187 w11 += 2;
188 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
189 w12 += 2;
190 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
191 w13 += 2;
192 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
193 w14 += 2;
194 __m128 v15 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
195 w15 += 2;
196
197 // Transpose 2x2
198 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
199 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
200 // Transpose 4x4
201 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
202 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
203 // Transpose 2x2
204 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
205 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
206 // Transpose 4x4
207 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
208 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
209 // Transpose 2x2
210 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
211 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
212 // Transpose 4x4
213 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
214 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
215 // Transpose 2x2
216 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
217 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // e g f h from row 2, 3
218 // Transpose 4x4
219 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
220 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
221
222 _mm_store_ps(packed_w, v0);
223 _mm_store_ps(packed_w + 4, v4);
224 _mm_store_ps(packed_w + 8, v8);
225 _mm_store_ps(packed_w + 12, v12);
226 _mm_store_ps(packed_w + 16, v1);
227 _mm_store_ps(packed_w + 20, v5);
228 _mm_store_ps(packed_w + 24, v9);
229 _mm_store_ps(packed_w + 28, v13);
230 packed_w += 32;
231 }
232 if (k & 1) {
233 // Read blocks of 4x1
234 // a
235 // b
236 // c
237 // d
238 __m128 v0 = _mm_load_ss(w0); w0 += 1;
239 __m128 v1 = _mm_load_ss(w1); w1 += 1;
240 __m128 v2 = _mm_load_ss(w2); w2 += 1;
241 __m128 v3 = _mm_load_ss(w3); w3 += 1;
242 __m128 v4 = _mm_load_ss(w4); w4 += 1;
243 __m128 v5 = _mm_load_ss(w5); w5 += 1;
244 __m128 v6 = _mm_load_ss(w6); w6 += 1;
245 __m128 v7 = _mm_load_ss(w7); w7 += 1;
246 __m128 v8 = _mm_load_ss(w8); w8 += 1;
247 __m128 v9 = _mm_load_ss(w9); w9 += 1;
248 __m128 v10 = _mm_load_ss(w10); w10 += 1;
249 __m128 v11 = _mm_load_ss(w11); w11 += 1;
250 __m128 v12 = _mm_load_ss(w12); w12 += 1;
251 __m128 v13 = _mm_load_ss(w13); w13 += 1;
252 __m128 v14 = _mm_load_ss(w14); w14 += 1;
253 __m128 v15 = _mm_load_ss(w15); w15 += 1;
254
255 // Transpose 2x2
256 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
257 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
258 // Transpose 4x4
259 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
260 // Transpose 2x2
261 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
262 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
263 // Transpose 4x4
264 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
265 // Transpose 2x2
266 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
267 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
268 // Transpose 4x4
269 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
270 // Transpose 2x2
271 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
272 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // c d from row 2, 3
273 // Transpose 4x4
274 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
275
276 _mm_store_ps(packed_w, v0);
277 _mm_store_ps(packed_w + 4, v4);
278 _mm_store_ps(packed_w + 8, v8);
279 _mm_store_ps(packed_w + 12, v12);
280 packed_w += 16;
281 }
282 }
283 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
284 w0 = w15;
285 }
286
287 // NC remainder (1..15)
288 if XNN_UNLIKELY(n != 0) {
289 assert(n >= 1);
290 assert(n <= 15);
291 if XNN_LIKELY(b != NULL) {
292 size_t nb = n;
293 do {
294 *packed_w++ = *b++;
295 } while (--nb != 0);
296 packed_w += (16 - n);
297 } else {
298 const __m256 vzero = _mm256_setzero_ps();
299 _mm256_store_ps(packed_w, vzero);
300 _mm256_store_ps(packed_w + 8, vzero);
301 packed_w += 16;
302 }
303
304 // NR remainder has less than 16 rows so last row is not loaded
305 // For SR=4 the
306 const float* w1 = w0 + kc;
307 if XNN_UNPREDICTABLE(n < 2) {
308 w1 = w0;
309 }
310 const float* w2 = w1 + kc;
311 if XNN_UNPREDICTABLE(n <= 2) {
312 w2 = w1;
313 }
314 const float* w3 = w2 + kc;
315 if XNN_UNPREDICTABLE(n < 4) {
316 w3 = w2;
317 }
318 const float* w4 = w3 + kc;
319 if XNN_UNPREDICTABLE(n <= 4) {
320 w4 = w3;
321 }
322 const float* w5 = w4 + kc;
323 if XNN_UNPREDICTABLE(n < 6) {
324 w5 = w4;
325 }
326 const float* w6 = w5 + kc;
327 if XNN_UNPREDICTABLE(n <= 6) {
328 w6 = w5;
329 }
330 const float* w7 = w6 + kc;
331 if XNN_UNPREDICTABLE(n < 8) {
332 w7 = w6;
333 }
334 const float* w8 = w7 + kc;
335 if XNN_UNPREDICTABLE(n <= 8) {
336 w8 = w7;
337 }
338 const float* w9 = w8 + kc;
339 if XNN_UNPREDICTABLE(n < 10) {
340 w9 = w8;
341 }
342 const float* w10 = w9 + kc;
343 if XNN_UNPREDICTABLE(n <= 10) {
344 w10 = w9;
345 }
346 const float* w11 = w10 + kc;
347 if XNN_UNPREDICTABLE(n < 12) {
348 w11 = w10;
349 }
350 const float* w12 = w11 + kc;
351 if XNN_UNPREDICTABLE(n <= 12) {
352 w12 = w11;
353 }
354 const float* w13 = w12 + kc;
355 if XNN_UNPREDICTABLE(n < 14) {
356 w13 = w12;
357 }
358 const float* w14 = w13 + kc;
359 if XNN_UNPREDICTABLE(n <= 14) {
360 w14 = w13;
361 }
362
363 // KC main loop multiple of 4
364 size_t k = kc;
365 for (; k >= 4; k -= 4) {
366 // Read blocks of 4x4
367 // a b c d
368 // e f g h
369 // i j k l
370 // m n o p
371 // Load first 4 rows of N into low part of each register
372 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
373 w0 += 4;
374 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
375 w1 += 4;
376 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
377 w2 += 4;
378 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
379 w3 += 4;
380 __m256 v8x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w8));
381 w8 += 4;
382 __m256 v9x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w9));
383 w9 += 4;
384 __m256 v10x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w10));
385 w10 += 4;
386 // castps leaves upper 128 bits undefined, so zero them.
387 __m256 v11x0123 = _mm256_zextps128_ps256(_mm_loadu_ps(w11));
388 w11 += 4;
389 // Load next 4 rows of N into the high part of each register
390 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
391 w4 += 4;
392 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
393 w5 += 4;
394 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
395 w6 += 4;
396 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
397 w7 += 4;
398 v8x0123 = _mm256_insertf128_ps(v8x0123, _mm_loadu_ps(w12), 1);
399 w12 += 4;
400 v9x0123 = _mm256_insertf128_ps(v9x0123, _mm_loadu_ps(w13), 1);
401 w13 += 4;
402 v10x0123 = _mm256_insertf128_ps(v10x0123, _mm_loadu_ps(w14), 1);
403 w14 += 4;
404
405 // Transpose 2x2
406 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
407 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
408 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
409 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
410 const __m256 vtmp8x0123 = _mm256_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
411 const __m256 vtmp9x0123 = _mm256_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
412 const __m256 vtmp10x0123 = _mm256_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
413 const __m256 vtmp11x0123 = _mm256_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
414 // Transpose 4x4
415 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
416 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
417 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
418 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
419 v8x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // a e i m from row 0, 1
420 v9x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // b f j n from row 0, 1
421 v10x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // c g k o from row 2, 3
422 v11x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // d h l p from row 2, 3
423
424 _mm256_store_ps(packed_w, v0x0123);
425 _mm256_store_ps(packed_w + 8, v8x0123);
426 _mm256_store_ps(packed_w + 16, v1x0123);
427 _mm256_store_ps(packed_w + 24, v9x0123);
428 _mm256_store_ps(packed_w + 32, v2x0123);
429 _mm256_store_ps(packed_w + 40, v10x0123);
430 _mm256_store_ps(packed_w + 48, v3x0123);
431 _mm256_store_ps(packed_w + 56, v11x0123);
432 packed_w += 64;
433 }
434
435 // KC remainder (1..3)
436 if XNN_UNLIKELY(k != 0) {
437 assert(k >= 1);
438 assert(k <= 3);
439 if (k & 2) {
440 // Read blocks of 4x2
441 // a b
442 // c d
443 // e f
444 // g h
445 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
446 w0 += 2;
447 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
448 w1 += 2;
449 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
450 w2 += 2;
451 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
452 w3 += 2;
453 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
454 w4 += 2;
455 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
456 w5 += 2;
457 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
458 w6 += 2;
459 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
460 w7 += 2;
461 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
462 w8 += 2;
463 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
464 w9 += 2;
465 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
466 w10 += 2;
467 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
468 w11 += 2;
469 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
470 w12 += 2;
471 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
472 w13 += 2;
473 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
474 w14 += 2;
475
476 // Transpose 2x2
477 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
478 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
479 // Transpose 4x4
480 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
481 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
482 // Transpose 2x2
483 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
484 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
485 // Transpose 4x4
486 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
487 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
488 // Transpose 2x2
489 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
490 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
491 // Transpose 4x4
492 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
493 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
494 // Transpose 2x2
495 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
496 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // e g f h from row 2, 3
497 // Transpose 4x4
498 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
499 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
500
501 _mm_store_ps(packed_w, v0);
502 _mm_store_ps(packed_w + 4, v4);
503 _mm_store_ps(packed_w + 8, v8);
504 _mm_store_ps(packed_w + 12, v12);
505 _mm_store_ps(packed_w + 16, v1);
506 _mm_store_ps(packed_w + 20, v5);
507 _mm_store_ps(packed_w + 24, v9);
508 _mm_store_ps(packed_w + 28, v13);
509 packed_w += 32;
510 }
511 if (k & 1) {
512 // Read blocks of 4x1
513 // a
514 // b
515 // c
516 // d
517 __m128 v0 = _mm_load_ss(w0); w0 += 1;
518 __m128 v1 = _mm_load_ss(w1); w1 += 1;
519 __m128 v2 = _mm_load_ss(w2); w2 += 1;
520 __m128 v3 = _mm_load_ss(w3); w3 += 1;
521 __m128 v4 = _mm_load_ss(w4); w4 += 1;
522 __m128 v5 = _mm_load_ss(w5); w5 += 1;
523 __m128 v6 = _mm_load_ss(w6); w6 += 1;
524 __m128 v7 = _mm_load_ss(w7); w7 += 1;
525 __m128 v8 = _mm_load_ss(w8); w8 += 1;
526 __m128 v9 = _mm_load_ss(w9); w9 += 1;
527 __m128 v10 = _mm_load_ss(w10); w10 += 1;
528 __m128 v11 = _mm_load_ss(w11); w11 += 1;
529 __m128 v12 = _mm_load_ss(w12); w12 += 1;
530 __m128 v13 = _mm_load_ss(w13); w13 += 1;
531 __m128 v14 = _mm_load_ss(w14); w14 += 1;
532
533 // Transpose 2x2
534 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
535 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
536 // Transpose 4x4
537 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
538 // Transpose 2x2
539 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
540 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
541 // Transpose 4x4
542 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
543 // Transpose 2x2
544 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
545 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
546 // Transpose 4x4
547 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
548 // Transpose 2x2
549 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
550 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // c d from row 2, 3
551 // Transpose 4x4
552 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
553
554 _mm_store_ps(packed_w, v0);
555 _mm_store_ps(packed_w + 4, v4);
556 _mm_store_ps(packed_w + 8, v8);
557 _mm_store_ps(packed_w + 12, v12);
558 packed_w += 16;
559 }
560 }
561 }
562 weights += nc * kc;
563 } while (--g != 0);
564 }