Debian Codegraph @

src/x32-packw/gen/x32-packw-x16-gemm-goi-avx-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/intrinsics-polyfill.h"
18 #include "xnnpack/packw.h"
19 #include "xnnpack/prefetch.h"
20
21
22 void xnn_x32_packw_gemm_goi_ukernel_x16__avx_u4_prfm(
23 size_t g,
24 size_t nc,
25 size_t kc,
26 size_t nr,
27 size_t kr,
28 size_t sr,
29 const uint32_t* weights,
30 const uint32_t* bias,
31 const void* scale,
32 uint32_t* packed_weights,
33 size_t extra_bytes,
34 const void* params)
35 {
36 assert(g != 0);
37 assert(nc != 0);
38 assert(kc != 0);
39 assert(nr == 16); // This kernel is for NR=16
40 assert(kr == 1);
41 assert(sr == 1);
42 assert(weights != NULL);
43 assert(packed_weights != NULL);
44
45 const float* b = (const float*) bias;
46 float* packed_w = (float*) packed_weights;
47 do {
48 // NC main loop multiple of 16
49 const float* w0 = (const float*) weights;
50 size_t n = nc;
51
52 for (; n >= 16; n -= 16) {
53 if XNN_LIKELY(b != NULL) {
54 const __m256 vb0 = _mm256_loadu_ps(b);
55 const __m256 vb8 = _mm256_loadu_ps(b + 8);
56 _mm256_store_ps(packed_w, vb0);
57 _mm256_store_ps(packed_w + 8, vb8);
58 b += 16;
59 } else {
60 const __m256 vzero = _mm256_setzero_ps();
61 _mm256_store_ps(packed_w, vzero);
62 _mm256_store_ps(packed_w + 8, vzero);
63 }
64 packed_w += 16;
65
66 const float* w1 = w0 + kc;
67 const float* w2 = w1 + kc;
68 const float* w3 = w2 + kc;
69 const float* w4 = w3 + kc;
70 const float* w5 = w4 + kc;
71 const float* w6 = w5 + kc;
72 const float* w7 = w6 + kc;
73 const float* w8 = w7 + kc;
74 const float* w9 = w8 + kc;
75 const float* w10 = w9 + kc;
76 const float* w11 = w10 + kc;
77 const float* w12 = w11 + kc;
78 const float* w13 = w12 + kc;
79 const float* w14 = w13 + kc;
80 const float* w15 = w14 + kc;
81 xnn_prefetch_to_l1((const int8_t*) w0);
82 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
83 xnn_prefetch_to_l1((const int8_t*) w1);
84 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
85 xnn_prefetch_to_l1((const int8_t*) w2);
86 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
87 xnn_prefetch_to_l1((const int8_t*) w3);
88 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
89 xnn_prefetch_to_l1((const int8_t*) w4);
90 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
91 xnn_prefetch_to_l1((const int8_t*) w5);
92 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
93 xnn_prefetch_to_l1((const int8_t*) w6);
94 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
95 xnn_prefetch_to_l1((const int8_t*) w7);
96 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
97 xnn_prefetch_to_l1((const int8_t*) w8);
98 xnn_prefetch_to_l1((const int8_t*) w8 + 64);
99 xnn_prefetch_to_l1((const int8_t*) w9);
100 xnn_prefetch_to_l1((const int8_t*) w9 + 64);
101 xnn_prefetch_to_l1((const int8_t*) w10);
102 xnn_prefetch_to_l1((const int8_t*) w10 + 64);
103 xnn_prefetch_to_l1((const int8_t*) w11);
104 xnn_prefetch_to_l1((const int8_t*) w11 + 64);
105 xnn_prefetch_to_l1((const int8_t*) w12);
106 xnn_prefetch_to_l1((const int8_t*) w12 + 64);
107 xnn_prefetch_to_l1((const int8_t*) w13);
108 xnn_prefetch_to_l1((const int8_t*) w13 + 64);
109 xnn_prefetch_to_l1((const int8_t*) w14);
110 xnn_prefetch_to_l1((const int8_t*) w14 + 64);
111 xnn_prefetch_to_l1((const int8_t*) w15);
112 xnn_prefetch_to_l1((const int8_t*) w15 + 64);
113
114 // KC main loop multiple of 4
115 size_t k = kc;
116 for (; k >= 4; k -= 4) {
117 // Read blocks of 4x4
118 // a b c d
119 // e f g h
120 // i j k l
121 // m n o p
122 // Load first 4 rows of N into low part of each register
123 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
124 w0 += 4;
125 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
126 w1 += 4;
127 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
128 w2 += 4;
129 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
130 w3 += 4;
131 __m256 v8x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w8));
132 w8 += 4;
133 __m256 v9x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w9));
134 w9 += 4;
135 __m256 v10x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w10));
136 w10 += 4;
137 __m256 v11x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w11));
138 w11 += 4;
139 // Load next 4 rows of N into the high part of each register
140 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
141 w4 += 4;
142 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
143 w5 += 4;
144 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
145 w6 += 4;
146 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
147 w7 += 4;
148 v8x0123 = _mm256_insertf128_ps(v8x0123, _mm_loadu_ps(w12), 1);
149 w12 += 4;
150 v9x0123 = _mm256_insertf128_ps(v9x0123, _mm_loadu_ps(w13), 1);
151 w13 += 4;
152 v10x0123 = _mm256_insertf128_ps(v10x0123, _mm_loadu_ps(w14), 1);
153 w14 += 4;
154 v11x0123 = _mm256_insertf128_ps(v11x0123, _mm_loadu_ps(w15), 1);
155 w15 += 4;
156
157 // Transpose 2x2
158 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
159 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
160 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
161 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
162 const __m256 vtmp8x0123 = _mm256_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
163 const __m256 vtmp9x0123 = _mm256_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
164 const __m256 vtmp10x0123 = _mm256_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
165 const __m256 vtmp11x0123 = _mm256_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
166 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
167 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
168 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
169 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
170 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
171 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
172 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
173 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
174 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
175 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
176 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
177 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
178 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
179 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
180 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
181 xnn_prefetch_to_l1((const int8_t*) w15 + 128);
182 // Transpose 4x4
183 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
184 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
185 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
186 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
187 v8x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // a e i m from row 0, 1
188 v9x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // b f j n from row 0, 1
189 v10x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // c g k o from row 2, 3
190 v11x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // d h l p from row 2, 3
191
192 _mm256_store_ps(packed_w, v0x0123);
193 _mm256_store_ps(packed_w + 8, v8x0123);
194 _mm256_store_ps(packed_w + 16, v1x0123);
195 _mm256_store_ps(packed_w + 24, v9x0123);
196 _mm256_store_ps(packed_w + 32, v2x0123);
197 _mm256_store_ps(packed_w + 40, v10x0123);
198 _mm256_store_ps(packed_w + 48, v3x0123);
199 _mm256_store_ps(packed_w + 56, v11x0123);
200 packed_w += 64;
201 }
202
203 // KC remainder (1..3)
204 if XNN_UNLIKELY(k != 0) {
205 assert(k >= 1);
206 assert(k <= 3);
207 if (k & 2) {
208 // Read blocks of 4x2
209 // a b
210 // c d
211 // e f
212 // g h
213 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
214 w0 += 2;
215 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
216 w1 += 2;
217 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
218 w2 += 2;
219 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
220 w3 += 2;
221 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
222 w4 += 2;
223 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
224 w5 += 2;
225 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
226 w6 += 2;
227 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
228 w7 += 2;
229 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
230 w8 += 2;
231 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
232 w9 += 2;
233 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
234 w10 += 2;
235 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
236 w11 += 2;
237 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
238 w12 += 2;
239 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
240 w13 += 2;
241 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
242 w14 += 2;
243 __m128 v15 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w15));
244 w15 += 2;
245
246 // Transpose 2x2
247 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
248 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
249 // Transpose 4x4
250 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
251 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
252 // Transpose 2x2
253 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
254 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
255 // Transpose 4x4
256 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
257 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
258 // Transpose 2x2
259 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
260 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
261 // Transpose 4x4
262 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
263 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
264 // Transpose 2x2
265 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
266 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // e g f h from row 2, 3
267 // Transpose 4x4
268 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
269 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
270
271 _mm_store_ps(packed_w, v0);
272 _mm_store_ps(packed_w + 4, v4);
273 _mm_store_ps(packed_w + 8, v8);
274 _mm_store_ps(packed_w + 12, v12);
275 _mm_store_ps(packed_w + 16, v1);
276 _mm_store_ps(packed_w + 20, v5);
277 _mm_store_ps(packed_w + 24, v9);
278 _mm_store_ps(packed_w + 28, v13);
279 packed_w += 32;
280 }
281 if (k & 1) {
282 // Read blocks of 4x1
283 // a
284 // b
285 // c
286 // d
287 __m128 v0 = _mm_load_ss(w0); w0 += 1;
288 __m128 v1 = _mm_load_ss(w1); w1 += 1;
289 __m128 v2 = _mm_load_ss(w2); w2 += 1;
290 __m128 v3 = _mm_load_ss(w3); w3 += 1;
291 __m128 v4 = _mm_load_ss(w4); w4 += 1;
292 __m128 v5 = _mm_load_ss(w5); w5 += 1;
293 __m128 v6 = _mm_load_ss(w6); w6 += 1;
294 __m128 v7 = _mm_load_ss(w7); w7 += 1;
295 __m128 v8 = _mm_load_ss(w8); w8 += 1;
296 __m128 v9 = _mm_load_ss(w9); w9 += 1;
297 __m128 v10 = _mm_load_ss(w10); w10 += 1;
298 __m128 v11 = _mm_load_ss(w11); w11 += 1;
299 __m128 v12 = _mm_load_ss(w12); w12 += 1;
300 __m128 v13 = _mm_load_ss(w13); w13 += 1;
301 __m128 v14 = _mm_load_ss(w14); w14 += 1;
302 __m128 v15 = _mm_load_ss(w15); w15 += 1;
303
304 // Transpose 2x2
305 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
306 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
307 // Transpose 4x4
308 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
309 // Transpose 2x2
310 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
311 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
312 // Transpose 4x4
313 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
314 // Transpose 2x2
315 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
316 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
317 // Transpose 4x4
318 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
319 // Transpose 2x2
320 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
321 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v15); // c d from row 2, 3
322 // Transpose 4x4
323 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
324
325 _mm_store_ps(packed_w, v0);
326 _mm_store_ps(packed_w + 4, v4);
327 _mm_store_ps(packed_w + 8, v8);
328 _mm_store_ps(packed_w + 12, v12);
329 packed_w += 16;
330 }
331 }
332 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
333 w0 = w15;
334 }
335
336 // NC remainder (1..15)
337 if XNN_UNLIKELY(n != 0) {
338 assert(n >= 1);
339 assert(n <= 15);
340 if XNN_LIKELY(b != NULL) {
341 size_t nb = n;
342 do {
343 *packed_w++ = *b++;
344 } while (--nb != 0);
345 packed_w += (16 - n);
346 } else {
347 const __m256 vzero = _mm256_setzero_ps();
348 _mm256_store_ps(packed_w, vzero);
349 _mm256_store_ps(packed_w + 8, vzero);
350 packed_w += 16;
351 }
352
353 // NR remainder has less than 16 rows so last row is not loaded
354 // For SR=4 the
355 const float* w1 = w0 + kc;
356 if XNN_UNPREDICTABLE(n < 2) {
357 w1 = w0;
358 }
359 const float* w2 = w1 + kc;
360 if XNN_UNPREDICTABLE(n <= 2) {
361 w2 = w1;
362 }
363 const float* w3 = w2 + kc;
364 if XNN_UNPREDICTABLE(n < 4) {
365 w3 = w2;
366 }
367 const float* w4 = w3 + kc;
368 if XNN_UNPREDICTABLE(n <= 4) {
369 w4 = w3;
370 }
371 const float* w5 = w4 + kc;
372 if XNN_UNPREDICTABLE(n < 6) {
373 w5 = w4;
374 }
375 const float* w6 = w5 + kc;
376 if XNN_UNPREDICTABLE(n <= 6) {
377 w6 = w5;
378 }
379 const float* w7 = w6 + kc;
380 if XNN_UNPREDICTABLE(n < 8) {
381 w7 = w6;
382 }
383 const float* w8 = w7 + kc;
384 if XNN_UNPREDICTABLE(n <= 8) {
385 w8 = w7;
386 }
387 const float* w9 = w8 + kc;
388 if XNN_UNPREDICTABLE(n < 10) {
389 w9 = w8;
390 }
391 const float* w10 = w9 + kc;
392 if XNN_UNPREDICTABLE(n <= 10) {
393 w10 = w9;
394 }
395 const float* w11 = w10 + kc;
396 if XNN_UNPREDICTABLE(n < 12) {
397 w11 = w10;
398 }
399 const float* w12 = w11 + kc;
400 if XNN_UNPREDICTABLE(n <= 12) {
401 w12 = w11;
402 }
403 const float* w13 = w12 + kc;
404 if XNN_UNPREDICTABLE(n < 14) {
405 w13 = w12;
406 }
407 const float* w14 = w13 + kc;
408 if XNN_UNPREDICTABLE(n <= 14) {
409 w14 = w13;
410 }
411
412 // KC main loop multiple of 4
413 size_t k = kc;
414 for (; k >= 4; k -= 4) {
415 // Read blocks of 4x4
416 // a b c d
417 // e f g h
418 // i j k l
419 // m n o p
420 // Load first 4 rows of N into low part of each register
421 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
422 w0 += 4;
423 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
424 w1 += 4;
425 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
426 w2 += 4;
427 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
428 w3 += 4;
429 __m256 v8x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w8));
430 w8 += 4;
431 __m256 v9x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w9));
432 w9 += 4;
433 __m256 v10x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w10));
434 w10 += 4;
435 // castps leaves upper 128 bits undefined, so zero them.
436 __m256 v11x0123 = _mm256_zextps128_ps256(_mm_loadu_ps(w11));
437 w11 += 4;
438 // Load next 4 rows of N into the high part of each register
439 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
440 w4 += 4;
441 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
442 w5 += 4;
443 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
444 w6 += 4;
445 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
446 w7 += 4;
447 v8x0123 = _mm256_insertf128_ps(v8x0123, _mm_loadu_ps(w12), 1);
448 w12 += 4;
449 v9x0123 = _mm256_insertf128_ps(v9x0123, _mm_loadu_ps(w13), 1);
450 w13 += 4;
451 v10x0123 = _mm256_insertf128_ps(v10x0123, _mm_loadu_ps(w14), 1);
452 w14 += 4;
453
454 // Transpose 2x2
455 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
456 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
457 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
458 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
459 const __m256 vtmp8x0123 = _mm256_unpacklo_ps(v8x0123, v9x0123); // a e b f from row 0, 1
460 const __m256 vtmp9x0123 = _mm256_unpacklo_ps(v10x0123, v11x0123); // i m j n from row 2, 3
461 const __m256 vtmp10x0123 = _mm256_unpackhi_ps(v8x0123, v9x0123); // c g d h from row 0, 1
462 const __m256 vtmp11x0123 = _mm256_unpackhi_ps(v10x0123, v11x0123); // k o l p from row 2, 3
463 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
464 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
465 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
466 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
467 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
468 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
469 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
470 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
471 xnn_prefetch_to_l1((const int8_t*) w8 + 128);
472 xnn_prefetch_to_l1((const int8_t*) w9 + 128);
473 xnn_prefetch_to_l1((const int8_t*) w10 + 128);
474 xnn_prefetch_to_l1((const int8_t*) w11 + 128);
475 xnn_prefetch_to_l1((const int8_t*) w12 + 128);
476 xnn_prefetch_to_l1((const int8_t*) w13 + 128);
477 xnn_prefetch_to_l1((const int8_t*) w14 + 128);
478 // Transpose 4x4
479 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
480 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
481 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
482 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
483 v8x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // a e i m from row 0, 1
484 v9x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp8x0123), _mm256_castps_pd(vtmp9x0123))); // b f j n from row 0, 1
485 v10x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // c g k o from row 2, 3
486 v11x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp10x0123), _mm256_castps_pd(vtmp11x0123))); // d h l p from row 2, 3
487
488 _mm256_store_ps(packed_w, v0x0123);
489 _mm256_store_ps(packed_w + 8, v8x0123);
490 _mm256_store_ps(packed_w + 16, v1x0123);
491 _mm256_store_ps(packed_w + 24, v9x0123);
492 _mm256_store_ps(packed_w + 32, v2x0123);
493 _mm256_store_ps(packed_w + 40, v10x0123);
494 _mm256_store_ps(packed_w + 48, v3x0123);
495 _mm256_store_ps(packed_w + 56, v11x0123);
496 packed_w += 64;
497 }
498
499 // KC remainder (1..3)
500 if XNN_UNLIKELY(k != 0) {
501 assert(k >= 1);
502 assert(k <= 3);
503 if (k & 2) {
504 // Read blocks of 4x2
505 // a b
506 // c d
507 // e f
508 // g h
509 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
510 w0 += 2;
511 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
512 w1 += 2;
513 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
514 w2 += 2;
515 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
516 w3 += 2;
517 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
518 w4 += 2;
519 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
520 w5 += 2;
521 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
522 w6 += 2;
523 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
524 w7 += 2;
525 __m128 v8 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w8));
526 w8 += 2;
527 __m128 v9 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w9));
528 w9 += 2;
529 __m128 v10 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w10));
530 w10 += 2;
531 __m128 v11 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w11));
532 w11 += 2;
533 __m128 v12 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w12));
534 w12 += 2;
535 __m128 v13 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w13));
536 w13 += 2;
537 __m128 v14 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w14));
538 w14 += 2;
539
540 // Transpose 2x2
541 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
542 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
543 // Transpose 4x4
544 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
545 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
546 // Transpose 2x2
547 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
548 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
549 // Transpose 4x4
550 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
551 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
552 // Transpose 2x2
553 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a c b d from row 0, 1
554 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // e g f h from row 2, 3
555 // Transpose 4x4
556 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a c e g from row 0, 1
557 v9 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // b d f h from row 0, 1
558 // Transpose 2x2
559 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a c b d from row 0, 1
560 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // e g f h from row 2, 3
561 // Transpose 4x4
562 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a c e g from row 0, 1
563 v13 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // b d f h from row 0, 1
564
565 _mm_store_ps(packed_w, v0);
566 _mm_store_ps(packed_w + 4, v4);
567 _mm_store_ps(packed_w + 8, v8);
568 _mm_store_ps(packed_w + 12, v12);
569 _mm_store_ps(packed_w + 16, v1);
570 _mm_store_ps(packed_w + 20, v5);
571 _mm_store_ps(packed_w + 24, v9);
572 _mm_store_ps(packed_w + 28, v13);
573 packed_w += 32;
574 }
575 if (k & 1) {
576 // Read blocks of 4x1
577 // a
578 // b
579 // c
580 // d
581 __m128 v0 = _mm_load_ss(w0); w0 += 1;
582 __m128 v1 = _mm_load_ss(w1); w1 += 1;
583 __m128 v2 = _mm_load_ss(w2); w2 += 1;
584 __m128 v3 = _mm_load_ss(w3); w3 += 1;
585 __m128 v4 = _mm_load_ss(w4); w4 += 1;
586 __m128 v5 = _mm_load_ss(w5); w5 += 1;
587 __m128 v6 = _mm_load_ss(w6); w6 += 1;
588 __m128 v7 = _mm_load_ss(w7); w7 += 1;
589 __m128 v8 = _mm_load_ss(w8); w8 += 1;
590 __m128 v9 = _mm_load_ss(w9); w9 += 1;
591 __m128 v10 = _mm_load_ss(w10); w10 += 1;
592 __m128 v11 = _mm_load_ss(w11); w11 += 1;
593 __m128 v12 = _mm_load_ss(w12); w12 += 1;
594 __m128 v13 = _mm_load_ss(w13); w13 += 1;
595 __m128 v14 = _mm_load_ss(w14); w14 += 1;
596
597 // Transpose 2x2
598 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
599 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
600 // Transpose 4x4
601 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
602 // Transpose 2x2
603 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
604 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
605 // Transpose 4x4
606 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
607 // Transpose 2x2
608 const __m128 vtmp8 = _mm_unpacklo_ps(v8, v9); // a b from row 0, 1
609 const __m128 vtmp9 = _mm_unpacklo_ps(v10, v11); // c d from row 2, 3
610 // Transpose 4x4
611 v8 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp8), _mm_castps_pd(vtmp9))); // a b c d from row 0, 1
612 // Transpose 2x2
613 const __m128 vtmp12 = _mm_unpacklo_ps(v12, v13); // a b from row 0, 1
614 const __m128 vtmp13 = _mm_unpacklo_ps(v14, v14); // c d from row 2, 3
615 // Transpose 4x4
616 v12 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp12), _mm_castps_pd(vtmp13))); // a b c d from row 0, 1
617
618 _mm_store_ps(packed_w, v0);
619 _mm_store_ps(packed_w + 4, v4);
620 _mm_store_ps(packed_w + 8, v8);
621 _mm_store_ps(packed_w + 12, v12);
622 packed_w += 16;
623 }
624 }
625 }
626 weights += nc * kc;
627 } while (--g != 0);
628 }