Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-avx-u4-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/intrinsics-polyfill.h"
18 #include "xnnpack/packw.h"
19 #include "xnnpack/prefetch.h"
20
21
22 void xnn_x32_packw_gemm_goi_ukernel_x8__avx_u4_prfm(
23 size_t g,
24 size_t nc,
25 size_t kc,
26 size_t nr,
27 size_t kr,
28 size_t sr,
29 const uint32_t* weights,
30 const uint32_t* bias,
31 const void* scale,
32 uint32_t* packed_weights,
33 size_t extra_bytes,
34 const void* params)
35 {
36 assert(g != 0);
37 assert(nc != 0);
38 assert(kc != 0);
39 assert(nr == 8); // This kernel is for NR=8
40 assert(kr == 1);
41 assert(sr == 1);
42 assert(weights != NULL);
43 assert(packed_weights != NULL);
44
45 const float* b = (const float*) bias;
46 float* packed_w = (float*) packed_weights;
47 do {
48 // NC main loop multiple of 8
49 const float* w0 = (const float*) weights;
50 size_t n = nc;
51
52 for (; n >= 8; n -= 8) {
53 if XNN_LIKELY(b != NULL) {
54 const __m256 vb0 = _mm256_loadu_ps(b);
55 _mm256_store_ps(packed_w, vb0);
56 b += 8;
57 } else {
58 const __m256 vzero = _mm256_setzero_ps();
59 _mm256_store_ps(packed_w, vzero);
60 }
61 packed_w += 8;
62
63 const float* w1 = w0 + kc;
64 const float* w2 = w1 + kc;
65 const float* w3 = w2 + kc;
66 const float* w4 = w3 + kc;
67 const float* w5 = w4 + kc;
68 const float* w6 = w5 + kc;
69 const float* w7 = w6 + kc;
70 xnn_prefetch_to_l1((const int8_t*) w0);
71 xnn_prefetch_to_l1((const int8_t*) w0 + 64);
72 xnn_prefetch_to_l1((const int8_t*) w1);
73 xnn_prefetch_to_l1((const int8_t*) w1 + 64);
74 xnn_prefetch_to_l1((const int8_t*) w2);
75 xnn_prefetch_to_l1((const int8_t*) w2 + 64);
76 xnn_prefetch_to_l1((const int8_t*) w3);
77 xnn_prefetch_to_l1((const int8_t*) w3 + 64);
78 xnn_prefetch_to_l1((const int8_t*) w4);
79 xnn_prefetch_to_l1((const int8_t*) w4 + 64);
80 xnn_prefetch_to_l1((const int8_t*) w5);
81 xnn_prefetch_to_l1((const int8_t*) w5 + 64);
82 xnn_prefetch_to_l1((const int8_t*) w6);
83 xnn_prefetch_to_l1((const int8_t*) w6 + 64);
84 xnn_prefetch_to_l1((const int8_t*) w7);
85 xnn_prefetch_to_l1((const int8_t*) w7 + 64);
86
87 // KC main loop multiple of 4
88 size_t k = kc;
89 for (; k >= 4; k -= 4) {
90 // Read blocks of 4x4
91 // a b c d
92 // e f g h
93 // i j k l
94 // m n o p
95 // Load first 4 rows of N into low part of each register
96 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
97 w0 += 4;
98 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
99 w1 += 4;
100 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
101 w2 += 4;
102 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
103 w3 += 4;
104 // Load next 4 rows of N into the high part of each register
105 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
106 w4 += 4;
107 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
108 w5 += 4;
109 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
110 w6 += 4;
111 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
112 w7 += 4;
113
114 // Transpose 2x2
115 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
116 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
117 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
118 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
119 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
120 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
121 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
122 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
123 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
124 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
125 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
126 xnn_prefetch_to_l1((const int8_t*) w7 + 128);
127 // Transpose 4x4
128 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
129 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
130 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
131 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
132
133 _mm256_store_ps(packed_w, v0x0123);
134 _mm256_store_ps(packed_w + 8, v1x0123);
135 _mm256_store_ps(packed_w + 16, v2x0123);
136 _mm256_store_ps(packed_w + 24, v3x0123);
137 packed_w += 32;
138 }
139
140 // KC remainder (1..3)
141 if XNN_UNLIKELY(k != 0) {
142 assert(k >= 1);
143 assert(k <= 3);
144 if (k & 2) {
145 // Read blocks of 4x2
146 // a b
147 // c d
148 // e f
149 // g h
150 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
151 w0 += 2;
152 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
153 w1 += 2;
154 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
155 w2 += 2;
156 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
157 w3 += 2;
158 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
159 w4 += 2;
160 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
161 w5 += 2;
162 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
163 w6 += 2;
164 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
165 w7 += 2;
166
167 // Transpose 2x2
168 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
169 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
170 // Transpose 4x4
171 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
172 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
173 // Transpose 2x2
174 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
175 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
176 // Transpose 4x4
177 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
178 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
179
180 _mm_store_ps(packed_w, v0);
181 _mm_store_ps(packed_w + 4, v4);
182 _mm_store_ps(packed_w + 8, v1);
183 _mm_store_ps(packed_w + 12, v5);
184 packed_w += 16;
185 }
186 if (k & 1) {
187 // Read blocks of 4x1
188 // a
189 // b
190 // c
191 // d
192 __m128 v0 = _mm_load_ss(w0); w0 += 1;
193 __m128 v1 = _mm_load_ss(w1); w1 += 1;
194 __m128 v2 = _mm_load_ss(w2); w2 += 1;
195 __m128 v3 = _mm_load_ss(w3); w3 += 1;
196 __m128 v4 = _mm_load_ss(w4); w4 += 1;
197 __m128 v5 = _mm_load_ss(w5); w5 += 1;
198 __m128 v6 = _mm_load_ss(w6); w6 += 1;
199 __m128 v7 = _mm_load_ss(w7); w7 += 1;
200
201 // Transpose 2x2
202 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
203 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
204 // Transpose 4x4
205 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
206 // Transpose 2x2
207 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
208 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
209 // Transpose 4x4
210 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
211
212 _mm_store_ps(packed_w, v0);
213 _mm_store_ps(packed_w + 4, v4);
214 packed_w += 8;
215 }
216 }
217 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
218 w0 = w7;
219 }
220
221 // NC remainder (1..7)
222 if XNN_UNLIKELY(n != 0) {
223 assert(n >= 1);
224 assert(n <= 7);
225 if XNN_LIKELY(b != NULL) {
226 size_t nb = n;
227 do {
228 *packed_w++ = *b++;
229 } while (--nb != 0);
230 packed_w += (8 - n);
231 } else {
232 const __m256 vzero = _mm256_setzero_ps();
233 _mm256_store_ps(packed_w, vzero);
234 packed_w += 8;
235 }
236
237 // NR remainder has less than 8 rows so last row is not loaded
238 // For SR=4 the
239 const float* w1 = w0 + kc;
240 if XNN_UNPREDICTABLE(n < 2) {
241 w1 = w0;
242 }
243 const float* w2 = w1 + kc;
244 if XNN_UNPREDICTABLE(n <= 2) {
245 w2 = w1;
246 }
247 const float* w3 = w2 + kc;
248 if XNN_UNPREDICTABLE(n < 4) {
249 w3 = w2;
250 }
251 const float* w4 = w3 + kc;
252 if XNN_UNPREDICTABLE(n <= 4) {
253 w4 = w3;
254 }
255 const float* w5 = w4 + kc;
256 if XNN_UNPREDICTABLE(n < 6) {
257 w5 = w4;
258 }
259 const float* w6 = w5 + kc;
260 if XNN_UNPREDICTABLE(n <= 6) {
261 w6 = w5;
262 }
263
264 // KC main loop multiple of 4
265 size_t k = kc;
266 for (; k >= 4; k -= 4) {
267 // Read blocks of 4x4
268 // a b c d
269 // e f g h
270 // i j k l
271 // m n o p
272 // Load first 4 rows of N into low part of each register
273 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
274 w0 += 4;
275 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
276 w1 += 4;
277 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
278 w2 += 4;
279 // castps leaves upper 128 bits undefined, so zero them.
280 __m256 v3x0123 = _mm256_zextps128_ps256(_mm_loadu_ps(w3));
281 w3 += 4;
282 // Load next 4 rows of N into the high part of each register
283 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
284 w4 += 4;
285 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
286 w5 += 4;
287 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
288 w6 += 4;
289
290 // Transpose 2x2
291 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
292 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
293 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
294 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
295 xnn_prefetch_to_l1((const int8_t*) w0 + 128);
296 xnn_prefetch_to_l1((const int8_t*) w1 + 128);
297 xnn_prefetch_to_l1((const int8_t*) w2 + 128);
298 xnn_prefetch_to_l1((const int8_t*) w3 + 128);
299 xnn_prefetch_to_l1((const int8_t*) w4 + 128);
300 xnn_prefetch_to_l1((const int8_t*) w5 + 128);
301 xnn_prefetch_to_l1((const int8_t*) w6 + 128);
302 // Transpose 4x4
303 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
304 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
305 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
306 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
307
308 _mm256_store_ps(packed_w, v0x0123);
309 _mm256_store_ps(packed_w + 8, v1x0123);
310 _mm256_store_ps(packed_w + 16, v2x0123);
311 _mm256_store_ps(packed_w + 24, v3x0123);
312 packed_w += 32;
313 }
314
315 // KC remainder (1..3)
316 if XNN_UNLIKELY(k != 0) {
317 assert(k >= 1);
318 assert(k <= 3);
319 if (k & 2) {
320 // Read blocks of 4x2
321 // a b
322 // c d
323 // e f
324 // g h
325 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
326 w0 += 2;
327 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
328 w1 += 2;
329 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
330 w2 += 2;
331 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
332 w3 += 2;
333 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
334 w4 += 2;
335 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
336 w5 += 2;
337 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
338 w6 += 2;
339
340 // Transpose 2x2
341 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
342 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
343 // Transpose 4x4
344 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
345 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
346 // Transpose 2x2
347 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
348 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v6); // e g f h from row 2, 3
349 // Transpose 4x4
350 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
351 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
352
353 _mm_store_ps(packed_w, v0);
354 _mm_store_ps(packed_w + 4, v4);
355 _mm_store_ps(packed_w + 8, v1);
356 _mm_store_ps(packed_w + 12, v5);
357 packed_w += 16;
358 }
359 if (k & 1) {
360 // Read blocks of 4x1
361 // a
362 // b
363 // c
364 // d
365 __m128 v0 = _mm_load_ss(w0); w0 += 1;
366 __m128 v1 = _mm_load_ss(w1); w1 += 1;
367 __m128 v2 = _mm_load_ss(w2); w2 += 1;
368 __m128 v3 = _mm_load_ss(w3); w3 += 1;
369 __m128 v4 = _mm_load_ss(w4); w4 += 1;
370 __m128 v5 = _mm_load_ss(w5); w5 += 1;
371 __m128 v6 = _mm_load_ss(w6); w6 += 1;
372
373 // Transpose 2x2
374 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
375 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
376 // Transpose 4x4
377 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
378 // Transpose 2x2
379 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
380 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v6); // c d from row 2, 3
381 // Transpose 4x4
382 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
383
384 _mm_store_ps(packed_w, v0);
385 _mm_store_ps(packed_w + 4, v4);
386 packed_w += 8;
387 }
388 }
389 }
390 weights += nc * kc;
391 } while (--g != 0);
392 }