Debian Codegraph @

src/x32-packw/gen/x32-packw-x8-gemm-goi-avx-u4.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2023 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <immintrin.h>
16
17 #include "xnnpack/intrinsics-polyfill.h"
18 #include "xnnpack/packw.h"
19
20
21 void xnn_x32_packw_gemm_goi_ukernel_x8__avx_u4(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 8); // This kernel is for NR=8
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(weights != NULL);
42 assert(packed_weights != NULL);
43
44 const float* b = (const float*) bias;
45 float* packed_w = (float*) packed_weights;
46 do {
47 // NC main loop multiple of 8
48 const float* w0 = (const float*) weights;
49 size_t n = nc;
50
51 for (; n >= 8; n -= 8) {
52 if XNN_LIKELY(b != NULL) {
53 const __m256 vb0 = _mm256_loadu_ps(b);
54 _mm256_store_ps(packed_w, vb0);
55 b += 8;
56 } else {
57 const __m256 vzero = _mm256_setzero_ps();
58 _mm256_store_ps(packed_w, vzero);
59 }
60 packed_w += 8;
61
62 const float* w1 = w0 + kc;
63 const float* w2 = w1 + kc;
64 const float* w3 = w2 + kc;
65 const float* w4 = w3 + kc;
66 const float* w5 = w4 + kc;
67 const float* w6 = w5 + kc;
68 const float* w7 = w6 + kc;
69
70 // KC main loop multiple of 4
71 size_t k = kc;
72 for (; k >= 4; k -= 4) {
73 // Read blocks of 4x4
74 // a b c d
75 // e f g h
76 // i j k l
77 // m n o p
78 // Load first 4 rows of N into low part of each register
79 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
80 w0 += 4;
81 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
82 w1 += 4;
83 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
84 w2 += 4;
85 __m256 v3x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w3));
86 w3 += 4;
87 // Load next 4 rows of N into the high part of each register
88 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
89 w4 += 4;
90 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
91 w5 += 4;
92 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
93 w6 += 4;
94 v3x0123 = _mm256_insertf128_ps(v3x0123, _mm_loadu_ps(w7), 1);
95 w7 += 4;
96
97 // Transpose 2x2
98 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
99 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
100 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
101 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
102 // Transpose 4x4
103 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
104 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
105 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
106 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
107
108 _mm256_store_ps(packed_w, v0x0123);
109 _mm256_store_ps(packed_w + 8, v1x0123);
110 _mm256_store_ps(packed_w + 16, v2x0123);
111 _mm256_store_ps(packed_w + 24, v3x0123);
112 packed_w += 32;
113 }
114
115 // KC remainder (1..3)
116 if XNN_UNLIKELY(k != 0) {
117 assert(k >= 1);
118 assert(k <= 3);
119 if (k & 2) {
120 // Read blocks of 4x2
121 // a b
122 // c d
123 // e f
124 // g h
125 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
126 w0 += 2;
127 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
128 w1 += 2;
129 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
130 w2 += 2;
131 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
132 w3 += 2;
133 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
134 w4 += 2;
135 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
136 w5 += 2;
137 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
138 w6 += 2;
139 __m128 v7 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w7));
140 w7 += 2;
141
142 // Transpose 2x2
143 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
144 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
145 // Transpose 4x4
146 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
147 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
148 // Transpose 2x2
149 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
150 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // e g f h from row 2, 3
151 // Transpose 4x4
152 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
153 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
154
155 _mm_store_ps(packed_w, v0);
156 _mm_store_ps(packed_w + 4, v4);
157 _mm_store_ps(packed_w + 8, v1);
158 _mm_store_ps(packed_w + 12, v5);
159 packed_w += 16;
160 }
161 if (k & 1) {
162 // Read blocks of 4x1
163 // a
164 // b
165 // c
166 // d
167 __m128 v0 = _mm_load_ss(w0); w0 += 1;
168 __m128 v1 = _mm_load_ss(w1); w1 += 1;
169 __m128 v2 = _mm_load_ss(w2); w2 += 1;
170 __m128 v3 = _mm_load_ss(w3); w3 += 1;
171 __m128 v4 = _mm_load_ss(w4); w4 += 1;
172 __m128 v5 = _mm_load_ss(w5); w5 += 1;
173 __m128 v6 = _mm_load_ss(w6); w6 += 1;
174 __m128 v7 = _mm_load_ss(w7); w7 += 1;
175
176 // Transpose 2x2
177 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
178 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
179 // Transpose 4x4
180 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
181 // Transpose 2x2
182 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
183 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v7); // c d from row 2, 3
184 // Transpose 4x4
185 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
186
187 _mm_store_ps(packed_w, v0);
188 _mm_store_ps(packed_w + 4, v4);
189 packed_w += 8;
190 }
191 }
192 packed_w = (float*) ((uintptr_t) packed_w + extra_bytes);
193 w0 = w7;
194 }
195
196 // NC remainder (1..7)
197 if XNN_UNLIKELY(n != 0) {
198 assert(n >= 1);
199 assert(n <= 7);
200 if XNN_LIKELY(b != NULL) {
201 size_t nb = n;
202 do {
203 *packed_w++ = *b++;
204 } while (--nb != 0);
205 packed_w += (8 - n);
206 } else {
207 const __m256 vzero = _mm256_setzero_ps();
208 _mm256_store_ps(packed_w, vzero);
209 packed_w += 8;
210 }
211
212 // NR remainder has less than 8 rows so last row is not loaded
213 // For SR=4 the
214 const float* w1 = w0 + kc;
215 if XNN_UNPREDICTABLE(n < 2) {
216 w1 = w0;
217 }
218 const float* w2 = w1 + kc;
219 if XNN_UNPREDICTABLE(n <= 2) {
220 w2 = w1;
221 }
222 const float* w3 = w2 + kc;
223 if XNN_UNPREDICTABLE(n < 4) {
224 w3 = w2;
225 }
226 const float* w4 = w3 + kc;
227 if XNN_UNPREDICTABLE(n <= 4) {
228 w4 = w3;
229 }
230 const float* w5 = w4 + kc;
231 if XNN_UNPREDICTABLE(n < 6) {
232 w5 = w4;
233 }
234 const float* w6 = w5 + kc;
235 if XNN_UNPREDICTABLE(n <= 6) {
236 w6 = w5;
237 }
238
239 // KC main loop multiple of 4
240 size_t k = kc;
241 for (; k >= 4; k -= 4) {
242 // Read blocks of 4x4
243 // a b c d
244 // e f g h
245 // i j k l
246 // m n o p
247 // Load first 4 rows of N into low part of each register
248 __m256 v0x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w0));
249 w0 += 4;
250 __m256 v1x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w1));
251 w1 += 4;
252 __m256 v2x0123 = _mm256_castps128_ps256(_mm_loadu_ps(w2));
253 w2 += 4;
254 // castps leaves upper 128 bits undefined, so zero them.
255 __m256 v3x0123 = _mm256_zextps128_ps256(_mm_loadu_ps(w3));
256 w3 += 4;
257 // Load next 4 rows of N into the high part of each register
258 v0x0123 = _mm256_insertf128_ps(v0x0123, _mm_loadu_ps(w4), 1);
259 w4 += 4;
260 v1x0123 = _mm256_insertf128_ps(v1x0123, _mm_loadu_ps(w5), 1);
261 w5 += 4;
262 v2x0123 = _mm256_insertf128_ps(v2x0123, _mm_loadu_ps(w6), 1);
263 w6 += 4;
264
265 // Transpose 2x2
266 const __m256 vtmp0x0123 = _mm256_unpacklo_ps(v0x0123, v1x0123); // a e b f from row 0, 1
267 const __m256 vtmp1x0123 = _mm256_unpacklo_ps(v2x0123, v3x0123); // i m j n from row 2, 3
268 const __m256 vtmp2x0123 = _mm256_unpackhi_ps(v0x0123, v1x0123); // c g d h from row 0, 1
269 const __m256 vtmp3x0123 = _mm256_unpackhi_ps(v2x0123, v3x0123); // k o l p from row 2, 3
270 // Transpose 4x4
271 v0x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // a e i m from row 0, 1
272 v1x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp0x0123), _mm256_castps_pd(vtmp1x0123))); // b f j n from row 0, 1
273 v2x0123 = _mm256_castpd_ps(_mm256_unpacklo_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // c g k o from row 2, 3
274 v3x0123 = _mm256_castpd_ps(_mm256_unpackhi_pd(_mm256_castps_pd(vtmp2x0123), _mm256_castps_pd(vtmp3x0123))); // d h l p from row 2, 3
275
276 _mm256_store_ps(packed_w, v0x0123);
277 _mm256_store_ps(packed_w + 8, v1x0123);
278 _mm256_store_ps(packed_w + 16, v2x0123);
279 _mm256_store_ps(packed_w + 24, v3x0123);
280 packed_w += 32;
281 }
282
283 // KC remainder (1..3)
284 if XNN_UNLIKELY(k != 0) {
285 assert(k >= 1);
286 assert(k <= 3);
287 if (k & 2) {
288 // Read blocks of 4x2
289 // a b
290 // c d
291 // e f
292 // g h
293 __m128 v0 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w0));
294 w0 += 2;
295 __m128 v1 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w1));
296 w1 += 2;
297 __m128 v2 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w2));
298 w2 += 2;
299 __m128 v3 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w3));
300 w3 += 2;
301 __m128 v4 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w4));
302 w4 += 2;
303 __m128 v5 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w5));
304 w5 += 2;
305 __m128 v6 = _mm_castsi128_ps(_mm_loadl_epi64((const __m128i*) w6));
306 w6 += 2;
307
308 // Transpose 2x2
309 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a c b d from row 0, 1
310 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // e g f h from row 2, 3
311 // Transpose 4x4
312 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a c e g from row 0, 1
313 v1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // b d f h from row 0, 1
314 // Transpose 2x2
315 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a c b d from row 0, 1
316 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v6); // e g f h from row 2, 3
317 // Transpose 4x4
318 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a c e g from row 0, 1
319 v5 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // b d f h from row 0, 1
320
321 _mm_store_ps(packed_w, v0);
322 _mm_store_ps(packed_w + 4, v4);
323 _mm_store_ps(packed_w + 8, v1);
324 _mm_store_ps(packed_w + 12, v5);
325 packed_w += 16;
326 }
327 if (k & 1) {
328 // Read blocks of 4x1
329 // a
330 // b
331 // c
332 // d
333 __m128 v0 = _mm_load_ss(w0); w0 += 1;
334 __m128 v1 = _mm_load_ss(w1); w1 += 1;
335 __m128 v2 = _mm_load_ss(w2); w2 += 1;
336 __m128 v3 = _mm_load_ss(w3); w3 += 1;
337 __m128 v4 = _mm_load_ss(w4); w4 += 1;
338 __m128 v5 = _mm_load_ss(w5); w5 += 1;
339 __m128 v6 = _mm_load_ss(w6); w6 += 1;
340
341 // Transpose 2x2
342 const __m128 vtmp0 = _mm_unpacklo_ps(v0, v1); // a b from row 0, 1
343 const __m128 vtmp1 = _mm_unpacklo_ps(v2, v3); // c d from row 2, 3
344 // Transpose 4x4
345 v0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp0), _mm_castps_pd(vtmp1))); // a b c d from row 0, 1
346 // Transpose 2x2
347 const __m128 vtmp4 = _mm_unpacklo_ps(v4, v5); // a b from row 0, 1
348 const __m128 vtmp5 = _mm_unpacklo_ps(v6, v6); // c d from row 2, 3
349 // Transpose 4x4
350 v4 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(vtmp4), _mm_castps_pd(vtmp5))); // a b c d from row 0, 1
351
352 _mm_store_ps(packed_w, v0);
353 _mm_store_ps(packed_w + 4, v4);
354 packed_w += 8;
355 }
356 }
357 }
358 weights += nc * kc;
359 } while (--g != 0);
360 }