Debian Codegraph @

src/x32-packw/gen/x32-packw-x32-gemm-gio-avx-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/gio-avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include <immintrin.h>
15
16 #include "xnnpack/intrinsics-polyfill.h"
17 #include "xnnpack/packw.h"
18
19
20 void xnn_x32_packw_gemm_gio_ukernel_x32__avx_u8(
21 size_t g,
22 size_t nc,
23 size_t kc,
24 size_t nr,
25 size_t kr,
26 size_t sr,
27 size_t k_stride,
28 const uint32_t* weights,
29 const uint32_t* bias,
30 const void* scale,
31 uint32_t* packed_weights,
32 size_t extra_bytes,
33 const void* params)
34 {
35 assert(g != 0);
36 assert(nc != 0);
37 assert(kc != 0);
38 assert(nr == 32); // This kernel is for NR=32
39 assert(kr == 1);
40 assert(sr == 1);
41 assert(k_stride != 0);
42 assert(weights != NULL);
43 assert(packed_weights != NULL);
44 static const int32_t mask_table[64] = {
45 -1, -1, -1, -1, -1, -1, -1, -1,
46 -1, -1, -1, -1, -1, -1, -1, -1,
47 -1, -1, -1, -1, -1, -1, -1, -1,
48 -1, -1, -1, -1, -1, -1, -1, -1,
49 0, 0, 0, 0, 0, 0, 0, 0,
50 0, 0, 0, 0, 0, 0, 0, 0,
51 0, 0, 0, 0, 0, 0, 0, 0,
52 0, 0, 0, 0, 0, 0, 0, 0,
53 };
54
55 const __m256 vzero = _mm256_setzero_ps();
56 const float* b = (const float*) bias;
57 float* packed_w = (float*) packed_weights;
58 do {
59 // NC main loop multiple of 32
60 const float* w = (const float*) weights;
61 size_t n = nc;
62
63 for (; n >= 32; n -= 32) {
64 if XNN_LIKELY(b != NULL) {
65 const __m256 vb0 = _mm256_loadu_ps(b + 0);
66 const __m256 vb8 = _mm256_loadu_ps(b + 8);
67 const __m256 vb16 = _mm256_loadu_ps(b + 16);
68 const __m256 vb24 = _mm256_loadu_ps(b + 24);
69 _mm256_store_ps(packed_w + 0, vb0);
70 _mm256_store_ps(packed_w + 8, vb8);
71 _mm256_store_ps(packed_w + 16, vb16);
72 _mm256_store_ps(packed_w + 24, vb24);
73 b += 32;
74 } else {
75 _mm256_store_ps(packed_w + 0, vzero);
76 _mm256_store_ps(packed_w + 8, vzero);
77 _mm256_store_ps(packed_w + 16, vzero);
78 _mm256_store_ps(packed_w + 24, vzero);
79 }
80 packed_w += 32;
81
82 size_t k = kc;
83 // KC main loop 8x32
84 for (; k >= 8; k -= 8) {
85 const __m256 v0_0 = _mm256_loadu_ps(w + 0 + 0 * k_stride);
86 const __m256 v8_0 = _mm256_loadu_ps(w + 8 + 0 * k_stride);
87 const __m256 v16_0 = _mm256_loadu_ps(w + 16 + 0 * k_stride);
88 const __m256 v24_0 = _mm256_loadu_ps(w + 24 + 0 * k_stride);
89 const __m256 v0_1 = _mm256_loadu_ps(w + 0 + 1 * k_stride);
90 const __m256 v8_1 = _mm256_loadu_ps(w + 8 + 1 * k_stride);
91 const __m256 v16_1 = _mm256_loadu_ps(w + 16 + 1 * k_stride);
92 const __m256 v24_1 = _mm256_loadu_ps(w + 24 + 1 * k_stride);
93 const __m256 v0_2 = _mm256_loadu_ps(w + 0 + 2 * k_stride);
94 const __m256 v8_2 = _mm256_loadu_ps(w + 8 + 2 * k_stride);
95 const __m256 v16_2 = _mm256_loadu_ps(w + 16 + 2 * k_stride);
96 const __m256 v24_2 = _mm256_loadu_ps(w + 24 + 2 * k_stride);
97 const __m256 v0_3 = _mm256_loadu_ps(w + 0 + 3 * k_stride);
98 const __m256 v8_3 = _mm256_loadu_ps(w + 8 + 3 * k_stride);
99 const __m256 v16_3 = _mm256_loadu_ps(w + 16 + 3 * k_stride);
100 const __m256 v24_3 = _mm256_loadu_ps(w + 24 + 3 * k_stride);
101 const __m256 v0_4 = _mm256_loadu_ps(w + 0 + 4 * k_stride);
102 const __m256 v8_4 = _mm256_loadu_ps(w + 8 + 4 * k_stride);
103 const __m256 v16_4 = _mm256_loadu_ps(w + 16 + 4 * k_stride);
104 const __m256 v24_4 = _mm256_loadu_ps(w + 24 + 4 * k_stride);
105 const __m256 v0_5 = _mm256_loadu_ps(w + 0 + 5 * k_stride);
106 const __m256 v8_5 = _mm256_loadu_ps(w + 8 + 5 * k_stride);
107 const __m256 v16_5 = _mm256_loadu_ps(w + 16 + 5 * k_stride);
108 const __m256 v24_5 = _mm256_loadu_ps(w + 24 + 5 * k_stride);
109 const __m256 v0_6 = _mm256_loadu_ps(w + 0 + 6 * k_stride);
110 const __m256 v8_6 = _mm256_loadu_ps(w + 8 + 6 * k_stride);
111 const __m256 v16_6 = _mm256_loadu_ps(w + 16 + 6 * k_stride);
112 const __m256 v24_6 = _mm256_loadu_ps(w + 24 + 6 * k_stride);
113 const __m256 v0_7 = _mm256_loadu_ps(w + 0 + 7 * k_stride);
114 const __m256 v8_7 = _mm256_loadu_ps(w + 8 + 7 * k_stride);
115 const __m256 v16_7 = _mm256_loadu_ps(w + 16 + 7 * k_stride);
116 const __m256 v24_7 = _mm256_loadu_ps(w + 24 + 7 * k_stride);
117 _mm256_store_ps(packed_w + 0, v0_0);
118 _mm256_store_ps(packed_w + 8, v8_0);
119 _mm256_store_ps(packed_w + 16, v16_0);
120 _mm256_store_ps(packed_w + 24, v24_0);
121 _mm256_store_ps(packed_w + 32, v0_1);
122 _mm256_store_ps(packed_w + 40, v8_1);
123 _mm256_store_ps(packed_w + 48, v16_1);
124 _mm256_store_ps(packed_w + 56, v24_1);
125 _mm256_store_ps(packed_w + 64, v0_2);
126 _mm256_store_ps(packed_w + 72, v8_2);
127 _mm256_store_ps(packed_w + 80, v16_2);
128 _mm256_store_ps(packed_w + 88, v24_2);
129 _mm256_store_ps(packed_w + 96, v0_3);
130 _mm256_store_ps(packed_w + 104, v8_3);
131 _mm256_store_ps(packed_w + 112, v16_3);
132 _mm256_store_ps(packed_w + 120, v24_3);
133 _mm256_store_ps(packed_w + 128, v0_4);
134 _mm256_store_ps(packed_w + 136, v8_4);
135 _mm256_store_ps(packed_w + 144, v16_4);
136 _mm256_store_ps(packed_w + 152, v24_4);
137 _mm256_store_ps(packed_w + 160, v0_5);
138 _mm256_store_ps(packed_w + 168, v8_5);
139 _mm256_store_ps(packed_w + 176, v16_5);
140 _mm256_store_ps(packed_w + 184, v24_5);
141 _mm256_store_ps(packed_w + 192, v0_6);
142 _mm256_store_ps(packed_w + 200, v8_6);
143 _mm256_store_ps(packed_w + 208, v16_6);
144 _mm256_store_ps(packed_w + 216, v24_6);
145 _mm256_store_ps(packed_w + 224, v0_7);
146 _mm256_store_ps(packed_w + 232, v8_7);
147 _mm256_store_ps(packed_w + 240, v16_7);
148 _mm256_store_ps(packed_w + 248, v24_7);
149 w += k_stride * 8;
150 packed_w += 256;
151 }
152
153 // KC remainder loop
154 for (; k > 0; --k) {
155 const __m256 v0 = _mm256_loadu_ps(w + 0);
156 const __m256 v8 = _mm256_loadu_ps(w + 8);
157 const __m256 v16 = _mm256_loadu_ps(w + 16);
158 const __m256 v24 = _mm256_loadu_ps(w + 24);
159 _mm256_store_ps(packed_w + 0, v0);
160 _mm256_store_ps(packed_w + 8, v8);
161 _mm256_store_ps(packed_w + 16, v16);
162 _mm256_store_ps(packed_w + 24, v24);
163 w += k_stride;
164 packed_w += 32;
165 }
166 w = w - kc * k_stride + 32; // Advance to next column of 32 floats
167 }
168
169 // NC remainder (1..31)
170 if XNN_UNLIKELY(n != 0) {
171 assert(n >= 1);
172 assert(n <= 31);
173 const __m256i vmask0 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);
174 const __m256i vmask8 = _mm256_loadu_si256((const __m256i*) &mask_table[40 - n]);
175 const __m256i vmask16 = _mm256_loadu_si256((const __m256i*) &mask_table[48 - n]);
176 const __m256i vmask24 = _mm256_loadu_si256((const __m256i*) &mask_table[56 - n]);
177
178 if XNN_LIKELY(b != NULL) {
179 const __m256 vb0 = _mm256_maskload_ps(b + 0, vmask0);
180 const __m256 vb8 = _mm256_maskload_ps(b + 8, vmask8);
181 const __m256 vb16 = _mm256_maskload_ps(b + 16, vmask16);
182 const __m256 vb24 = _mm256_maskload_ps(b + 24, vmask24);
183 _mm256_store_ps(packed_w + 0, vb0);
184 _mm256_store_ps(packed_w + 8, vb8);
185 _mm256_store_ps(packed_w + 16, vb16);
186 _mm256_store_ps(packed_w + 24, vb24);
187 b += n;
188 } else {
189 _mm256_store_ps(packed_w + 0, vzero);
190 _mm256_store_ps(packed_w + 8, vzero);
191 _mm256_store_ps(packed_w + 16, vzero);
192 _mm256_store_ps(packed_w + 24, vzero);
193 }
194 packed_w += 32;
195
196 // KC main loop
197 for (size_t k = kc; k > 0; --k) {
198 const __m256 v0 = _mm256_maskload_ps(w + 0, vmask0);
199 const __m256 v8 = _mm256_maskload_ps(w + 8, vmask8);
200 const __m256 v16 = _mm256_maskload_ps(w + 16, vmask16);
201 const __m256 v24 = _mm256_maskload_ps(w + 24, vmask24);
202 _mm256_store_ps(packed_w + 0, v0);
203 _mm256_store_ps(packed_w + 8, v8);
204 _mm256_store_ps(packed_w + 16, v16);
205 _mm256_store_ps(packed_w + 24, v24);
206 w += k_stride;
207 packed_w += 32;
208 }
209 }
210 weights += nc * kc;
211 } while (--g != 0);
212 }