Debian Codegraph @

src/x32-packw/gen/x32-packw-x32-gemm-gio-avx-u8-prfm.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/gio-avx.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 Google LLC
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10 #include <assert.h>
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include <immintrin.h>
15
16 #include "xnnpack/intrinsics-polyfill.h"
17 #include "xnnpack/packw.h"
18 #include "xnnpack/prefetch.h"
19
20
21 void xnn_x32_packw_gemm_gio_ukernel_x32__avx_u8_prfm(
22 size_t g,
23 size_t nc,
24 size_t kc,
25 size_t nr,
26 size_t kr,
27 size_t sr,
28 size_t k_stride,
29 const uint32_t* weights,
30 const uint32_t* bias,
31 const void* scale,
32 uint32_t* packed_weights,
33 size_t extra_bytes,
34 const void* params)
35 {
36 assert(g != 0);
37 assert(nc != 0);
38 assert(kc != 0);
39 assert(nr == 32); // This kernel is for NR=32
40 assert(kr == 1);
41 assert(sr == 1);
42 assert(k_stride != 0);
43 assert(weights != NULL);
44 assert(packed_weights != NULL);
45 static const int32_t mask_table[64] = {
46 -1, -1, -1, -1, -1, -1, -1, -1,
47 -1, -1, -1, -1, -1, -1, -1, -1,
48 -1, -1, -1, -1, -1, -1, -1, -1,
49 -1, -1, -1, -1, -1, -1, -1, -1,
50 0, 0, 0, 0, 0, 0, 0, 0,
51 0, 0, 0, 0, 0, 0, 0, 0,
52 0, 0, 0, 0, 0, 0, 0, 0,
53 0, 0, 0, 0, 0, 0, 0, 0,
54 };
55
56 const __m256 vzero = _mm256_setzero_ps();
57 const float* b = (const float*) bias;
58 float* packed_w = (float*) packed_weights;
59 do {
60 // NC main loop multiple of 32
61 const float* w = (const float*) weights;
62 size_t n = nc;
63
64 for (; n >= 32; n -= 32) {
65 if XNN_LIKELY(b != NULL) {
66 const __m256 vb0 = _mm256_loadu_ps(b + 0);
67 const __m256 vb8 = _mm256_loadu_ps(b + 8);
68 const __m256 vb16 = _mm256_loadu_ps(b + 16);
69 const __m256 vb24 = _mm256_loadu_ps(b + 24);
70 _mm256_store_ps(packed_w + 0, vb0);
71 _mm256_store_ps(packed_w + 8, vb8);
72 _mm256_store_ps(packed_w + 16, vb16);
73 _mm256_store_ps(packed_w + 24, vb24);
74 b += 32;
75 } else {
76 _mm256_store_ps(packed_w + 0, vzero);
77 _mm256_store_ps(packed_w + 8, vzero);
78 _mm256_store_ps(packed_w + 16, vzero);
79 _mm256_store_ps(packed_w + 24, vzero);
80 }
81 packed_w += 32;
82
83 size_t k = kc;
84 // KC main loop 8x32
85 for (; k >= 8; k -= 8) {
86 const __m256 v0_0 = _mm256_loadu_ps(w + 0 + 0 * k_stride);
87 const __m256 v8_0 = _mm256_loadu_ps(w + 8 + 0 * k_stride);
88 const __m256 v16_0 = _mm256_loadu_ps(w + 16 + 0 * k_stride);
89 const __m256 v24_0 = _mm256_loadu_ps(w + 24 + 0 * k_stride);
90 const __m256 v0_1 = _mm256_loadu_ps(w + 0 + 1 * k_stride);
91 const __m256 v8_1 = _mm256_loadu_ps(w + 8 + 1 * k_stride);
92 const __m256 v16_1 = _mm256_loadu_ps(w + 16 + 1 * k_stride);
93 const __m256 v24_1 = _mm256_loadu_ps(w + 24 + 1 * k_stride);
94 const __m256 v0_2 = _mm256_loadu_ps(w + 0 + 2 * k_stride);
95 const __m256 v8_2 = _mm256_loadu_ps(w + 8 + 2 * k_stride);
96 const __m256 v16_2 = _mm256_loadu_ps(w + 16 + 2 * k_stride);
97 const __m256 v24_2 = _mm256_loadu_ps(w + 24 + 2 * k_stride);
98 const __m256 v0_3 = _mm256_loadu_ps(w + 0 + 3 * k_stride);
99 const __m256 v8_3 = _mm256_loadu_ps(w + 8 + 3 * k_stride);
100 const __m256 v16_3 = _mm256_loadu_ps(w + 16 + 3 * k_stride);
101 const __m256 v24_3 = _mm256_loadu_ps(w + 24 + 3 * k_stride);
102 const __m256 v0_4 = _mm256_loadu_ps(w + 0 + 4 * k_stride);
103 const __m256 v8_4 = _mm256_loadu_ps(w + 8 + 4 * k_stride);
104 const __m256 v16_4 = _mm256_loadu_ps(w + 16 + 4 * k_stride);
105 const __m256 v24_4 = _mm256_loadu_ps(w + 24 + 4 * k_stride);
106 const __m256 v0_5 = _mm256_loadu_ps(w + 0 + 5 * k_stride);
107 const __m256 v8_5 = _mm256_loadu_ps(w + 8 + 5 * k_stride);
108 const __m256 v16_5 = _mm256_loadu_ps(w + 16 + 5 * k_stride);
109 const __m256 v24_5 = _mm256_loadu_ps(w + 24 + 5 * k_stride);
110 const __m256 v0_6 = _mm256_loadu_ps(w + 0 + 6 * k_stride);
111 const __m256 v8_6 = _mm256_loadu_ps(w + 8 + 6 * k_stride);
112 const __m256 v16_6 = _mm256_loadu_ps(w + 16 + 6 * k_stride);
113 const __m256 v24_6 = _mm256_loadu_ps(w + 24 + 6 * k_stride);
114 const __m256 v0_7 = _mm256_loadu_ps(w + 0 + 7 * k_stride);
115 const __m256 v8_7 = _mm256_loadu_ps(w + 8 + 7 * k_stride);
116 const __m256 v16_7 = _mm256_loadu_ps(w + 16 + 7 * k_stride);
117 const __m256 v24_7 = _mm256_loadu_ps(w + 24 + 7 * k_stride);
118 xnn_prefetch_to_l1((const int8_t*) w + 960);
119 xnn_prefetch_to_l1((const int8_t*) w + 1024);
120 xnn_prefetch_to_l1((const int8_t*) w + 960);
121 xnn_prefetch_to_l1((const int8_t*) w + 1024);
122 xnn_prefetch_to_l1((const int8_t*) w + 960);
123 xnn_prefetch_to_l1((const int8_t*) w + 1024);
124 xnn_prefetch_to_l1((const int8_t*) w + 960);
125 xnn_prefetch_to_l1((const int8_t*) w + 1024);
126 xnn_prefetch_to_l1((const int8_t*) w + 960);
127 xnn_prefetch_to_l1((const int8_t*) w + 1024);
128 xnn_prefetch_to_l1((const int8_t*) w + 960);
129 xnn_prefetch_to_l1((const int8_t*) w + 1024);
130 xnn_prefetch_to_l1((const int8_t*) w + 960);
131 xnn_prefetch_to_l1((const int8_t*) w + 1024);
132 xnn_prefetch_to_l1((const int8_t*) w + 960);
133 xnn_prefetch_to_l1((const int8_t*) w + 1024);
134 _mm256_store_ps(packed_w + 0, v0_0);
135 _mm256_store_ps(packed_w + 8, v8_0);
136 _mm256_store_ps(packed_w + 16, v16_0);
137 _mm256_store_ps(packed_w + 24, v24_0);
138 _mm256_store_ps(packed_w + 32, v0_1);
139 _mm256_store_ps(packed_w + 40, v8_1);
140 _mm256_store_ps(packed_w + 48, v16_1);
141 _mm256_store_ps(packed_w + 56, v24_1);
142 _mm256_store_ps(packed_w + 64, v0_2);
143 _mm256_store_ps(packed_w + 72, v8_2);
144 _mm256_store_ps(packed_w + 80, v16_2);
145 _mm256_store_ps(packed_w + 88, v24_2);
146 _mm256_store_ps(packed_w + 96, v0_3);
147 _mm256_store_ps(packed_w + 104, v8_3);
148 _mm256_store_ps(packed_w + 112, v16_3);
149 _mm256_store_ps(packed_w + 120, v24_3);
150 _mm256_store_ps(packed_w + 128, v0_4);
151 _mm256_store_ps(packed_w + 136, v8_4);
152 _mm256_store_ps(packed_w + 144, v16_4);
153 _mm256_store_ps(packed_w + 152, v24_4);
154 _mm256_store_ps(packed_w + 160, v0_5);
155 _mm256_store_ps(packed_w + 168, v8_5);
156 _mm256_store_ps(packed_w + 176, v16_5);
157 _mm256_store_ps(packed_w + 184, v24_5);
158 _mm256_store_ps(packed_w + 192, v0_6);
159 _mm256_store_ps(packed_w + 200, v8_6);
160 _mm256_store_ps(packed_w + 208, v16_6);
161 _mm256_store_ps(packed_w + 216, v24_6);
162 _mm256_store_ps(packed_w + 224, v0_7);
163 _mm256_store_ps(packed_w + 232, v8_7);
164 _mm256_store_ps(packed_w + 240, v16_7);
165 _mm256_store_ps(packed_w + 248, v24_7);
166 w += k_stride * 8;
167 packed_w += 256;
168 }
169
170 // KC remainder loop
171 for (; k > 0; --k) {
172 const __m256 v0 = _mm256_loadu_ps(w + 0);
173 const __m256 v8 = _mm256_loadu_ps(w + 8);
174 const __m256 v16 = _mm256_loadu_ps(w + 16);
175 const __m256 v24 = _mm256_loadu_ps(w + 24);
176 xnn_prefetch_to_l1((const int8_t*) w + 960);
177 xnn_prefetch_to_l1((const int8_t*) w + 1024);
178 _mm256_store_ps(packed_w + 0, v0);
179 _mm256_store_ps(packed_w + 8, v8);
180 _mm256_store_ps(packed_w + 16, v16);
181 _mm256_store_ps(packed_w + 24, v24);
182 w += k_stride;
183 packed_w += 32;
184 }
185 w = w - kc * k_stride + 32; // Advance to next column of 32 floats
186 }
187
188 // NC remainder (1..31)
189 if XNN_UNLIKELY(n != 0) {
190 assert(n >= 1);
191 assert(n <= 31);
192 const __m256i vmask0 = _mm256_loadu_si256((const __m256i*) &mask_table[32 - n]);
193 const __m256i vmask8 = _mm256_loadu_si256((const __m256i*) &mask_table[40 - n]);
194 const __m256i vmask16 = _mm256_loadu_si256((const __m256i*) &mask_table[48 - n]);
195 const __m256i vmask24 = _mm256_loadu_si256((const __m256i*) &mask_table[56 - n]);
196
197 if XNN_LIKELY(b != NULL) {
198 const __m256 vb0 = _mm256_maskload_ps(b + 0, vmask0);
199 const __m256 vb8 = _mm256_maskload_ps(b + 8, vmask8);
200 const __m256 vb16 = _mm256_maskload_ps(b + 16, vmask16);
201 const __m256 vb24 = _mm256_maskload_ps(b + 24, vmask24);
202 _mm256_store_ps(packed_w + 0, vb0);
203 _mm256_store_ps(packed_w + 8, vb8);
204 _mm256_store_ps(packed_w + 16, vb16);
205 _mm256_store_ps(packed_w + 24, vb24);
206 b += n;
207 } else {
208 _mm256_store_ps(packed_w + 0, vzero);
209 _mm256_store_ps(packed_w + 8, vzero);
210 _mm256_store_ps(packed_w + 16, vzero);
211 _mm256_store_ps(packed_w + 24, vzero);
212 }
213 packed_w += 32;
214
215 // KC main loop
216 for (size_t k = kc; k > 0; --k) {
217 const __m256 v0 = _mm256_maskload_ps(w + 0, vmask0);
218 const __m256 v8 = _mm256_maskload_ps(w + 8, vmask8);
219 const __m256 v16 = _mm256_maskload_ps(w + 16, vmask16);
220 const __m256 v24 = _mm256_maskload_ps(w + 24, vmask24);
221 _mm256_store_ps(packed_w + 0, v0);
222 _mm256_store_ps(packed_w + 8, v8);
223 _mm256_store_ps(packed_w + 16, v16);
224 _mm256_store_ps(packed_w + 24, v24);
225 w += k_stride;
226 packed_w += 32;
227 }
228 }
229 weights += nc * kc;
230 } while (--g != 0);
231 }