Debian Codegraph @

src/x32-packw/gen/x32-packw-x4v-gemm-goi-rvv-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/rvv.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 SiFive, Inc.
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <riscv_vector.h>
16
17 #include "xnnpack/packw.h"
18
19 void xnn_x32_packw_gemm_goi_ukernel_x4v__rvv_u8(
20 size_t g,
21 size_t nc,
22 size_t kc,
23 size_t nr,
24 size_t kr,
25 size_t sr,
26 const uint32_t* weights,
27 const uint32_t* bias,
28 const void* scale,
29 uint32_t* packed_weights,
30 size_t extra_bytes,
31 const void* params)
32 {
33 assert(g != 0);
34 assert(nc != 0);
35 assert(kc != 0);
36 assert(nr == __riscv_vsetvlmax_e32m4());
37 assert(kr == 1);
38 assert(sr == 1);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 uint32_t* out = packed_weights;
43 const uint32_t* b = bias;
44 size_t kc_bstride = kc * 4;
45
46 do {
47 const uint32_t* w0 = weights;
48 size_t n = nc;
49 // NC main loop: process multiple of NR
50 for (;n >= nr; n -= nr) {
51 // Pack nr bias at begining of tile
52 size_t vlmax = __riscv_vsetvlmax_e32m4();
53 vuint32m4_t v_bias;
54 if XNN_LIKELY(b != NULL) {
55 v_bias = __riscv_vle32_v_u32m4(b, vlmax); b += nr;
56 } else {
57 v_bias = __riscv_vmv_v_x_u32m4(0, vlmax);
58 }
59 __riscv_vse32_v_u32m4(out, v_bias, vlmax); out += nr;
60
61 uint32_t* out0 = out;
62 size_t k = kc;
63 // vlsseg8, LMUL must <= 1
64 vlmax = __riscv_vsetvlmax_e32m1();
65 // Pack 8 x nr weights
66 for (; k >= 8; k -= 8) {
67 uint32_t* out1 = out0 + nr;
68 uint32_t* out2 = out1 + nr;
69 uint32_t* out3 = out2 + nr;
70 uint32_t* out4 = out3 + nr;
71 uint32_t* out5 = out4 + nr;
72 uint32_t* out6 = out5 + nr;
73 uint32_t* out7 = out6 + nr;
74 // When vlsseg8, LMUL is contraint to 1. We need to use multiple of load & store.
75 const uint32_t* w_ptr = w0;
76 size_t remaining_n = nr;
77 do {
78 vuint32m1x8_t v_w_m1x8 = __riscv_vlsseg8e32_v_u32m1x8(w_ptr, kc_bstride, vlmax);
79 w_ptr += kc * vlmax;
80 vuint32m1_t v_w0 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 0);
81 __riscv_vse32_v_u32m1(out0, v_w0, vlmax); out0 += vlmax;
82 vuint32m1_t v_w1 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 1);
83 __riscv_vse32_v_u32m1(out1, v_w1, vlmax); out1 += vlmax;
84 vuint32m1_t v_w2 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 2);
85 __riscv_vse32_v_u32m1(out2, v_w2, vlmax); out2 += vlmax;
86 vuint32m1_t v_w3 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 3);
87 __riscv_vse32_v_u32m1(out3, v_w3, vlmax); out3 += vlmax;
88 vuint32m1_t v_w4 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 4);
89 __riscv_vse32_v_u32m1(out4, v_w4, vlmax); out4 += vlmax;
90 vuint32m1_t v_w5 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 5);
91 __riscv_vse32_v_u32m1(out5, v_w5, vlmax); out5 += vlmax;
92 vuint32m1_t v_w6 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 6);
93 __riscv_vse32_v_u32m1(out6, v_w6, vlmax); out6 += vlmax;
94 vuint32m1_t v_w7 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 7);
95 __riscv_vse32_v_u32m1(out7, v_w7, vlmax); out7 += vlmax;
96 remaining_n -= vlmax;
97 } while(remaining_n > 0);
98 out0 = out7;
99 w0 += 8;
100 }
101 // vlsseg4, LMUL must <= 2
102 vlmax = __riscv_vsetvlmax_e32m2();
103 // Pack 4 x nr weights
104 for (; k >= 4; k -= 4) {
105 uint32_t* out1 = out0 + nr;
106 uint32_t* out2 = out1 + nr;
107 uint32_t* out3 = out2 + nr;
108 // When vlsseg4, LMUL is contraint to 2. We need to use multiple of load & store.
109 const uint32_t* w_ptr = w0;
110 size_t remaining_n = nr;
111 do {
112 vuint32m2x4_t v_w_m2x4 = __riscv_vlsseg4e32_v_u32m2x4(w_ptr, kc_bstride, vlmax);
113 w_ptr += kc * vlmax;
114 vuint32m2_t v_w0 = __riscv_vget_v_u32m2x4_u32m2(v_w_m2x4, 0);
115 __riscv_vse32_v_u32m2(out0, v_w0, vlmax); out0 += vlmax;
116 vuint32m2_t v_w1 = __riscv_vget_v_u32m2x4_u32m2(v_w_m2x4, 1);
117 __riscv_vse32_v_u32m2(out1, v_w1, vlmax); out1 += vlmax;
118 vuint32m2_t v_w2 = __riscv_vget_v_u32m2x4_u32m2(v_w_m2x4, 2);
119 __riscv_vse32_v_u32m2(out2, v_w2, vlmax); out2 += vlmax;
120 vuint32m2_t v_w3 = __riscv_vget_v_u32m2x4_u32m2(v_w_m2x4, 3);
121 __riscv_vse32_v_u32m2(out3, v_w3, vlmax); out3 += vlmax;
122 remaining_n -= vlmax;
123 } while(remaining_n > 0);
124 out0 = out3;
125 w0 += 4;
126 }
127 vlmax = __riscv_vsetvlmax_e32m4();
128 // Pack nr weights
129 for (; k >= 1; k -= 1) {
130 vuint32m4_t v_w = __riscv_vlse32_v_u32m4(w0, kc_bstride, vlmax);
131 __riscv_vse32_v_u32m4(out0, v_w, vlmax);
132 out0 += vlmax;
133 w0 += 1;
134 }
135 out = (uint32_t*) ((uintptr_t) out0 + extra_bytes);
136 w0 += (nr - 1) * kc;
137 }
138 // NC remainder: process n < NR
139 if (n > 0) {
140 // Pack nr bias at begining of tile
141 size_t vl = __riscv_vsetvl_e32m4(n);
142 vuint32m4_t v_bias;
143 if XNN_LIKELY(b != NULL) {
144 v_bias = __riscv_vle32_v_u32m4(b, vl); b += vl;
145 } else {
146 v_bias = __riscv_vmv_v_x_u32m4(0, vl);
147 }
148 __riscv_vse32_v_u32m4(out, v_bias, vl); out += nr;
149
150 size_t vlmax;
151 uint32_t* out0 = out;
152 size_t k = kc;
153 // vlsseg8, LMUL must <= 1
154 vlmax = __riscv_vsetvlmax_e32m1();
155 // Pack 8 x n weights
156 for (; k >= 8; k -= 8) {
157 uint32_t* out1 = out0 + nr;
158 uint32_t* out2 = out1 + nr;
159 uint32_t* out3 = out2 + nr;
160 uint32_t* out4 = out3 + nr;
161 uint32_t* out5 = out4 + nr;
162 uint32_t* out6 = out5 + nr;
163 uint32_t* out7 = out6 + nr;
164 // When vlsseg8, LMUL is contraint to 1. We need to use multiple of load & store.
165 const uint32_t* w_ptr = w0;
166 unsigned char remaining_blocks = 4;
167 size_t remaining_n = n;
168 do {
169 size_t vl;
170 if XNN_LIKELY(remaining_n >= vlmax) {
171 vl = vlmax;
172 } else {
173 vl = __riscv_vsetvl_e32m1(remaining_n);
174 }
175 vuint32m1x8_t v_w_m1x8 = __riscv_vlsseg8e32_v_u32m1x8(w_ptr, kc_bstride, vl);
176 w_ptr += kc * vl;
177 vuint32m1_t v_w0 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 0);
178 __riscv_vse32_v_u32m1(out0, v_w0, vl); out0 += vlmax;
179 vuint32m1_t v_w1 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 1);
180 __riscv_vse32_v_u32m1(out1, v_w1, vl); out1 += vlmax;
181 vuint32m1_t v_w2 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 2);
182 __riscv_vse32_v_u32m1(out2, v_w2, vl); out2 += vlmax;
183 vuint32m1_t v_w3 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 3);
184 __riscv_vse32_v_u32m1(out3, v_w3, vl); out3 += vlmax;
185 vuint32m1_t v_w4 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 4);
186 __riscv_vse32_v_u32m1(out4, v_w4, vl); out4 += vlmax;
187 vuint32m1_t v_w5 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 5);
188 __riscv_vse32_v_u32m1(out5, v_w5, vl); out5 += vlmax;
189 vuint32m1_t v_w6 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 6);
190 __riscv_vse32_v_u32m1(out6, v_w6, vl); out6 += vlmax;
191 vuint32m1_t v_w7 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 7);
192 __riscv_vse32_v_u32m1(out7, v_w7, vl); out7 += vlmax;
193 remaining_n -= vl;
194 remaining_blocks--;
195 } while(remaining_n > 0);
196 out0 = out7 + remaining_blocks * vlmax;
197 w0 += 8;
198 }
199 // vlsseg4, LMUL must <= 2
200 vlmax = __riscv_vsetvlmax_e32m2();
201 // Pack 4 x n weights
202 for (; k >= 4; k -= 4) {
203 uint32_t* out1 = out0 + nr;
204 uint32_t* out2 = out1 + nr;
205 uint32_t* out3 = out2 + nr;
206 // When vlsseg4, LMUL is contraint to 2. We need to use multiple of load & store.
207 const uint32_t* w_ptr = w0;
208 unsigned char remaining_blocks = 2;
209 size_t remaining_n = n;
210 do {
211 size_t vl;
212 if XNN_LIKELY(remaining_n >= vlmax) {
213 vl = vlmax;
214 } else {
215 vl = __riscv_vsetvl_e32m2(remaining_n);
216 }
217 vuint32m2x4_t v_w_m2x4 = __riscv_vlsseg4e32_v_u32m2x4(w_ptr, kc_bstride, vl);
218 w_ptr += kc * vl;
219 vuint32m2_t v_w0 = __riscv_vget_v_u32m2x4_u32m2(v_w_m2x4, 0);
220 __riscv_vse32_v_u32m2(out0, v_w0, vl); out0 += vlmax;
221 vuint32m2_t v_w1 = __riscv_vget_v_u32m2x4_u32m2(v_w_m2x4, 1);
222 __riscv_vse32_v_u32m2(out1, v_w1, vl); out1 += vlmax;
223 vuint32m2_t v_w2 = __riscv_vget_v_u32m2x4_u32m2(v_w_m2x4, 2);
224 __riscv_vse32_v_u32m2(out2, v_w2, vl); out2 += vlmax;
225 vuint32m2_t v_w3 = __riscv_vget_v_u32m2x4_u32m2(v_w_m2x4, 3);
226 __riscv_vse32_v_u32m2(out3, v_w3, vl); out3 += vlmax;
227 remaining_n -= vl;
228 remaining_blocks--;
229 } while(remaining_n > 0);
230 out0 = out3 + remaining_blocks * vlmax;
231 w0 += 4;
232 }
233 vlmax = __riscv_vsetvlmax_e32m4();
234 vl = __riscv_vsetvl_e32m4(n);
235 // Pack n weights
236 for (; k >= 1; k -= 1) {
237 vuint32m4_t v_w = __riscv_vlse32_v_u32m4(w0, kc_bstride, vl);
238 __riscv_vse32_v_u32m4(out0, v_w, vl);
239 out0 += vlmax;
240 w0 += 1;
241 }
242 out = (uint32_t*) ((uintptr_t) out0 + extra_bytes);
243 w0 += (nr - 1) * kc;
244 }
245 weights += nc * kc;
246 } while (--g != 0);
247 }