Debian Codegraph @

src/x32-packw/gen/x32-packw-x1v-gemm-goi-rvv-u8.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/rvv.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 SiFive, Inc.
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <riscv_vector.h>
16
17 #include "xnnpack/packw.h"
18
19 void xnn_x32_packw_gemm_goi_ukernel_x1v__rvv_u8(
20 size_t g,
21 size_t nc,
22 size_t kc,
23 size_t nr,
24 size_t kr,
25 size_t sr,
26 const uint32_t* weights,
27 const uint32_t* bias,
28 const void* scale,
29 uint32_t* packed_weights,
30 size_t extra_bytes,
31 const void* params)
32 {
33 assert(g != 0);
34 assert(nc != 0);
35 assert(kc != 0);
36 assert(nr == __riscv_vsetvlmax_e32m1());
37 assert(kr == 1);
38 assert(sr == 1);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 uint32_t* out = packed_weights;
43 const uint32_t* b = bias;
44 size_t kc_bstride = kc * 4;
45
46 do {
47 const uint32_t* w0 = weights;
48 size_t n = nc;
49 // NC main loop: process multiple of NR
50 for (;n >= nr; n -= nr) {
51 // Pack nr bias at begining of tile
52 size_t vlmax = __riscv_vsetvlmax_e32m1();
53 vuint32m1_t v_bias;
54 if XNN_LIKELY(b != NULL) {
55 v_bias = __riscv_vle32_v_u32m1(b, vlmax); b += nr;
56 } else {
57 v_bias = __riscv_vmv_v_x_u32m1(0, vlmax);
58 }
59 __riscv_vse32_v_u32m1(out, v_bias, vlmax); out += nr;
60
61 uint32_t* out0 = out;
62 size_t k = kc;
63 // vlsseg8, LMUL must <= 1
64 vlmax = __riscv_vsetvlmax_e32m1();
65 // Pack 8 x nr weights
66 for (; k >= 8; k -= 8) {
67 uint32_t* out1 = out0 + nr;
68 uint32_t* out2 = out1 + nr;
69 uint32_t* out3 = out2 + nr;
70 uint32_t* out4 = out3 + nr;
71 uint32_t* out5 = out4 + nr;
72 uint32_t* out6 = out5 + nr;
73 uint32_t* out7 = out6 + nr;
74 // 1 load & store can pack 8 x nr weights
75 vuint32m1x8_t v_w_m1x8 = __riscv_vlsseg8e32_v_u32m1x8(w0, kc_bstride, vlmax);
76 vuint32m1_t v_w0 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 0);
77 __riscv_vse32_v_u32m1(out0, v_w0, vlmax);
78 vuint32m1_t v_w1 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 1);
79 __riscv_vse32_v_u32m1(out1, v_w1, vlmax);
80 vuint32m1_t v_w2 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 2);
81 __riscv_vse32_v_u32m1(out2, v_w2, vlmax);
82 vuint32m1_t v_w3 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 3);
83 __riscv_vse32_v_u32m1(out3, v_w3, vlmax);
84 vuint32m1_t v_w4 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 4);
85 __riscv_vse32_v_u32m1(out4, v_w4, vlmax);
86 vuint32m1_t v_w5 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 5);
87 __riscv_vse32_v_u32m1(out5, v_w5, vlmax);
88 vuint32m1_t v_w6 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 6);
89 __riscv_vse32_v_u32m1(out6, v_w6, vlmax);
90 vuint32m1_t v_w7 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 7);
91 __riscv_vse32_v_u32m1(out7, v_w7, vlmax);
92 out0 = out7 + vlmax;
93 w0 += 8;
94 }
95 // vlsseg4, LMUL must <= 2
96 vlmax = __riscv_vsetvlmax_e32m1();
97 // Pack 4 x nr weights
98 for (; k >= 4; k -= 4) {
99 uint32_t* out1 = out0 + nr;
100 uint32_t* out2 = out1 + nr;
101 uint32_t* out3 = out2 + nr;
102 // 1 load & store can pack 4 x nr weights
103 vuint32m1x4_t v_w_m1x4 = __riscv_vlsseg4e32_v_u32m1x4(w0, kc_bstride, vlmax);
104 vuint32m1_t v_w0 = __riscv_vget_v_u32m1x4_u32m1(v_w_m1x4, 0);
105 __riscv_vse32_v_u32m1(out0, v_w0, vlmax);
106 vuint32m1_t v_w1 = __riscv_vget_v_u32m1x4_u32m1(v_w_m1x4, 1);
107 __riscv_vse32_v_u32m1(out1, v_w1, vlmax);
108 vuint32m1_t v_w2 = __riscv_vget_v_u32m1x4_u32m1(v_w_m1x4, 2);
109 __riscv_vse32_v_u32m1(out2, v_w2, vlmax);
110 vuint32m1_t v_w3 = __riscv_vget_v_u32m1x4_u32m1(v_w_m1x4, 3);
111 __riscv_vse32_v_u32m1(out3, v_w3, vlmax);
112 out0 = out3 + vlmax;
113 w0 += 4;
114 }
115 vlmax = __riscv_vsetvlmax_e32m1();
116 // Pack nr weights
117 for (; k >= 1; k -= 1) {
118 vuint32m1_t v_w = __riscv_vlse32_v_u32m1(w0, kc_bstride, vlmax);
119 __riscv_vse32_v_u32m1(out0, v_w, vlmax);
120 out0 += vlmax;
121 w0 += 1;
122 }
123 out = (uint32_t*) ((uintptr_t) out0 + extra_bytes);
124 w0 += (nr - 1) * kc;
125 }
126 // NC remainder: process n < NR
127 if (n > 0) {
128 // Pack nr bias at begining of tile
129 size_t vl = __riscv_vsetvl_e32m1(n);
130 vuint32m1_t v_bias;
131 if XNN_LIKELY(b != NULL) {
132 v_bias = __riscv_vle32_v_u32m1(b, vl); b += vl;
133 } else {
134 v_bias = __riscv_vmv_v_x_u32m1(0, vl);
135 }
136 __riscv_vse32_v_u32m1(out, v_bias, vl); out += nr;
137
138 size_t vlmax;
139 uint32_t* out0 = out;
140 size_t k = kc;
141 // vlsseg8, LMUL must <= 1
142 vlmax = __riscv_vsetvlmax_e32m1();
143 // Pack 8 x n weights
144 for (; k >= 8; k -= 8) {
145 uint32_t* out1 = out0 + nr;
146 uint32_t* out2 = out1 + nr;
147 uint32_t* out3 = out2 + nr;
148 uint32_t* out4 = out3 + nr;
149 uint32_t* out5 = out4 + nr;
150 uint32_t* out6 = out5 + nr;
151 uint32_t* out7 = out6 + nr;
152 // 1 load & store can pack 8 x n value
153 size_t vl = __riscv_vsetvl_e32m1(n);
154 vuint32m1x8_t v_w_m1x8 = __riscv_vlsseg8e32_v_u32m1x8(w0, kc_bstride, vl);
155 vuint32m1_t v_w0 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 0);
156 __riscv_vse32_v_u32m1(out0, v_w0, vl);
157 vuint32m1_t v_w1 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 1);
158 __riscv_vse32_v_u32m1(out1, v_w1, vl);
159 vuint32m1_t v_w2 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 2);
160 __riscv_vse32_v_u32m1(out2, v_w2, vl);
161 vuint32m1_t v_w3 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 3);
162 __riscv_vse32_v_u32m1(out3, v_w3, vl);
163 vuint32m1_t v_w4 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 4);
164 __riscv_vse32_v_u32m1(out4, v_w4, vl);
165 vuint32m1_t v_w5 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 5);
166 __riscv_vse32_v_u32m1(out5, v_w5, vl);
167 vuint32m1_t v_w6 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 6);
168 __riscv_vse32_v_u32m1(out6, v_w6, vl);
169 vuint32m1_t v_w7 = __riscv_vget_v_u32m1x8_u32m1(v_w_m1x8, 7);
170 __riscv_vse32_v_u32m1(out7, v_w7, vl);
171 out0 = out7 + vlmax;
172 w0 += 8;
173 }
174 // vlsseg4, LMUL must <= 2
175 vlmax = __riscv_vsetvlmax_e32m1();
176 // Pack 4 x n weights
177 for (; k >= 4; k -= 4) {
178 uint32_t* out1 = out0 + nr;
179 uint32_t* out2 = out1 + nr;
180 uint32_t* out3 = out2 + nr;
181 // 1 load & store can pack 4 x nr weights
182 size_t vl = __riscv_vsetvl_e32m1(n);
183 vuint32m1x4_t v_w_m1x4 = __riscv_vlsseg4e32_v_u32m1x4(w0, kc_bstride, vl);
184 vuint32m1_t v_w0 = __riscv_vget_v_u32m1x4_u32m1(v_w_m1x4, 0);
185 __riscv_vse32_v_u32m1(out0, v_w0, vl);
186 vuint32m1_t v_w1 = __riscv_vget_v_u32m1x4_u32m1(v_w_m1x4, 1);
187 __riscv_vse32_v_u32m1(out1, v_w1, vl);
188 vuint32m1_t v_w2 = __riscv_vget_v_u32m1x4_u32m1(v_w_m1x4, 2);
189 __riscv_vse32_v_u32m1(out2, v_w2, vl);
190 vuint32m1_t v_w3 = __riscv_vget_v_u32m1x4_u32m1(v_w_m1x4, 3);
191 __riscv_vse32_v_u32m1(out3, v_w3, vl);
192 out0 = out3 + vlmax;
193 w0 += 4;
194 }
195 vlmax = __riscv_vsetvlmax_e32m1();
196 vl = __riscv_vsetvl_e32m1(n);
197 // Pack n weights
198 for (; k >= 1; k -= 1) {
199 vuint32m1_t v_w = __riscv_vlse32_v_u32m1(w0, kc_bstride, vl);
200 __riscv_vse32_v_u32m1(out0, v_w, vl);
201 out0 += vlmax;
202 w0 += 1;
203 }
204 out = (uint32_t*) ((uintptr_t) out0 + extra_bytes);
205 w0 += (nr - 1) * kc;
206 }
207 weights += nc * kc;
208 } while (--g != 0);
209 }