Debian Codegraph @

src/x32-packw/gen/x32-packw-x4v-gemm-goi-rvv-u2.c

raw
1 // Auto-generated file. Do not edit!
2 // Template: src/x32-packw/rvv.c.in
3 // Generator: tools/xngen
4 //
5 // Copyright 2024 SiFive, Inc.
6 //
7 // This source code is licensed under the BSD-style license found in the
8 // LICENSE file in the root directory of this source tree.
9
10
11 #include <assert.h>
12 #include <stddef.h>
13 #include <stdint.h>
14
15 #include <riscv_vector.h>
16
17 #include "xnnpack/packw.h"
18
19 void xnn_x32_packw_gemm_goi_ukernel_x4v__rvv_u2(
20 size_t g,
21 size_t nc,
22 size_t kc,
23 size_t nr,
24 size_t kr,
25 size_t sr,
26 const uint32_t* weights,
27 const uint32_t* bias,
28 const void* scale,
29 uint32_t* packed_weights,
30 size_t extra_bytes,
31 const void* params)
32 {
33 assert(g != 0);
34 assert(nc != 0);
35 assert(kc != 0);
36 assert(nr == __riscv_vsetvlmax_e32m4());
37 assert(kr == 1);
38 assert(sr == 1);
39 assert(weights != NULL);
40 assert(packed_weights != NULL);
41
42 uint32_t* out = packed_weights;
43 const uint32_t* b = bias;
44 size_t kc_bstride = kc * 4;
45
46 do {
47 const uint32_t* w0 = weights;
48 size_t n = nc;
49 // NC main loop: process multiple of NR
50 for (;n >= nr; n -= nr) {
51 // Pack nr bias at begining of tile
52 size_t vlmax = __riscv_vsetvlmax_e32m4();
53 vuint32m4_t v_bias;
54 if XNN_LIKELY(b != NULL) {
55 v_bias = __riscv_vle32_v_u32m4(b, vlmax); b += nr;
56 } else {
57 v_bias = __riscv_vmv_v_x_u32m4(0, vlmax);
58 }
59 __riscv_vse32_v_u32m4(out, v_bias, vlmax); out += nr;
60
61 uint32_t* out0 = out;
62 size_t k = kc;
63 // vlsseg2, LMUL must <= 4
64 vlmax = __riscv_vsetvlmax_e32m4();
65 // Pack 2 x nr weights
66 for (; k >= 2; k -= 2) {
67 uint32_t* out1 = out0 + nr;
68 // 1 load & store can pack 2 x nr weights
69 vuint32m4x2_t v_w_m4x2 = __riscv_vlsseg2e32_v_u32m4x2(w0, kc_bstride, vlmax);
70 vuint32m4_t v_w0 = __riscv_vget_v_u32m4x2_u32m4(v_w_m4x2, 0);
71 __riscv_vse32_v_u32m4(out0, v_w0, vlmax);
72 vuint32m4_t v_w1 = __riscv_vget_v_u32m4x2_u32m4(v_w_m4x2, 1);
73 __riscv_vse32_v_u32m4(out1, v_w1, vlmax);
74 out0 = out1 + vlmax;
75 w0 += 2;
76 }
77 vlmax = __riscv_vsetvlmax_e32m4();
78 // Pack nr weights
79 for (; k >= 1; k -= 1) {
80 vuint32m4_t v_w = __riscv_vlse32_v_u32m4(w0, kc_bstride, vlmax);
81 __riscv_vse32_v_u32m4(out0, v_w, vlmax);
82 out0 += vlmax;
83 w0 += 1;
84 }
85 out = (uint32_t*) ((uintptr_t) out0 + extra_bytes);
86 w0 += (nr - 1) * kc;
87 }
88 // NC remainder: process n < NR
89 if (n > 0) {
90 // Pack nr bias at begining of tile
91 size_t vl = __riscv_vsetvl_e32m4(n);
92 vuint32m4_t v_bias;
93 if XNN_LIKELY(b != NULL) {
94 v_bias = __riscv_vle32_v_u32m4(b, vl); b += vl;
95 } else {
96 v_bias = __riscv_vmv_v_x_u32m4(0, vl);
97 }
98 __riscv_vse32_v_u32m4(out, v_bias, vl); out += nr;
99
100 size_t vlmax;
101 uint32_t* out0 = out;
102 size_t k = kc;
103 // vlsseg2, LMUL must <= 4
104 vlmax = __riscv_vsetvlmax_e32m4();
105 // Pack 2 x n weights
106 for (; k >= 2; k -= 2) {
107 uint32_t* out1 = out0 + nr;
108 // 1 load & store can pack 2 x nr weights
109 size_t vl = __riscv_vsetvl_e32m4(n);
110 vuint32m4x2_t v_w_m4x2 = __riscv_vlsseg2e32_v_u32m4x2(w0, kc_bstride, vl);
111 vuint32m4_t v_w0 = __riscv_vget_v_u32m4x2_u32m4(v_w_m4x2, 0);
112 __riscv_vse32_v_u32m4(out0, v_w0, vl);
113 vuint32m4_t v_w1 = __riscv_vget_v_u32m4x2_u32m4(v_w_m4x2, 1);
114 __riscv_vse32_v_u32m4(out1, v_w1, vl);
115 out0 = out1 + vlmax;
116 w0 += 2;
117 }
118 vlmax = __riscv_vsetvlmax_e32m4();
119 vl = __riscv_vsetvl_e32m4(n);
120 // Pack n weights
121 for (; k >= 1; k -= 1) {
122 vuint32m4_t v_w = __riscv_vlse32_v_u32m4(w0, kc_bstride, vl);
123 __riscv_vse32_v_u32m4(out0, v_w, vl);
124 out0 += vlmax;
125 w0 += 1;
126 }
127 out = (uint32_t*) ((uintptr_t) out0 + extra_bytes);
128 w0 += (nr - 1) * kc;
129 }
130 weights += nc * kc;
131 } while (--g != 0);
132 }