Debian Codegraph @

src/3rdparty/pcre2/deps/sljit/sljit_src/sljitNativeRISCV_common.c

raw
1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #if !(defined SLJIT_CONFIG_RISCV_ATOMIC) && defined(__riscv_atomic)
28 /* Auto detect atomic instruction support. */
29 #define SLJIT_CONFIG_RISCV_ATOMIC 200
30 #endif /* !SLJIT_CONFIG_RISCV_ATOMIC && __riscv_atomic */
31
32 /* SLJIT_CONFIG_RISCV_ATOMIC enables/disables atomic instruction
33 support. Non-zero values represents the highest version of the feature
34 that is supported by the CPU.
35 Allowed values: 0 - disabled, 200 - 2.00 */
36 #if (defined SLJIT_CONFIG_RISCV_ATOMIC && SLJIT_CONFIG_RISCV_ATOMIC != 0)
37 #if SLJIT_CONFIG_RISCV_ATOMIC != 200
38 #error "Unsupported value for SLJIT_CONFIG_RISCV_ATOMIC"
39 #endif
40 #define RISCV_HAS_ATOMIC(x) ((SLJIT_CONFIG_RISCV_ATOMIC) >= (x))
41 #define RISCV_ATOMIC_INFO "a"
42 #else /* !SLJIT_CONFIG_RISCV_ATOMIC || SLJIT_CONFIG_RISCV_ATOMIC == 0 */
43 #define RISCV_HAS_ATOMIC(x) 0
44 #define RISCV_ATOMIC_INFO ""
45 #endif /* SLJIT_CONFIG_RISCV_ATOMIC && SLJIT_CONFIG_RISCV_ATOMIC != 0 */
46
47 #if !(defined SLJIT_CONFIG_RISCV_COMPRESSED) && defined(__riscv_compressed)
48 /* Auto detect compressed instruction support. */
49 #define SLJIT_CONFIG_RISCV_COMPRESSED 200
50 #endif /* !SLJIT_CONFIG_RISCV_COMPRESSED && __riscv_compressed */
51
52 /* SLJIT_CONFIG_RISCV_COMPRESSED enables/disables compressed instruction
53 support. Non-zero values represents the highest version of the feature
54 that is supported by the CPU.
55 Allowed values: 0 - disabled, 200 - 2.00 */
56 #if (defined SLJIT_CONFIG_RISCV_COMPRESSED && SLJIT_CONFIG_RISCV_COMPRESSED != 0)
57 #if SLJIT_CONFIG_RISCV_COMPRESSED != 200
58 #error "Unsupported value for SLJIT_CONFIG_RISCV_COMPRESSED"
59 #endif
60 #define RISCV_HAS_COMPRESSED(x) ((SLJIT_CONFIG_RISCV_COMPRESSED) >= (x))
61 #define RISCV_COMPRESSED_CHECK(x) (x)
62 #define RISCV_COMPRESSED_INFO "c"
63 #else /* !SLJIT_CONFIG_RISCV_COMPRESSED || SLJIT_CONFIG_RISCV_COMPRESSED == 0 */
64 #define RISCV_HAS_COMPRESSED(x) 0
65 #define RISCV_COMPRESSED_CHECK(x) 0
66 #define RISCV_COMPRESSED_INFO ""
67 #endif /* SLJIT_CONFIG_RISCV_COMPRESSED && SLJIT_CONFIG_RISCV_COMPRESSED != 0 */
68
69 #if !(defined SLJIT_CONFIG_RISCV_VECTOR) && defined(__riscv_vector)
70 /* Auto detect vector instruction support. */
71 #define SLJIT_CONFIG_RISCV_VECTOR 100
72 #endif /* !SLJIT_CONFIG_RISCV_VECTOR && __riscv_vector */
73
74 /* SLJIT_CONFIG_RISCV_VECTOR enables/disables vector instruction
75 support. Non-zero values represents the highest version of the feature
76 that is supported by the CPU.
77 Allowed values: 0 - disabled, 100 - 1.00 */
78 #if (defined SLJIT_CONFIG_RISCV_VECTOR && SLJIT_CONFIG_RISCV_VECTOR != 0)
79 #if SLJIT_CONFIG_RISCV_VECTOR != 100
80 #error "Unsupported value for SLJIT_CONFIG_RISCV_VECTOR"
81 #endif
82 #define RISCV_HAS_VECTOR(x) ((SLJIT_CONFIG_RISCV_VECTOR) >= (x))
83 #define RISCV_VECTOR_INFO "v"
84 #else /* !SLJIT_CONFIG_RISCV_VECTOR || SLJIT_CONFIG_RISCV_VECTOR == 0 */
85 #define RISCV_HAS_VECTOR(x) 0
86 #define RISCV_VECTOR_INFO ""
87 #endif /* SLJIT_CONFIG_RISCV_VECTOR && SLJIT_CONFIG_RISCV_VECTOR != 0 */
88
89 #if !(defined SLJIT_CONFIG_RISCV_BITMANIP_A) && defined(__riscv_zba)
90 /* Auto detect bit manipulation extension A instruction support. */
91 #define SLJIT_CONFIG_RISCV_BITMANIP_A 93
92 #endif /* !SLJIT_CONFIG_RISCV_BITMANIP_A && __riscv_zba */
93
94 /* SLJIT_CONFIG_RISCV_BITMANIP_A enables/disables bit manipulation extension A
95 instruction support. Non-zero values represents the highest version of the
96 feature that is supported by the CPU.
97 Allowed values: 0 - disabled, 93 - 0.93 */
98 #if (defined SLJIT_CONFIG_RISCV_BITMANIP_A && SLJIT_CONFIG_RISCV_BITMANIP_A != 0)
99 #if SLJIT_CONFIG_RISCV_BITMANIP_A != 93
100 #error "Unsupported value for SLJIT_CONFIG_RISCV_BITMANIP_A"
101 #endif
102 #define RISCV_HAS_BITMANIP_A(x) ((SLJIT_CONFIG_RISCV_BITMANIP_A) >= (x))
103 #define RISCV_BITMANIP_A_INFO "_zba"
104 #else /* !SLJIT_CONFIG_RISCV_BITMANIP_A || SLJIT_CONFIG_RISCV_BITMANIP_A == 0 */
105 #define RISCV_HAS_BITMANIP_A(x) 0
106 #define RISCV_BITMANIP_A_INFO ""
107 #endif /* SLJIT_CONFIG_RISCV_BITMANIP_A && SLJIT_CONFIG_RISCV_BITMANIP_A != 0 */
108
109 #if !(defined SLJIT_CONFIG_RISCV_BITMANIP_B) && defined(__riscv_zbb)
110 /* Auto detect bit manipulation extension B instruction support. */
111 #define SLJIT_CONFIG_RISCV_BITMANIP_B 93
112 #endif /* !SLJIT_CONFIG_RISCV_BITMANIP_B && __riscv_zbb */
113
114 /* SLJIT_CONFIG_RISCV_BITMANIP_B enables/disables bit manipulation extension B
115 instruction support. Non-zero values represents the highest version of the
116 feature that is supported by the CPU.
117 Allowed values: 0 - disabled, 93 - 0.93 */
118 #if (defined SLJIT_CONFIG_RISCV_BITMANIP_B && SLJIT_CONFIG_RISCV_BITMANIP_B != 0)
119 #if SLJIT_CONFIG_RISCV_BITMANIP_B != 93
120 #error "Unsupported value for SLJIT_CONFIG_RISCV_BITMANIP_B"
121 #endif
122 #define RISCV_HAS_BITMANIP_B(x) ((SLJIT_CONFIG_RISCV_BITMANIP_B) >= (x))
123 #define RISCV_BITMANIP_B_INFO "_zbb"
124 #else /* !SLJIT_CONFIG_RISCV_BITMANIP_B || SLJIT_CONFIG_RISCV_BITMANIP_B == 0 */
125 #define RISCV_HAS_BITMANIP_B(x) 0
126 #define RISCV_BITMANIP_B_INFO ""
127 #endif /* SLJIT_CONFIG_RISCV_BITMANIP_B && SLJIT_CONFIG_RISCV_BITMANIP_B != 0 */
128
129 #if !(defined SLJIT_CONFIG_RISCV_ICOND) && defined(__riscv_zicond)
130 /* Auto detect integer conditional instruction support. */
131 #define SLJIT_CONFIG_RISCV_ICOND 100
132 #endif /* !SLJIT_CONFIG_RISCV_ICOND && __riscv_zicond */
133
134 /* SLJIT_CONFIG_RISCV_ICOND enables/disables integer conditional
135 instruction support. Non-zero values represents the highest version of the
136 feature that is supported by the CPU.
137 Allowed values: 0 - disabled, 100 - 1.00 */
138 #if (defined SLJIT_CONFIG_RISCV_ICOND && SLJIT_CONFIG_RISCV_ICOND != 0)
139 #if SLJIT_CONFIG_RISCV_ICOND != 100
140 #error "Unsupported value for SLJIT_CONFIG_RISCV_ICOND"
141 #endif
142 #define RISCV_HAS_ICOND(x) ((SLJIT_CONFIG_RISCV_ICOND) >= (x))
143 #define RISCV_ICOND_INFO "_zicond"
144 #else /* !SLJIT_CONFIG_RISCV_ICOND || SLJIT_CONFIG_RISCV_ICOND == 0 */
145 #define RISCV_HAS_ICOND(x) 0
146 #define RISCV_ICOND_INFO ""
147 #endif /* SLJIT_CONFIG_RISCV_ICOND && SLJIT_CONFIG_RISCV_ICOND != 0 */
148
149 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
150 #define RISCV_CHECK_COMPRESSED_JUMP(jump, diff, unit) \
151 (!((jump)->flags & IS_CALL) && (diff) >= (JUMP16_MIN / SSIZE_OF(unit)) && (diff) <= (JUMP16_MAX / SSIZE_OF(unit)))
152 #else /* !SLJIT_CONFIG_RISCV_64 */
153 #define RISCV_CHECK_COMPRESSED_JUMP(jump, diff, unit) \
154 ((diff) >= (JUMP16_MIN / SSIZE_OF(unit)) && (diff) <= (JUMP16_MAX / SSIZE_OF(unit)))
155 #endif /* SLJIT_CONFIG_RISCV_64 */
156
157 SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
158 {
159 /* The arch string is not entirely correct since 'g' contains 'a'. */
160 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
161 return "RISC-V-32 (rv32g" RISCV_ATOMIC_INFO RISCV_COMPRESSED_INFO RISCV_VECTOR_INFO RISCV_BITMANIP_A_INFO RISCV_BITMANIP_B_INFO RISCV_ICOND_INFO ")" SLJIT_CPUINFO;
162 #else /* !SLJIT_CONFIG_RISCV_32 */
163 return "RISC-V-64 (rv64g" RISCV_ATOMIC_INFO RISCV_COMPRESSED_INFO RISCV_VECTOR_INFO RISCV_BITMANIP_A_INFO RISCV_BITMANIP_B_INFO RISCV_ICOND_INFO ")" SLJIT_CPUINFO;
164 #endif /* SLJIT_CONFIG_RISCV_32 */
165 }
166
167 /* Most instructions are 32 bit long on RISCV.
168 Code is processed as 16 bit units. */
169 typedef sljit_u32 sljit_ins;
170
171 #define TMP_REG1 (SLJIT_NUMBER_OF_REGISTERS + 2)
172 #define TMP_REG2 (SLJIT_NUMBER_OF_REGISTERS + 3)
173 #define TMP_REG3 (SLJIT_NUMBER_OF_REGISTERS + 4)
174 #define TMP_ZERO 0
175
176 /* Flags are kept in volatile registers. */
177 #define EQUAL_FLAG (SLJIT_NUMBER_OF_REGISTERS + 5)
178 #define RETURN_ADDR_REG TMP_REG2
179 #define OTHER_FLAG (SLJIT_NUMBER_OF_REGISTERS + 6)
180
181 #define TMP_FREG1 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
182 #define TMP_FREG2 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2)
183
184 #define TMP_VREG1 (SLJIT_NUMBER_OF_VECTOR_REGISTERS + 1)
185 #define TMP_VREG2 (SLJIT_NUMBER_OF_VECTOR_REGISTERS + 2)
186
187 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 7] = {
188 0, 10, 11, 12, 13, 14, 15, 16, 17, 29, 30, 31, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 9, 8, 2, 6, 1, 7, 5, 28
189 };
190
191 static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = {
192 0, 10, 11, 12, 13, 14, 15, 16, 17, 2, 3, 4, 5, 6, 7, 28, 29, 30, 31, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 9, 8, 0, 1,
193 };
194
195 static const sljit_u8 vreg_map[SLJIT_NUMBER_OF_VECTOR_REGISTERS + 3] = {
196 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
197 };
198
199 /* --------------------------------------------------------------------- */
200 /* Instruction forms */
201 /* --------------------------------------------------------------------- */
202
203 #define RD(rd) ((sljit_ins)reg_map[rd] << 7)
204 #define RS1(rs1) ((sljit_ins)reg_map[rs1] << 15)
205 #define RS2(rs2) ((sljit_ins)reg_map[rs2] << 20)
206 #define FRD(rd) ((sljit_ins)freg_map[rd] << 7)
207 #define FRS1(rs1) ((sljit_ins)freg_map[rs1] << 15)
208 #define FRS2(rs2) ((sljit_ins)freg_map[rs2] << 20)
209 #define VRD(rd) ((sljit_ins)vreg_map[rd] << 7)
210 #define VRS1(rs1) ((sljit_ins)vreg_map[rs1] << 15)
211 #define VRS2(rs2) ((sljit_ins)vreg_map[rs2] << 20)
212 #define IMM_I(imm) ((sljit_ins)(imm) << 20)
213 #define IMM_S(imm) ((((sljit_ins)(imm) & 0xfe0) << 20) | (((sljit_ins)(imm) & 0x1f) << 7))
214 #define C_RD(rd) ((sljit_u16)((sljit_u16)reg_map[rd] << 7))
215 #define C_RS2(rd) ((sljit_u16)((sljit_u16)reg_map[rd] << 2))
216 #define C_FRD(rd) ((sljit_u16)((sljit_u16)freg_map[rd] << 7))
217 #define C_FRS2(rd) ((sljit_u16)((sljit_u16)freg_map[rd] << 2))
218 #define C_RS1_R3(rs1) ((sljit_u16)((sljit_u16)(reg_map[rs1] & 0x7) << 7))
219 #define C_RS2_R3(rs2) ((sljit_u16)((sljit_u16)(reg_map[rs2] & 0x7) << 2))
220 #define C_FRS2_R3(rs2) ((sljit_u16)((sljit_u16)(freg_map[rs2] & 0x7) << 2))
221 #define C_IS_R3(r) ((reg_map[r] & 0x18) == 0x08)
222 #define C_IS_FR3(r) ((freg_map[r] & 0x18) == 0x08)
223 #define C_IMM_I(imm) ((sljit_u16)((((imm) & 0x1f) << 2) | (((imm) & 0x20) << 7)))
224 #define C_LD32_SP(imm) ((sljit_u16)((((imm) & 0x1c) << 2) | (((imm) & 0x20) << 7) | (((imm) & 0xc0) >> 4)))
225 #define C_ST32_SP(imm) ((sljit_u16)((((imm) & 0x3c) << 7) | (((imm) & 0xc0) << 1)))
226 #define C_LD64_SP(imm) ((sljit_u16)((((imm) & 0x18) << 2) | (((imm) & 0x20) << 7) | (((imm) & 0x1c0) >> 4)))
227 #define C_ST64_SP(imm) ((sljit_u16)((((imm) & 0x38) << 7) | (((imm) & 0x1c0) << 1)))
228 #define C_MEM32(imm) ((sljit_u16)((((imm) & 0x4) << 4) | (((imm) & 0x38) << 7) | (((imm) & 0x40) >> 1)))
229 #define C_MEM64(imm) ((sljit_u16)((((imm) & 0x38) << 7) | (((imm) & 0xc0) >> 1)))
230 #define C_BRN16(imm) ((sljit_u16)((((imm) & 0x6) << 2) | (((imm) & 0x18) << 7) | (((imm) & 0x20) >> 3) | (((imm) & 0xc0) >> 1) | (((imm) & 0x100) << 4)))
231 #define C_JMP16(imm) ((sljit_u16)((((imm) & 0xb40) << 1) | (((imm) & 0xe) << 2) | (((imm) & 0x10) << 7) | (((imm) & 0x20) >> 3) | (((imm) & 0x80) >> 1) | (((imm) & 0x400) >> 2)))
232
233 /* Represents funct(i) parts of the instructions. */
234 #define OPC(o) ((sljit_ins)(o))
235 #define C_OPC(o1, o2) ((sljit_u16)((o1) | ((o2) << 13)))
236 #define F3(f) ((sljit_ins)(f) << 12)
237 #define F12(f) ((sljit_ins)(f) << 20)
238 #define F7(f) ((sljit_ins)(f) << 25)
239
240 /* Vector instruction types. */
241 #define OPFVF (F3(0x5) | OPC(0x57))
242 #define OPFVV (F3(0x1) | OPC(0x57))
243 #define OPIVI (F3(0x3) | OPC(0x57))
244 #define OPIVV (F3(0x0) | OPC(0x57))
245 #define OPIVX (F3(0x4) | OPC(0x57))
246 #define OPMVV (F3(0x2) | OPC(0x57))
247 #define OPMVX (F3(0x6) | OPC(0x57))
248
249 #define ADD (F7(0x0) | F3(0x0) | OPC(0x33))
250 #define ADDI (F3(0x0) | OPC(0x13))
251 #define AND (F7(0x0) | F3(0x7) | OPC(0x33))
252 #define ANDI (F3(0x7) | OPC(0x13))
253 #define AUIPC (OPC(0x17))
254 #define BEQ (F3(0x0) | OPC(0x63))
255 #define BNE (F3(0x1) | OPC(0x63))
256 #define BLT (F3(0x4) | OPC(0x63))
257 #define BGE (F3(0x5) | OPC(0x63))
258 #define BLTU (F3(0x6) | OPC(0x63))
259 #define BGEU (F3(0x7) | OPC(0x63))
260 /* C_*: compressed */
261 #define C_ADD (C_OPC(0x2, 0x4) | (sljit_u16)(1 << 12))
262 #define C_ADDI (C_OPC(0x1, 0x0))
263 #define C_ADDIW (C_OPC(0x1, 0x1))
264 #define C_ADDW (C_OPC(0x1, 0x4) | (sljit_u16)(7 << 10) | (sljit_u16)(1 << 5))
265 #define C_ADDI16SP (C_OPC(0x1, 0x3) | (sljit_u16)(2 << 7))
266 #define C_AND (C_OPC(0x1, 0x4) | (sljit_u16)(3 << 10) | (sljit_u16)(3 << 5))
267 #define C_ANDI (C_OPC(0x1, 0x4) | (sljit_u16)(2 << 10))
268 #define C_BEQZ (C_OPC(0x1, 0x6))
269 #define C_BNEZ (C_OPC(0x1, 0x7))
270 #define C_EBREAK (C_OPC(0x2, 0x4) | (sljit_u16)(1 << 12))
271 #define C_FLDSP (C_OPC(0x2, 0x1))
272 #define C_FLWSP (C_OPC(0x2, 0x3))
273 #define C_FSD (C_OPC(0x0, 0x5))
274 #define C_FSDSP (C_OPC(0x2, 0x5))
275 #define C_FSW (C_OPC(0x0, 0x7))
276 #define C_FSWSP (C_OPC(0x2, 0x7))
277 #define C_J (C_OPC(0x1, 0x5))
278 #define C_JR (C_OPC(0x2, 0x4))
279 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
280 #define C_JAL (C_OPC(0x1, 0x1))
281 #endif
282 #define C_JALR (C_OPC(0x2, 0x4) | (sljit_u16)(1 << 12))
283 #define C_LI (C_OPC(0x1, 0x2))
284 #define C_LUI (C_OPC(0x1, 0x3))
285 #define C_LDSP (C_OPC(0x2, 0x3))
286 #define C_LWSP (C_OPC(0x2, 0x2))
287 #define C_MV (C_OPC(0x2, 0x4))
288 #define C_NOP (C_OPC(0x1, 0x0))
289 #define C_SD (C_OPC(0x0, 0x7))
290 #define C_SDSP (C_OPC(0x2, 0x7))
291 #define C_SLLI (C_OPC(0x2, 0x0))
292 #define C_OR (C_OPC(0x1, 0x4) | (sljit_u16)(3 << 10) | (sljit_u16)(2 << 5))
293 #define C_SRAI (C_OPC(0x1, 0x4) | (sljit_u16)(1 << 10))
294 #define C_SRLI (C_OPC(0x1, 0x4) | (sljit_u16)(0 << 10))
295 #define C_SUB (C_OPC(0x1, 0x4) | (sljit_u16)(3 << 10) | (sljit_u16)(0 << 5))
296 #define C_SW (C_OPC(0x0, 0x6))
297 #define C_SWSP (C_OPC(0x2, 0x6))
298 #define C_XOR (C_OPC(0x1, 0x4) | (sljit_u16)(3 << 10) | (sljit_u16)(1 << 5))
299 /* CLZ / CTZ: zbb */
300 #define CLZ (F7(0x30) | F3(0x1) | OPC(0x13))
301 #define CTZ (F7(0x30) | F12(0x1) | F3(0x1) | OPC(0x13))
302 #define CZERO_EQZ (F7(0x7) | F3(0x5) | OPC(0x33))
303 #define CZERO_NEZ (F7(0x7) | F3(0x7) | OPC(0x33))
304 #define DIV (F7(0x1) | F3(0x4) | OPC(0x33))
305 #define DIVU (F7(0x1) | F3(0x5) | OPC(0x33))
306 #define EBREAK (F12(0x1) | F3(0x0) | OPC(0x73))
307 #define FADD_S (F7(0x0) | F3(0x7) | OPC(0x53))
308 #define FDIV_S (F7(0xc) | F3(0x7) | OPC(0x53))
309 #define FENCE (F3(0x0) | OPC(0xf))
310 #define FEQ_S (F7(0x50) | F3(0x2) | OPC(0x53))
311 #define FLD (F3(0x3) | OPC(0x7))
312 #define FLE_S (F7(0x50) | F3(0x0) | OPC(0x53))
313 #define FLT_S (F7(0x50) | F3(0x1) | OPC(0x53))
314 /* These conversion opcodes are partly defined. */
315 #define FCVT_S_D (F7(0x20) | OPC(0x53))
316 #define FCVT_S_W (F7(0x68) | OPC(0x53))
317 #define FCVT_S_WU (F7(0x68) | F12(0x1) | OPC(0x53))
318 #define FCVT_W_S (F7(0x60) | F3(0x1) | OPC(0x53))
319 #define FMUL_S (F7(0x8) | F3(0x7) | OPC(0x53))
320 #define FMV_X_W (F7(0x70) | F3(0x0) | OPC(0x53))
321 #define FMV_W_X (F7(0x78) | F3(0x0) | OPC(0x53))
322 #define FSD (F3(0x3) | OPC(0x27))
323 #define FSGNJ_S (F7(0x10) | F3(0x0) | OPC(0x53))
324 #define FSGNJN_S (F7(0x10) | F3(0x1) | OPC(0x53))
325 #define FSGNJX_S (F7(0x10) | F3(0x2) | OPC(0x53))
326 #define FSUB_S (F7(0x4) | F3(0x7) | OPC(0x53))
327 #define FSW (F3(0x2) | OPC(0x27))
328 #define JAL (OPC(0x6f))
329 #define JALR (F3(0x0) | OPC(0x67))
330 #define LD (F3(0x3) | OPC(0x3))
331 #define LUI (OPC(0x37))
332 #define LW (F3(0x2) | OPC(0x3))
333 /* LR: atomic */
334 #define LR (F7(0x8) | OPC(0x2f))
335 #define MAX (F7(0x5) | F3(0x6) | OPC(0x33))
336 #define MAXU (F7(0x5) | F3(0x7) | OPC(0x33))
337 #define MIN (F7(0x5) | F3(0x4) | OPC(0x33))
338 #define MINU (F7(0x5) | F3(0x5) | OPC(0x33))
339 #define MUL (F7(0x1) | F3(0x0) | OPC(0x33))
340 #define MULH (F7(0x1) | F3(0x1) | OPC(0x33))
341 #define MULHU (F7(0x1) | F3(0x3) | OPC(0x33))
342 #define NOP ADDI
343 #define OR (F7(0x0) | F3(0x6) | OPC(0x33))
344 #define ORI (F3(0x6) | OPC(0x13))
345 #define REM (F7(0x1) | F3(0x6) | OPC(0x33))
346 #define REMU (F7(0x1) | F3(0x7) | OPC(0x33))
347 /* REV8 / ROL / ROR / RORI: zbb */
348 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
349 #define REV8 (F12(0x698) | F3(0x5) | OPC(0x13))
350 #elif (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
351 #define REV8 (F12(0x6b8) | F3(0x5) | OPC(0x13))
352 #endif /* SLJIT_CONFIG_RISCV_32 */
353 #define ROL (F7(0x30) | F3(0x1) | OPC(0x33))
354 #define ROR (F7(0x30) | F3(0x5) | OPC(0x33))
355 #define RORI (F7(0x30) | F3(0x5) | OPC(0x13))
356 /* SC: atomic */
357 #define SC (F7(0xc) | OPC(0x2f))
358 #define SD (F3(0x3) | OPC(0x23))
359 /* SEXTB / SEXTH: zbb */
360 #define SEXTB (F7(0x30) | F12(0x4) | F3(0x1) | OPC(0x13))
361 #define SEXTH (F7(0x30) | F12(0x5) | F3(0x1) | OPC(0x13))
362 /* SH1ADD / SH2ADD / SH3ADD: zba */
363 #define SH1ADD (F7(0x10) | F3(0x2) | OPC(0x33))
364 #define SH2ADD (F7(0x10) | F3(0x4) | OPC(0x33))
365 #define SH3ADD (F7(0x10) | F3(0x6) | OPC(0x33))
366 #define SLL (F7(0x0) | F3(0x1) | OPC(0x33))
367 #define SLLI (F3(0x1) | OPC(0x13))
368 #define SLT (F7(0x0) | F3(0x2) | OPC(0x33))
369 #define SLTI (F3(0x2) | OPC(0x13))
370 #define SLTU (F7(0x0) | F3(0x3) | OPC(0x33))
371 #define SLTUI (F3(0x3) | OPC(0x13))
372 #define SRL (F7(0x0) | F3(0x5) | OPC(0x33))
373 #define SRLI (F3(0x5) | OPC(0x13))
374 #define SRA (F7(0x20) | F3(0x5) | OPC(0x33))
375 #define SRAI (F7(0x20) | F3(0x5) | OPC(0x13))
376 #define SUB (F7(0x20) | F3(0x0) | OPC(0x33))
377 #define SW (F3(0x2) | OPC(0x23))
378 /* V*: vector */
379 #define VAND_VV (F7(0x13) | OPIVV)
380 #define VFMV_FS (F7(0x21) | OPFVV)
381 #define VFMV_SF (F7(0x21) | OPFVF)
382 #define VFMV_VF (F7(0x2f) | OPFVF)
383 #define VFWCVT_FFV (F7(0x25) | (0xc << 15) | OPFVV)
384 #define VL (F7(0x1) | OPC(0x7))
385 #define VMSLE_VI (F7(0x3b) | OPIVI)
386 #define VMV_SX (F7(0x21) | OPMVX)
387 #define VMV_VI (F7(0x2f) | OPIVI)
388 #define VMV_VV (F7(0x2f) | OPIVV)
389 #define VMV_VX (F7(0x2f) | OPIVX)
390 #define VMV_XS (F7(0x21) | OPMVV)
391 #define VOR_VV (F7(0x15) | OPIVV)
392 #define VSETVLI (F7(0x0) | F3(0x7) | OPC(0x57))
393 #define VSETIVLI (F7(0x60) | F3(0x7) | OPC(0x57))
394 #define VS (F7(0x1) | OPC(0x27))
395 #define VSLIDEDOWN_VX (F7(0x1f) | OPIVX)
396 #define VSLIDEDOWN_VI (F7(0x1f) | OPIVI)
397 #define VSLIDEUP_VX (F7(0x1d) | OPIVX)
398 #define VSLIDEUP_VI (F7(0x1d) | OPIVI)
399 #define VRGATHER_VI (F7(0x19) | OPIVI)
400 #define VRGATHER_VV (F7(0x19) | OPIVV)
401 #define VXOR_VV (F7(0x17) | OPIVV)
402 #define VZEXT_VF2 (F7(0x25) | (0x6 << 15) | OPMVV)
403 #define VZEXT_VF4 (F7(0x25) | (0x4 << 15) | OPMVV)
404 #define VZEXT_VF8 (F7(0x25) | (0x2 << 15) | OPMVV)
405 #define XOR (F7(0x0) | F3(0x4) | OPC(0x33))
406 #define XORI (F3(0x4) | OPC(0x13))
407 /* ZEXTH: zbb */
408 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
409 #define ZEXTH (F7(0x4) | F3(0x4) | OPC(0x33))
410 #else /* SLJIT_CONFIG_RISCV_64 */
411 #define ZEXTH (F7(0x4) | F3(0x4) | OPC(0x3B))
412 #endif /* SLJIT_CONFIG_RISCV_32 */
413
414 #define SIMM_MAX (0x7ff)
415 #define SIMM_MIN (-0x800)
416 #define SIMM16_MAX (0x1f)
417 #define SIMM16_MIN (-0x20)
418 #define BRANCH_MAX (0xfff)
419 #define BRANCH_MIN (-0x1000)
420 #define BRANCH16_MAX (0xff)
421 #define BRANCH16_MIN (-0x100)
422 #define JUMP_MAX (0xfffff)
423 #define JUMP_MIN (-0x100000)
424 #define JUMP16_MAX SIMM_MAX
425 #define JUMP16_MIN SIMM_MIN
426
427 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
428 #define S32_MAX (0x7ffff7ffl)
429 #define S32_MIN (-0x80000000l)
430 #define S44_MAX (0x7fffffff7ffl)
431 #define S52_MAX (0x7ffffffffffffl)
432 #endif /* SLJIT_CONFIG_RISCV_64 */
433
434 #define C_ADDI_W(word) (C_ADDI | (sljit_u16)((word) << 10))
435 #define C_SUB_W(word) (C_SUB | (sljit_u16)((word) << 9))
436
437 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
438 #define BRANCH_LENGTH ((sljit_ins)(3 * sizeof(sljit_ins)) << 7)
439 #define BRANCH16_LENGTH C_BRN16(5 * sizeof(sljit_u16))
440 #else /* !SLJIT_CONFIG_RISCV_32 */
441 #define BRANCH_LENGTH ((sljit_ins)(7 * sizeof(sljit_ins)) << 7)
442 #define BRANCH16_LENGTH C_BRN16(13 * sizeof(sljit_u16))
443 #endif /* SLJIT_CONFIG_RISCV_32 */
444
445 static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins)
446 {
447 sljit_u16 *ptr = (sljit_u16*)ensure_buf(compiler, sizeof(sljit_ins));
448 FAIL_IF(!ptr);
449 ptr[0] = (sljit_u16)ins;
450 ptr[1] = (sljit_u16)(ins >> 16);
451 compiler->size += 2;
452 return SLJIT_SUCCESS;
453 }
454
455 static sljit_s32 push_inst16(struct sljit_compiler *compiler, sljit_u16 ins)
456 {
457 sljit_u16 *ptr = (sljit_u16*)ensure_buf(compiler, sizeof(sljit_u16));
458 FAIL_IF(!ptr);
459 *ptr = ins;
460 compiler->size++;
461 return SLJIT_SUCCESS;
462 }
463
464 static sljit_s32 push_imm_s_inst(struct sljit_compiler *compiler, sljit_ins ins, sljit_sw imm)
465 {
466 return push_inst(compiler, ins | IMM_S(imm));
467 }
468
469 static SLJIT_INLINE sljit_u16* detect_jump_type(struct sljit_jump *jump, sljit_u16 *code_ptr, sljit_u16 *code, sljit_sw executable_offset)
470 {
471 sljit_sw diff, cond_diff;
472 sljit_uw target_addr;
473 sljit_uw jump_addr = (sljit_uw)code_ptr;
474 sljit_uw orig_addr = jump->addr;
475 sljit_ins ins;
476 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
477 sljit_s32 jalr_offset = JUMP_MAX_SIZE - 1;
478 #endif /* SLJIT_CONFIG_RISCV_64 */
479 SLJIT_UNUSED_ARG(executable_offset);
480
481 jump->addr = jump_addr;
482 if (jump->flags & SLJIT_REWRITABLE_JUMP)
483 goto exit;
484
485 if (jump->flags & JUMP_ADDR)
486 target_addr = jump->u.target;
487 else {
488 SLJIT_ASSERT(jump->u.label != NULL);
489 target_addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code + jump->u.label->size, executable_offset);
490
491 if (jump->u.label->size > orig_addr)
492 jump_addr = (sljit_uw)(code + orig_addr);
493 }
494
495 diff = (sljit_sw)target_addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_addr, executable_offset);
496
497 if (jump->flags & IS_COND) {
498 cond_diff = diff + SSIZE_OF(ins);
499
500 if (RISCV_COMPRESSED_CHECK((jump->flags & IS_COND16))) {
501 SLJIT_ASSERT((code_ptr[-1] & 0xe003) == C_BEQZ || (code_ptr[-1] & 0xe003) == C_BNEZ);
502
503 cond_diff = diff + SSIZE_OF(u16);
504
505 if (diff >= BRANCH16_MIN && diff <= BRANCH16_MAX) {
506 code_ptr--;
507 code_ptr[0] = (sljit_u16)((code_ptr[0] & 0xe383) ^ 0x2000);
508 jump->flags |= PATCH_B | PATCH_16;
509 jump->addr = (sljit_uw)code_ptr;
510 return code_ptr;
511 }
512 }
513
514 if (cond_diff >= BRANCH_MIN && cond_diff <= BRANCH_MAX) {
515 if (RISCV_COMPRESSED_CHECK((jump->flags & IS_COND16))) {
516 /* Converting 16 bit branch to 32 bit branch. */
517 code_ptr--;
518 code_ptr[1] = (sljit_u16)(((code_ptr[0] & 0x300) >> 8) | 0x4);
519 code_ptr[0] = (sljit_u16)((BNE ^ ((code_ptr[0] & 0x2000) >> 1)) | ((code_ptr[0] & 0x80) << 8));
520 } else {
521 code_ptr -= 2;
522 code_ptr[0] = (sljit_u16)((code_ptr[0] & 0xf07f) ^ 0x1000);
523 code_ptr[1] = (sljit_u16)(code_ptr[1] & 0x1ff);
524 }
525
526 jump->flags |= PATCH_B;
527 jump->addr = (sljit_uw)code_ptr;
528 return code_ptr + 1;
529 }
530 }
531
532 if (RISCV_HAS_COMPRESSED(200) && RISCV_CHECK_COMPRESSED_JUMP(jump, diff, u8)) {
533 /* A conditional instruction has larger max offset
534 than a 16 bit jump instruction. */
535 SLJIT_ASSERT(!(jump->flags & IS_COND));
536 jump->flags |= PATCH_J | PATCH_16;
537 return code_ptr;
538 }
539
540 if (diff >= JUMP_MIN && diff <= JUMP_MAX) {
541 if (jump->flags & IS_COND) {
542 if (RISCV_COMPRESSED_CHECK((jump->flags & IS_COND16)))
543 code_ptr[-1] ^= (sljit_u16)(BRANCH16_LENGTH ^ C_BRN16(3 * sizeof(sljit_u16)));
544 else
545 code_ptr[-2] ^= (sljit_u16)(BRANCH_LENGTH ^ (2 * sizeof(sljit_ins) << 7));
546 }
547
548 jump->flags |= PATCH_J;
549 return code_ptr + 1;
550 }
551
552 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
553 if (diff >= S32_MIN && diff <= S32_MAX) {
554 if (jump->flags & IS_COND) {
555 if (RISCV_COMPRESSED_CHECK((jump->flags & IS_COND16)))
556 code_ptr[-1] ^= (sljit_u16)(BRANCH16_LENGTH ^ C_BRN16(5 * sizeof(sljit_u16)));
557 else
558 code_ptr[-2] ^= (sljit_u16)(BRANCH_LENGTH ^ (3 * sizeof(sljit_ins) << 7));
559 }
560
561 jump->flags |= PATCH_REL32;
562 jalr_offset = 3;
563 } else if (target_addr <= (sljit_uw)S32_MAX) {
564 if (jump->flags & IS_COND) {
565 if (RISCV_COMPRESSED_CHECK((jump->flags & IS_COND16)))
566 code_ptr[-1] ^= (sljit_u16)(BRANCH16_LENGTH ^ C_BRN16(5 * sizeof(sljit_u16)));
567 else
568 code_ptr[-2] ^= (sljit_u16)(BRANCH_LENGTH ^ (3 * sizeof(sljit_ins) << 7));
569 }
570
571 jump->flags |= PATCH_ABS32;
572 jalr_offset = 3;
573 } else if (target_addr <= S44_MAX) {
574 if (jump->flags & IS_COND) {
575 if (RISCV_COMPRESSED_CHECK((jump->flags & IS_COND16)))
576 code_ptr[-1] ^= (sljit_u16)(BRANCH16_LENGTH ^ C_BRN16(9 * sizeof(sljit_u16)));
577 else
578 code_ptr[-2] ^= (sljit_u16)(BRANCH_LENGTH ^ (5 * sizeof(sljit_ins) << 7));
579 }
580
581 jump->flags |= PATCH_ABS44;
582 jalr_offset = 7;
583 } else if (target_addr <= S52_MAX) {
584 if (jump->flags & IS_COND) {
585 if (RISCV_COMPRESSED_CHECK((jump->flags & IS_COND16)))
586 code_ptr[-1] ^= (sljit_u16)(BRANCH16_LENGTH ^ C_BRN16(11 * sizeof(sljit_u16)));
587 else
588 code_ptr[-2] ^= (sljit_u16)(BRANCH_LENGTH ^ (6 * sizeof(sljit_ins) << 7));
589 }
590
591 jump->flags |= PATCH_ABS52;
592 jalr_offset = 9;
593 }
594 #endif /* SLJIT_CONFIG_RISCV_64 */
595
596 exit:
597 ins = JALR | RS1(TMP_REG1) | IMM_I(0);
598 if (jump->flags & IS_CALL)
599 ins |= RD(RETURN_ADDR_REG);
600
601 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
602 code_ptr[2] = (sljit_u16)ins;
603 code_ptr[3] = (sljit_u16)(ins >> 16);
604 return code_ptr + 3;
605 #else /* !SLJIT_CONFIG_RISCV_32 */
606 code_ptr += jalr_offset;
607 code_ptr[-1] = (sljit_u16)ins;
608 code_ptr[0] = (sljit_u16)(ins >> 16);
609 return code_ptr;
610 #endif /* SLJIT_CONFIG_RISCV_32 */
611 }
612
613 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
614
615 static SLJIT_INLINE sljit_sw mov_addr_get_length(struct sljit_jump *jump, sljit_u16 *code_ptr, sljit_u16 *code, sljit_sw executable_offset)
616 {
617 sljit_uw addr;
618 sljit_uw jump_addr = (sljit_uw)code_ptr;
619 sljit_sw diff;
620 SLJIT_UNUSED_ARG(executable_offset);
621
622 SLJIT_ASSERT(jump->flags < (JUMP_MAX_SIZE << JUMP_SIZE_SHIFT));
623 if (jump->flags & JUMP_ADDR)
624 addr = jump->u.target;
625 else {
626 addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code + jump->u.label->size, executable_offset);
627
628 if (jump->u.label->size > jump->addr)
629 jump_addr = (sljit_uw)(code + jump->addr);
630 }
631
632 diff = (sljit_sw)addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_addr, executable_offset);
633
634 if (diff >= S32_MIN && diff <= S32_MAX) {
635 SLJIT_ASSERT(jump->flags >= ((sljit_uw)3 << JUMP_SIZE_SHIFT));
636 jump->flags |= PATCH_REL32;
637 return 3;
638 }
639
640 if (addr <= S32_MAX) {
641 SLJIT_ASSERT(jump->flags >= ((sljit_uw)3 << JUMP_SIZE_SHIFT));
642 jump->flags |= PATCH_ABS32;
643 return 3;
644 }
645
646 if (addr <= S44_MAX) {
647 SLJIT_ASSERT(jump->flags >= ((sljit_uw)7 << JUMP_SIZE_SHIFT));
648 jump->flags |= PATCH_ABS44;
649 return 7;
650 }
651
652 if (addr <= S52_MAX) {
653 SLJIT_ASSERT(jump->flags >= ((sljit_uw)9 << JUMP_SIZE_SHIFT));
654 jump->flags |= PATCH_ABS52;
655 return 9;
656 }
657
658 SLJIT_ASSERT(jump->flags >= ((sljit_uw)11 << JUMP_SIZE_SHIFT));
659 return 11;
660 }
661
662 #endif /* SLJIT_CONFIG_RISCV_64 */
663
664 static SLJIT_INLINE void load_addr_to_reg(struct sljit_jump *jump, sljit_sw executable_offset)
665 {
666 sljit_uw flags = jump->flags;
667 sljit_uw addr = (flags & JUMP_ADDR) ? jump->u.target : jump->u.label->u.addr;
668 sljit_ins ins;
669 sljit_u16 *buf = (sljit_u16 *)jump->addr;
670 sljit_u32 reg = (flags & JUMP_MOV_ADDR) ? *buf : TMP_REG1;
671 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
672 sljit_sw high;
673 #endif /* SLJIT_CONFIG_RISCV_64 */
674 SLJIT_UNUSED_ARG(executable_offset);
675
676 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
677 if (flags & PATCH_REL32) {
678 addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET(buf, executable_offset);
679
680 SLJIT_ASSERT((sljit_sw)addr >= S32_MIN && (sljit_sw)addr <= S32_MAX);
681
682 if ((addr & 0x800) != 0)
683 addr += 0x1000;
684
685 ins = AUIPC | RD(reg) | (sljit_ins)((sljit_sw)addr & ~0xfff);
686 buf[0] = (sljit_u16)ins;
687 buf[1] = (sljit_u16)(ins >> 16);
688
689 if (!(flags & JUMP_MOV_ADDR)) {
690 ins = JALR | RS1(reg) | IMM_I(addr);
691 if (jump->flags & IS_CALL)
692 ins |= RD(RETURN_ADDR_REG);
693 } else
694 ins = ADDI | RD(reg) | RS1(reg) | IMM_I(addr);
695
696 buf[2] = (sljit_u16)ins;
697 buf[3] = (sljit_u16)(ins >> 16);
698 return;
699 }
700 #endif /* SLJIT_CONFIG_RISCV_64 */
701
702 if ((addr & 0x800) != 0)
703 addr += 0x1000;
704
705 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
706 ins = LUI | RD(reg) | (sljit_ins)((sljit_sw)addr & ~0xfff);
707 buf[0] = (sljit_u16)ins;
708 buf[1] = (sljit_u16)(ins >> 16);
709 #else /* !SLJIT_CONFIG_RISCV_32 */
710
711 if (flags & PATCH_ABS32) {
712 SLJIT_ASSERT(addr <= S32_MAX);
713 ins = LUI | RD(reg) | (sljit_ins)((sljit_sw)addr & ~0xfff);
714 buf[0] = (sljit_u16)ins;
715 buf[1] = (sljit_u16)(ins >> 16);
716 } else if (flags & PATCH_ABS44) {
717 high = (sljit_sw)addr >> 12;
718 SLJIT_ASSERT((sljit_uw)high <= 0x7fffffff);
719
720 if (high > S32_MAX) {
721 SLJIT_ASSERT((high & 0x800) != 0);
722 ins = LUI | RD(reg) | (sljit_ins)0x80000000u;
723 buf[0] = (sljit_u16)ins;
724 buf[1] = (sljit_u16)(ins >> 16);
725 ins = XORI | RD(reg) | RS1(reg) | IMM_I(high);
726 buf[2] = (sljit_u16)ins;
727 buf[3] = (sljit_u16)(ins >> 16);
728 } else {
729 if ((high & 0x800) != 0)
730 high += 0x1000;
731
732 ins = LUI | RD(reg) | (sljit_ins)(high & ~0xfff);
733 buf[0] = (sljit_u16)ins;
734 buf[1] = (sljit_u16)(ins >> 16);
735 ins = ADDI | RD(reg) | RS1(reg) | IMM_I(high);
736 buf[2] = (sljit_u16)ins;
737 buf[3] = (sljit_u16)(ins >> 16);
738 }
739
740 ins = SLLI | RD(reg) | RS1(reg) | IMM_I(12);
741 buf[4] = (sljit_u16)ins;
742 buf[5] = (sljit_u16)(ins >> 16);
743 buf += 4;
744 } else {
745 high = (sljit_sw)addr >> 32;
746
747 if ((addr & 0x80000000l) != 0)
748 high = ~high;
749
750 if (flags & PATCH_ABS52) {
751 SLJIT_ASSERT(addr <= S52_MAX);
752 ins = LUI | RD(TMP_REG3) | (sljit_ins)(high << 12);
753 buf[0] = (sljit_u16)ins;
754 buf[1] = (sljit_u16)(ins >> 16);
755 } else {
756 if ((high & 0x800) != 0)
757 high += 0x1000;
758 ins = LUI | RD(TMP_REG3) | (sljit_ins)(high & ~0xfff);
759 buf[0] = (sljit_u16)ins;
760 buf[1] = (sljit_u16)(ins >> 16);
761 ins = ADDI | RD(TMP_REG3) | RS1(TMP_REG3) | IMM_I(high);
762 buf[2] = (sljit_u16)ins;
763 buf[3] = (sljit_u16)(ins >> 16);
764 buf += 2;
765 }
766
767 ins = LUI | RD(reg) | (sljit_ins)((sljit_sw)addr & ~0xfff);
768 buf[2] = (sljit_u16)ins;
769 buf[3] = (sljit_u16)(ins >> 16);
770 ins = SLLI | RD(TMP_REG3) | RS1(TMP_REG3) | IMM_I((flags & PATCH_ABS52) ? 20 : 32);
771 buf[4] = (sljit_u16)ins;
772 buf[5] = (sljit_u16)(ins >> 16);
773 ins = XOR | RD(reg) | RS1(reg) | RS2(TMP_REG3);
774 buf[6] = (sljit_u16)ins;
775 buf[7] = (sljit_u16)(ins >> 16);
776 buf += 6;
777 }
778 #endif /* !SLJIT_CONFIG_RISCV_32 */
779
780 if (!(flags & JUMP_MOV_ADDR)) {
781 ins = JALR | RS1(reg) | IMM_I(addr);
782 if (jump->flags & IS_CALL)
783 ins |= RD(RETURN_ADDR_REG);
784 } else
785 ins = ADDI | RD(reg) | RS1(reg) | IMM_I(addr);
786
787 buf[2] = (sljit_u16)ins;
788 buf[3] = (sljit_u16)(ins >> 16);
789 }
790
791 static SLJIT_INLINE sljit_u16 *process_extended_label(sljit_u16 *code_ptr, struct sljit_extended_label *ext_label)
792 {
793 SLJIT_ASSERT(ext_label->label.u.index == SLJIT_LABEL_ALIGNED);
794 return (sljit_u16*)((sljit_uw)code_ptr & ~(ext_label->data));
795 }
796
797 static void reduce_code_size(struct sljit_compiler *compiler)
798 {
799 struct sljit_label *label;
800 struct sljit_jump *jump;
801 struct sljit_const *const_;
802 SLJIT_NEXT_DEFINE_TYPES;
803 sljit_uw total_size;
804 sljit_uw size_reduce = 0;
805 sljit_sw diff, cond_size, cond_diff;
806
807 label = compiler->labels;
808 jump = compiler->jumps;
809 const_ = compiler->consts;
810 SLJIT_NEXT_INIT_TYPES();
811
812 while (1) {
813 SLJIT_GET_NEXT_MIN();
814
815 if (next_min_addr == SLJIT_MAX_ADDRESS)
816 break;
817
818 if (next_min_addr == next_label_size) {
819 label->size -= size_reduce;
820
821 label = label->next;
822 next_label_size = SLJIT_GET_NEXT_SIZE(label);
823 }
824
825 if (next_min_addr == next_const_addr) {
826 const_->addr -= size_reduce;
827 const_ = const_->next;
828 next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
829 continue;
830 }
831
832 if (next_min_addr != next_jump_addr)
833 continue;
834
835 jump->addr -= size_reduce;
836 if (!(jump->flags & JUMP_MOV_ADDR)) {
837 total_size = JUMP_MAX_SIZE;
838
839 if (!(jump->flags & SLJIT_REWRITABLE_JUMP)) {
840 if (jump->flags & JUMP_ADDR) {
841 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
842 if (jump->u.target <= S32_MAX)
843 total_size = 4;
844 else if (jump->u.target <= S44_MAX)
845 total_size = 8;
846 else if (jump->u.target <= S52_MAX)
847 total_size = 10;
848 #endif /* SLJIT_CONFIG_RISCV_64 */
849 } else {
850 /* Unit size: instruction. */
851 diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
852 if (jump->u.label->size > jump->addr) {
853 SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
854 diff -= (sljit_sw)size_reduce;
855 }
856
857 cond_size = RISCV_COMPRESSED_CHECK((jump->flags & IS_COND16) != 0);
858 cond_diff = diff + 2 - cond_size;
859
860 if (RISCV_COMPRESSED_CHECK(cond_size) && diff >= (BRANCH16_MIN / SSIZE_OF(u16)) && diff <= (BRANCH16_MAX / SSIZE_OF(u16)))
861 total_size = 0;
862 else if ((jump->flags & IS_COND) && cond_diff <= (BRANCH_MAX / SSIZE_OF(u16)) && cond_diff >= (BRANCH_MIN / SSIZE_OF(u16)))
863 total_size = (sljit_uw)cond_size;
864 else if (RISCV_HAS_COMPRESSED(200) && RISCV_CHECK_COMPRESSED_JUMP(jump, diff, u16))
865 total_size = 1;
866 else if (diff >= (JUMP_MIN / SSIZE_OF(u16)) && diff <= (JUMP_MAX / SSIZE_OF(u16)))
867 total_size = 2;
868 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
869 else if (diff >= (S32_MIN / SSIZE_OF(u16)) && diff <= (S32_MAX / SSIZE_OF(u16)))
870 total_size = 4;
871 #endif /* SLJIT_CONFIG_RISCV_64 */
872 }
873 }
874
875 size_reduce += JUMP_MAX_SIZE - total_size;
876 jump->flags |= total_size << JUMP_SIZE_SHIFT;
877 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
878 } else {
879 total_size = 11;
880
881 if (!(jump->flags & JUMP_ADDR)) {
882 /* Real size minus 1. Unit size: 16 bit. */
883 diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
884 if (jump->u.label->size > jump->addr) {
885 SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
886 diff -= (sljit_sw)size_reduce;
887 }
888
889 if (diff >= (S32_MIN / SSIZE_OF(u16)) && diff <= (S32_MAX / SSIZE_OF(u16)))
890 total_size = 3;
891 } else if (jump->u.target < S32_MAX)
892 total_size = 3;
893 else if (jump->u.target < S44_MAX)
894 total_size = 7;
895 else if (jump->u.target <= S52_MAX)
896 total_size = 9;
897
898 size_reduce += 11 - total_size;
899 jump->flags |= total_size << JUMP_SIZE_SHIFT;
900 #endif /* !SLJIT_CONFIG_RISCV_64 */
901 }
902
903 jump = jump->next;
904 next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
905 }
906
907 compiler->size -= size_reduce;
908 }
909
910 SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler, sljit_s32 options, void *exec_allocator_data)
911 {
912 struct sljit_memory_fragment *buf;
913 sljit_u16 *code;
914 sljit_u16 *code_ptr;
915 sljit_u16 *buf_ptr;
916 sljit_u16 *buf_end;
917 sljit_uw half_count;
918 SLJIT_NEXT_DEFINE_TYPES;
919 sljit_sw executable_offset;
920 sljit_uw addr;
921 sljit_ins ins;
922
923 struct sljit_label *label;
924 struct sljit_jump *jump;
925 struct sljit_const *const_;
926
927 CHECK_ERROR_PTR();
928 CHECK_PTR(check_sljit_generate_code(compiler, options));
929
930 reduce_code_size(compiler);
931
932 code = (sljit_u16 *)allocate_executable_memory(compiler->size * sizeof(sljit_u16), options, exec_allocator_data, &executable_offset);
933 PTR_FAIL_WITH_EXEC_IF(code);
934
935 reverse_buf(compiler);
936 buf = compiler->buf;
937
938 code_ptr = code;
939 half_count = 0;
940 label = compiler->labels;
941 jump = compiler->jumps;
942 const_ = compiler->consts;
943 SLJIT_NEXT_INIT_TYPES();
944 SLJIT_GET_NEXT_MIN();
945
946 do {
947 buf_ptr = (sljit_u16*)buf->memory;
948 buf_end = buf_ptr + (buf->used_size >> 1);
949 do {
950 *code_ptr = *buf_ptr++;
951 if (next_min_addr == half_count) {
952 SLJIT_ASSERT(!label || label->size >= half_count);
953 SLJIT_ASSERT(!jump || jump->addr >= half_count);
954 SLJIT_ASSERT(!const_ || const_->addr >= half_count);
955
956 /* These structures are ordered by their address. */
957 if (next_min_addr == next_label_size) {
958 if (label->u.index >= SLJIT_LABEL_ALIGNED) {
959 code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);
960 *code_ptr = buf_ptr[-1];
961 }
962
963 label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
964 label->size = (sljit_uw)(code_ptr - code);
965 label = label->next;
966 next_label_size = SLJIT_GET_NEXT_SIZE(label);
967 }
968
969 if (next_min_addr == next_jump_addr) {
970 if (!(jump->flags & JUMP_MOV_ADDR)) {
971 half_count = half_count - 1 + (jump->flags >> JUMP_SIZE_SHIFT);
972 code_ptr = detect_jump_type(jump, code_ptr, code, executable_offset);
973 SLJIT_ASSERT((jump->flags & PATCH_B) || ((sljit_uw)code_ptr - jump->addr < (jump->flags >> JUMP_SIZE_SHIFT) * sizeof(sljit_ins)));
974 } else {
975 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
976 half_count += 3;
977 jump->addr = (sljit_uw)code_ptr;
978 code_ptr += 3;
979 #else /* !SLJIT_CONFIG_RISCV_32 */
980 half_count += jump->flags >> JUMP_SIZE_SHIFT;
981 addr = (sljit_uw)code_ptr;
982 code_ptr += mov_addr_get_length(jump, code_ptr, code, executable_offset);
983 jump->addr = addr;
984 #endif /* SLJIT_CONFIG_RISCV_32 */
985 }
986 jump = jump->next;
987 next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
988 } else if (next_min_addr == next_const_addr) {
989 const_->addr = (sljit_uw)code_ptr;
990 const_ = const_->next;
991 next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
992 }
993
994 SLJIT_GET_NEXT_MIN();
995 }
996 code_ptr++;
997 half_count++;
998 } while (buf_ptr < buf_end);
999
1000 buf = buf->next;
1001 } while (buf);
1002
1003 if (label && label->size == half_count) {
1004 if (label->u.index >= SLJIT_LABEL_ALIGNED)
1005 code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);
1006
1007 label->u.addr = (sljit_uw)code_ptr;
1008 label->size = (sljit_uw)(code_ptr - code);
1009 label = label->next;
1010 }
1011
1012 SLJIT_ASSERT(!label);
1013 SLJIT_ASSERT(!jump);
1014 SLJIT_ASSERT(!const_);
1015 SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size);
1016
1017 jump = compiler->jumps;
1018 while (jump) {
1019 do {
1020 if (!(jump->flags & (PATCH_B | PATCH_J)) || (jump->flags & JUMP_MOV_ADDR)) {
1021 load_addr_to_reg(jump, executable_offset);
1022 break;
1023 }
1024
1025 addr = (jump->flags & JUMP_ADDR) ? jump->u.target : jump->u.label->u.addr;
1026 buf_ptr = (sljit_u16 *)jump->addr;
1027 addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset);
1028
1029 if (jump->flags & PATCH_B) {
1030 SLJIT_ASSERT(RISCV_HAS_COMPRESSED(200) || !(jump->flags & PATCH_16));
1031 SLJIT_ASSERT((sljit_sw)addr >= BRANCH_MIN && (sljit_sw)addr <= BRANCH_MAX);
1032
1033 if (RISCV_COMPRESSED_CHECK(jump->flags & PATCH_16)) {
1034 buf_ptr[0] |= C_BRN16(addr);
1035 break;
1036 }
1037
1038 buf_ptr[0] |= (sljit_u16)(((addr & 0x800) >> 4) | ((addr & 0x1e) << 7));
1039 buf_ptr[1] |= (sljit_u16)(((addr & 0x7e0) << 4) | ((addr & 0x1000) << 3));
1040 break;
1041 }
1042
1043 SLJIT_ASSERT(RISCV_HAS_COMPRESSED(200) || !(jump->flags & PATCH_16));
1044 if (RISCV_COMPRESSED_CHECK(jump->flags & PATCH_16)) {
1045 SLJIT_ASSERT((sljit_sw)addr >= JUMP16_MIN && (sljit_sw)addr <= JUMP16_MAX);
1046 #if defined SLJIT_CONFIG_RISCV_32
1047 ins = ((jump->flags & IS_CALL) ? C_JAL : C_J) | C_JMP16(addr);
1048 #else /* !SLJIT_CONFIG_RISCV_32 */
1049 SLJIT_ASSERT(!(jump->flags & IS_CALL));
1050 ins = C_J | C_JMP16(addr);
1051 #endif /* SLJIT_CONFIG_RISCV_32 */
1052 buf_ptr[0] = (sljit_u16)ins;
1053 break;
1054 }
1055
1056 SLJIT_ASSERT((sljit_sw)addr >= JUMP_MIN && (sljit_sw)addr <= JUMP_MAX);
1057 addr = (addr & 0xff000) | ((addr & 0x800) << 9) | ((addr & 0x7fe) << 20) | ((addr & 0x100000) << 11);
1058 ins = JAL | RD((jump->flags & IS_CALL) ? RETURN_ADDR_REG : TMP_ZERO) | (sljit_ins)addr;
1059 buf_ptr[0] = (sljit_u16)ins;
1060 buf_ptr[1] = (sljit_u16)(ins >> 16);
1061 } while (0);
1062
1063 jump = jump->next;
1064 }
1065
1066 compiler->error = SLJIT_ERR_COMPILED;
1067 compiler->executable_offset = executable_offset;
1068 compiler->executable_size = (sljit_uw)(code_ptr - code) * sizeof(sljit_ins);
1069
1070 code = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
1071 code_ptr = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
1072
1073 SLJIT_CACHE_FLUSH(code, code_ptr);
1074 SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1);
1075 return code;
1076 }
1077
1078 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
1079 {
1080 switch (feature_type) {
1081 case SLJIT_HAS_FPU:
1082 #ifdef SLJIT_IS_FPU_AVAILABLE
1083 return (SLJIT_IS_FPU_AVAILABLE) != 0;
1084 #elif defined(__riscv_float_abi_soft)
1085 return 0;
1086 #else /* !SLJIT_IS_FPU_AVAILABLE && !__riscv_float_abi_soft */
1087 return 1;
1088 #endif /* SLJIT_IS_FPU_AVAILABLE */
1089 case SLJIT_HAS_ZERO_REGISTER:
1090 case SLJIT_HAS_COPY_F32:
1091 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1092 case SLJIT_HAS_COPY_F64:
1093 #endif /* !SLJIT_CONFIG_RISCV_64 */
1094 case SLJIT_HAS_ATOMIC:
1095 return RISCV_HAS_ATOMIC(200) ? 1 : 0;
1096 case SLJIT_HAS_MEMORY_BARRIER:
1097 return 1;
1098 case SLJIT_HAS_CLZ:
1099 case SLJIT_HAS_CTZ:
1100 case SLJIT_HAS_REV:
1101 case SLJIT_HAS_ROT:
1102 return RISCV_HAS_BITMANIP_B(93) ? 1 : 0;
1103 case SLJIT_HAS_CMOV:
1104 return RISCV_HAS_ICOND(100) ? 2 : 0;
1105 case SLJIT_HAS_SIMD:
1106 return RISCV_HAS_VECTOR(100) ? 1 : 0;
1107 default:
1108 return 0;
1109 }
1110 }
1111
1112 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
1113 {
1114 switch (type) {
1115 case SLJIT_UNORDERED_OR_EQUAL:
1116 case SLJIT_ORDERED_NOT_EQUAL:
1117 return 2;
1118
1119 case SLJIT_UNORDERED:
1120 case SLJIT_ORDERED:
1121 return 1;
1122 }
1123
1124 return 0;
1125 }
1126
1127 /* --------------------------------------------------------------------- */
1128 /* Entry, exit */
1129 /* --------------------------------------------------------------------- */
1130
1131 /* Creates an index in data_transfer_insts array. */
1132 #define LOAD_DATA 0x01
1133 #define WORD_DATA 0x00
1134 #define BYTE_DATA 0x02
1135 #define HALF_DATA 0x04
1136 #define INT_DATA 0x06
1137 #define SIGNED_DATA 0x08
1138 /* Separates integer and floating point registers */
1139 #define GPR_REG 0x0f
1140 #define DOUBLE_DATA 0x10
1141 #define SINGLE_DATA 0x12
1142
1143 #define MEM_MASK 0x1f
1144
1145 #define ARG_TEST 0x00020
1146 #define ALT_KEEP_CACHE 0x00040
1147 #define CUMULATIVE_OP 0x00080
1148 #define IMM_OP 0x00100
1149 #define MOVE_OP 0x00200
1150 #define SRC2_IMM 0x00400
1151
1152 #define UNUSED_DEST 0x00800
1153 #define REG_DEST 0x01000
1154 #define REG1_SOURCE 0x02000
1155 #define REG2_SOURCE 0x04000
1156 #define SLOW_SRC1 0x08000
1157 #define SLOW_SRC2 0x10000
1158 #define SLOW_DEST 0x20000
1159 #define MEM_USE_TMP2 0x40000
1160
1161 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1162 #define STACK_STORE SW
1163 #define STACK_LOAD LW
1164 #define C_STACK_OFFSET_CHECK(offset) ((offset) <= 0xfc)
1165 #define C_STACK_LOAD(offset) (C_LWSP | C_LD32_SP(offset))
1166 #define C_STACK_STORE(offset) (C_SWSP | C_ST32_SP(offset))
1167 #else /* !SLJIT_CONFIG_RISCV_32 */
1168 #define STACK_STORE SD
1169 #define STACK_LOAD LD
1170 #define C_STACK_OFFSET_CHECK(offset) ((offset) <= 0x1f8)
1171 #define C_STACK_LOAD(offset) (C_LDSP | C_LD64_SP(offset))
1172 #define C_STACK_STORE(offset) (C_SDSP | C_ST64_SP(offset))
1173 #endif /* SLJIT_CONFIG_RISCV_32 */
1174
1175 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1176 #include "sljitNativeRISCV_32.c"
1177 #else /* !SLJIT_CONFIG_RISCV_32 */
1178 #include "sljitNativeRISCV_64.c"
1179 #endif /* SLJIT_CONFIG_RISCV_32 */
1180
1181 #define STACK_MAX_DISTANCE (-SIMM_MIN)
1182
1183 static sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw);
1184
1185 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
1186 sljit_s32 options, sljit_s32 arg_types,
1187 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1188 {
1189 sljit_s32 fscratches = ENTER_GET_FLOAT_REGS(scratches);
1190 sljit_s32 fsaveds = ENTER_GET_FLOAT_REGS(saveds);
1191 sljit_s32 i, tmp, offset;
1192 sljit_s32 saved_arg_count = SLJIT_KEPT_SAVEDS_COUNT(options);
1193
1194 CHECK_ERROR();
1195 CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, local_size));
1196 set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
1197
1198 scratches = ENTER_GET_REGS(scratches);
1199 saveds = ENTER_GET_REGS(saveds);
1200 local_size += GET_SAVED_REGISTERS_SIZE(scratches, saveds - saved_arg_count, 1);
1201 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1202 if (fsaveds > 0 || fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG) {
1203 if ((local_size & SSIZE_OF(sw)) != 0)
1204 local_size += SSIZE_OF(sw);
1205 local_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
1206 }
1207 #else /* !SLJIT_CONFIG_RISCV_32 */
1208 local_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
1209 #endif /* SLJIT_CONFIG_RISCV_32 */
1210 local_size = (local_size + SLJIT_LOCALS_OFFSET + 15) & ~0xf;
1211 compiler->local_size = local_size;
1212
1213 if (local_size <= STACK_MAX_DISTANCE) {
1214 /* Frequent case. */
1215 offset = local_size - SSIZE_OF(sw);
1216 local_size = -local_size;
1217 if (RISCV_HAS_COMPRESSED(200) && local_size >= -0x200) {
1218 SLJIT_ASSERT((local_size & 0x200) != 0 && (local_size & 0xf) == 0);
1219 FAIL_IF(push_inst16(compiler, C_ADDI16SP | (sljit_u16)(((local_size & 0x10) << 2) | ((local_size & 0x20) >> 3) | ((local_size & 0x40) >> 1) | ((local_size & 0x180) >> 4) | (1 << 12))));
1220 } else
1221 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RS1(SLJIT_SP) | IMM_I(local_size)));
1222 local_size = 0;
1223 } else {
1224 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RS1(SLJIT_SP) | IMM_I(STACK_MAX_DISTANCE)));
1225 local_size -= STACK_MAX_DISTANCE;
1226
1227 if (local_size > STACK_MAX_DISTANCE)
1228 FAIL_IF(load_immediate(compiler, TMP_REG1, local_size, TMP_REG3));
1229 offset = STACK_MAX_DISTANCE - SSIZE_OF(sw);
1230 }
1231
1232 if (RISCV_HAS_COMPRESSED(200) && C_STACK_OFFSET_CHECK(offset))
1233 FAIL_IF(push_inst16(compiler, C_STACK_STORE(offset) | C_RS2(RETURN_ADDR_REG)));
1234 else
1235 FAIL_IF(push_imm_s_inst(compiler, STACK_STORE | RS1(SLJIT_SP) | RS2(RETURN_ADDR_REG), offset));
1236
1237 tmp = SLJIT_S0 - saveds;
1238 for (i = SLJIT_S0 - saved_arg_count; i > tmp; i--) {
1239 offset -= SSIZE_OF(sw);
1240 if (RISCV_HAS_COMPRESSED(200) && C_STACK_OFFSET_CHECK(offset))
1241 FAIL_IF(push_inst16(compiler, C_STACK_STORE(offset) | C_RS2(i)));
1242 else
1243 FAIL_IF(push_imm_s_inst(compiler, STACK_STORE | RS1(SLJIT_SP) | RS2(i), offset));
1244 }
1245
1246 for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
1247 offset -= SSIZE_OF(sw);
1248 if (RISCV_HAS_COMPRESSED(200) && C_STACK_OFFSET_CHECK(offset))
1249 FAIL_IF(push_inst16(compiler, C_STACK_STORE(offset) | C_RS2(i)));
1250 else
1251 FAIL_IF(push_imm_s_inst(compiler, STACK_STORE | RS1(SLJIT_SP) | RS2(i), offset));
1252 }
1253
1254 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1255 /* This alignment is valid because offset is not used after storing FPU regs. */
1256 if ((offset & SSIZE_OF(sw)) != 0)
1257 offset -= SSIZE_OF(sw);
1258 #endif /* SLJIT_CONFIG_RISCV_32 */
1259
1260 tmp = SLJIT_FS0 - fsaveds;
1261 for (i = SLJIT_FS0; i > tmp; i--) {
1262 offset -= SSIZE_OF(f64);
1263 if (RISCV_HAS_COMPRESSED(200) && offset <= 0x1f8)
1264 FAIL_IF(push_inst16(compiler, C_FSDSP | C_FRS2(i) | C_ST64_SP(offset)));
1265 else
1266 FAIL_IF(push_imm_s_inst(compiler, FSD | RS1(SLJIT_SP) | FRS2(i), offset));
1267 }
1268
1269 for (i = fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
1270 offset -= SSIZE_OF(f64);
1271 if (RISCV_HAS_COMPRESSED(200) && offset <= 0x1f8)
1272 FAIL_IF(push_inst16(compiler, C_FSDSP | C_FRS2(i) | C_ST64_SP(offset)));
1273 else
1274 FAIL_IF(push_imm_s_inst(compiler, FSD | RS1(SLJIT_SP) | FRS2(i), offset));
1275 }
1276
1277 if (local_size > STACK_MAX_DISTANCE)
1278 FAIL_IF(push_inst(compiler, SUB | RD(SLJIT_SP) | RS1(SLJIT_SP) | RS2(TMP_REG1)));
1279 else if (local_size > 0)
1280 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RS1(SLJIT_SP) | IMM_I(-local_size)));
1281
1282 if (options & SLJIT_ENTER_REG_ARG)
1283 return SLJIT_SUCCESS;
1284
1285 arg_types >>= SLJIT_ARG_SHIFT;
1286 saved_arg_count = 0;
1287 tmp = SLJIT_R0;
1288
1289 while (arg_types > 0) {
1290 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64) {
1291 if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
1292 if (RISCV_HAS_COMPRESSED(200))
1293 FAIL_IF(push_inst16(compiler, C_MV | C_RD(SLJIT_S0 - saved_arg_count) | C_RS2(tmp)));
1294 else
1295 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_S0 - saved_arg_count) | RS1(tmp) | IMM_I(0)));
1296 saved_arg_count++;
1297 }
1298 tmp++;
1299 }
1300
1301 arg_types >>= SLJIT_ARG_SHIFT;
1302 }
1303
1304 return SLJIT_SUCCESS;
1305 }
1306
1307 #undef STACK_MAX_DISTANCE
1308
1309 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
1310 sljit_s32 options, sljit_s32 arg_types,
1311 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1312 {
1313 sljit_s32 fscratches = ENTER_GET_FLOAT_REGS(scratches);
1314 sljit_s32 fsaveds = ENTER_GET_FLOAT_REGS(saveds);
1315
1316 CHECK_ERROR();
1317 CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, local_size));
1318 set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
1319
1320 scratches = ENTER_GET_REGS(scratches);
1321 saveds = ENTER_GET_REGS(saveds);
1322 local_size += GET_SAVED_REGISTERS_SIZE(scratches, saveds - SLJIT_KEPT_SAVEDS_COUNT(options), 1);
1323 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1324 if (fsaveds > 0 || fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG) {
1325 if ((local_size & SSIZE_OF(sw)) != 0)
1326 local_size += SSIZE_OF(sw);
1327 local_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
1328 }
1329 #else /* !SLJIT_CONFIG_RISCV_32 */
1330 local_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
1331 #endif /* SLJIT_CONFIG_RISCV_32 */
1332 compiler->local_size = (local_size + SLJIT_LOCALS_OFFSET + 15) & ~0xf;
1333
1334 return SLJIT_SUCCESS;
1335 }
1336
1337 #define STACK_MAX_DISTANCE (-SIMM_MIN - 16)
1338
1339 static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler, sljit_s32 is_return_to)
1340 {
1341 sljit_s32 i, tmp, offset;
1342 sljit_s32 local_size = compiler->local_size;
1343
1344 if (local_size > STACK_MAX_DISTANCE) {
1345 local_size -= STACK_MAX_DISTANCE;
1346
1347 if (local_size > STACK_MAX_DISTANCE) {
1348 FAIL_IF(load_immediate(compiler, TMP_REG2, local_size, TMP_REG3));
1349 FAIL_IF(push_inst(compiler, ADD | RD(SLJIT_SP) | RS1(SLJIT_SP) | RS2(TMP_REG2)));
1350 } else
1351 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RS1(SLJIT_SP) | IMM_I(local_size)));
1352
1353 local_size = STACK_MAX_DISTANCE;
1354 }
1355
1356 SLJIT_ASSERT(local_size > 0);
1357
1358 offset = local_size - SSIZE_OF(sw);
1359 if (!is_return_to) {
1360 if (RISCV_HAS_COMPRESSED(200) && C_STACK_OFFSET_CHECK(offset))
1361 FAIL_IF(push_inst16(compiler, C_STACK_LOAD(offset) | C_RD(RETURN_ADDR_REG)));
1362 else
1363 FAIL_IF(push_inst(compiler, STACK_LOAD | RD(RETURN_ADDR_REG) | RS1(SLJIT_SP) | IMM_I(offset)));
1364 }
1365
1366 tmp = SLJIT_S0 - compiler->saveds;
1367 for (i = SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options); i > tmp; i--) {
1368 offset -= SSIZE_OF(sw);
1369 if (RISCV_HAS_COMPRESSED(200) && C_STACK_OFFSET_CHECK(offset))
1370 FAIL_IF(push_inst16(compiler, C_STACK_LOAD(offset) | C_RD(i)));
1371 else
1372 FAIL_IF(push_inst(compiler, STACK_LOAD | RD(i) | RS1(SLJIT_SP) | IMM_I(offset)));
1373 }
1374
1375 for (i = compiler->scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
1376 offset -= SSIZE_OF(sw);
1377 if (RISCV_HAS_COMPRESSED(200) && C_STACK_OFFSET_CHECK(offset))
1378 FAIL_IF(push_inst16(compiler, C_STACK_LOAD(offset) | C_RD(i)));
1379 else
1380 FAIL_IF(push_inst(compiler, STACK_LOAD | RD(i) | RS1(SLJIT_SP) | IMM_I(offset)));
1381 }
1382
1383 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1384 /* This alignment is valid because offset is not used after storing FPU regs. */
1385 if ((offset & SSIZE_OF(sw)) != 0)
1386 offset -= SSIZE_OF(sw);
1387 #endif /* SLJIT_CONFIG_RISCV_32 */
1388
1389 tmp = SLJIT_FS0 - compiler->fsaveds;
1390 for (i = SLJIT_FS0; i > tmp; i--) {
1391 offset -= SSIZE_OF(f64);
1392 if (RISCV_HAS_COMPRESSED(200) && offset <= 0x1f8)
1393 FAIL_IF(push_inst16(compiler, C_FLDSP | C_FRD(i) | C_LD64_SP(offset)));
1394 else
1395 FAIL_IF(push_inst(compiler, FLD | FRD(i) | RS1(SLJIT_SP) | IMM_I(offset)));
1396 }
1397
1398 for (i = compiler->fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
1399 offset -= SSIZE_OF(f64);
1400 if (RISCV_HAS_COMPRESSED(200) && offset <= 0x1f8)
1401 FAIL_IF(push_inst16(compiler, C_FLDSP | C_FRD(i) | C_LD64_SP(offset)));
1402 else
1403 FAIL_IF(push_inst(compiler, FLD | FRD(i) | RS1(SLJIT_SP) | IMM_I(offset)));
1404 }
1405
1406 if (RISCV_HAS_COMPRESSED(200) && local_size <= 0x1f0) {
1407 SLJIT_ASSERT((local_size & 0xf) == 0);
1408 return push_inst16(compiler, C_ADDI16SP | (sljit_u16)(((local_size & 0x10) << 2) | ((local_size & 0x20) >> 3) | ((local_size & 0x40) >> 1) | ((local_size & 0x180) >> 4)));
1409 }
1410
1411 return push_inst(compiler, ADDI | RD(SLJIT_SP) | RS1(SLJIT_SP) | IMM_I(local_size));
1412 }
1413
1414 #undef STACK_MAX_DISTANCE
1415
1416 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
1417 {
1418 CHECK_ERROR();
1419 CHECK(check_sljit_emit_return_void(compiler));
1420
1421 FAIL_IF(emit_stack_frame_release(compiler, 0));
1422 return push_inst(compiler, JALR | RD(TMP_ZERO) | RS1(RETURN_ADDR_REG) | IMM_I(0));
1423 }
1424
1425 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_to(struct sljit_compiler *compiler,
1426 sljit_s32 src, sljit_sw srcw)
1427 {
1428 CHECK_ERROR();
1429 CHECK(check_sljit_emit_return_to(compiler, src, srcw));
1430
1431 if (src & SLJIT_MEM) {
1432 ADJUST_LOCAL_OFFSET(src, srcw);
1433 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src, srcw));
1434 src = TMP_REG1;
1435 srcw = 0;
1436 } else if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
1437 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(src) | IMM_I(0)));
1438 src = TMP_REG1;
1439 srcw = 0;
1440 }
1441
1442 FAIL_IF(emit_stack_frame_release(compiler, 1));
1443
1444 SLJIT_SKIP_CHECKS(compiler);
1445 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1446 }
1447
1448 /* --------------------------------------------------------------------- */
1449 /* Operators */
1450 /* --------------------------------------------------------------------- */
1451
1452 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1453 #define ARCH_32_64(a, b) a
1454 #else /* !SLJIT_CONFIG_RISCV_32 */
1455 #define ARCH_32_64(a, b) b
1456 #endif /* SLJIT_CONFIG_RISCV_32 */
1457
1458 static const sljit_ins data_transfer_insts[16 + 4] = {
1459 /* u w s */ ARCH_32_64(F3(0x2) | OPC(0x23) /* sw */, F3(0x3) | OPC(0x23) /* sd */),
1460 /* u w l */ ARCH_32_64(F3(0x2) | OPC(0x3) /* lw */, F3(0x3) | OPC(0x3) /* ld */),
1461 /* u b s */ F3(0x0) | OPC(0x23) /* sb */,
1462 /* u b l */ F3(0x4) | OPC(0x3) /* lbu */,
1463 /* u h s */ F3(0x1) | OPC(0x23) /* sh */,
1464 /* u h l */ F3(0x5) | OPC(0x3) /* lhu */,
1465 /* u i s */ F3(0x2) | OPC(0x23) /* sw */,
1466 /* u i l */ ARCH_32_64(F3(0x2) | OPC(0x3) /* lw */, F3(0x6) | OPC(0x3) /* lwu */),
1467
1468 /* s w s */ ARCH_32_64(F3(0x2) | OPC(0x23) /* sw */, F3(0x3) | OPC(0x23) /* sd */),
1469 /* s w l */ ARCH_32_64(F3(0x2) | OPC(0x3) /* lw */, F3(0x3) | OPC(0x3) /* ld */),
1470 /* s b s */ F3(0x0) | OPC(0x23) /* sb */,
1471 /* s b l */ F3(0x0) | OPC(0x3) /* lb */,
1472 /* s h s */ F3(0x1) | OPC(0x23) /* sh */,
1473 /* s h l */ F3(0x1) | OPC(0x3) /* lh */,
1474 /* s i s */ F3(0x2) | OPC(0x23) /* sw */,
1475 /* s i l */ F3(0x2) | OPC(0x3) /* lw */,
1476
1477 /* d s */ F3(0x3) | OPC(0x27) /* fsd */,
1478 /* d l */ F3(0x3) | OPC(0x7) /* fld */,
1479 /* s s */ F3(0x2) | OPC(0x27) /* fsw */,
1480 /* s l */ F3(0x2) | OPC(0x7) /* flw */,
1481 };
1482
1483 #undef ARCH_32_64
1484
1485 static sljit_s32 push_mem_inst(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 base, sljit_sw offset)
1486 {
1487 sljit_ins ins;
1488 sljit_s32 signed_flags;
1489 sljit_s32 load_flags;
1490
1491 SLJIT_ASSERT(FAST_IS_REG(base) && offset <= 0xfff && offset >= SIMM_MIN);
1492
1493 flags &= MEM_MASK;
1494
1495 if (RISCV_HAS_COMPRESSED(200) && offset >= 0) {
1496 if (base == SLJIT_SP) {
1497 signed_flags = flags | SIGNED_DATA;
1498
1499 if ((offset & 0x3) == 0 && offset <= 0xfc) {
1500 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1501 if ((signed_flags == (SIGNED_DATA | LOAD_DATA) || signed_flags == (INT_DATA | SIGNED_DATA | LOAD_DATA)))
1502 return push_inst16(compiler, C_LWSP | C_RD(reg) | C_LD32_SP(offset));
1503 if ((signed_flags == SIGNED_DATA || signed_flags == (INT_DATA | SIGNED_DATA)))
1504 return push_inst16(compiler, C_SWSP | C_RS2(reg) | C_ST32_SP(offset));
1505 if (flags == (SINGLE_DATA | LOAD_DATA))
1506 return push_inst16(compiler, C_FLWSP | C_FRD(reg) | C_LD32_SP(offset));
1507 if (flags == SINGLE_DATA)
1508 return push_inst16(compiler, C_FSWSP | C_FRS2(reg) | C_ST32_SP(offset));
1509 #else /* !SLJIT_CONFIG_RISCV_32 */
1510 if (flags == (INT_DATA | SIGNED_DATA | LOAD_DATA))
1511 return push_inst16(compiler, C_LWSP | C_RD(reg) | C_LD32_SP(offset));
1512 if (signed_flags == (INT_DATA | SIGNED_DATA))
1513 return push_inst16(compiler, C_SWSP | C_RS2(reg) | C_ST32_SP(offset));
1514 #endif /* SLJIT_CONFIG_RISCV_32 */
1515 }
1516
1517 if ((offset & 0x7) == 0 && offset <= 0x1f8) {
1518 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1519 if (signed_flags == (SIGNED_DATA | LOAD_DATA))
1520 return push_inst16(compiler, C_LDSP | C_RD(reg) | C_LD64_SP(offset));
1521 if (signed_flags == SIGNED_DATA)
1522 return push_inst16(compiler, C_SDSP | C_RS2(reg) | C_ST64_SP(offset));
1523 #endif /* SLJIT_CONFIG_RISCV_64 */
1524 if (flags == (DOUBLE_DATA | LOAD_DATA))
1525 return push_inst16(compiler, C_FLDSP | C_FRD(reg) | C_LD64_SP(offset));
1526 if (flags == DOUBLE_DATA)
1527 return push_inst16(compiler, C_FSDSP | C_FRS2(reg) | C_ST64_SP(offset));
1528 }
1529 } else if (C_IS_R3(base) && (flags <= GPR_REG ? C_IS_R3(reg) : C_IS_FR3(reg))) {
1530 SLJIT_COMPILE_ASSERT(LOAD_DATA == 1, load_data_bit_error);
1531 signed_flags = flags | LOAD_DATA | SIGNED_DATA;
1532 load_flags = flags | LOAD_DATA;
1533
1534 if ((offset & 0x3) == 0 && offset <= 0x7c) {
1535 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1536 if (signed_flags == (SIGNED_DATA | LOAD_DATA) || signed_flags == (INT_DATA | SIGNED_DATA | LOAD_DATA))
1537 return push_inst16(compiler, (C_SW ^ (sljit_u16)((flags & LOAD_DATA) << 15)) | C_RS1_R3(base) | C_RS2_R3(reg) | C_MEM32(offset));
1538 if (load_flags == (SINGLE_DATA | LOAD_DATA))
1539 return push_inst16(compiler, (C_FSW ^ (sljit_u16)((flags & LOAD_DATA) << 15)) | C_RS1_R3(base) | C_FRS2_R3(reg) | C_MEM32(offset));
1540 #else /* !SLJIT_CONFIG_RISCV_32 */
1541 if (load_flags == (INT_DATA | SIGNED_DATA | LOAD_DATA))
1542 return push_inst16(compiler, (C_SW ^ (sljit_u16)((flags & LOAD_DATA) << 15)) | C_RS1_R3(base) | C_RS2_R3(reg) | C_MEM32(offset));
1543 #endif /* SLJIT_CONFIG_RISCV_32 */
1544 }
1545
1546 if ((offset & 0x7) == 0 && offset <= 0xf8) {
1547 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1548 if (signed_flags == (SIGNED_DATA | LOAD_DATA))
1549 return push_inst16(compiler, (C_SD ^ (sljit_u16)((flags & LOAD_DATA) << 15)) | C_RS1_R3(base) | C_RS2_R3(reg) | C_MEM64(offset));
1550 #endif /* SLJIT_CONFIG_RISCV_64 */
1551 if (load_flags == (DOUBLE_DATA | LOAD_DATA))
1552 return push_inst16(compiler, (C_FSD ^ (sljit_u16)((flags & LOAD_DATA) << 15)) | C_RS1_R3(base) | C_FRS2_R3(reg) | C_MEM64(offset));
1553 }
1554 }
1555 }
1556
1557 ins = data_transfer_insts[flags] | RS1(base);
1558 if (flags & LOAD_DATA)
1559 ins |= (flags <= GPR_REG ? RD(reg) : FRD(reg)) | IMM_I(offset);
1560 else
1561 ins |= (flags <= GPR_REG ? RS2(reg) : FRS2(reg)) | IMM_S(offset);
1562
1563 return push_inst(compiler, ins);
1564 }
1565
1566 /* Can perform an operation using at most 1 instruction. */
1567 static sljit_s32 getput_arg_fast(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw)
1568 {
1569 SLJIT_ASSERT(arg & SLJIT_MEM);
1570
1571 if (!(arg & OFFS_REG_MASK) && argw <= SIMM_MAX && argw >= SIMM_MIN) {
1572 /* Works for both absoulte and relative addresses. */
1573 if (SLJIT_UNLIKELY(flags & ARG_TEST))
1574 return 1;
1575
1576 FAIL_IF(push_mem_inst(compiler, flags, reg, arg & REG_MASK, argw));
1577 return -1;
1578 }
1579 return 0;
1580 }
1581
1582 #define TO_ARGW_HI(argw) (((argw) & ~0xfff) + (((argw) & 0x800) ? 0x1000 : 0))
1583
1584 /* See getput_arg below.
1585 Note: can_cache is called only for binary operators. */
1586 static sljit_s32 can_cache(sljit_s32 arg, sljit_sw argw, sljit_s32 next_arg, sljit_sw next_argw)
1587 {
1588 SLJIT_ASSERT((arg & SLJIT_MEM) && (next_arg & SLJIT_MEM));
1589
1590 /* Simple operation except for updates. */
1591 if (arg & OFFS_REG_MASK) {
1592 argw &= 0x3;
1593 next_argw &= 0x3;
1594 if (argw && argw == next_argw && (arg == next_arg || (arg & OFFS_REG_MASK) == (next_arg & OFFS_REG_MASK)))
1595 return 1;
1596 return 0;
1597 }
1598
1599 if (arg == next_arg) {
1600 if (((next_argw - argw) <= SIMM_MAX && (next_argw - argw) >= SIMM_MIN)
1601 || TO_ARGW_HI(argw) == TO_ARGW_HI(next_argw))
1602 return 1;
1603 return 0;
1604 }
1605
1606 return 0;
1607 }
1608
1609 /* Emit the necessary instructions. See can_cache above. */
1610 static sljit_s32 getput_arg(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw, sljit_s32 next_arg, sljit_sw next_argw)
1611 {
1612 sljit_s32 base = arg & REG_MASK;
1613 sljit_s32 tmp_r = (flags & MEM_USE_TMP2) ? TMP_REG2 : TMP_REG1;
1614 sljit_sw offset, argw_hi;
1615 sljit_ins ins = ADD;
1616
1617 SLJIT_ASSERT(arg & SLJIT_MEM);
1618 if (!(next_arg & SLJIT_MEM)) {
1619 next_arg = 0;
1620 next_argw = 0;
1621 }
1622
1623 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1624 argw &= 0x3;
1625
1626 if (RISCV_HAS_BITMANIP_A(93)) {
1627 switch (argw) {
1628 case 1:
1629 ins = SH1ADD;
1630 break;
1631 case 2:
1632 ins = SH2ADD;
1633 break;
1634 case 3:
1635 ins = SH3ADD;
1636 break;
1637 }
1638 FAIL_IF(push_inst(compiler, ins | RD(tmp_r) | RS1(OFFS_REG(arg)) | RS2(base)));
1639 return push_mem_inst(compiler, flags, reg, tmp_r, 0);
1640 }
1641
1642 /* Using the cache. */
1643 if (argw == compiler->cache_argw) {
1644 if (arg == compiler->cache_arg)
1645 return push_mem_inst(compiler, flags, reg, TMP_REG3, 0);
1646
1647 if ((SLJIT_MEM | (arg & OFFS_REG_MASK)) == compiler->cache_arg) {
1648 if (arg == next_arg && argw == (next_argw & 0x3)) {
1649 compiler->cache_arg = arg;
1650 compiler->cache_argw = argw;
1651 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG3) | RS1(TMP_REG3) | RS2(base)));
1652 return push_mem_inst(compiler, flags, reg, TMP_REG3, 0);
1653 }
1654 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(base) | RS2(TMP_REG3)));
1655 return push_mem_inst(compiler, flags, reg, tmp_r, 0);
1656 }
1657 }
1658
1659 if (SLJIT_UNLIKELY(argw)) {
1660 compiler->cache_arg = SLJIT_MEM | (arg & OFFS_REG_MASK);
1661 compiler->cache_argw = argw;
1662 FAIL_IF(push_inst(compiler, SLLI | RD(TMP_REG3) | RS1(OFFS_REG(arg)) | IMM_I(argw)));
1663 }
1664
1665 if (arg == next_arg && argw == (next_argw & 0x3)) {
1666 compiler->cache_arg = arg;
1667 compiler->cache_argw = argw;
1668 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG3) | RS1(base) | RS2(!argw ? OFFS_REG(arg) : TMP_REG3)));
1669 tmp_r = TMP_REG3;
1670 }
1671 else
1672 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(base) | RS2(!argw ? OFFS_REG(arg) : TMP_REG3)));
1673
1674 return push_mem_inst(compiler, flags, reg, tmp_r, 0);
1675 }
1676
1677 if (compiler->cache_arg == arg && argw - compiler->cache_argw <= SIMM_MAX && argw - compiler->cache_argw >= SIMM_MIN)
1678 return push_mem_inst(compiler, flags, reg, TMP_REG3, argw - compiler->cache_argw);
1679
1680 if (compiler->cache_arg == SLJIT_MEM && (argw - compiler->cache_argw <= SIMM_MAX) && (argw - compiler->cache_argw >= SIMM_MIN)) {
1681 offset = argw - compiler->cache_argw;
1682 } else {
1683 compiler->cache_arg = SLJIT_MEM;
1684
1685 argw_hi = TO_ARGW_HI(argw);
1686
1687 if (next_arg && next_argw - argw <= SIMM_MAX && next_argw - argw >= SIMM_MIN && argw_hi != TO_ARGW_HI(next_argw)) {
1688 FAIL_IF(load_immediate(compiler, TMP_REG3, argw, tmp_r));
1689 compiler->cache_argw = argw;
1690 offset = 0;
1691 } else {
1692 FAIL_IF(load_immediate(compiler, TMP_REG3, argw_hi, tmp_r));
1693 compiler->cache_argw = argw_hi;
1694 offset = argw & 0xfff;
1695 argw = argw_hi;
1696 }
1697 }
1698
1699 if (!base)
1700 return push_mem_inst(compiler, flags, reg, TMP_REG3, offset);
1701
1702 if (arg == next_arg && next_argw - argw <= SIMM_MAX && next_argw - argw >= SIMM_MIN) {
1703 compiler->cache_arg = arg;
1704 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG3) | RS1(TMP_REG3) | RS2(base)));
1705 return push_mem_inst(compiler, flags, reg, TMP_REG3, offset);
1706 }
1707
1708 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(TMP_REG3) | RS2(base)));
1709 return push_mem_inst(compiler, flags, reg, tmp_r, offset);
1710 }
1711
1712 static sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw)
1713 {
1714 sljit_s32 base = arg & REG_MASK;
1715 sljit_s32 tmp_r = TMP_REG1;
1716
1717 if (getput_arg_fast(compiler, flags, reg, arg, argw))
1718 return compiler->error;
1719
1720 if ((flags & MEM_MASK) <= GPR_REG && (flags & LOAD_DATA))
1721 tmp_r = reg;
1722
1723 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1724 argw &= 0x3;
1725
1726 if (SLJIT_UNLIKELY(argw)) {
1727 FAIL_IF(push_inst(compiler, SLLI | RD(tmp_r) | RS1(OFFS_REG(arg)) | IMM_I(argw)));
1728 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(tmp_r) | RS2(base)));
1729 }
1730 else
1731 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(base) | RS2(OFFS_REG(arg))));
1732
1733 argw = 0;
1734 } else {
1735 FAIL_IF(load_immediate(compiler, tmp_r, TO_ARGW_HI(argw), TMP_REG3));
1736
1737 if (base != 0)
1738 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(tmp_r) | RS2(base)));
1739 }
1740
1741 return push_mem_inst(compiler, flags, reg, tmp_r, argw & 0xfff);
1742 }
1743
1744 static SLJIT_INLINE sljit_s32 emit_op_mem2(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg1, sljit_sw arg1w, sljit_s32 arg2, sljit_sw arg2w)
1745 {
1746 if (getput_arg_fast(compiler, flags, reg, arg1, arg1w))
1747 return compiler->error;
1748 return getput_arg(compiler, flags, reg, arg1, arg1w, arg2, arg2w);
1749 }
1750
1751 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1752 #define WORD 0
1753 #define WORD_32 0
1754 #define IMM_EXTEND(v) (IMM_I(v))
1755 #else /* !SLJIT_CONFIG_RISCV_32 */
1756 #define WORD word
1757 #define WORD_32 0x08
1758 #define IMM_EXTEND(v) (IMM_I((op & SLJIT_32) ? (v) : (32 + (v))))
1759 #endif /* SLJIT_CONFIG_RISCV_32 */
1760
1761 static sljit_s32 emit_add(struct sljit_compiler *compiler, sljit_ins word, sljit_s32 dst, sljit_s32 src1, sljit_sw src2)
1762 {
1763 SLJIT_UNUSED_ARG(word);
1764
1765 if (dst == src2) {
1766 src2 = src1;
1767 src1 = dst;
1768 }
1769
1770 if (RISCV_HAS_COMPRESSED(200) && dst == src1) {
1771 if (WORD == 0 && src2 != 0)
1772 return push_inst16(compiler, C_ADD | C_RD(dst) | C_RS2(src2));
1773
1774 if (WORD == 0x8 && C_IS_R3(dst) && C_IS_R3(src2))
1775 return push_inst16(compiler, C_ADDW | C_RS1_R3(dst) | C_RS2_R3(src2));
1776 }
1777
1778 return push_inst(compiler, ADD | WORD | RD(dst) | RS1(src1) | RS2(src2));
1779 }
1780
1781 static sljit_s32 emit_clz_ctz(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw src)
1782 {
1783 sljit_s32 is_clz = (GET_OPCODE(op) == SLJIT_CLZ);
1784 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1785 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
1786 sljit_ins word_size = (op & SLJIT_32) ? 32 : 64;
1787 #else /* !SLJIT_CONFIG_RISCV_64 */
1788 sljit_ins word_size = 32;
1789 #endif /* SLJIT_CONFIG_RISCV_64 */
1790
1791 SLJIT_ASSERT(WORD == 0 || WORD == 0x8);
1792
1793 /* The OTHER_FLAG is the counter. */
1794 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(OTHER_FLAG) | RS1(TMP_ZERO) | IMM_I(word_size)));
1795
1796 /* The TMP_REG2 is the next value. */
1797 if (src != TMP_REG2)
1798 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(TMP_REG2) | RS1(src) | IMM_I(0)));
1799
1800 FAIL_IF(push_inst(compiler, BEQ | RS1(TMP_REG2) | RS2(TMP_ZERO) | ((sljit_ins)((is_clz ? 4 : 5) * SSIZE_OF(ins)) << 7) | ((sljit_ins)(8 * SSIZE_OF(ins)) << 20)));
1801
1802 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(OTHER_FLAG) | RS1(TMP_ZERO) | IMM_I(0)));
1803 if (!is_clz) {
1804 FAIL_IF(push_inst(compiler, ANDI | RD(TMP_REG1) | RS1(TMP_REG2) | IMM_I(1)));
1805 FAIL_IF(push_inst(compiler, BNE | RS1(TMP_REG1) | RS2(TMP_ZERO) | ((sljit_ins)(2 * SSIZE_OF(ins)) << 7) | ((sljit_ins)(8 * SSIZE_OF(ins)) << 20)));
1806 } else
1807 FAIL_IF(push_inst(compiler, BLT | RS1(TMP_REG2) | RS2(TMP_ZERO) | ((sljit_ins)(2 * SSIZE_OF(ins)) << 7) | ((sljit_ins)(8 * SSIZE_OF(ins)) << 20)));
1808
1809 /* The TMP_REG1 is the next shift. */
1810 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(TMP_REG1) | RS1(TMP_ZERO) | IMM_I(word_size)));
1811
1812 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(EQUAL_FLAG) | RS1(TMP_REG2) | IMM_I(0)));
1813 FAIL_IF(push_inst(compiler, SRLI | WORD | RD(TMP_REG1) | RS1(TMP_REG1) | IMM_I(1)));
1814
1815 FAIL_IF(push_inst(compiler, (is_clz ? SRL : SLL) | WORD | RD(TMP_REG2) | RS1(EQUAL_FLAG) | RS2(TMP_REG1)));
1816 FAIL_IF(push_inst(compiler, BNE | RS1(TMP_REG2) | RS2(TMP_ZERO) | ((sljit_ins)0xfe000e80 - ((2 * SSIZE_OF(ins)) << 7))));
1817 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(TMP_REG2) | RS1(TMP_REG1) | IMM_I(-1)));
1818 FAIL_IF(push_inst(compiler, (is_clz ? SRL : SLL) | WORD | RD(TMP_REG2) | RS1(EQUAL_FLAG) | RS2(TMP_REG2)));
1819 FAIL_IF(push_inst(compiler, OR | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(TMP_REG1)));
1820 FAIL_IF(push_inst(compiler, BEQ | RS1(TMP_REG2) | RS2(TMP_ZERO) | ((sljit_ins)0xfe000e80 - ((5 * SSIZE_OF(ins)) << 7))));
1821
1822 return push_inst(compiler, ADDI | WORD | RD(dst) | RS1(OTHER_FLAG) | IMM_I(0));
1823 }
1824
1825 static sljit_s32 emit_rev(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw src)
1826 {
1827 SLJIT_UNUSED_ARG(op);
1828
1829 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1830 if (!(op & SLJIT_32)) {
1831 FAIL_IF(push_inst(compiler, LUI | RD(OTHER_FLAG) | 0x10000));
1832 FAIL_IF(push_inst(compiler, SRLI | RD(TMP_REG1) | RS1(src) | IMM_I(32)));
1833 FAIL_IF(push_inst(compiler, ADDI | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | IMM_I(0xfff)));
1834 FAIL_IF(push_inst(compiler, SLLI | RD(dst) | RS1(src) | IMM_I(32)));
1835 FAIL_IF(push_inst(compiler, SLLI | RD(EQUAL_FLAG) | RS1(OTHER_FLAG) | IMM_I(32)));
1836 FAIL_IF(push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1)));
1837 FAIL_IF(push_inst(compiler, OR | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(EQUAL_FLAG)));
1838
1839 FAIL_IF(push_inst(compiler, SRLI | RD(TMP_REG1) | RS1(dst) | IMM_I(16)));
1840 FAIL_IF(push_inst(compiler, AND | RD(dst) | RS1(dst) | RS2(OTHER_FLAG)));
1841 FAIL_IF(push_inst(compiler, AND | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(OTHER_FLAG)));
1842 FAIL_IF(push_inst(compiler, SLLI | RD(EQUAL_FLAG) | RS1(OTHER_FLAG) | IMM_I(8)));
1843 FAIL_IF(push_inst(compiler, SLLI | RD(dst) | RS1(dst) | IMM_I(16)));
1844 FAIL_IF(push_inst(compiler, XOR | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(EQUAL_FLAG)));
1845 FAIL_IF(push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1)));
1846
1847 FAIL_IF(push_inst(compiler, SRLI | RD(TMP_REG1) | RS1(dst) | IMM_I(8)));
1848 FAIL_IF(push_inst(compiler, AND | RD(dst) | RS1(dst) | RS2(OTHER_FLAG)));
1849 FAIL_IF(push_inst(compiler, AND | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(OTHER_FLAG)));
1850 FAIL_IF(push_inst(compiler, SLLI | RD(dst) | RS1(dst) | IMM_I(8)));
1851 return push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1));
1852 }
1853 #endif /* SLJIT_CONFIG_RISCV_64 */
1854
1855 FAIL_IF(push_inst(compiler, SRLI | WORD_32 | RD(TMP_REG1) | RS1(src) | IMM_I(16)));
1856 FAIL_IF(push_inst(compiler, LUI | RD(OTHER_FLAG) | 0xff0000));
1857 FAIL_IF(push_inst(compiler, SLLI | WORD_32 | RD(dst) | RS1(src) | IMM_I(16)));
1858 FAIL_IF(push_inst(compiler, ORI | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | IMM_I(0xff)));
1859 FAIL_IF(push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1)));
1860
1861 FAIL_IF(push_inst(compiler, SRLI | WORD_32 | RD(TMP_REG1) | RS1(dst) | IMM_I(8)));
1862 FAIL_IF(push_inst(compiler, AND | RD(dst) | RS1(dst) | RS2(OTHER_FLAG)));
1863 FAIL_IF(push_inst(compiler, AND | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(OTHER_FLAG)));
1864 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1865 FAIL_IF(push_inst(compiler, SLLI | (GET_OPCODE(op) == SLJIT_REV_S32 ? WORD_32 : 0) | RD(dst) | RS1(dst) | IMM_I(8)));
1866 #else /* !SLJIT_CONFIG_RISCV_64 */
1867 FAIL_IF(push_inst(compiler, SLLI | RD(dst) | RS1(dst) | IMM_I(8)));
1868 #endif /* SLJIT_CONFIG_RISCV_64 */
1869 return push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1));
1870 }
1871
1872 static sljit_s32 emit_rev16(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw src)
1873 {
1874 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1875 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
1876 sljit_ins word_size = (op & SLJIT_32) ? 32 : 64;
1877 #else /* !SLJIT_CONFIG_RISCV_64 */
1878 sljit_ins word_size = 32;
1879 #endif /* SLJIT_CONFIG_RISCV_64 */
1880
1881 FAIL_IF(push_inst(compiler, SRLI | WORD | RD(TMP_REG1) | RS1(src) | IMM_I(8)));
1882 FAIL_IF(push_inst(compiler, SLLI | WORD | RD(dst) | RS1(src) | IMM_I(word_size - 8)));
1883 FAIL_IF(push_inst(compiler, ANDI | RD(TMP_REG1) | RS1(TMP_REG1) | IMM_I(0xff)));
1884 FAIL_IF(push_inst(compiler, (GET_OPCODE(op) == SLJIT_REV_U16 ? SRLI : SRAI) | WORD | RD(dst) | RS1(dst) | IMM_I(word_size - 16)));
1885 return push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1));
1886 }
1887
1888 #define EMIT_LOGICAL(op_imm, op_c_imm, op_reg, op_c_reg) \
1889 if (flags & SRC2_IMM) { \
1890 if (op & SLJIT_SET_Z) \
1891 FAIL_IF(push_inst(compiler, op_imm | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(src2))); \
1892 if (!(flags & UNUSED_DEST)) { \
1893 if (op_c_imm != 0 && RISCV_HAS_COMPRESSED(200) && dst == src1 && src2 <= SIMM16_MAX && src2 >= SIMM16_MIN && C_IS_R3(dst)) \
1894 FAIL_IF(push_inst16(compiler, op_c_imm | C_RS1_R3(dst) | C_IMM_I(src2))); \
1895 else \
1896 FAIL_IF(push_inst(compiler, op_imm | RD(dst) | RS1(src1) | IMM_I(src2))); \
1897 } \
1898 } else { \
1899 if (dst == src2) { \
1900 src2 = src1; \
1901 src1 = dst; \
1902 } \
1903 \
1904 if (op & SLJIT_SET_Z) \
1905 FAIL_IF(push_inst(compiler, op_reg | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2))); \
1906 if (!(flags & UNUSED_DEST)) { \
1907 if (RISCV_HAS_COMPRESSED(200) && dst == src1 && C_IS_R3(dst) && C_IS_R3(src2)) \
1908 FAIL_IF(push_inst16(compiler, op_c_reg | C_RS1_R3(dst) | C_RS2_R3(src2))); \
1909 else \
1910 FAIL_IF(push_inst(compiler, op_reg | RD(dst) | RS1(src1) | RS2(src2))); \
1911 } \
1912 }
1913
1914 static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 flags,
1915 sljit_s32 dst, sljit_s32 src1, sljit_sw src2)
1916 {
1917 sljit_s32 is_overflow, is_carry, carry_src_r, is_handled, reg;
1918 sljit_ins op_imm, op_reg;
1919 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1920 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
1921 #endif /* SLJIT_CONFIG_RISCV_64 */
1922
1923 SLJIT_ASSERT(WORD == 0 || WORD == 0x8);
1924
1925 switch (GET_OPCODE(op)) {
1926 case SLJIT_MOV:
1927 SLJIT_ASSERT(src1 == TMP_ZERO && !(flags & SRC2_IMM));
1928 if (dst == src2)
1929 return SLJIT_SUCCESS;
1930
1931 if (!RISCV_HAS_COMPRESSED(200))
1932 return push_inst(compiler, ADDI | RD(dst) | RS1(src2) | IMM_I(0));
1933
1934 /* Revert the x0 to immediate 0. */
1935 if (src2 == 0)
1936 return push_inst16(compiler, C_LI | C_RD(dst));
1937 return push_inst16(compiler, C_MV | C_RD(dst) | C_RS2(src2));
1938
1939 case SLJIT_MOV_U8:
1940 SLJIT_ASSERT(src1 == TMP_ZERO && !(flags & SRC2_IMM));
1941 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE))
1942 return push_inst(compiler, ANDI | RD(dst) | RS1(src2) | IMM_I(0xff));
1943 SLJIT_ASSERT(dst == src2);
1944 return SLJIT_SUCCESS;
1945
1946 case SLJIT_MOV_S8:
1947 if (RISCV_HAS_BITMANIP_B(93))
1948 return push_inst(compiler, SEXTB | RD(dst) | RS1(src2));
1949
1950 SLJIT_ASSERT(src1 == TMP_ZERO && !(flags & SRC2_IMM));
1951 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) {
1952 FAIL_IF(push_inst(compiler, SLLI | WORD | RD(dst) | RS1(src2) | IMM_EXTEND(24)));
1953 return push_inst(compiler, SRAI | WORD | RD(dst) | RS1(dst) | IMM_EXTEND(24));
1954 }
1955 SLJIT_ASSERT(dst == src2);
1956 return SLJIT_SUCCESS;
1957
1958 case SLJIT_MOV_U16:
1959 if (RISCV_HAS_BITMANIP_B(93))
1960 return push_inst(compiler, ZEXTH | RD(dst) | RS1(src2));
1961
1962 SLJIT_ASSERT(src1 == TMP_ZERO && !(flags & SRC2_IMM));
1963 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) {
1964 FAIL_IF(push_inst(compiler, SLLI | WORD | RD(dst) | RS1(src2) | IMM_EXTEND(16)));
1965 return push_inst(compiler, SRLI | WORD | RD(dst) | RS1(dst) | IMM_EXTEND(16));
1966 }
1967 SLJIT_ASSERT(dst == src2);
1968 return SLJIT_SUCCESS;
1969
1970 case SLJIT_MOV_S16:
1971 if (RISCV_HAS_BITMANIP_B(93))
1972 return push_inst(compiler, SEXTH | RD(dst) | RS1(src2));
1973
1974 SLJIT_ASSERT(src1 == TMP_ZERO && !(flags & SRC2_IMM));
1975 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) {
1976 FAIL_IF(push_inst(compiler, SLLI | WORD | RD(dst) | RS1(src2) | IMM_EXTEND(16)));
1977 return push_inst(compiler, SRAI | WORD | RD(dst) | RS1(dst) | IMM_EXTEND(16));
1978 }
1979 SLJIT_ASSERT(dst == src2);
1980 return SLJIT_SUCCESS;
1981
1982 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1983 case SLJIT_MOV_U32:
1984 SLJIT_ASSERT(src1 == TMP_ZERO && !(flags & SRC2_IMM));
1985 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) {
1986 FAIL_IF(push_inst(compiler, SLLI | RD(dst) | RS1(src2) | IMM_I(32)));
1987 return push_inst(compiler, SRLI | RD(dst) | RS1(dst) | IMM_I(32));
1988 }
1989 SLJIT_ASSERT(dst == src2);
1990 return SLJIT_SUCCESS;
1991
1992 case SLJIT_MOV_S32:
1993 SLJIT_ASSERT(src1 == TMP_ZERO && !(flags & SRC2_IMM));
1994 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) {
1995 if (RISCV_HAS_COMPRESSED(200) && dst == src2)
1996 return push_inst16(compiler, C_ADDIW | C_RD(dst));
1997 return push_inst(compiler, ADDI | 0x8 | RD(dst) | RS1(src2) | IMM_I(0));
1998 }
1999 SLJIT_ASSERT(dst == src2);
2000 return SLJIT_SUCCESS;
2001 #endif /* SLJIT_CONFIG_RISCV_64 */
2002
2003 case SLJIT_CLZ:
2004 case SLJIT_CTZ:
2005 SLJIT_ASSERT(src1 == TMP_ZERO && !(flags & SRC2_IMM));
2006 if (RISCV_HAS_BITMANIP_B(93))
2007 return push_inst(compiler, ((GET_OPCODE(op) == SLJIT_CLZ) ? CLZ : CTZ) | WORD | RD(dst) | RS1(src2));
2008
2009 return emit_clz_ctz(compiler, op, dst, src2);
2010
2011 case SLJIT_REV:
2012 case SLJIT_REV_S32:
2013 case SLJIT_REV_U32:
2014 SLJIT_ASSERT(src1 == TMP_ZERO && !(flags & SRC2_IMM));
2015 if (RISCV_HAS_BITMANIP_B(93)) {
2016 #if defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64
2017 FAIL_IF(push_inst(compiler, REV8 | RD(dst) | RS1(src2)));
2018 if ((op & SLJIT_32) || GET_OPCODE(op) != SLJIT_REV)
2019 return push_inst(compiler, (GET_OPCODE(op) == SLJIT_REV_U32 ? SRLI : SRAI)| RD(dst) | RS1(dst) | IMM_I(32));
2020 return SLJIT_SUCCESS;
2021 #else /* !SLJIT_CONFIG_RISCV_64 */
2022 return push_inst(compiler, REV8 | RD(dst) | RS1(src2));
2023 #endif /* SLJIT_CONFIG_RISCV_64 */
2024 }
2025
2026 return emit_rev(compiler, op, dst, src2);
2027
2028 case SLJIT_REV_U16:
2029 case SLJIT_REV_S16:
2030 SLJIT_ASSERT(src1 == TMP_ZERO && !(flags & SRC2_IMM));
2031 if (RISCV_HAS_BITMANIP_B(93)) {
2032 FAIL_IF(push_inst(compiler, REV8 | RD(dst) | RS1(src2)));
2033 #if defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64
2034 return push_inst(compiler, (GET_OPCODE(op) == SLJIT_REV_U16 ? SRLI : SRAI)| RD(dst) | RS1(dst) | IMM_I(48));
2035 #else /* !SLJIT_CONFIG_RISCV_64 */
2036 return push_inst(compiler, (GET_OPCODE(op) == SLJIT_REV_U16 ? SRLI : SRAI) | RD(dst) | RS1(dst) | IMM_I(16));
2037 #endif /* SLJIT_CONFIG_RISCV_64 */
2038 }
2039
2040 return emit_rev16(compiler, op, dst, src2);
2041
2042 case SLJIT_ADD:
2043 /* Overflow computation (both add and sub): overflow = src1_sign ^ src2_sign ^ result_sign ^ carry_flag */
2044 is_overflow = GET_FLAG_TYPE(op) == SLJIT_OVERFLOW;
2045 carry_src_r = GET_FLAG_TYPE(op) == SLJIT_CARRY;
2046
2047 if (flags & SRC2_IMM) {
2048 if (is_overflow) {
2049 if (src2 >= 0) {
2050 if (RISCV_HAS_COMPRESSED(200))
2051 FAIL_IF(push_inst16(compiler, C_MV | C_RD(EQUAL_FLAG) | C_RS2(src1)));
2052 else
2053 FAIL_IF(push_inst(compiler, ADDI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(0)));
2054 } else
2055 FAIL_IF(push_inst(compiler, XORI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(-1)));
2056 }
2057 else if (op & SLJIT_SET_Z)
2058 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(src2)));
2059
2060 /* Only the zero flag is needed. */
2061 if (!(flags & UNUSED_DEST) || (op & VARIABLE_FLAG_MASK)) {
2062 SLJIT_ASSERT(src2 != 0);
2063 if (RISCV_HAS_COMPRESSED(200) && dst == src1 && src2 <= SIMM16_MAX && src2 >= SIMM16_MIN)
2064 FAIL_IF(push_inst16(compiler, C_ADDI_W(WORD) | C_RD(dst) | C_IMM_I(src2)));
2065 else
2066 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(src2)));
2067 }
2068 } else {
2069 if (is_overflow)
2070 FAIL_IF(push_inst(compiler, XOR | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
2071 else if (op & SLJIT_SET_Z)
2072 FAIL_IF(push_inst(compiler, ADD | WORD | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
2073
2074 if (is_overflow || carry_src_r != 0) {
2075 if (src1 != dst)
2076 carry_src_r = (sljit_s32)src1;
2077 else if (src2 != dst)
2078 carry_src_r = (sljit_s32)src2;
2079 else {
2080 FAIL_IF(push_inst(compiler, ADDI | RD(OTHER_FLAG) | RS1(src1) | IMM_I(0)));
2081 carry_src_r = OTHER_FLAG;
2082 }
2083 }
2084
2085 /* Only the zero flag is needed. */
2086 if (!(flags & UNUSED_DEST) || (op & VARIABLE_FLAG_MASK))
2087 FAIL_IF(emit_add(compiler, WORD, dst, src1, src2));
2088 }
2089
2090 /* Carry is zero if a + b >= a or a + b >= b, otherwise it is 1. */
2091 if (is_overflow || carry_src_r != 0) {
2092 if (flags & SRC2_IMM)
2093 FAIL_IF(push_inst(compiler, SLTUI | RD(OTHER_FLAG) | RS1(dst) | IMM_I(src2)));
2094 else
2095 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(dst) | RS2(carry_src_r)));
2096 }
2097
2098 if (!is_overflow)
2099 return SLJIT_SUCCESS;
2100
2101 FAIL_IF(push_inst(compiler, XOR | RD(TMP_REG1) | RS1(dst) | RS2(EQUAL_FLAG)));
2102 if (op & SLJIT_SET_Z) {
2103 if (RISCV_HAS_COMPRESSED(200))
2104 FAIL_IF(push_inst16(compiler, C_MV | C_RD(EQUAL_FLAG) | C_RS2(dst)));
2105 else
2106 FAIL_IF(push_inst(compiler, ADDI | RD(EQUAL_FLAG) | RS1(dst) | IMM_I(0)));
2107 }
2108 FAIL_IF(push_inst(compiler, SRLI | WORD | RD(TMP_REG1) | RS1(TMP_REG1) | IMM_EXTEND(31)));
2109 return push_inst(compiler, XOR | RD(OTHER_FLAG) | RS1(TMP_REG1) | RS2(OTHER_FLAG));
2110
2111 case SLJIT_ADDC:
2112 carry_src_r = GET_FLAG_TYPE(op) == SLJIT_CARRY;
2113
2114 if (flags & SRC2_IMM) {
2115 SLJIT_ASSERT(src2 != 0);
2116 if (RISCV_HAS_COMPRESSED(200) && dst == src1 && src2 <= SIMM16_MAX && src2 >= SIMM16_MIN)
2117 FAIL_IF(push_inst16(compiler, C_ADDI_W(WORD) | C_RD(dst) | C_IMM_I(src2)));
2118 else
2119 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(src2)));
2120 } else {
2121 if (carry_src_r != 0) {
2122 if (src1 != dst)
2123 carry_src_r = (sljit_s32)src1;
2124 else if (src2 != dst)
2125 carry_src_r = (sljit_s32)src2;
2126 else {
2127 if (RISCV_HAS_COMPRESSED(200))
2128 FAIL_IF(push_inst16(compiler, C_MV | C_RD(EQUAL_FLAG) | C_RS2(src1)));
2129 else
2130 FAIL_IF(push_inst(compiler, ADDI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(0)));
2131 carry_src_r = EQUAL_FLAG;
2132 }
2133 }
2134
2135 FAIL_IF(emit_add(compiler, WORD, dst, src1, src2));
2136 }
2137
2138 /* Carry is zero if a + b >= a or a + b >= b, otherwise it is 1. */
2139 if (carry_src_r != 0) {
2140 if (flags & SRC2_IMM)
2141 FAIL_IF(push_inst(compiler, SLTUI | RD(EQUAL_FLAG) | RS1(dst) | IMM_I(src2)));
2142 else
2143 FAIL_IF(push_inst(compiler, SLTU | RD(EQUAL_FLAG) | RS1(dst) | RS2(carry_src_r)));
2144 }
2145
2146 if (RISCV_HAS_COMPRESSED(200) && WORD == 0)
2147 FAIL_IF(push_inst16(compiler, C_ADD | C_RD(dst) | C_RS2(OTHER_FLAG)));
2148 else
2149 FAIL_IF(push_inst(compiler, ADD | WORD | RD(dst) | RS1(dst) | RS2(OTHER_FLAG)));
2150
2151 if (carry_src_r == 0)
2152 return SLJIT_SUCCESS;
2153
2154 /* Set ULESS_FLAG (dst == 0) && (OTHER_FLAG == 1). */
2155 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(dst) | RS2(OTHER_FLAG)));
2156 /* Set carry flag. */
2157 return push_inst(compiler, OR | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(EQUAL_FLAG));
2158
2159 case SLJIT_SUB:
2160 if ((flags & SRC2_IMM) && src2 == SIMM_MIN) {
2161 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG2) | RS1(TMP_ZERO) | IMM_I(src2)));
2162 src2 = TMP_REG2;
2163 flags &= ~SRC2_IMM;
2164 }
2165
2166 is_handled = 0;
2167
2168 if (flags & SRC2_IMM) {
2169 if (GET_FLAG_TYPE(op) == SLJIT_LESS) {
2170 FAIL_IF(push_inst(compiler, SLTUI | RD(OTHER_FLAG) | RS1(src1) | IMM_I(src2)));
2171 is_handled = 1;
2172 } else if (GET_FLAG_TYPE(op) == SLJIT_SIG_LESS) {
2173 FAIL_IF(push_inst(compiler, SLTI | RD(OTHER_FLAG) | RS1(src1) | IMM_I(src2)));
2174 is_handled = 1;
2175 }
2176 }
2177
2178 if (!is_handled && GET_FLAG_TYPE(op) >= SLJIT_LESS && GET_FLAG_TYPE(op) <= SLJIT_SIG_LESS_EQUAL) {
2179 is_handled = 1;
2180
2181 if (flags & SRC2_IMM) {
2182 reg = (src1 == TMP_REG1) ? TMP_REG2 : TMP_REG1;
2183 if (RISCV_HAS_COMPRESSED(200) && src2 <= SIMM16_MAX && src2 >= SIMM16_MIN)
2184 FAIL_IF(push_inst16(compiler, C_LI | C_RD(reg) | C_IMM_I(src2)));
2185 else
2186 FAIL_IF(push_inst(compiler, ADDI | RD(reg) | RS1(TMP_ZERO) | IMM_I(src2)));
2187
2188 src2 = reg;
2189 flags &= ~SRC2_IMM;
2190 }
2191
2192 switch (GET_FLAG_TYPE(op)) {
2193 case SLJIT_LESS:
2194 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(src1) | RS2(src2)));
2195 break;
2196 case SLJIT_GREATER:
2197 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(src2) | RS2(src1)));
2198 break;
2199 case SLJIT_SIG_LESS:
2200 FAIL_IF(push_inst(compiler, SLT | RD(OTHER_FLAG) | RS1(src1) | RS2(src2)));
2201 break;
2202 case SLJIT_SIG_GREATER:
2203 FAIL_IF(push_inst(compiler, SLT | RD(OTHER_FLAG) | RS1(src2) | RS2(src1)));
2204 break;
2205 }
2206 }
2207
2208 if (is_handled) {
2209 if (flags & SRC2_IMM) {
2210 if (op & SLJIT_SET_Z)
2211 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(-src2)));
2212 if (!(flags & UNUSED_DEST)) {
2213 src2 = -src2;
2214 if (RISCV_HAS_COMPRESSED(200) && dst == src1 && src2 <= SIMM16_MAX && src2 >= SIMM16_MIN)
2215 return push_inst16(compiler, C_ADDI_W(WORD) | C_RD(dst) | C_IMM_I(src2));
2216
2217 return push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(src2));
2218 }
2219 } else {
2220 if (op & SLJIT_SET_Z)
2221 FAIL_IF(push_inst(compiler, SUB | WORD | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
2222 if (!(flags & UNUSED_DEST)) {
2223 if (RISCV_HAS_COMPRESSED(200) && dst == src1 && C_IS_R3(dst) && C_IS_R3(src2))
2224 return push_inst16(compiler, C_SUB_W(WORD) | C_RS1_R3(dst) | C_RS2_R3(src2));
2225
2226 return push_inst(compiler, SUB | WORD | RD(dst) | RS1(src1) | RS2(src2));
2227 }
2228 }
2229 return SLJIT_SUCCESS;
2230 }
2231
2232 is_overflow = GET_FLAG_TYPE(op) == SLJIT_OVERFLOW;
2233 is_carry = GET_FLAG_TYPE(op) == SLJIT_CARRY;
2234
2235 if (flags & SRC2_IMM) {
2236 if (is_overflow) {
2237 if (src2 >= 0) {
2238 if (RISCV_HAS_COMPRESSED(200))
2239 FAIL_IF(push_inst16(compiler, C_MV | C_RD(EQUAL_FLAG) | C_RS2(src1)));
2240 else
2241 FAIL_IF(push_inst(compiler, ADDI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(0)));
2242 } else
2243 FAIL_IF(push_inst(compiler, XORI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(-1)));
2244 } else if (op & SLJIT_SET_Z)
2245 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(-src2)));
2246
2247 if (is_overflow || is_carry)
2248 FAIL_IF(push_inst(compiler, SLTUI | RD(OTHER_FLAG) | RS1(src1) | IMM_I(src2)));
2249
2250 /* Only the zero flag is needed. */
2251 if (!(flags & UNUSED_DEST) || (op & VARIABLE_FLAG_MASK)) {
2252 src2 = -src2;
2253 if (RISCV_HAS_COMPRESSED(200) && dst == src1 && src2 <= SIMM16_MAX && src2 >= SIMM16_MIN)
2254 FAIL_IF(push_inst16(compiler, C_ADDI_W(WORD) | C_RD(dst) | C_IMM_I(src2)));
2255 else
2256 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(src2)));
2257 }
2258 } else {
2259 if (is_overflow)
2260 FAIL_IF(push_inst(compiler, XOR | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
2261 else if (op & SLJIT_SET_Z)
2262 FAIL_IF(push_inst(compiler, SUB | WORD | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
2263
2264 if (is_overflow || is_carry)
2265 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(src1) | RS2(src2)));
2266
2267 /* Only the zero flag is needed. */
2268 if (!(flags & UNUSED_DEST) || (op & VARIABLE_FLAG_MASK)) {
2269 if (RISCV_HAS_COMPRESSED(200) && dst == src1 && C_IS_R3(dst) && C_IS_R3(src2))
2270 FAIL_IF(push_inst16(compiler, C_SUB_W(WORD) | C_RS1_R3(dst) | C_RS2_R3(src2)));
2271 else
2272 FAIL_IF(push_inst(compiler, SUB | WORD | RD(dst) | RS1(src1) | RS2(src2)));
2273 }
2274 }
2275
2276 if (!is_overflow)
2277 return SLJIT_SUCCESS;
2278
2279 FAIL_IF(push_inst(compiler, XOR | RD(TMP_REG1) | RS1(dst) | RS2(EQUAL_FLAG)));
2280 if (op & SLJIT_SET_Z) {
2281 if (RISCV_HAS_COMPRESSED(200))
2282 FAIL_IF(push_inst16(compiler, C_MV | C_RD(EQUAL_FLAG) | C_RS2(dst)));
2283 else
2284 FAIL_IF(push_inst(compiler, ADDI | RD(EQUAL_FLAG) | RS1(dst) | IMM_I(0)));
2285 }
2286 FAIL_IF(push_inst(compiler, SRLI | WORD | RD(TMP_REG1) | RS1(TMP_REG1) | IMM_EXTEND(31)));
2287 return push_inst(compiler, XOR | RD(OTHER_FLAG) | RS1(TMP_REG1) | RS2(OTHER_FLAG));
2288
2289 case SLJIT_SUBC:
2290 if ((flags & SRC2_IMM) && src2 == SIMM_MIN) {
2291 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG2) | RS1(TMP_ZERO) | IMM_I(src2)));
2292 src2 = TMP_REG2;
2293 flags &= ~SRC2_IMM;
2294 }
2295
2296 is_carry = GET_FLAG_TYPE(op) == SLJIT_CARRY;
2297
2298 if (flags & SRC2_IMM) {
2299 if (is_carry)
2300 FAIL_IF(push_inst(compiler, SLTUI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(src2)));
2301
2302 src2 = -src2;
2303 if (RISCV_HAS_COMPRESSED(200) && dst == src1 && src2 <= SIMM16_MAX && src2 >= SIMM16_MIN)
2304 FAIL_IF(push_inst16(compiler, C_ADDI_W(WORD) | C_RD(dst) | C_IMM_I(src2)));
2305 else
2306 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(src2)));
2307 } else {
2308 if (is_carry)
2309 FAIL_IF(push_inst(compiler, SLTU | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
2310
2311 if (RISCV_HAS_COMPRESSED(200) && dst == src1 && C_IS_R3(dst) && C_IS_R3(src2))
2312 FAIL_IF(push_inst16(compiler, C_SUB_W(WORD) | C_RS1_R3(dst) | C_RS2_R3(src2)));
2313 else
2314 FAIL_IF(push_inst(compiler, SUB | WORD | RD(dst) | RS1(src1) | RS2(src2)));
2315 }
2316
2317 if (is_carry)
2318 FAIL_IF(push_inst(compiler, SLTU | RD(TMP_REG1) | RS1(dst) | RS2(OTHER_FLAG)));
2319
2320 FAIL_IF(push_inst(compiler, SUB | WORD | RD(dst) | RS1(dst) | RS2(OTHER_FLAG)));
2321
2322 if (!is_carry)
2323 return SLJIT_SUCCESS;
2324
2325 return push_inst(compiler, OR | RD(OTHER_FLAG) | RS1(EQUAL_FLAG) | RS2(TMP_REG1));
2326
2327 case SLJIT_MUL:
2328 SLJIT_ASSERT(!(flags & SRC2_IMM));
2329
2330 if (GET_FLAG_TYPE(op) != SLJIT_OVERFLOW)
2331 return push_inst(compiler, MUL | WORD | RD(dst) | RS1(src1) | RS2(src2));
2332
2333 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2334 if (word) {
2335 FAIL_IF(push_inst(compiler, MUL | RD(OTHER_FLAG) | RS1(src1) | RS2(src2)));
2336 FAIL_IF(push_inst(compiler, MUL | 0x8 | RD(dst) | RS1(src1) | RS2(src2)));
2337 return push_inst(compiler, SUB | RD(OTHER_FLAG) | RS1(dst) | RS2(OTHER_FLAG));
2338 }
2339 #endif /* SLJIT_CONFIG_RISCV_64 */
2340
2341 FAIL_IF(push_inst(compiler, MULH | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
2342 FAIL_IF(push_inst(compiler, MUL | RD(dst) | RS1(src1) | RS2(src2)));
2343 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2344 FAIL_IF(push_inst(compiler, SRAI | RD(OTHER_FLAG) | RS1(dst) | IMM_I(31)));
2345 #else /* !SLJIT_CONFIG_RISCV_32 */
2346 FAIL_IF(push_inst(compiler, SRAI | RD(OTHER_FLAG) | RS1(dst) | IMM_I(63)));
2347 #endif /* SLJIT_CONFIG_RISCV_32 */
2348 return push_inst(compiler, SUB | RD(OTHER_FLAG) | RS1(EQUAL_FLAG) | RS2(OTHER_FLAG));
2349
2350 case SLJIT_AND:
2351 EMIT_LOGICAL(ANDI, C_ANDI, AND, C_AND);
2352 return SLJIT_SUCCESS;
2353
2354 case SLJIT_OR:
2355 EMIT_LOGICAL(ORI, 0, OR, C_OR);
2356 return SLJIT_SUCCESS;
2357
2358 case SLJIT_XOR:
2359 EMIT_LOGICAL(XORI, 0, XOR, C_XOR);
2360 return SLJIT_SUCCESS;
2361
2362 case SLJIT_SHL:
2363 case SLJIT_MSHL:
2364 op_imm = SLLI;
2365 op_reg = SLL;
2366 break;
2367
2368 case SLJIT_LSHR:
2369 case SLJIT_MLSHR:
2370 op_imm = SRLI;
2371 op_reg = SRL;
2372 break;
2373
2374 case SLJIT_ASHR:
2375 case SLJIT_MASHR:
2376 op_imm = SRAI;
2377 op_reg = SRA;
2378 break;
2379
2380 case SLJIT_ROTL:
2381 case SLJIT_ROTR:
2382 if (flags & SRC2_IMM) {
2383 SLJIT_ASSERT(src2 != 0);
2384
2385 if (RISCV_HAS_BITMANIP_B(93)) {
2386 if (GET_OPCODE(op) == SLJIT_ROTL) {
2387 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2388 src2 = ((op & SLJIT_32) ? 32 : 64) - src2;
2389 #else /* !SLJIT_CONFIG_RISCV_64 */
2390 src2 = 32 - src2;
2391 #endif /* SLJIT_CONFIG_RISCV_64 */
2392 }
2393 return push_inst(compiler, RORI | WORD | RD(dst) | RS1(src1) | IMM_I(src2));
2394 }
2395
2396 op_imm = (GET_OPCODE(op) == SLJIT_ROTL) ? SLLI : SRLI;
2397 FAIL_IF(push_inst(compiler, op_imm | WORD | RD(OTHER_FLAG) | RS1(src1) | IMM_I(src2)));
2398
2399 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2400 src2 = ((op & SLJIT_32) ? 32 : 64) - src2;
2401 #else /* !SLJIT_CONFIG_RISCV_64 */
2402 src2 = 32 - src2;
2403 #endif /* SLJIT_CONFIG_RISCV_64 */
2404 op_imm = (GET_OPCODE(op) == SLJIT_ROTL) ? SRLI : SLLI;
2405 FAIL_IF(push_inst(compiler, op_imm | WORD | RD(dst) | RS1(src1) | IMM_I(src2)));
2406 return push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(OTHER_FLAG));
2407 }
2408
2409 if (RISCV_HAS_BITMANIP_B(93))
2410 return push_inst(compiler, (GET_OPCODE(op) == SLJIT_ROTL ? ROL : ROR) | WORD | RD(dst) | RS1(src1) | RS2(src2));
2411
2412 if (src2 == TMP_ZERO) {
2413 if (dst != src1)
2414 return push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(0));
2415 return SLJIT_SUCCESS;
2416 }
2417
2418 FAIL_IF(push_inst(compiler, SUB | WORD | RD(EQUAL_FLAG) | RS1(TMP_ZERO) | RS2(src2)));
2419 op_reg = (GET_OPCODE(op) == SLJIT_ROTL) ? SLL : SRL;
2420 FAIL_IF(push_inst(compiler, op_reg | WORD | RD(OTHER_FLAG) | RS1(src1) | RS2(src2)));
2421 op_reg = (GET_OPCODE(op) == SLJIT_ROTL) ? SRL : SLL;
2422 FAIL_IF(push_inst(compiler, op_reg | WORD | RD(dst) | RS1(src1) | RS2(EQUAL_FLAG)));
2423 return push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(OTHER_FLAG));
2424 default:
2425 SLJIT_UNREACHABLE();
2426 return SLJIT_SUCCESS;
2427 }
2428
2429 if (flags & SRC2_IMM) {
2430 if (op & SLJIT_SET_Z)
2431 FAIL_IF(push_inst(compiler, op_imm | WORD | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(src2)));
2432
2433 if (flags & UNUSED_DEST)
2434 return SLJIT_SUCCESS;
2435
2436 if (RISCV_HAS_COMPRESSED(200) && WORD == 0 && dst == src1) {
2437 if (op_imm == SLLI)
2438 return push_inst16(compiler, C_SLLI | C_RD(dst) | C_IMM_I(src2));
2439
2440 if (C_IS_R3(dst))
2441 return push_inst16(compiler, (op_imm == SRLI ? C_SRLI : C_SRAI) | C_RS1_R3(dst) | C_IMM_I(src2));
2442 }
2443
2444 return push_inst(compiler, op_imm | WORD | RD(dst) | RS1(src1) | IMM_I(src2));
2445 }
2446
2447 if (op & SLJIT_SET_Z)
2448 FAIL_IF(push_inst(compiler, op_reg | WORD | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
2449
2450 if (flags & UNUSED_DEST)
2451 return SLJIT_SUCCESS;
2452 return push_inst(compiler, op_reg | WORD | RD(dst) | RS1(src1) | RS2(src2));
2453 }
2454
2455 #undef IMM_EXTEND
2456
2457 static sljit_s32 emit_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 flags,
2458 sljit_s32 dst, sljit_sw dstw,
2459 sljit_s32 src1, sljit_sw src1w,
2460 sljit_s32 src2, sljit_sw src2w)
2461 {
2462 /* arg1 goes to TMP_REG1 or src reg
2463 arg2 goes to TMP_REG2, imm or src reg
2464 TMP_REG3 can be used for caching
2465 result goes to TMP_REG2, so put result can use TMP_REG1 and TMP_REG3. */
2466 sljit_s32 dst_r = TMP_REG2;
2467 sljit_s32 src1_r;
2468 sljit_sw src2_r = 0;
2469 sljit_s32 src2_tmp_reg = (GET_OPCODE(op) >= SLJIT_OP2_BASE && FAST_IS_REG(src1)) ? TMP_REG1 : TMP_REG2;
2470
2471 if (!(flags & ALT_KEEP_CACHE)) {
2472 compiler->cache_arg = 0;
2473 compiler->cache_argw = 0;
2474 }
2475
2476 if (dst == 0) {
2477 SLJIT_ASSERT(HAS_FLAGS(op));
2478 flags |= UNUSED_DEST;
2479 dst = TMP_REG2;
2480 }
2481 else if (FAST_IS_REG(dst)) {
2482 dst_r = dst;
2483 flags |= REG_DEST;
2484 if (flags & MOVE_OP)
2485 src2_tmp_reg = dst_r;
2486 }
2487 else if ((dst & SLJIT_MEM) && !getput_arg_fast(compiler, flags | ARG_TEST, TMP_REG1, dst, dstw))
2488 flags |= SLOW_DEST;
2489
2490 if (flags & IMM_OP) {
2491 if (src2 == SLJIT_IMM && src2w != 0 && src2w <= SIMM_MAX && src2w >= SIMM_MIN) {
2492 flags |= SRC2_IMM;
2493 src2_r = src2w;
2494 }
2495 else if ((flags & CUMULATIVE_OP) && src1 == SLJIT_IMM && src1w != 0 && src1w <= SIMM_MAX && src1w >= SIMM_MIN) {
2496 flags |= SRC2_IMM;
2497 src2_r = src1w;
2498
2499 /* And swap arguments. */
2500 src1 = src2;
2501 src1w = src2w;
2502 src2 = SLJIT_IMM;
2503 /* src2w = src2_r unneeded. */
2504 }
2505 }
2506
2507 /* Source 1. */
2508 if (FAST_IS_REG(src1)) {
2509 src1_r = src1;
2510 flags |= REG1_SOURCE;
2511 } else if (src1 == SLJIT_IMM) {
2512 if (src1w) {
2513 FAIL_IF(load_immediate(compiler, TMP_REG1, src1w, TMP_REG3));
2514 src1_r = TMP_REG1;
2515 }
2516 else
2517 src1_r = TMP_ZERO;
2518 } else {
2519 if (getput_arg_fast(compiler, flags | LOAD_DATA, TMP_REG1, src1, src1w))
2520 FAIL_IF(compiler->error);
2521 else
2522 flags |= SLOW_SRC1;
2523 src1_r = TMP_REG1;
2524 }
2525
2526 /* Source 2. */
2527 if (FAST_IS_REG(src2)) {
2528 src2_r = src2;
2529 flags |= REG2_SOURCE;
2530 if ((flags & (REG_DEST | MOVE_OP)) == MOVE_OP)
2531 dst_r = (sljit_s32)src2_r;
2532 } else if (src2 == SLJIT_IMM) {
2533 if (!(flags & SRC2_IMM)) {
2534 if (src2w) {
2535 FAIL_IF(load_immediate(compiler, src2_tmp_reg, src2w, TMP_REG3));
2536 src2_r = src2_tmp_reg;
2537 } else {
2538 src2_r = TMP_ZERO;
2539 if (flags & MOVE_OP) {
2540 if (dst & SLJIT_MEM)
2541 dst_r = 0;
2542 else
2543 op = SLJIT_MOV;
2544 }
2545 }
2546 }
2547 } else {
2548 if (getput_arg_fast(compiler, flags | LOAD_DATA, src2_tmp_reg, src2, src2w))
2549 FAIL_IF(compiler->error);
2550 else
2551 flags |= SLOW_SRC2;
2552 src2_r = src2_tmp_reg;
2553 }
2554
2555 if ((flags & (SLOW_SRC1 | SLOW_SRC2)) == (SLOW_SRC1 | SLOW_SRC2)) {
2556 SLJIT_ASSERT(src2_r == TMP_REG2);
2557 if ((flags & SLOW_DEST) && !can_cache(src2, src2w, src1, src1w) && can_cache(src2, src2w, dst, dstw)) {
2558 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG1, src1, src1w, src2, src2w));
2559 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA | MEM_USE_TMP2, TMP_REG2, src2, src2w, dst, dstw));
2560 } else {
2561 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG2, src2, src2w, src1, src1w));
2562 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG1, src1, src1w, dst, dstw));
2563 }
2564 }
2565 else if (flags & SLOW_SRC1)
2566 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG1, src1, src1w, dst, dstw));
2567 else if (flags & SLOW_SRC2)
2568 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA | ((src1_r == TMP_REG1) ? MEM_USE_TMP2 : 0), src2_tmp_reg, src2, src2w, dst, dstw));
2569
2570 FAIL_IF(emit_single_op(compiler, op, flags, dst_r, src1_r, src2_r));
2571
2572 if (dst & SLJIT_MEM) {
2573 if (!(flags & SLOW_DEST)) {
2574 getput_arg_fast(compiler, flags, dst_r, dst, dstw);
2575 return compiler->error;
2576 }
2577 return getput_arg(compiler, flags, dst_r, dst, dstw, 0, 0);
2578 }
2579
2580 return SLJIT_SUCCESS;
2581 }
2582
2583 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
2584 {
2585 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2586 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
2587
2588 SLJIT_ASSERT(word == 0 || word == 0x8);
2589 #endif /* SLJIT_CONFIG_RISCV_64 */
2590
2591 CHECK_ERROR();
2592 CHECK(check_sljit_emit_op0(compiler, op));
2593
2594 switch (GET_OPCODE(op)) {
2595 case SLJIT_BREAKPOINT:
2596 if (RISCV_HAS_COMPRESSED(200))
2597 return push_inst16(compiler, C_EBREAK);
2598 return push_inst(compiler, EBREAK);
2599 case SLJIT_NOP:
2600 if (RISCV_HAS_COMPRESSED(200))
2601 return push_inst16(compiler, C_NOP);
2602 return push_inst(compiler, NOP);
2603 case SLJIT_LMUL_UW:
2604 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(SLJIT_R1) | IMM_I(0)));
2605 FAIL_IF(push_inst(compiler, MULHU | RD(SLJIT_R1) | RS1(SLJIT_R0) | RS2(SLJIT_R1)));
2606 return push_inst(compiler, MUL | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(TMP_REG1));
2607 case SLJIT_LMUL_SW:
2608 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(SLJIT_R1) | IMM_I(0)));
2609 FAIL_IF(push_inst(compiler, MULH | RD(SLJIT_R1) | RS1(SLJIT_R0) | RS2(SLJIT_R1)));
2610 return push_inst(compiler, MUL | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(TMP_REG1));
2611 case SLJIT_DIVMOD_UW:
2612 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(SLJIT_R0) | IMM_I(0)));
2613 FAIL_IF(push_inst(compiler, DIVU | WORD | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(SLJIT_R1)));
2614 return push_inst(compiler, REMU | WORD | RD(SLJIT_R1) | RS1(TMP_REG1) | RS2(SLJIT_R1));
2615 case SLJIT_DIVMOD_SW:
2616 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(SLJIT_R0) | IMM_I(0)));
2617 FAIL_IF(push_inst(compiler, DIV | WORD | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(SLJIT_R1)));
2618 return push_inst(compiler, REM | WORD | RD(SLJIT_R1) | RS1(TMP_REG1) | RS2(SLJIT_R1));
2619 case SLJIT_DIV_UW:
2620 return push_inst(compiler, DIVU | WORD | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(SLJIT_R1));
2621 case SLJIT_DIV_SW:
2622 return push_inst(compiler, DIV | WORD | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(SLJIT_R1));
2623 case SLJIT_MEMORY_BARRIER:
2624 return push_inst(compiler, FENCE | 0x0ff00000);
2625 case SLJIT_ENDBR:
2626 case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
2627 return SLJIT_SUCCESS;
2628 }
2629
2630 return SLJIT_SUCCESS;
2631 }
2632
2633 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
2634 sljit_s32 dst, sljit_sw dstw,
2635 sljit_s32 src, sljit_sw srcw)
2636 {
2637 sljit_s32 flags = 0;
2638
2639 CHECK_ERROR();
2640 CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
2641 ADJUST_LOCAL_OFFSET(dst, dstw);
2642 ADJUST_LOCAL_OFFSET(src, srcw);
2643
2644 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2645 if (op & SLJIT_32)
2646 flags = INT_DATA | SIGNED_DATA;
2647 #endif /* SLJIT_CONFIG_RISCV_64 */
2648
2649 switch (GET_OPCODE(op)) {
2650 case SLJIT_MOV:
2651 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2652 case SLJIT_MOV_U32:
2653 case SLJIT_MOV_S32:
2654 case SLJIT_MOV32:
2655 #endif /* SLJIT_CONFIG_RISCV_32 */
2656 case SLJIT_MOV_P:
2657 return emit_op(compiler, SLJIT_MOV, WORD_DATA | MOVE_OP, dst, dstw, TMP_ZERO, 0, src, srcw);
2658
2659 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2660 case SLJIT_MOV_U32:
2661 return emit_op(compiler, SLJIT_MOV_U32, INT_DATA | MOVE_OP, dst, dstw, TMP_ZERO, 0, src, (src == SLJIT_IMM) ? (sljit_u32)srcw : srcw);
2662
2663 case SLJIT_MOV_S32:
2664 /* Logical operators have no W variant, so sign extended input is necessary for them. */
2665 case SLJIT_MOV32:
2666 return emit_op(compiler, SLJIT_MOV_S32, INT_DATA | SIGNED_DATA | MOVE_OP, dst, dstw, TMP_ZERO, 0, src, (src == SLJIT_IMM) ? (sljit_s32)srcw : srcw);
2667 #endif /* SLJIT_CONFIG_RISCV_64 */
2668
2669 case SLJIT_MOV_U8:
2670 return emit_op(compiler, op, BYTE_DATA | MOVE_OP, dst, dstw, TMP_ZERO, 0, src, (src == SLJIT_IMM) ? (sljit_u8)srcw : srcw);
2671
2672 case SLJIT_MOV_S8:
2673 return emit_op(compiler, op, BYTE_DATA | SIGNED_DATA | MOVE_OP, dst, dstw, TMP_ZERO, 0, src, (src == SLJIT_IMM) ? (sljit_s8)srcw : srcw);
2674
2675 case SLJIT_MOV_U16:
2676 return emit_op(compiler, op, HALF_DATA | MOVE_OP, dst, dstw, TMP_ZERO, 0, src, (src == SLJIT_IMM) ? (sljit_u16)srcw : srcw);
2677
2678 case SLJIT_MOV_S16:
2679 return emit_op(compiler, op, HALF_DATA | SIGNED_DATA | MOVE_OP, dst, dstw, TMP_ZERO, 0, src, (src == SLJIT_IMM) ? (sljit_s16)srcw : srcw);
2680
2681 case SLJIT_CLZ:
2682 case SLJIT_CTZ:
2683 case SLJIT_REV:
2684 return emit_op(compiler, op, flags, dst, dstw, TMP_ZERO, 0, src, srcw);
2685
2686 case SLJIT_REV_U16:
2687 case SLJIT_REV_S16:
2688 return emit_op(compiler, op, HALF_DATA, dst, dstw, TMP_ZERO, 0, src, srcw);
2689
2690 case SLJIT_REV_U32:
2691 case SLJIT_REV_S32:
2692 return emit_op(compiler, op | SLJIT_32, INT_DATA, dst, dstw, TMP_ZERO, 0, src, srcw);
2693 }
2694
2695 SLJIT_UNREACHABLE();
2696 return SLJIT_SUCCESS;
2697 }
2698
2699 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
2700 sljit_s32 dst, sljit_sw dstw,
2701 sljit_s32 src1, sljit_sw src1w,
2702 sljit_s32 src2, sljit_sw src2w)
2703 {
2704 sljit_s32 flags = 0;
2705
2706 CHECK_ERROR();
2707 CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
2708 ADJUST_LOCAL_OFFSET(dst, dstw);
2709 ADJUST_LOCAL_OFFSET(src1, src1w);
2710 ADJUST_LOCAL_OFFSET(src2, src2w);
2711
2712 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2713 if (op & SLJIT_32) {
2714 flags |= INT_DATA | SIGNED_DATA;
2715 if (src1 == SLJIT_IMM)
2716 src1w = (sljit_s32)src1w;
2717 if (src2 == SLJIT_IMM)
2718 src2w = (sljit_s32)src2w;
2719 }
2720 #endif /* SLJIT_CONFIG_RISCV_64 */
2721
2722 switch (GET_OPCODE(op)) {
2723 case SLJIT_ADD:
2724 case SLJIT_ADDC:
2725 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
2726 return emit_op(compiler, op, flags | CUMULATIVE_OP | IMM_OP, dst, dstw, src1, src1w, src2, src2w);
2727
2728 case SLJIT_SUB:
2729 case SLJIT_SUBC:
2730 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
2731 return emit_op(compiler, op, flags | IMM_OP, dst, dstw, src1, src1w, src2, src2w);
2732
2733 case SLJIT_MUL:
2734 compiler->status_flags_state = 0;
2735 return emit_op(compiler, op, flags | CUMULATIVE_OP, dst, dstw, src1, src1w, src2, src2w);
2736
2737 case SLJIT_AND:
2738 case SLJIT_OR:
2739 case SLJIT_XOR:
2740 return emit_op(compiler, op, flags | CUMULATIVE_OP | IMM_OP, dst, dstw, src1, src1w, src2, src2w);
2741
2742 case SLJIT_SHL:
2743 case SLJIT_MSHL:
2744 case SLJIT_LSHR:
2745 case SLJIT_MLSHR:
2746 case SLJIT_ASHR:
2747 case SLJIT_MASHR:
2748 case SLJIT_ROTL:
2749 case SLJIT_ROTR:
2750 if (src2 == SLJIT_IMM) {
2751 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2752 src2w &= 0x1f;
2753 #else /* !SLJIT_CONFIG_RISCV_32 */
2754 if (op & SLJIT_32)
2755 src2w &= 0x1f;
2756 else
2757 src2w &= 0x3f;
2758 #endif /* SLJIT_CONFIG_RISCV_32 */
2759 }
2760
2761 return emit_op(compiler, op, flags | IMM_OP, dst, dstw, src1, src1w, src2, src2w);
2762 }
2763
2764 SLJIT_UNREACHABLE();
2765 return SLJIT_SUCCESS;
2766 }
2767
2768 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
2769 sljit_s32 src1, sljit_sw src1w,
2770 sljit_s32 src2, sljit_sw src2w)
2771 {
2772 CHECK_ERROR();
2773 CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
2774
2775 SLJIT_SKIP_CHECKS(compiler);
2776 return sljit_emit_op2(compiler, op, 0, 0, src1, src1w, src2, src2w);
2777 }
2778
2779 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
2780 sljit_s32 dst_reg,
2781 sljit_s32 src1, sljit_sw src1w,
2782 sljit_s32 src2, sljit_sw src2w)
2783 {
2784 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2785 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
2786 #endif /* SLJIT_CONFIG_RISCV_64 */
2787
2788 CHECK_ERROR();
2789 CHECK(check_sljit_emit_op2r(compiler, op, dst_reg, src1, src1w, src2, src2w));
2790
2791 SLJIT_ASSERT(WORD == 0 || WORD == 0x8);
2792
2793 switch (GET_OPCODE(op)) {
2794 case SLJIT_MULADD:
2795 SLJIT_SKIP_CHECKS(compiler);
2796 FAIL_IF(sljit_emit_op2(compiler, SLJIT_MUL | (op & SLJIT_32), TMP_REG2, 0, src1, src1w, src2, src2w));
2797 return push_inst(compiler, ADD | WORD | RD(dst_reg) | RS1(dst_reg) | RS2(TMP_REG2));
2798 }
2799
2800 return SLJIT_SUCCESS;
2801 }
2802
2803 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
2804 sljit_s32 dst_reg,
2805 sljit_s32 src1_reg,
2806 sljit_s32 src2_reg,
2807 sljit_s32 src3, sljit_sw src3w)
2808 {
2809 sljit_s32 is_left;
2810 sljit_ins ins1, ins2, ins3;
2811 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2812 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
2813 sljit_s32 inp_flags = ((op & SLJIT_32) ? INT_DATA : WORD_DATA) | LOAD_DATA;
2814 sljit_sw bit_length = (op & SLJIT_32) ? 32 : 64;
2815 #else /* !SLJIT_CONFIG_RISCV_64 */
2816 sljit_s32 inp_flags = WORD_DATA | LOAD_DATA;
2817 sljit_sw bit_length = 32;
2818 #endif /* SLJIT_CONFIG_RISCV_64 */
2819
2820 SLJIT_ASSERT(WORD == 0 || WORD == 0x8);
2821
2822 CHECK_ERROR();
2823 CHECK(check_sljit_emit_shift_into(compiler, op, dst_reg, src1_reg, src2_reg, src3, src3w));
2824
2825 is_left = (GET_OPCODE(op) == SLJIT_SHL || GET_OPCODE(op) == SLJIT_MSHL);
2826
2827 if (src1_reg == src2_reg) {
2828 SLJIT_SKIP_CHECKS(compiler);
2829 return sljit_emit_op2(compiler, (is_left ? SLJIT_ROTL : SLJIT_ROTR) | (op & SLJIT_32), dst_reg, 0, src1_reg, 0, src3, src3w);
2830 }
2831
2832 ADJUST_LOCAL_OFFSET(src3, src3w);
2833
2834 if (src3 == SLJIT_IMM) {
2835 src3w &= bit_length - 1;
2836
2837 if (src3w == 0)
2838 return SLJIT_SUCCESS;
2839
2840 if (is_left) {
2841 ins1 = SLLI | WORD | IMM_I(src3w);
2842 src3w = bit_length - src3w;
2843 ins2 = SRLI | WORD | IMM_I(src3w);
2844 } else {
2845 ins1 = SRLI | WORD | IMM_I(src3w);
2846 src3w = bit_length - src3w;
2847 ins2 = SLLI | WORD | IMM_I(src3w);
2848 }
2849
2850 FAIL_IF(push_inst(compiler, ins1 | RD(dst_reg) | RS1(src1_reg)));
2851 FAIL_IF(push_inst(compiler, ins2 | RD(TMP_REG1) | RS1(src2_reg)));
2852 return push_inst(compiler, OR | RD(dst_reg) | RS1(dst_reg) | RS2(TMP_REG1));
2853 }
2854
2855 if (src3 & SLJIT_MEM) {
2856 FAIL_IF(emit_op_mem(compiler, inp_flags, TMP_REG2, src3, src3w));
2857 src3 = TMP_REG2;
2858 } else if (dst_reg == src3) {
2859 push_inst(compiler, ADDI | WORD | RD(TMP_REG2) | RS1(src3) | IMM_I(0));
2860 src3 = TMP_REG2;
2861 }
2862
2863 if (is_left) {
2864 ins1 = SLL;
2865 ins2 = SRLI;
2866 ins3 = SRL;
2867 } else {
2868 ins1 = SRL;
2869 ins2 = SLLI;
2870 ins3 = SLL;
2871 }
2872
2873 FAIL_IF(push_inst(compiler, ins1 | WORD | RD(dst_reg) | RS1(src1_reg) | RS2(src3)));
2874
2875 if (!(op & SLJIT_SHIFT_INTO_NON_ZERO)) {
2876 FAIL_IF(push_inst(compiler, ins2 | WORD | RD(TMP_REG1) | RS1(src2_reg) | IMM_I(1)));
2877 FAIL_IF(push_inst(compiler, XORI | RD(TMP_REG2) | RS1(src3) | IMM_I((sljit_ins)bit_length - 1)));
2878 src2_reg = TMP_REG1;
2879 } else
2880 FAIL_IF(push_inst(compiler, SUB | WORD | RD(TMP_REG2) | RS1(TMP_ZERO) | RS2(src3)));
2881
2882 FAIL_IF(push_inst(compiler, ins3 | WORD | RD(TMP_REG1) | RS1(src2_reg) | RS2(TMP_REG2)));
2883 return push_inst(compiler, OR | RD(dst_reg) | RS1(dst_reg) | RS2(TMP_REG1));
2884 }
2885
2886 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2_shift(struct sljit_compiler *compiler, sljit_s32 op,
2887 sljit_s32 dst, sljit_sw dstw,
2888 sljit_s32 src1, sljit_sw src1w,
2889 sljit_s32 src2, sljit_sw src2w,
2890 sljit_sw shift_arg)
2891 {
2892 sljit_s32 dst_r, tmp_r;
2893 sljit_ins ins;
2894
2895 CHECK_ERROR();
2896 CHECK(check_sljit_emit_op2_shift(compiler, op, dst, dstw, src1, src1w, src2, src2w, shift_arg));
2897 ADJUST_LOCAL_OFFSET(dst, dstw);
2898 ADJUST_LOCAL_OFFSET(src1, src1w);
2899 ADJUST_LOCAL_OFFSET(src2, src2w);
2900
2901 shift_arg &= (sljit_sw)((sizeof(sljit_sw) * 8) - 1);
2902
2903 if (src2 == SLJIT_IMM) {
2904 src2w = src2w << shift_arg;
2905 shift_arg = 0;
2906 }
2907
2908 if (shift_arg == 0) {
2909 SLJIT_SKIP_CHECKS(compiler);
2910 return sljit_emit_op2(compiler, GET_OPCODE(op), dst, dstw, src1, src1w, src2, src2w);
2911 }
2912
2913 if (src2 & SLJIT_MEM) {
2914 FAIL_IF(emit_op_mem2(compiler, WORD_DATA | LOAD_DATA, TMP_REG2, src2, src2w, src1, src1w));
2915 src2 = TMP_REG2;
2916 }
2917
2918 if (src1 == SLJIT_IMM) {
2919 FAIL_IF(load_immediate(compiler, TMP_REG1, src1w, TMP_REG3));
2920 src1 = TMP_REG1;
2921 } else if (src1 & SLJIT_MEM) {
2922 FAIL_IF(emit_op_mem2(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src1, src1w, dst, dstw));
2923 src1 = TMP_REG1;
2924 }
2925
2926 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
2927 ins = 0;
2928
2929 if (RISCV_HAS_BITMANIP_A(93)) {
2930 switch (shift_arg) {
2931 case 1:
2932 ins = SH1ADD;
2933 break;
2934 case 2:
2935 ins = SH2ADD;
2936 break;
2937 case 3:
2938 ins = SH3ADD;
2939 break;
2940 }
2941 }
2942
2943 if (ins == 0) {
2944 tmp_r = (src1 == TMP_REG1) ? TMP_REG2 : TMP_REG1;
2945 FAIL_IF(push_inst(compiler, SLLI | RD(tmp_r) | RS1(src2) | IMM_I(shift_arg)));
2946 FAIL_IF(push_inst(compiler, ADD | RD(dst_r) | RS1(src1) | RS2(tmp_r)));
2947 } else {
2948 FAIL_IF(push_inst(compiler, ins | RD(dst_r) | RS1(src2) | RS2(src1)));
2949 }
2950
2951 if (dst & SLJIT_MEM)
2952 return emit_op_mem2(compiler, WORD_DATA, dst_r, dst, dstw, 0, 0);
2953 return SLJIT_SUCCESS;
2954 }
2955
2956 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
2957 sljit_s32 src, sljit_sw srcw)
2958 {
2959 CHECK_ERROR();
2960 CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
2961 ADJUST_LOCAL_OFFSET(src, srcw);
2962
2963 switch (op) {
2964 case SLJIT_FAST_RETURN:
2965 if (FAST_IS_REG(src)) {
2966 if (src != RETURN_ADDR_REG)
2967 FAIL_IF(push_inst(compiler, ADDI | RD(RETURN_ADDR_REG) | RS1(src) | IMM_I(0)));
2968 } else
2969 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, RETURN_ADDR_REG, src, srcw));
2970
2971 return push_inst(compiler, JALR | RD(TMP_ZERO) | RS1(RETURN_ADDR_REG) | IMM_I(0));
2972 case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
2973 return SLJIT_SUCCESS;
2974 case SLJIT_PREFETCH_L1:
2975 case SLJIT_PREFETCH_L2:
2976 case SLJIT_PREFETCH_L3:
2977 case SLJIT_PREFETCH_ONCE:
2978 return SLJIT_SUCCESS;
2979 }
2980
2981 return SLJIT_SUCCESS;
2982 }
2983
2984 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
2985 sljit_s32 dst, sljit_sw dstw)
2986 {
2987 sljit_s32 dst_r;
2988
2989 CHECK_ERROR();
2990 CHECK(check_sljit_emit_op_dst(compiler, op, dst, dstw));
2991 ADJUST_LOCAL_OFFSET(dst, dstw);
2992
2993 switch (op) {
2994 case SLJIT_FAST_ENTER:
2995 if (FAST_IS_REG(dst)) {
2996 if (dst == RETURN_ADDR_REG)
2997 return SLJIT_SUCCESS;
2998 return push_inst(compiler, ADDI | RD(dst) | RS1(RETURN_ADDR_REG) | IMM_I(0));
2999 }
3000
3001 SLJIT_ASSERT(RETURN_ADDR_REG == TMP_REG2);
3002 break;
3003 case SLJIT_GET_RETURN_ADDRESS:
3004 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
3005 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, dst_r, SLJIT_MEM1(SLJIT_SP), compiler->local_size - SSIZE_OF(sw)));
3006 break;
3007 }
3008
3009 if (dst & SLJIT_MEM)
3010 return emit_op_mem(compiler, WORD_DATA, TMP_REG2, dst, dstw);
3011
3012 return SLJIT_SUCCESS;
3013 }
3014
3015 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
3016 {
3017 CHECK_REG_INDEX(check_sljit_get_register_index(type, reg));
3018
3019 if (type == SLJIT_GP_REGISTER)
3020 return reg_map[reg];
3021
3022 if (type == SLJIT_FLOAT_REGISTER)
3023 return freg_map[reg];
3024
3025 return vreg_map[reg];
3026 }
3027
3028 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
3029 void *instruction, sljit_u32 size)
3030 {
3031 SLJIT_UNUSED_ARG(size);
3032
3033 CHECK_ERROR();
3034 CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
3035
3036 if (size == 2)
3037 return push_inst16(compiler, *(sljit_u16*)instruction);
3038
3039 return push_inst(compiler, *(sljit_ins*)instruction);
3040 }
3041
3042 /* --------------------------------------------------------------------- */
3043 /* Floating point operators */
3044 /* --------------------------------------------------------------------- */
3045
3046 #define FLOAT_DATA(op) (DOUBLE_DATA | ((op & SLJIT_32) >> 7))
3047 #define FMT(op) ((sljit_ins)((op & SLJIT_32) ^ SLJIT_32) << 17)
3048
3049 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
3050 sljit_s32 dst, sljit_sw dstw,
3051 sljit_s32 src, sljit_sw srcw)
3052 {
3053 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
3054 # define flags (sljit_u32)0
3055 #else /* !SLJIT_CONFIG_RISCV_32 */
3056 sljit_u32 flags = ((sljit_u32)(GET_OPCODE(op) == SLJIT_CONV_SW_FROM_F64)) << 21;
3057 #endif /* SLJIT_CONFIG_RISCV_32 */
3058 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
3059
3060 if (src & SLJIT_MEM) {
3061 FAIL_IF(emit_op_mem2(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src, srcw, dst, dstw));
3062 src = TMP_FREG1;
3063 }
3064
3065 FAIL_IF(push_inst(compiler, FCVT_W_S | FMT(op) | flags | RD(dst_r) | FRS1(src)));
3066
3067 /* Store the integer value from a VFP register. */
3068 if (dst & SLJIT_MEM) {
3069 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
3070 return emit_op_mem2(compiler, WORD_DATA, TMP_REG2, dst, dstw, 0, 0);
3071 #else /* !SLJIT_CONFIG_RISCV_32 */
3072 return emit_op_mem2(compiler, flags ? WORD_DATA : INT_DATA, TMP_REG2, dst, dstw, 0, 0);
3073 #endif /* SLJIT_CONFIG_RISCV_32 */
3074 }
3075 return SLJIT_SUCCESS;
3076
3077 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
3078 # undef flags
3079 #endif /* SLJIT_CONFIG_RISCV_32 */
3080 }
3081
3082 static sljit_s32 sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler *compiler, sljit_ins ins,
3083 sljit_s32 dst, sljit_sw dstw,
3084 sljit_s32 src, sljit_sw srcw)
3085 {
3086 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
3087
3088 if (src & SLJIT_MEM) {
3089 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
3090 FAIL_IF(emit_op_mem2(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src, srcw, dst, dstw));
3091 #else /* SLJIT_CONFIG_RISCV_32 */
3092 FAIL_IF(emit_op_mem2(compiler, ((ins & (1 << 21)) ? WORD_DATA : INT_DATA) | LOAD_DATA, TMP_REG1, src, srcw, dst, dstw));
3093 #endif /* !SLJIT_CONFIG_RISCV_32 */
3094 src = TMP_REG1;
3095 } else if (src == SLJIT_IMM) {
3096 FAIL_IF(load_immediate(compiler, TMP_REG1, srcw, TMP_REG3));
3097 src = TMP_REG1;
3098 }
3099
3100 FAIL_IF(push_inst(compiler, ins | FRD(dst_r) | RS1(src)));
3101
3102 if (dst & SLJIT_MEM)
3103 return emit_op_mem2(compiler, DOUBLE_DATA | ((sljit_s32)(~ins >> 24) & 0x2), TMP_FREG1, dst, dstw, 0, 0);
3104 return SLJIT_SUCCESS;
3105 }
3106
3107 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
3108 sljit_s32 dst, sljit_sw dstw,
3109 sljit_s32 src, sljit_sw srcw)
3110 {
3111 sljit_ins ins = FCVT_S_W | FMT(op);
3112
3113 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
3114 if (op & SLJIT_32)
3115 ins |= F3(0x7);
3116 #else /* !SLJIT_CONFIG_RISCV_32 */
3117 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_SW)
3118 ins |= (1 << 21);
3119 else if (src == SLJIT_IMM)
3120 srcw = (sljit_s32)srcw;
3121
3122 if (op != SLJIT_CONV_F64_FROM_S32)
3123 ins |= F3(0x7);
3124 #endif /* SLJIT_CONFIG_RISCV_32 */
3125
3126 return sljit_emit_fop1_conv_f64_from_w(compiler, ins, dst, dstw, src, srcw);
3127 }
3128
3129 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_uw(struct sljit_compiler *compiler, sljit_s32 op,
3130 sljit_s32 dst, sljit_sw dstw,
3131 sljit_s32 src, sljit_sw srcw)
3132 {
3133 sljit_ins ins = FCVT_S_WU | FMT(op);
3134
3135 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
3136 if (op & SLJIT_32)
3137 ins |= F3(0x7);
3138 #else /* !SLJIT_CONFIG_RISCV_32 */
3139 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_UW)
3140 ins |= (1 << 21);
3141 else if (src == SLJIT_IMM)
3142 srcw = (sljit_u32)srcw;
3143
3144 if (op != SLJIT_CONV_F64_FROM_S32)
3145 ins |= F3(0x7);
3146 #endif /* SLJIT_CONFIG_RISCV_32 */
3147
3148 return sljit_emit_fop1_conv_f64_from_w(compiler, ins, dst, dstw, src, srcw);
3149 }
3150
3151 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
3152 sljit_s32 src1, sljit_sw src1w,
3153 sljit_s32 src2, sljit_sw src2w)
3154 {
3155 sljit_ins inst;
3156
3157 if (src1 & SLJIT_MEM) {
3158 FAIL_IF(emit_op_mem2(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, src2, src2w));
3159 src1 = TMP_FREG1;
3160 }
3161
3162 if (src2 & SLJIT_MEM) {
3163 FAIL_IF(emit_op_mem2(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, 0, 0));
3164 src2 = TMP_FREG2;
3165 }
3166
3167 switch (GET_FLAG_TYPE(op)) {
3168 case SLJIT_F_EQUAL:
3169 case SLJIT_ORDERED_EQUAL:
3170 inst = FEQ_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src2);
3171 break;
3172 case SLJIT_F_LESS:
3173 case SLJIT_ORDERED_LESS:
3174 inst = FLT_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src2);
3175 break;
3176 case SLJIT_ORDERED_GREATER:
3177 inst = FLT_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src2) | FRS2(src1);
3178 break;
3179 case SLJIT_F_GREATER:
3180 case SLJIT_UNORDERED_OR_GREATER:
3181 inst = FLE_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src2);
3182 break;
3183 case SLJIT_UNORDERED_OR_LESS:
3184 inst = FLE_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src2) | FRS2(src1);
3185 break;
3186 case SLJIT_UNORDERED_OR_EQUAL:
3187 FAIL_IF(push_inst(compiler, FLT_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src2)));
3188 FAIL_IF(push_inst(compiler, FLT_S | FMT(op) | RD(TMP_REG1) | FRS1(src2) | FRS2(src1)));
3189 inst = OR | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(TMP_REG1);
3190 break;
3191 default: /* SLJIT_UNORDERED */
3192 if (src1 == src2) {
3193 inst = FEQ_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src1);
3194 break;
3195 }
3196 FAIL_IF(push_inst(compiler, FEQ_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src1)));
3197 FAIL_IF(push_inst(compiler, FEQ_S | FMT(op) | RD(TMP_REG1) | FRS1(src2) | FRS2(src2)));
3198 inst = AND | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(TMP_REG1);
3199 break;
3200 }
3201
3202 return push_inst(compiler, inst);
3203 }
3204
3205 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
3206 sljit_s32 dst, sljit_sw dstw,
3207 sljit_s32 src, sljit_sw srcw)
3208 {
3209 sljit_s32 dst_r;
3210
3211 CHECK_ERROR();
3212 compiler->cache_arg = 0;
3213 compiler->cache_argw = 0;
3214
3215 SLJIT_COMPILE_ASSERT((SLJIT_32 == 0x100) && !(DOUBLE_DATA & 0x2), float_transfer_bit_error);
3216 SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
3217
3218 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32)
3219 op ^= SLJIT_32;
3220
3221 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
3222
3223 if (src & SLJIT_MEM) {
3224 FAIL_IF(emit_op_mem2(compiler, FLOAT_DATA(op) | LOAD_DATA, dst_r, src, srcw, dst, dstw));
3225 src = dst_r;
3226 }
3227
3228 switch (GET_OPCODE(op)) {
3229 case SLJIT_MOV_F64:
3230 if (src != dst_r) {
3231 if (!(dst & SLJIT_MEM))
3232 FAIL_IF(push_inst(compiler, FSGNJ_S | FMT(op) | FRD(dst_r) | FRS1(src) | FRS2(src)));
3233 else
3234 dst_r = src;
3235 }
3236 break;
3237 case SLJIT_NEG_F64:
3238 FAIL_IF(push_inst(compiler, FSGNJN_S | FMT(op) | FRD(dst_r) | FRS1(src) | FRS2(src)));
3239 break;
3240 case SLJIT_ABS_F64:
3241 FAIL_IF(push_inst(compiler, FSGNJX_S | FMT(op) | FRD(dst_r) | FRS1(src) | FRS2(src)));
3242 break;
3243 case SLJIT_CONV_F64_FROM_F32:
3244 /* The SLJIT_32 bit is inverted because sljit_f32 needs to be loaded from the memory. */
3245 FAIL_IF(push_inst(compiler, FCVT_S_D | ((op & SLJIT_32) ? (1 << 25) : ((1 << 20) | F3(7))) | FRD(dst_r) | FRS1(src)));
3246 op ^= SLJIT_32;
3247 break;
3248 }
3249
3250 if (dst & SLJIT_MEM)
3251 return emit_op_mem2(compiler, FLOAT_DATA(op), dst_r, dst, dstw, 0, 0);
3252 return SLJIT_SUCCESS;
3253 }
3254
3255 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
3256 sljit_s32 dst, sljit_sw dstw,
3257 sljit_s32 src1, sljit_sw src1w,
3258 sljit_s32 src2, sljit_sw src2w)
3259 {
3260 sljit_s32 dst_r, flags = 0;
3261
3262 CHECK_ERROR();
3263 CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
3264 ADJUST_LOCAL_OFFSET(dst, dstw);
3265 ADJUST_LOCAL_OFFSET(src1, src1w);
3266 ADJUST_LOCAL_OFFSET(src2, src2w);
3267
3268 compiler->cache_arg = 0;
3269 compiler->cache_argw = 0;
3270
3271 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG2;
3272
3273 if (src1 & SLJIT_MEM) {
3274 if (getput_arg_fast(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w)) {
3275 FAIL_IF(compiler->error);
3276 src1 = TMP_FREG1;
3277 } else
3278 flags |= SLOW_SRC1;
3279 }
3280
3281 if (src2 & SLJIT_MEM) {
3282 if (getput_arg_fast(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w)) {
3283 FAIL_IF(compiler->error);
3284 src2 = TMP_FREG2;
3285 } else
3286 flags |= SLOW_SRC2;
3287 }
3288
3289 if ((flags & (SLOW_SRC1 | SLOW_SRC2)) == (SLOW_SRC1 | SLOW_SRC2)) {
3290 if ((dst & SLJIT_MEM) && !can_cache(src1, src1w, src2, src2w) && can_cache(src1, src1w, dst, dstw)) {
3291 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, src1, src1w));
3292 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, dst, dstw));
3293 } else {
3294 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, src2, src2w));
3295 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, dst, dstw));
3296 }
3297 }
3298 else if (flags & SLOW_SRC1)
3299 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, dst, dstw));
3300 else if (flags & SLOW_SRC2)
3301 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, dst, dstw));
3302
3303 if (flags & SLOW_SRC1)
3304 src1 = TMP_FREG1;
3305 if (flags & SLOW_SRC2)
3306 src2 = TMP_FREG2;
3307
3308 switch (GET_OPCODE(op)) {
3309 case SLJIT_ADD_F64:
3310 FAIL_IF(push_inst(compiler, FADD_S | FMT(op) | FRD(dst_r) | FRS1(src1) | FRS2(src2)));
3311 break;
3312
3313 case SLJIT_SUB_F64:
3314 FAIL_IF(push_inst(compiler, FSUB_S | FMT(op) | FRD(dst_r) | FRS1(src1) | FRS2(src2)));
3315 break;
3316
3317 case SLJIT_MUL_F64:
3318 FAIL_IF(push_inst(compiler, FMUL_S | FMT(op) | FRD(dst_r) | FRS1(src1) | FRS2(src2)));
3319 break;
3320
3321 case SLJIT_DIV_F64:
3322 FAIL_IF(push_inst(compiler, FDIV_S | FMT(op) | FRD(dst_r) | FRS1(src1) | FRS2(src2)));
3323 break;
3324
3325 case SLJIT_COPYSIGN_F64:
3326 return push_inst(compiler, FSGNJ_S | FMT(op) | FRD(dst_r) | FRS1(src1) | FRS2(src2));
3327 }
3328
3329 if (dst_r != dst)
3330 FAIL_IF(emit_op_mem2(compiler, FLOAT_DATA(op), TMP_FREG2, dst, dstw, 0, 0));
3331
3332 return SLJIT_SUCCESS;
3333 }
3334
3335 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset32(struct sljit_compiler *compiler,
3336 sljit_s32 freg, sljit_f32 value)
3337 {
3338 union {
3339 sljit_s32 imm;
3340 sljit_f32 value;
3341 } u;
3342
3343 CHECK_ERROR();
3344 CHECK(check_sljit_emit_fset32(compiler, freg, value));
3345
3346 u.value = value;
3347
3348 if (u.imm == 0)
3349 return push_inst(compiler, FMV_W_X | RS1(TMP_ZERO) | FRD(freg));
3350
3351 FAIL_IF(load_immediate(compiler, TMP_REG1, u.imm, TMP_REG3));
3352 return push_inst(compiler, FMV_W_X | RS1(TMP_REG1) | FRD(freg));
3353 }
3354
3355 /* --------------------------------------------------------------------- */
3356 /* Conditional instructions */
3357 /* --------------------------------------------------------------------- */
3358
3359 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
3360 {
3361 struct sljit_label *label;
3362
3363 CHECK_ERROR_PTR();
3364 CHECK_PTR(check_sljit_emit_label(compiler));
3365
3366 if (compiler->last_label && compiler->last_label->size == compiler->size)
3367 return compiler->last_label;
3368
3369 label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
3370 PTR_FAIL_IF(!label);
3371 set_label(label, compiler);
3372 return label;
3373 }
3374
3375 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_aligned_label(struct sljit_compiler *compiler,
3376 sljit_s32 alignment, struct sljit_read_only_buffer *buffers)
3377 {
3378 sljit_uw mask = 0, i;
3379 struct sljit_label *label = NULL;
3380 struct sljit_label *next_label;
3381 struct sljit_extended_label *ext_label;
3382
3383 CHECK_ERROR_PTR();
3384 CHECK_PTR(check_sljit_emit_aligned_label(compiler, alignment, buffers));
3385
3386 sljit_reset_read_only_buffers(buffers);
3387
3388 if (RISCV_HAS_COMPRESSED(200)) {
3389 if (alignment <= SLJIT_LABEL_ALIGN_2) {
3390 SLJIT_SKIP_CHECKS(compiler);
3391 label = sljit_emit_label(compiler);
3392 PTR_FAIL_IF(!label);
3393 } else {
3394 /* The used space is filled with NOPs. */
3395 mask = ((sljit_uw)1 << alignment) - sizeof(sljit_u16);
3396
3397 for (i = (mask >> 1); i != 0; i--)
3398 PTR_FAIL_IF(push_inst16(compiler, C_NOP));
3399 }
3400 } else {
3401 if (alignment <= SLJIT_LABEL_ALIGN_4) {
3402 SLJIT_SKIP_CHECKS(compiler);
3403 label = sljit_emit_label(compiler);
3404 PTR_FAIL_IF(!label);
3405 } else {
3406 /* The used space is filled with NOPs. */
3407 mask = ((sljit_uw)1 << alignment) - sizeof(sljit_ins);
3408
3409 for (i = (mask >> 2); i != 0; i--)
3410 PTR_FAIL_IF(push_inst(compiler, NOP));
3411 }
3412 }
3413
3414 if (label == NULL) {
3415 ext_label = (struct sljit_extended_label*)ensure_abuf(compiler, sizeof(struct sljit_extended_label));
3416 PTR_FAIL_IF(!ext_label);
3417 set_extended_label(ext_label, compiler, SLJIT_LABEL_ALIGNED, mask);
3418 label = &ext_label->label;
3419 }
3420
3421 if (buffers == NULL)
3422 return label;
3423
3424 next_label = label;
3425
3426 while (1) {
3427 buffers->u.label = next_label;
3428
3429 if (RISCV_HAS_COMPRESSED(200)) {
3430 for (i = (buffers->size + 1) >> 1; i > 0; i--)
3431 PTR_FAIL_IF(push_inst16(compiler, C_NOP));
3432 } else {
3433 for (i = (buffers->size + 3) >> 2; i > 0; i--)
3434 PTR_FAIL_IF(push_inst(compiler, NOP));
3435 }
3436
3437 buffers = buffers->next;
3438
3439 if (buffers == NULL)
3440 break;
3441
3442 SLJIT_SKIP_CHECKS(compiler);
3443 next_label = sljit_emit_label(compiler);
3444 PTR_FAIL_IF(!next_label);
3445 }
3446
3447 return label;
3448 }
3449
3450 static sljit_ins get_jump_instruction(sljit_s32 type)
3451 {
3452 switch (type) {
3453 case SLJIT_EQUAL:
3454 return BNE | RS1(EQUAL_FLAG) | RS2(TMP_ZERO);
3455 case SLJIT_NOT_EQUAL:
3456 return BEQ | RS1(EQUAL_FLAG) | RS2(TMP_ZERO);
3457 case SLJIT_LESS:
3458 case SLJIT_GREATER:
3459 case SLJIT_SIG_LESS:
3460 case SLJIT_SIG_GREATER:
3461 case SLJIT_OVERFLOW:
3462 case SLJIT_CARRY:
3463 case SLJIT_ATOMIC_NOT_STORED:
3464 case SLJIT_F_EQUAL:
3465 case SLJIT_ORDERED_EQUAL:
3466 case SLJIT_ORDERED_NOT_EQUAL:
3467 case SLJIT_F_LESS:
3468 case SLJIT_ORDERED_LESS:
3469 case SLJIT_ORDERED_GREATER:
3470 case SLJIT_F_LESS_EQUAL:
3471 case SLJIT_ORDERED_LESS_EQUAL:
3472 case SLJIT_ORDERED_GREATER_EQUAL:
3473 case SLJIT_ORDERED:
3474 return BEQ | RS1(OTHER_FLAG) | RS2(TMP_ZERO);
3475 break;
3476 case SLJIT_GREATER_EQUAL:
3477 case SLJIT_LESS_EQUAL:
3478 case SLJIT_SIG_GREATER_EQUAL:
3479 case SLJIT_SIG_LESS_EQUAL:
3480 case SLJIT_NOT_OVERFLOW:
3481 case SLJIT_NOT_CARRY:
3482 case SLJIT_ATOMIC_STORED:
3483 case SLJIT_F_NOT_EQUAL:
3484 case SLJIT_UNORDERED_OR_NOT_EQUAL:
3485 case SLJIT_UNORDERED_OR_EQUAL:
3486 case SLJIT_F_GREATER_EQUAL:
3487 case SLJIT_UNORDERED_OR_GREATER_EQUAL:
3488 case SLJIT_UNORDERED_OR_LESS_EQUAL:
3489 case SLJIT_F_GREATER:
3490 case SLJIT_UNORDERED_OR_GREATER:
3491 case SLJIT_UNORDERED_OR_LESS:
3492 case SLJIT_UNORDERED:
3493 return BNE | RS1(OTHER_FLAG) | RS2(TMP_ZERO);
3494 default:
3495 /* Not conditional branch. */
3496 return 0;
3497 }
3498 }
3499
3500 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
3501 {
3502 struct sljit_jump *jump;
3503 sljit_ins inst;
3504
3505 CHECK_ERROR_PTR();
3506 CHECK_PTR(check_sljit_emit_jump(compiler, type));
3507
3508 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3509 PTR_FAIL_IF(!jump);
3510 set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
3511 type &= 0xff;
3512
3513 inst = get_jump_instruction(type);
3514
3515 if (inst != 0) {
3516 PTR_FAIL_IF(push_inst(compiler, inst | BRANCH_LENGTH));
3517 jump->flags |= IS_COND;
3518 }
3519
3520 jump->addr = compiler->size;
3521
3522 if (type >= SLJIT_FAST_CALL)
3523 jump->flags |= IS_CALL;
3524
3525 PTR_FAIL_IF(push_inst16(compiler, 0));
3526
3527 /* Maximum number of instructions required for generating a constant. */
3528 compiler->size += JUMP_MAX_SIZE - 1;
3529 return jump;
3530 }
3531
3532 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
3533 sljit_s32 arg_types)
3534 {
3535 SLJIT_UNUSED_ARG(arg_types);
3536 CHECK_ERROR_PTR();
3537 CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
3538
3539 if (type & SLJIT_CALL_RETURN) {
3540 PTR_FAIL_IF(emit_stack_frame_release(compiler, 0));
3541 type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
3542 }
3543
3544 SLJIT_SKIP_CHECKS(compiler);
3545 return sljit_emit_jump(compiler, type);
3546 }
3547
3548 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_cmp(struct sljit_compiler *compiler, sljit_s32 type,
3549 sljit_s32 src1, sljit_sw src1w,
3550 sljit_s32 src2, sljit_sw src2w)
3551 {
3552 struct sljit_jump *jump;
3553 sljit_s32 flags;
3554 sljit_ins inst;
3555 sljit_s32 src2_tmp_reg = FAST_IS_REG(src1) ? TMP_REG1 : TMP_REG2;
3556
3557 CHECK_ERROR_PTR();
3558 CHECK_PTR(check_sljit_emit_cmp(compiler, type, src1, src1w, src2, src2w));
3559 ADJUST_LOCAL_OFFSET(src1, src1w);
3560 ADJUST_LOCAL_OFFSET(src2, src2w);
3561
3562 compiler->cache_arg = 0;
3563 compiler->cache_argw = 0;
3564 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
3565 flags = WORD_DATA | LOAD_DATA;
3566 #else /* !SLJIT_CONFIG_RISCV_32 */
3567 flags = ((type & SLJIT_32) ? INT_DATA : WORD_DATA) | LOAD_DATA;
3568 #endif /* SLJIT_CONFIG_RISCV_32 */
3569
3570 if (src1 & SLJIT_MEM) {
3571 PTR_FAIL_IF(emit_op_mem2(compiler, flags, TMP_REG1, src1, src1w, src2, src2w));
3572 src1 = TMP_REG1;
3573 }
3574
3575 if (src2 & SLJIT_MEM) {
3576 PTR_FAIL_IF(emit_op_mem2(compiler, flags | (src1 == TMP_REG1 ? MEM_USE_TMP2 : 0), src2_tmp_reg, src2, src2w, 0, 0));
3577 src2 = src2_tmp_reg;
3578 }
3579
3580 if (src1 == SLJIT_IMM) {
3581 if (src1w != 0) {
3582 PTR_FAIL_IF(load_immediate(compiler, TMP_REG1, src1w, TMP_REG3));
3583 src1 = TMP_REG1;
3584 }
3585 else
3586 src1 = TMP_ZERO;
3587 }
3588
3589 if (src2 == SLJIT_IMM) {
3590 if (src2w != 0) {
3591 PTR_FAIL_IF(load_immediate(compiler, src2_tmp_reg, src2w, TMP_REG3));
3592 src2 = src2_tmp_reg;
3593 }
3594 else
3595 src2 = TMP_ZERO;
3596 }
3597
3598 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3599 PTR_FAIL_IF(!jump);
3600 set_jump(jump, compiler, (sljit_u32)((type & SLJIT_REWRITABLE_JUMP) | IS_COND));
3601 type &= 0xff;
3602
3603 switch (type) {
3604 case SLJIT_EQUAL:
3605 if (RISCV_HAS_COMPRESSED(200)) {
3606 if (src1 == TMP_ZERO && C_IS_R3(src2)) {
3607 inst = C_BNEZ | C_RS1_R3(src2) | BRANCH16_LENGTH;
3608 jump->flags |= IS_COND16;
3609 break;
3610 }
3611
3612 if (src2 == TMP_ZERO && C_IS_R3(src1)) {
3613 inst = C_BNEZ | C_RS1_R3(src1) | BRANCH16_LENGTH;
3614 jump->flags |= IS_COND16;
3615 break;
3616 }
3617 }
3618
3619 inst = BNE | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
3620 break;
3621 case SLJIT_NOT_EQUAL:
3622 if (RISCV_HAS_COMPRESSED(200)) {
3623 if (src1 == TMP_ZERO && C_IS_R3(src2)) {
3624 inst = C_BEQZ | C_RS1_R3(src2) | BRANCH16_LENGTH;
3625 jump->flags |= IS_COND16;
3626 break;
3627 }
3628
3629 if (src2 == TMP_ZERO && C_IS_R3(src1)) {
3630 inst = C_BEQZ | C_RS1_R3(src1) | BRANCH16_LENGTH;
3631 jump->flags |= IS_COND16;
3632 break;
3633 }
3634 }
3635
3636 inst = BEQ | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
3637 break;
3638 case SLJIT_LESS:
3639 inst = BGEU | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
3640 break;
3641 case SLJIT_GREATER_EQUAL:
3642 inst = BLTU | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
3643 break;
3644 case SLJIT_GREATER:
3645 inst = BGEU | RS1(src2) | RS2(src1) | BRANCH_LENGTH;
3646 break;
3647 case SLJIT_LESS_EQUAL:
3648 inst = BLTU | RS1(src2) | RS2(src1) | BRANCH_LENGTH;
3649 break;
3650 case SLJIT_SIG_LESS:
3651 inst = BGE | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
3652 break;
3653 case SLJIT_SIG_GREATER_EQUAL:
3654 inst = BLT | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
3655 break;
3656 case SLJIT_SIG_GREATER:
3657 inst = BGE | RS1(src2) | RS2(src1) | BRANCH_LENGTH;
3658 break;
3659 default: /* SLJIT_SIG_LESS_EQUAL */
3660 inst = BLT | RS1(src2) | RS2(src1) | BRANCH_LENGTH;
3661 break;
3662 }
3663
3664 SLJIT_COMPILE_ASSERT((C_BEQZ & 0x2) == 0 && (C_BNEZ & 0x2) == 0, branch16_bit_error);
3665 if (RISCV_COMPRESSED_CHECK((inst & 0x2) == 0))
3666 PTR_FAIL_IF(push_inst16(compiler, (sljit_u16)inst));
3667 else
3668 PTR_FAIL_IF(push_inst(compiler, inst));
3669
3670 jump->addr = compiler->size;
3671 PTR_FAIL_IF(push_inst16(compiler, 0));
3672
3673 /* Maximum number of instructions required for generating a constant. */
3674 compiler->size += JUMP_MAX_SIZE - 1;
3675 return jump;
3676 }
3677
3678 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
3679 {
3680 struct sljit_jump *jump;
3681
3682 CHECK_ERROR();
3683 CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
3684
3685 if (src != SLJIT_IMM) {
3686 if (src & SLJIT_MEM) {
3687 ADJUST_LOCAL_OFFSET(src, srcw);
3688 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src, srcw));
3689 src = TMP_REG1;
3690 }
3691
3692 if (RISCV_HAS_COMPRESSED(200))
3693 return push_inst16(compiler, ((type >= SLJIT_FAST_CALL) ? C_JALR : C_JR) | C_RD(src));
3694
3695 return push_inst(compiler, JALR | RD((type >= SLJIT_FAST_CALL) ? RETURN_ADDR_REG : TMP_ZERO) | RS1(src) | IMM_I(0));
3696 }
3697
3698 /* These jumps are converted to jump/call instructions when possible. */
3699 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3700 FAIL_IF(!jump);
3701 set_jump(jump, compiler, JUMP_ADDR | ((type >= SLJIT_FAST_CALL) ? IS_CALL : 0));
3702 jump->u.target = (sljit_uw)srcw;
3703
3704 jump->addr = compiler->size;
3705 FAIL_IF(push_inst16(compiler, 0));
3706
3707 /* Maximum number of instructions required for generating a constant. */
3708 compiler->size += JUMP_MAX_SIZE - 1;
3709 return SLJIT_SUCCESS;
3710 }
3711
3712 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
3713 sljit_s32 arg_types,
3714 sljit_s32 src, sljit_sw srcw)
3715 {
3716 SLJIT_UNUSED_ARG(arg_types);
3717 CHECK_ERROR();
3718 CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
3719
3720 if (src & SLJIT_MEM) {
3721 ADJUST_LOCAL_OFFSET(src, srcw);
3722 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src, srcw));
3723 src = TMP_REG1;
3724 }
3725
3726 if (type & SLJIT_CALL_RETURN) {
3727 if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
3728 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(src) | IMM_I(0)));
3729 src = TMP_REG1;
3730 }
3731
3732 FAIL_IF(emit_stack_frame_release(compiler, 0));
3733 type = SLJIT_JUMP;
3734 }
3735
3736 SLJIT_SKIP_CHECKS(compiler);
3737 return sljit_emit_ijump(compiler, type, src, srcw);
3738 }
3739
3740 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
3741 sljit_s32 dst, sljit_sw dstw,
3742 sljit_s32 type)
3743 {
3744 sljit_s32 src_r, dst_r, invert;
3745 sljit_s32 saved_op = op;
3746 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
3747 sljit_s32 mem_type = WORD_DATA;
3748 #else /* !SLJIT_CONFIG_RISCV_32 */
3749 sljit_s32 mem_type = ((op & SLJIT_32) || op == SLJIT_MOV32) ? (INT_DATA | SIGNED_DATA) : WORD_DATA;
3750 #endif /* SLJIT_CONFIG_RISCV_32 */
3751
3752 CHECK_ERROR();
3753 CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
3754 ADJUST_LOCAL_OFFSET(dst, dstw);
3755
3756 op = GET_OPCODE(op);
3757 dst_r = (op < SLJIT_ADD && FAST_IS_REG(dst)) ? dst : TMP_REG2;
3758
3759 compiler->cache_arg = 0;
3760 compiler->cache_argw = 0;
3761
3762 if (op >= SLJIT_ADD && (dst & SLJIT_MEM))
3763 FAIL_IF(emit_op_mem2(compiler, mem_type | LOAD_DATA, TMP_REG1, dst, dstw, dst, dstw));
3764
3765 if (type < SLJIT_F_EQUAL) {
3766 src_r = OTHER_FLAG;
3767 invert = type & 0x1;
3768
3769 switch (type) {
3770 case SLJIT_EQUAL:
3771 case SLJIT_NOT_EQUAL:
3772 FAIL_IF(push_inst(compiler, SLTUI | RD(dst_r) | RS1(EQUAL_FLAG) | IMM_I(1)));
3773 src_r = dst_r;
3774 break;
3775 case SLJIT_OVERFLOW:
3776 case SLJIT_NOT_OVERFLOW:
3777 if (compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)) {
3778 src_r = OTHER_FLAG;
3779 break;
3780 }
3781 FAIL_IF(push_inst(compiler, SLTUI | RD(dst_r) | RS1(OTHER_FLAG) | IMM_I(1)));
3782 src_r = dst_r;
3783 invert ^= 0x1;
3784 break;
3785 case SLJIT_ATOMIC_STORED:
3786 case SLJIT_ATOMIC_NOT_STORED:
3787 invert ^= 0x1;
3788 break;
3789 }
3790 } else {
3791 invert = 0;
3792 src_r = OTHER_FLAG;
3793
3794 switch (type) {
3795 case SLJIT_F_NOT_EQUAL:
3796 case SLJIT_UNORDERED_OR_NOT_EQUAL:
3797 case SLJIT_UNORDERED_OR_EQUAL: /* Not supported. */
3798 case SLJIT_F_GREATER_EQUAL:
3799 case SLJIT_UNORDERED_OR_GREATER_EQUAL:
3800 case SLJIT_UNORDERED_OR_LESS_EQUAL:
3801 case SLJIT_F_GREATER:
3802 case SLJIT_UNORDERED_OR_GREATER:
3803 case SLJIT_UNORDERED_OR_LESS:
3804 case SLJIT_UNORDERED:
3805 invert = 1;
3806 break;
3807 }
3808 }
3809
3810 if (invert) {
3811 FAIL_IF(push_inst(compiler, XORI | RD(dst_r) | RS1(src_r) | IMM_I(1)));
3812 src_r = dst_r;
3813 }
3814
3815 if (op < SLJIT_ADD) {
3816 if (dst & SLJIT_MEM)
3817 return emit_op_mem(compiler, mem_type, src_r, dst, dstw);
3818
3819 if (src_r != dst_r)
3820 return push_inst(compiler, ADDI | RD(dst_r) | RS1(src_r) | IMM_I(0));
3821 return SLJIT_SUCCESS;
3822 }
3823
3824 mem_type |= CUMULATIVE_OP | IMM_OP | ALT_KEEP_CACHE;
3825
3826 if (dst & SLJIT_MEM)
3827 return emit_op(compiler, saved_op, mem_type, dst, dstw, TMP_REG1, 0, src_r, 0);
3828 return emit_op(compiler, saved_op, mem_type, dst, dstw, dst, dstw, src_r, 0);
3829 }
3830
3831 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
3832 sljit_s32 dst_reg,
3833 sljit_s32 src1, sljit_sw src1w,
3834 sljit_s32 src2_reg)
3835 {
3836 sljit_u16 *ptr;
3837 sljit_uw size;
3838 sljit_ins ins;
3839 sljit_s32 cond_is_1;
3840 sljit_s32 is_compare = (type & SLJIT_COMPARE_SELECT);
3841 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
3842 sljit_ins word = (sljit_ins)(type & SLJIT_32) >> 5;
3843 sljit_s32 inp_flags = ((type & SLJIT_32) ? INT_DATA : WORD_DATA) | LOAD_DATA;
3844 #else /* !SLJIT_CONFIG_RISCV_64 */
3845 sljit_s32 inp_flags = WORD_DATA | LOAD_DATA;
3846 #endif /* SLJIT_CONFIG_RISCV_64 */
3847
3848 SLJIT_ASSERT(WORD == 0 || WORD == 0x8);
3849
3850 CHECK_ERROR();
3851 CHECK(check_sljit_emit_select(compiler, type, dst_reg, src1, src1w, src2_reg));
3852
3853 ADJUST_LOCAL_OFFSET(src1, src1w);
3854
3855 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
3856 if (src1 == SLJIT_IMM && word)
3857 src1w = (sljit_s32)src1w;
3858 #endif /* SLJIT_CONFIG_RISCV_64 */
3859
3860 type &= ~(SLJIT_32 | SLJIT_COMPARE_SELECT);
3861
3862 if (is_compare || RISCV_HAS_ICOND(100)) {
3863 if (src1 & SLJIT_MEM) {
3864 FAIL_IF(emit_op_mem(compiler, inp_flags, TMP_REG1, src1, src1w));
3865 src1 = TMP_REG1;
3866 src1w = 0;
3867 } else if (src1 == SLJIT_IMM) {
3868 if (src1w == 0) {
3869 src1 = TMP_ZERO;
3870 } else {
3871 FAIL_IF(load_immediate(compiler, TMP_REG1, src1w, TMP_REG3));
3872 src1 = TMP_REG1;
3873 src1w = 0;
3874 }
3875 }
3876
3877 if (RISCV_HAS_BITMANIP_B(93) && is_compare) {
3878 switch (type) {
3879 case SLJIT_LESS:
3880 case SLJIT_LESS_EQUAL:
3881 return push_inst(compiler, MINU | RD(dst_reg) | RS1(src1) | RS2(src2_reg));
3882 case SLJIT_GREATER:
3883 case SLJIT_GREATER_EQUAL:
3884 return push_inst(compiler, MAXU | RD(dst_reg) | RS1(src1) | RS2(src2_reg));
3885 case SLJIT_SIG_LESS:
3886 case SLJIT_SIG_LESS_EQUAL:
3887 return push_inst(compiler, MIN | RD(dst_reg) | RS1(src1) | RS2(src2_reg));
3888 default:
3889 return push_inst(compiler, MAX | RD(dst_reg) | RS1(src1) | RS2(src2_reg));
3890 }
3891 }
3892
3893 if (RISCV_HAS_ICOND(100)) {
3894 if (is_compare) {
3895 cond_is_1 = 0;
3896
3897 switch (type) {
3898 case SLJIT_LESS:
3899 case SLJIT_LESS_EQUAL:
3900 cond_is_1 = 1;
3901 SLJIT_FALLTHROUGH
3902 case SLJIT_GREATER:
3903 case SLJIT_GREATER_EQUAL:
3904 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(src1) | RS2(src2_reg)));
3905 break;
3906 case SLJIT_SIG_LESS:
3907 case SLJIT_SIG_LESS_EQUAL:
3908 cond_is_1 = 1;
3909 SLJIT_FALLTHROUGH
3910 default:
3911 FAIL_IF(push_inst(compiler, SLT | RD(OTHER_FLAG) | RS1(src1) | RS2(src2_reg)));
3912 break;
3913 }
3914
3915 type = OTHER_FLAG;
3916 } else {
3917 /* BEQ instruction (type is inverted). */
3918 cond_is_1 = (get_jump_instruction(type) & F3(0x1)) == 0;
3919 type = (type == SLJIT_EQUAL || type == SLJIT_NOT_EQUAL) ? EQUAL_FLAG : OTHER_FLAG;
3920 }
3921
3922 if (src1 == TMP_ZERO)
3923 return push_inst(compiler, (cond_is_1 ? CZERO_NEZ : CZERO_EQZ) | RD(dst_reg) | RS1(src2_reg) | RS2(type));
3924
3925 FAIL_IF(push_inst(compiler, (cond_is_1 ? CZERO_EQZ : CZERO_NEZ) | RD(TMP_REG1) | RS1(src1) | RS2(type)));
3926 FAIL_IF(push_inst(compiler, (cond_is_1 ? CZERO_NEZ : CZERO_EQZ) | RD(dst_reg) | RS1(src2_reg) | RS2(type)));
3927 return push_inst(compiler, OR | RD(dst_reg) | RS1(dst_reg) | RS2(TMP_REG1));
3928 }
3929 }
3930
3931 if (dst_reg != src2_reg) {
3932 if (dst_reg == src1) {
3933 src1 = src2_reg;
3934 src1w = 0;
3935 src2_reg = dst_reg;
3936 if (!is_compare)
3937 type ^= 0x1;
3938 } else {
3939 if (ADDRESSING_DEPENDS_ON(src1, dst_reg)) {
3940 SLJIT_ASSERT(!(type & SLJIT_COMPARE_SELECT));
3941 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(dst_reg) | IMM_I(0)));
3942
3943 if ((src1 & REG_MASK) == dst_reg)
3944 src1 = (src1 & ~REG_MASK) | TMP_REG1;
3945
3946 if (OFFS_REG(src1) == dst_reg)
3947 src1 = (src1 & ~OFFS_REG_MASK) | TO_OFFS_REG(TMP_REG1);
3948 }
3949
3950 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst_reg) | RS1(src2_reg) | IMM_I(0)));
3951 }
3952 }
3953
3954 if (src1 == SLJIT_IMM && (src1w <= 0 && src1w >= -1)) {
3955 src1 = OTHER_FLAG;
3956
3957 if (type == SLJIT_EQUAL || type == SLJIT_NOT_EQUAL) {
3958 FAIL_IF(push_inst(compiler, SLTUI | RD(TMP_REG1) | RS1(EQUAL_FLAG) | IMM_I(1)));
3959 src1 = TMP_REG1;
3960 cond_is_1 = (type == SLJIT_EQUAL);
3961 } else {
3962 /* BEQ instruction (type is inverted). */
3963 cond_is_1 = (get_jump_instruction(type) & F3(0x1)) == 0;
3964 }
3965
3966 if (src1w == 0) {
3967 if (cond_is_1)
3968 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(src1) | IMM_I(-1)));
3969 else
3970 FAIL_IF(push_inst(compiler, SUB | RD(TMP_REG1) | RS1(TMP_ZERO) | RS2(src1)));
3971
3972 return push_inst(compiler, AND | RD(dst_reg) | RS1(dst_reg) | RS2(TMP_REG1));
3973 }
3974
3975 if (cond_is_1)
3976 FAIL_IF(push_inst(compiler, SUB | RD(TMP_REG1) | RS1(TMP_ZERO) | RS2(src1)));
3977 else
3978 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(src1) | IMM_I(-1)));
3979
3980 return push_inst(compiler, OR | RD(dst_reg) | RS1(dst_reg) | RS2(TMP_REG1));
3981 }
3982
3983 size = compiler->size;
3984
3985 ptr = (sljit_u16 *)ensure_buf(compiler, sizeof(sljit_ins));
3986 FAIL_IF(!ptr);
3987 compiler->size += 2;
3988
3989 if (src1 & SLJIT_MEM) {
3990 FAIL_IF(emit_op_mem(compiler, inp_flags, dst_reg, src1, src1w));
3991 } else if (src1 == SLJIT_IMM) {
3992 FAIL_IF(load_immediate(compiler, dst_reg, src1w, TMP_REG1));
3993 } else
3994 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst_reg) | RS1(src1) | IMM_I(0)));
3995
3996 size = compiler->size - size;
3997
3998 if (is_compare) {
3999 switch (type) {
4000 case SLJIT_LESS:
4001 case SLJIT_LESS_EQUAL:
4002 ins = BGEU;
4003 break;
4004 case SLJIT_GREATER:
4005 case SLJIT_GREATER_EQUAL:
4006 ins = BLTU;
4007 break;
4008 case SLJIT_SIG_LESS:
4009 case SLJIT_SIG_LESS_EQUAL:
4010 ins = BGE;
4011 break;
4012 default:
4013 ins = BLT;
4014 break;
4015 }
4016
4017 ins |= RS1(src1) | RS2(src2_reg);
4018 } else {
4019 ins = get_jump_instruction(type);
4020 }
4021
4022 ins |= (sljit_ins)((size & 0xf) << 8) | (sljit_ins)((size >> 4) << 25);
4023 ptr[0] = (sljit_u16)ins;
4024 ptr[1] = (sljit_u16)(ins >> 16);
4025 return SLJIT_SUCCESS;
4026 }
4027
4028 #undef WORD
4029
4030 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
4031 sljit_s32 dst_freg,
4032 sljit_s32 src1, sljit_sw src1w,
4033 sljit_s32 src2_freg)
4034 {
4035 sljit_u16 *ptr;
4036 sljit_ins ins;
4037 sljit_uw size;
4038
4039 CHECK_ERROR();
4040 CHECK(check_sljit_emit_fselect(compiler, type, dst_freg, src1, src1w, src2_freg));
4041
4042 ADJUST_LOCAL_OFFSET(src1, src1w);
4043
4044 if (dst_freg != src2_freg) {
4045 if (dst_freg == src1) {
4046 src1 = src2_freg;
4047 src1w = 0;
4048 type ^= 0x1;
4049 } else
4050 FAIL_IF(push_inst(compiler, FSGNJ_S | FMT(type) | FRD(dst_freg) | FRS1(src2_freg) | FRS2(src2_freg)));
4051 }
4052
4053 size = compiler->size;
4054
4055 ptr = (sljit_u16 *)ensure_buf(compiler, sizeof(sljit_ins));
4056 FAIL_IF(!ptr);
4057 compiler->size += 2;
4058
4059 if (src1 & SLJIT_MEM)
4060 FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(type) | LOAD_DATA, dst_freg, src1, src1w));
4061 else
4062 FAIL_IF(push_inst(compiler, FSGNJ_S | FMT(type) | FRD(dst_freg) | FRS1(src1) | FRS2(src1)));
4063
4064 size = compiler->size - size;
4065 ins = get_jump_instruction(type & ~SLJIT_32) | (sljit_ins)((size & 0xf) << 8) | (sljit_ins)((size >> 4) << 25);
4066 ptr[0] = (sljit_u16)ins;
4067 ptr[1] = (sljit_u16)(ins >> 16);
4068 return SLJIT_SUCCESS;
4069 }
4070
4071 #undef FLOAT_DATA
4072 #undef FMT
4073
4074 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
4075 sljit_s32 reg,
4076 sljit_s32 mem, sljit_sw memw)
4077 {
4078 sljit_s32 flags;
4079
4080 CHECK_ERROR();
4081 CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw));
4082
4083 if (!(reg & REG_PAIR_MASK))
4084 return sljit_emit_mem_unaligned(compiler, type, reg, mem, memw);
4085
4086 if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
4087 memw &= 0x3;
4088
4089 if (SLJIT_UNLIKELY(memw != 0)) {
4090 FAIL_IF(push_inst(compiler, SLLI | RD(TMP_REG1) | RS1(OFFS_REG(mem)) | IMM_I(memw)));
4091 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(mem & REG_MASK)));
4092 } else
4093 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG1) | RS1(mem & REG_MASK) | RS2(OFFS_REG(mem))));
4094
4095 mem = TMP_REG1;
4096 memw = 0;
4097 } else if (memw > SIMM_MAX - SSIZE_OF(sw) || memw < SIMM_MIN) {
4098 if (((memw + 0x800) & 0xfff) <= 0xfff - SSIZE_OF(sw)) {
4099 FAIL_IF(load_immediate(compiler, TMP_REG1, TO_ARGW_HI(memw), TMP_REG3));
4100 memw &= 0xfff;
4101 } else {
4102 FAIL_IF(load_immediate(compiler, TMP_REG1, memw, TMP_REG3));
4103 memw = 0;
4104 }
4105
4106 if (mem & REG_MASK)
4107 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(mem & REG_MASK)));
4108
4109 mem = TMP_REG1;
4110 } else {
4111 mem &= REG_MASK;
4112 memw &= 0xfff;
4113 }
4114
4115 SLJIT_ASSERT((memw >= 0 && memw <= SIMM_MAX - SSIZE_OF(sw)) || (memw > SIMM_MAX && memw <= 0xfff));
4116
4117 if (!(type & SLJIT_MEM_STORE) && mem == REG_PAIR_FIRST(reg)) {
4118 FAIL_IF(push_mem_inst(compiler, WORD_DATA | LOAD_DATA, REG_PAIR_SECOND(reg), mem, (memw + SSIZE_OF(sw)) & 0xfff));
4119 return push_mem_inst(compiler, WORD_DATA | LOAD_DATA, REG_PAIR_FIRST(reg), mem, memw);
4120 }
4121
4122 flags = WORD_DATA | (!(type & SLJIT_MEM_STORE) ? LOAD_DATA : 0);
4123
4124 FAIL_IF(push_mem_inst(compiler, flags, REG_PAIR_FIRST(reg), mem, memw));
4125 return push_mem_inst(compiler, flags, REG_PAIR_SECOND(reg), mem, (memw + SSIZE_OF(sw)) & 0xfff);
4126 }
4127
4128 #undef TO_ARGW_HI
4129
4130 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
4131 sljit_s32 dst_reg,
4132 sljit_s32 mem_reg)
4133 {
4134 sljit_ins ins;
4135
4136 CHECK_ERROR();
4137 CHECK(check_sljit_emit_atomic_load(compiler, op, dst_reg, mem_reg));
4138
4139 if (!RISCV_HAS_ATOMIC(200) || (op & SLJIT_ATOMIC_USE_CAS))
4140 return SLJIT_ERR_UNSUPPORTED;
4141
4142 switch (GET_OPCODE(op)) {
4143 case SLJIT_MOV:
4144 case SLJIT_MOV_P:
4145 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
4146 ins = LR | (3 << 12);
4147 break;
4148 #endif /* SLJIT_CONFIG_RISCV_64 */
4149 case SLJIT_MOV_S32:
4150 case SLJIT_MOV32:
4151 ins = LR | (2 << 12);
4152 break;
4153
4154 default:
4155 return SLJIT_ERR_UNSUPPORTED;
4156 }
4157
4158 if (op & SLJIT_ATOMIC_TEST)
4159 return SLJIT_SUCCESS;
4160
4161 return push_inst(compiler, ins | RD(dst_reg) | RS1(mem_reg));
4162 }
4163
4164 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
4165 sljit_s32 src_reg,
4166 sljit_s32 mem_reg,
4167 sljit_s32 temp_reg)
4168 {
4169 sljit_ins ins;
4170
4171 /* temp_reg == mem_reg is undefined so use another temp register */
4172 SLJIT_UNUSED_ARG(temp_reg);
4173
4174 CHECK_ERROR();
4175 CHECK(check_sljit_emit_atomic_store(compiler, op, src_reg, mem_reg, temp_reg));
4176
4177 if (!RISCV_HAS_ATOMIC(200) || (op & SLJIT_ATOMIC_USE_CAS))
4178 return SLJIT_ERR_UNSUPPORTED;
4179
4180 switch (GET_OPCODE(op)) {
4181 case SLJIT_MOV:
4182 case SLJIT_MOV_P:
4183 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
4184 ins = SC | (3 << 12);
4185 break;
4186 #endif /* SLJIT_CONFIG_RISCV_64 */
4187 case SLJIT_MOV_S32:
4188 case SLJIT_MOV32:
4189 ins = SC | (2 << 12);
4190 break;
4191
4192 default:
4193 return SLJIT_ERR_UNSUPPORTED;
4194 }
4195
4196 if (op & SLJIT_ATOMIC_TEST)
4197 return SLJIT_SUCCESS;
4198
4199 return push_inst(compiler, ins | RD(OTHER_FLAG) | RS1(mem_reg) | RS2(src_reg));
4200 }
4201
4202 static sljit_s32 sljit_max_vector_length = 0;
4203
4204 static void init_compiler(void)
4205 {
4206 sljit_sw vector_length;
4207 sljit_s32 vector_length_log2;
4208
4209 if (!RISCV_HAS_VECTOR(100))
4210 return;
4211
4212 __asm__ __volatile__ ("csrr %0, 0xc22" : "=r" (vector_length));
4213
4214 /* Probably something is wrong. */
4215 if (vector_length < (1 << 3))
4216 return;
4217
4218 vector_length_log2 = 3;
4219 if (vector_length_log2 < 6 && vector_length > (1 << vector_length_log2))
4220 vector_length_log2++;
4221 sljit_max_vector_length = vector_length_log2;
4222 }
4223
4224 /*
4225 SEW = Selected element width
4226 LMUL = Vector register group multiplier
4227
4228 VLMUL values (in binary):
4229 100 : reserved
4230 101 : 1/8
4231 110 : 1/4
4232 111 : 1/2
4233 000 : 1
4234 001 : 2
4235 010 : 4
4236 011 : 8
4237 */
4238
4239 static SLJIT_INLINE sljit_s32 sljit_emit_vsetivli(struct sljit_compiler *compiler, sljit_s32 type, sljit_ins vlmul)
4240 {
4241 sljit_ins elem_size = (sljit_ins)SLJIT_SIMD_GET_ELEM_SIZE(type);
4242 sljit_ins avl = (sljit_ins)1 << (SLJIT_SIMD_GET_REG_SIZE(type) - elem_size);
4243
4244 if (avl < 31)
4245 return push_inst(compiler, VSETIVLI | RD(TMP_REG1) | (elem_size << 23) | (vlmul << 20) | (avl << 15));
4246
4247 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(TMP_ZERO) | IMM_I(avl)));
4248 return push_inst(compiler, VSETVLI | RD(TMP_REG1) | (elem_size << 23) | (vlmul << 20) | RS1(TMP_REG1));
4249 }
4250
4251 static SLJIT_INLINE sljit_s32 sljit_emit_vsetivli_size(struct sljit_compiler *compiler, sljit_s32 reg_size, sljit_s32 elem_size)
4252 {
4253 sljit_ins avl = (sljit_ins)1 << (reg_size - elem_size);
4254
4255 if (avl < 31)
4256 return push_inst(compiler, VSETIVLI | RD(TMP_REG1) | ((sljit_ins)elem_size << 23) | (avl << 15));
4257
4258 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(TMP_ZERO) | IMM_I(avl)));
4259 return push_inst(compiler, VSETVLI | RD(TMP_REG1) | ((sljit_ins)elem_size << 23) | RS1(TMP_REG1));
4260 }
4261
4262 static sljit_s32 sljit_emit_vmem(struct sljit_compiler *compiler, sljit_ins ins, sljit_s32 elem_size, sljit_s32 mem, sljit_sw memw)
4263 {
4264 sljit_s32 base = mem & REG_MASK;
4265
4266 if (elem_size > 0)
4267 ins |= (1 << 14) | ((sljit_ins)elem_size << 12);
4268
4269 if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
4270 memw &= 0x3;
4271
4272 if (SLJIT_UNLIKELY(memw)) {
4273 FAIL_IF(push_inst(compiler, SLLI | RD(TMP_REG1) | RS1(OFFS_REG(mem)) | IMM_I(memw)));
4274 }
4275
4276 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG1) | RS1(base) | RS2(!memw ? OFFS_REG(mem) : TMP_REG1)));
4277 return push_inst(compiler, ins | RS1(TMP_REG1));
4278 }
4279
4280 if (memw == 0)
4281 return push_inst(compiler, ins | RS1(base));
4282
4283 if (memw <= SIMM_MAX && memw >= SIMM_MIN) {
4284 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(base) | IMM_I(memw)));
4285 return push_inst(compiler, ins | RS1(TMP_REG1));
4286 }
4287
4288 FAIL_IF(load_immediate(compiler, TMP_REG1, memw, TMP_REG3));
4289
4290 if (base != 0)
4291 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(base)));
4292
4293 return push_inst(compiler, ins | RS1(TMP_REG1));
4294 }
4295
4296 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
4297 sljit_s32 vreg,
4298 sljit_s32 srcdst, sljit_sw srcdstw)
4299 {
4300 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4301 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4302 sljit_ins ins;
4303
4304 CHECK_ERROR();
4305 CHECK(check_sljit_emit_simd_mov(compiler, type, vreg, srcdst, srcdstw));
4306
4307 ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
4308
4309 if (reg_size < 3 || reg_size > sljit_max_vector_length)
4310 return SLJIT_ERR_UNSUPPORTED;
4311
4312 if (type & SLJIT_SIMD_TEST)
4313 return SLJIT_SUCCESS;
4314
4315 if (elem_size > 3)
4316 elem_size = 3;
4317
4318 FAIL_IF(sljit_emit_vsetivli_size(compiler, reg_size, elem_size));
4319
4320 if (srcdst & SLJIT_MEM) {
4321 ins = (type & SLJIT_SIMD_STORE) ? VS : VL;
4322 return sljit_emit_vmem(compiler, ins | VRD(vreg), elem_size, srcdst, srcdstw);
4323 }
4324
4325 if (type & SLJIT_SIMD_STORE)
4326 ins = VRD(srcdst) | VRS1(vreg);
4327 else
4328 ins = VRD(vreg) | VRS1(srcdst);
4329
4330 return push_inst(compiler, VMV_VV | ins);
4331 }
4332
4333 static sljit_s32 sljit_simd_get_mem_flags(sljit_s32 elem_size)
4334 {
4335 switch (elem_size) {
4336 case 0:
4337 return BYTE_DATA;
4338 case 1:
4339 return HALF_DATA;
4340 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
4341 case 2:
4342 return INT_DATA;
4343 #endif /* SLJIT_CONFIG_RISCV_64 */
4344 default:
4345 return WORD_DATA;
4346 }
4347 }
4348
4349 static sljit_sw sljit_simd_get_imm(sljit_s32 elem_size, sljit_sw imm)
4350 {
4351 switch (elem_size) {
4352 case 0:
4353 return (sljit_s8)imm;
4354 case 1:
4355 return (sljit_s16)imm;
4356 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
4357 case 2:
4358 return (sljit_s32)imm;
4359 #endif /* SLJIT_CONFIG_RISCV_64 */
4360 default:
4361 return imm;
4362 }
4363 }
4364
4365 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
4366 sljit_s32 vreg,
4367 sljit_s32 src, sljit_sw srcw)
4368 {
4369 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4370 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4371 sljit_s32 flags;
4372
4373 CHECK_ERROR();
4374 CHECK(check_sljit_emit_simd_replicate(compiler, type, vreg, src, srcw));
4375
4376 ADJUST_LOCAL_OFFSET(src, srcw);
4377
4378 if (reg_size < 3 || reg_size > sljit_max_vector_length)
4379 return SLJIT_ERR_UNSUPPORTED;
4380
4381 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
4382 if ((type & SLJIT_SIMD_FLOAT) ? (elem_size < 2 || elem_size > 3) : elem_size > 2)
4383 return SLJIT_ERR_UNSUPPORTED;
4384 #else /* !SLJIT_CONFIG_RISCV_32 */
4385 if (((type & SLJIT_SIMD_FLOAT) && elem_size < 2) || elem_size > 3)
4386 return SLJIT_ERR_UNSUPPORTED;
4387 #endif /* SLJIT_CONFIG_RISCV_32 */
4388
4389 if (type & SLJIT_SIMD_TEST)
4390 return SLJIT_SUCCESS;
4391
4392 FAIL_IF(sljit_emit_vsetivli(compiler, type, 0));
4393
4394 if (type & SLJIT_SIMD_FLOAT) {
4395 if (src == SLJIT_IMM)
4396 return push_inst(compiler, VMV_VI | VRD(vreg) | ((sljit_ins)(srcw & 0x1f) << 15));
4397
4398 if (src & SLJIT_MEM) {
4399 flags = (elem_size == 2) ? SINGLE_DATA : DOUBLE_DATA;
4400 FAIL_IF(emit_op_mem(compiler, flags | LOAD_DATA, TMP_FREG1, src, srcw));
4401 src = TMP_FREG1;
4402 }
4403
4404 return push_inst(compiler, VFMV_VF | VRD(vreg) | FRS1(src));
4405 }
4406
4407 if (src == SLJIT_IMM) {
4408 srcw = sljit_simd_get_imm(elem_size, srcw);
4409
4410 if (srcw >= -0x10 && srcw <= 0xf)
4411 return push_inst(compiler, VMV_VI | VRD(vreg) | ((sljit_ins)(srcw & 0x1f) << 15));
4412
4413 FAIL_IF(load_immediate(compiler, TMP_REG1, srcw, TMP_REG3));
4414 src = TMP_REG1;
4415 } else if (src & SLJIT_MEM) {
4416 FAIL_IF(emit_op_mem(compiler, sljit_simd_get_mem_flags(elem_size) | LOAD_DATA, TMP_REG1, src, srcw));
4417 src = TMP_REG1;
4418 }
4419
4420 return push_inst(compiler, VMV_VX | VRD(vreg) | RS1(src));
4421 }
4422
4423 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
4424 sljit_s32 vreg, sljit_s32 lane_index,
4425 sljit_s32 srcdst, sljit_sw srcdstw)
4426 {
4427 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4428 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4429 sljit_s32 flags;
4430
4431 CHECK_ERROR();
4432 CHECK(check_sljit_emit_simd_lane_mov(compiler, type, vreg, lane_index, srcdst, srcdstw));
4433
4434 ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
4435
4436 if (reg_size < 3 || reg_size > sljit_max_vector_length)
4437 return SLJIT_ERR_UNSUPPORTED;
4438
4439 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
4440 if ((type & SLJIT_SIMD_FLOAT) ? (elem_size < 2 || elem_size > 3) : elem_size > 2)
4441 return SLJIT_ERR_UNSUPPORTED;
4442 #else /* !SLJIT_CONFIG_RISCV_32 */
4443 if (((type & SLJIT_SIMD_FLOAT) && elem_size < 2) || elem_size > 3)
4444 return SLJIT_ERR_UNSUPPORTED;
4445 #endif /* SLJIT_CONFIG_RISCV_32 */
4446
4447 if (type & SLJIT_SIMD_TEST)
4448 return SLJIT_SUCCESS;
4449
4450 if (type & SLJIT_SIMD_STORE) {
4451 FAIL_IF(push_inst(compiler, VSETIVLI | RD(TMP_REG1) | ((sljit_ins)elem_size << 23) | (1 << 15)));
4452
4453 if (lane_index > 0) {
4454 FAIL_IF(push_inst(compiler, VSLIDEDOWN_VI | VRD(TMP_VREG1) | ((sljit_ins)lane_index << 15) | VRS2(vreg)));
4455 vreg = TMP_VREG1;
4456 }
4457
4458 if (srcdst & SLJIT_MEM)
4459 return sljit_emit_vmem(compiler, VS | VRD(vreg), elem_size, srcdst, srcdstw);
4460
4461 if (type & SLJIT_SIMD_FLOAT)
4462 return push_inst(compiler, VFMV_FS | FRD(srcdst) | VRS2(vreg));
4463
4464 FAIL_IF(push_inst(compiler, VMV_XS | RD(srcdst) | VRS2(vreg)));
4465
4466 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
4467 if ((type & SLJIT_SIMD_LANE_SIGNED) || elem_size >= 2)
4468 return SLJIT_SUCCESS;
4469 #else /* !SLJIT_CONFIG_RISCV_32 */
4470 if ((type & SLJIT_SIMD_LANE_SIGNED) || elem_size >= 3 || (elem_size == 2 && (type & SLJIT_32)))
4471 return SLJIT_SUCCESS;
4472 #endif /* SLJIT_CONFIG_RISCV_32 */
4473
4474 if (elem_size == 0)
4475 return push_inst(compiler, ANDI | RD(srcdst) | RS1(srcdst) | IMM_I(0xff));
4476
4477 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
4478 flags = 16;
4479 #else /* !SLJIT_CONFIG_RISCV_32 */
4480 flags = (elem_size == 1) ? 48 : 32;
4481 #endif /* SLJIT_CONFIG_RISCV_32 */
4482
4483 FAIL_IF(push_inst(compiler, SLLI | RD(srcdst) | RS1(srcdst) | IMM_I(flags)));
4484 return push_inst(compiler, SRLI | RD(srcdst) | RS1(srcdst) | IMM_I(flags));
4485 }
4486
4487 if (type & SLJIT_SIMD_LANE_ZERO) {
4488 FAIL_IF(sljit_emit_vsetivli(compiler, type, 0));
4489 FAIL_IF(push_inst(compiler, VMV_VI | VRD(vreg)));
4490 }
4491
4492 if (srcdst & SLJIT_MEM) {
4493 FAIL_IF(push_inst(compiler, VSETIVLI | RD(TMP_REG1) | ((sljit_ins)elem_size << 23) | (1 << 15)));
4494 FAIL_IF(sljit_emit_vmem(compiler, VL | VRD(lane_index > 0 ? TMP_VREG1 : vreg), elem_size, srcdst, srcdstw));
4495
4496 if (lane_index == 0)
4497 return SLJIT_SUCCESS;
4498
4499 FAIL_IF(push_inst(compiler, VSETIVLI | RD(TMP_REG1) | ((sljit_ins)elem_size << 23) | ((sljit_ins)(lane_index + 1) << 15)));
4500 return push_inst(compiler, VSLIDEUP_VI | VRD(vreg) | ((sljit_ins)lane_index << 15) | VRS2(TMP_VREG1));
4501 }
4502
4503 if (!(type & SLJIT_SIMD_LANE_ZERO) || lane_index > 0)
4504 FAIL_IF(push_inst(compiler, VSETIVLI | RD(TMP_REG1) | ((sljit_ins)elem_size << 23) | ((sljit_ins)(lane_index + 1) << 15)));
4505
4506 if (type & SLJIT_SIMD_FLOAT) {
4507 FAIL_IF(push_inst(compiler, VFMV_SF | VRD(lane_index > 0 ? TMP_VREG1 : vreg) | FRS1(srcdst)));
4508
4509 if (lane_index == 0)
4510 return SLJIT_SUCCESS;
4511
4512 return push_inst(compiler, VSLIDEUP_VI | VRD(vreg) | ((sljit_ins)lane_index << 15) | VRS2(TMP_VREG1));
4513 }
4514
4515 if (srcdst == SLJIT_IMM) {
4516 srcdstw = sljit_simd_get_imm(elem_size, srcdstw);
4517 FAIL_IF(load_immediate(compiler, TMP_REG1, srcdstw, TMP_REG3));
4518 srcdst = TMP_REG1;
4519 }
4520
4521 FAIL_IF(push_inst(compiler, VMV_SX | VRD(lane_index > 0 ? TMP_VREG1 : vreg) | RS1(srcdst)));
4522
4523 if (lane_index == 0)
4524 return SLJIT_SUCCESS;
4525
4526 return push_inst(compiler, VSLIDEUP_VI | VRD(vreg) | ((sljit_ins)lane_index << 15) | VRS2(TMP_VREG1));
4527 }
4528
4529 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
4530 sljit_s32 vreg,
4531 sljit_s32 src, sljit_s32 src_lane_index)
4532 {
4533 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4534 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4535
4536 CHECK_ERROR();
4537 CHECK(check_sljit_emit_simd_lane_replicate(compiler, type, vreg, src, src_lane_index));
4538
4539 if (reg_size < 3 || reg_size > sljit_max_vector_length)
4540 return SLJIT_ERR_UNSUPPORTED;
4541
4542 if (((type & SLJIT_SIMD_FLOAT) && elem_size < 2) || elem_size > 3)
4543 return SLJIT_ERR_UNSUPPORTED;
4544
4545 if (type & SLJIT_SIMD_TEST)
4546 return SLJIT_SUCCESS;
4547
4548 FAIL_IF(sljit_emit_vsetivli(compiler, type, 0));
4549
4550 FAIL_IF(push_inst(compiler, VRGATHER_VI | VRD(vreg != src ? vreg : TMP_VREG1) | ((sljit_ins)src_lane_index << 15) | VRS2(src)));
4551 if (vreg == src)
4552 return push_inst(compiler, VMV_VV | VRD(vreg) | VRS1(TMP_VREG1));
4553 return SLJIT_SUCCESS;
4554 }
4555
4556 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
4557 sljit_s32 vreg,
4558 sljit_s32 src, sljit_sw srcw)
4559 {
4560 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4561 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4562 sljit_s32 elem2_size = SLJIT_SIMD_GET_ELEM2_SIZE(type);
4563 sljit_ins ins;
4564
4565 CHECK_ERROR();
4566 CHECK(check_sljit_emit_simd_extend(compiler, type, vreg, src, srcw));
4567
4568 ADJUST_LOCAL_OFFSET(src, srcw);
4569
4570 if (reg_size < 3 || reg_size > sljit_max_vector_length)
4571 return SLJIT_ERR_UNSUPPORTED;
4572
4573 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
4574 if ((type & SLJIT_SIMD_FLOAT) ? (elem_size < 2 || elem_size > 3) : elem_size > 2)
4575 return SLJIT_ERR_UNSUPPORTED;
4576 #else /* !SLJIT_CONFIG_RISCV_32 */
4577 if (((type & SLJIT_SIMD_FLOAT) && elem_size < 2) || elem_size > 3)
4578 return SLJIT_ERR_UNSUPPORTED;
4579 #endif /* SLJIT_CONFIG_RISCV_32 */
4580
4581 if (type & SLJIT_SIMD_TEST)
4582 return SLJIT_SUCCESS;
4583
4584 if ((src & SLJIT_MEM) || vreg == src) {
4585 ins = (sljit_ins)1 << (reg_size - elem2_size);
4586 FAIL_IF(push_inst(compiler, VSETIVLI | RD(TMP_REG1) | ((sljit_ins)elem_size << 23) | (ins << 15)));
4587
4588 if (src & SLJIT_MEM)
4589 FAIL_IF(sljit_emit_vmem(compiler, VL | VRD(TMP_VREG1), elem_size, src, srcw));
4590 else
4591 FAIL_IF(push_inst(compiler, VMV_VV | VRD(TMP_VREG1) | VRS1(src)));
4592
4593 src = TMP_VREG1;
4594 }
4595
4596 if (type & SLJIT_SIMD_FLOAT) {
4597 FAIL_IF(sljit_emit_vsetivli(compiler, type, 0x7));
4598 return push_inst(compiler, VFWCVT_FFV | VRD(vreg) | VRS2(src));
4599 }
4600
4601 ins = (sljit_ins)1 << (reg_size - elem2_size);
4602 FAIL_IF(push_inst(compiler, VSETIVLI | RD(TMP_REG1) | ((sljit_ins)elem2_size << 23) | (ins << 15)));
4603
4604 switch (elem2_size - elem_size) {
4605 case 1:
4606 ins = VZEXT_VF2;
4607 break;
4608 case 2:
4609 ins = VZEXT_VF4;
4610 break;
4611 default:
4612 ins = VZEXT_VF8;
4613 break;
4614 }
4615
4616 if (type & SLJIT_SIMD_EXTEND_SIGNED)
4617 ins |= 1 << 15;
4618
4619 return push_inst(compiler, ins | VRD(vreg) | VRS2(src));
4620 }
4621
4622 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
4623 sljit_s32 vreg,
4624 sljit_s32 dst, sljit_sw dstw)
4625 {
4626 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4627 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4628 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
4629
4630 CHECK_ERROR();
4631 CHECK(check_sljit_emit_simd_sign(compiler, type, vreg, dst, dstw));
4632
4633 ADJUST_LOCAL_OFFSET(dst, dstw);
4634
4635 if (reg_size < 3 || reg_size > sljit_max_vector_length)
4636 return SLJIT_ERR_UNSUPPORTED;
4637
4638 if (((type & SLJIT_SIMD_FLOAT) && elem_size < 2) || elem_size > 3)
4639 return SLJIT_ERR_UNSUPPORTED;
4640
4641 FAIL_IF(sljit_emit_vsetivli(compiler, type, 0));
4642 FAIL_IF(push_inst(compiler, VMV_VI | VRD(TMP_VREG1) | (0x0 << 15)));
4643 FAIL_IF(push_inst(compiler, VMSLE_VI | VRD(TMP_VREG1) | (0x1f << 15) | VRS2(vreg)));
4644
4645 FAIL_IF(sljit_emit_vsetivli_size(compiler, 2, 2));
4646 FAIL_IF(push_inst(compiler, VMV_XS | RD(dst_r) | VRS2(TMP_VREG1)));
4647
4648 if (dst & SLJIT_MEM)
4649 return emit_op_mem(compiler, (type & SLJIT_32) ? INT_DATA : WORD_DATA, dst_r, dst, dstw);
4650 return SLJIT_SUCCESS;
4651 }
4652
4653 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
4654 sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
4655 {
4656 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4657 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4658 sljit_ins ins = 0;
4659
4660 CHECK_ERROR();
4661 CHECK(check_sljit_emit_simd_op2(compiler, type, dst_vreg, src1_vreg, src2, src2w));
4662
4663 ADJUST_LOCAL_OFFSET(src2, src2w);
4664
4665 if (reg_size < 3 || reg_size > sljit_max_vector_length)
4666 return SLJIT_ERR_UNSUPPORTED;
4667
4668 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
4669 return SLJIT_ERR_UNSUPPORTED;
4670
4671 if (type & SLJIT_SIMD_TEST)
4672 return SLJIT_SUCCESS;
4673
4674 switch (SLJIT_SIMD_GET_OPCODE(type)) {
4675 case SLJIT_SIMD_OP2_AND:
4676 ins = VAND_VV;
4677 break;
4678 case SLJIT_SIMD_OP2_OR:
4679 ins = VOR_VV;
4680 break;
4681 case SLJIT_SIMD_OP2_XOR:
4682 ins = VXOR_VV;
4683 break;
4684 case SLJIT_SIMD_OP2_SHUFFLE:
4685 ins = VRGATHER_VV;
4686 elem_size = 0;
4687 break;
4688 }
4689
4690 if (elem_size > 3)
4691 elem_size = 3;
4692
4693 FAIL_IF(sljit_emit_vsetivli_size(compiler, reg_size, elem_size));
4694
4695 if (src2 & SLJIT_MEM) {
4696 FAIL_IF(sljit_emit_vmem(compiler, VL | VRD(TMP_VREG1), elem_size, src2, src2w));
4697 src2 = TMP_VREG1;
4698 }
4699
4700 if (SLJIT_SIMD_GET_OPCODE(type) != SLJIT_SIMD_OP2_SHUFFLE)
4701 return push_inst(compiler, ins | VRD(dst_vreg) | VRS1(src1_vreg) | VRS2(src2));
4702
4703 if (dst_vreg == src2) {
4704 FAIL_IF(push_inst(compiler, VMV_VV | VRD(TMP_VREG1) | VRS1(src2)));
4705 src2 = TMP_VREG1;
4706 }
4707
4708 if (dst_vreg == src1_vreg) {
4709 FAIL_IF(push_inst(compiler, VMV_VV | VRD(TMP_VREG2) | VRS1(src1_vreg)));
4710 src1_vreg = TMP_VREG2;
4711 }
4712
4713 return push_inst(compiler, ins | VRD(dst_vreg) | VRS1(src2) | VRS2(src1_vreg));
4714 }
4715
4716 SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 op,
4717 sljit_s32 dst, sljit_sw dstw,
4718 sljit_sw init_value)
4719 {
4720 struct sljit_const *const_;
4721 sljit_s32 dst_r;
4722 sljit_s32 mem_flags = WORD_DATA;
4723
4724 CHECK_ERROR_PTR();
4725 CHECK_PTR(check_sljit_emit_const(compiler, op, dst, dstw, init_value));
4726 ADJUST_LOCAL_OFFSET(dst, dstw);
4727
4728 const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
4729 PTR_FAIL_IF(!const_);
4730 set_const(const_, compiler);
4731
4732 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
4733
4734 switch (GET_OPCODE(op)) {
4735 case SLJIT_MOV_U8:
4736 if (init_value & 0x100)
4737 init_value |= 0xf00;
4738 else
4739 init_value &= 0xff;
4740
4741 PTR_FAIL_IF(push_inst(compiler, ADDI | RD(dst_r) | IMM_I(init_value)));
4742 mem_flags = BYTE_DATA;
4743 break;
4744
4745 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
4746 case SLJIT_MOV32:
4747 mem_flags = INT_DATA;
4748 SLJIT_FALLTHROUGH
4749 case SLJIT_MOV_S32:
4750 if ((init_value & 0x800) != 0)
4751 init_value ^= ~(sljit_sw)0xfff;
4752
4753 PTR_FAIL_IF(push_inst(compiler, LUI | RD(dst_r) | (sljit_ins)(init_value & ~(sljit_sw)0xfff)));
4754 PTR_FAIL_IF(push_inst(compiler, XORI | RD(dst_r) | RS1(dst_r) | IMM_I(init_value)));
4755 break;
4756 #endif /* SLJIT_CONFIG_RISCV_64 */
4757
4758 default:
4759 PTR_FAIL_IF(emit_const(compiler, dst_r, init_value, ADDI | RD(dst_r)));
4760 break;
4761 }
4762
4763 if (dst & SLJIT_MEM)
4764 PTR_FAIL_IF(emit_op_mem(compiler, mem_flags, TMP_REG2, dst, dstw));
4765
4766 return const_;
4767 }
4768
4769 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_op_addr(struct sljit_compiler *compiler, sljit_s32 op,
4770 sljit_s32 dst, sljit_sw dstw)
4771 {
4772 struct sljit_jump *jump;
4773 sljit_s32 dst_r, target_r;
4774 SLJIT_UNUSED_ARG(op);
4775
4776 CHECK_ERROR_PTR();
4777 CHECK_PTR(check_sljit_emit_op_addr(compiler, op, dst, dstw));
4778 ADJUST_LOCAL_OFFSET(dst, dstw);
4779
4780 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
4781
4782 if (op != SLJIT_ADD_ABS_ADDR)
4783 target_r = dst_r;
4784 else {
4785 target_r = TMP_REG1;
4786
4787 if (dst & SLJIT_MEM)
4788 PTR_FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG2, dst, dstw));
4789 }
4790
4791 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
4792 PTR_FAIL_IF(!jump);
4793 set_mov_addr(jump, compiler, 0);
4794
4795 PTR_FAIL_IF(push_inst16(compiler, (sljit_u16)target_r));
4796 compiler->size += JUMP_MAX_SIZE - 1;
4797
4798 if (op == SLJIT_ADD_ABS_ADDR) {
4799 if (RISCV_HAS_COMPRESSED(200))
4800 PTR_FAIL_IF(push_inst16(compiler, C_ADD | C_RD(dst_r) | C_RS2(TMP_REG1)));
4801 else
4802 PTR_FAIL_IF(push_inst(compiler, ADD | RD(dst_r) | RS1(dst_r) | RS2(TMP_REG1)));
4803 }
4804
4805 if (dst & SLJIT_MEM)
4806 PTR_FAIL_IF(emit_op_mem(compiler, WORD_DATA, TMP_REG2, dst, dstw));
4807
4808 return jump;
4809 }
4810
4811 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_s32 op, sljit_sw new_constant, sljit_sw executable_offset)
4812 {
4813 sljit_u16 *inst;
4814
4815 switch (GET_OPCODE(op)) {
4816 case SLJIT_MOV_U8:
4817 inst = (sljit_u16*)addr;
4818 SLJIT_ASSERT((inst[0] & 0x707f) == ADDI);
4819
4820 if (new_constant & 0x100)
4821 new_constant |= 0xf00;
4822 else
4823 new_constant &= 0xff;
4824
4825 SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0);
4826 inst[1] = (sljit_u16)(new_constant << 4) | (inst[1] & 0xf);
4827 SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1);
4828 inst = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
4829 SLJIT_CACHE_FLUSH(inst, inst + 2);
4830 return;
4831
4832 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
4833 case SLJIT_MOV32:
4834 case SLJIT_MOV_S32:
4835 inst = (sljit_u16*)addr;
4836 SLJIT_ASSERT((inst[0] & 0x7f) == LUI && (inst[2] & 0x707f) == XORI);
4837
4838 if ((new_constant & 0x800) != 0)
4839 new_constant ^= ~(sljit_sw)0xfff;
4840
4841 SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 0);
4842 inst[0] = (sljit_u16)((inst[0] & 0xfff) | (new_constant & 0xf000));
4843 inst[1] = (sljit_u16)(new_constant >> 16);
4844 inst[3] = (sljit_u16)(new_constant << 4) | (inst[3] & 0xf);
4845 SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 1);
4846 inst = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
4847 SLJIT_CACHE_FLUSH(inst, inst + 4);
4848 return;
4849 #endif /* SLJIT_CONFIG_RISCV_64 */
4850
4851 default:
4852 sljit_set_jump_addr(addr, (sljit_uw)new_constant, executable_offset);
4853 return;
4854 }
4855 }