Debian Codegraph @

src/3rdparty/pcre2/deps/sljit/sljit_src/sljitNativeARM_T2_32.c

raw
1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
28 {
29 #ifdef __SOFTFP__
30 return "ARM-Thumb2" SLJIT_CPUINFO " ABI:softfp";
31 #else
32 return "ARM-Thumb2" SLJIT_CPUINFO " ABI:hardfp";
33 #endif
34 }
35
36 /* Length of an instruction word. */
37 typedef sljit_u32 sljit_ins;
38
39 /* Last register + 1. */
40 #define TMP_REG1 (SLJIT_NUMBER_OF_REGISTERS + 2)
41 #define TMP_REG2 (SLJIT_NUMBER_OF_REGISTERS + 3)
42 #define TMP_PC (SLJIT_NUMBER_OF_REGISTERS + 4)
43
44 #define TMP_FREG1 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
45 #define TMP_FREG2 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2)
46
47 /* See sljit_emit_enter and sljit_emit_op0 if you want to change them. */
48 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 5] = {
49 0, 0, 1, 2, 3, 11, 10, 9, 8, 7, 6, 5, 4, 13, 12, 14, 15
50 };
51
52 static const sljit_u8 freg_map[((SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2) << 1) + 1] = {
53 0,
54 0, 1, 2, 3, 4, 5, 15, 14, 13, 12, 11, 10, 9, 8,
55 7, 6,
56 0, 1, 2, 3, 4, 5, 15, 14, 13, 12, 11, 10, 9, 8,
57 7, 6
58 };
59
60 static const sljit_u8 freg_ebit_map[((SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2) << 1) + 1] = {
61 0,
62 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
63 0, 0,
64 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
65 1, 1
66 };
67
68 #define COPY_BITS(src, from, to, bits) \
69 ((from >= to ? ((sljit_ins)(src) >> (from - to)) : ((sljit_ins)(src) << (to - from))) & (((1 << bits) - 1) << to))
70
71 #define NEGATE(uimm) ((sljit_uw)-(sljit_sw)(uimm))
72
73 /* Thumb16 encodings. */
74 #define RD3(rd) ((sljit_ins)reg_map[rd])
75 #define RN3(rn) ((sljit_ins)reg_map[rn] << 3)
76 #define RM3(rm) ((sljit_ins)reg_map[rm] << 6)
77 #define RDN3(rdn) ((sljit_ins)reg_map[rdn] << 8)
78 #define IMM3(imm) ((sljit_ins)imm << 6)
79 #define IMM8(imm) ((sljit_ins)imm)
80
81 /* Thumb16 helpers. */
82 #define SET_REGS44(rd, rn) \
83 (((sljit_ins)reg_map[rn] << 3) | ((sljit_ins)reg_map[rd] & 0x7) | (((sljit_ins)reg_map[rd] & 0x8) << 4))
84 #define IS_2_LO_REGS(reg1, reg2) \
85 (reg_map[reg1] <= 7 && reg_map[reg2] <= 7)
86 #define IS_3_LO_REGS(reg1, reg2, reg3) \
87 (reg_map[reg1] <= 7 && reg_map[reg2] <= 7 && reg_map[reg3] <= 7)
88
89 /* Thumb32 encodings. */
90 #define RM4(rm) ((sljit_ins)reg_map[rm])
91 #define RD4(rd) ((sljit_ins)reg_map[rd] << 8)
92 #define RT4(rt) ((sljit_ins)reg_map[rt] << 12)
93 #define RN4(rn) ((sljit_ins)reg_map[rn] << 16)
94
95 #define VM4(vm) (((sljit_ins)freg_map[vm]) | ((sljit_ins)freg_ebit_map[vm] << 5))
96 #define VD4(vd) (((sljit_ins)freg_map[vd] << 12) | ((sljit_ins)freg_ebit_map[vd] << 22))
97 #define VN4(vn) (((sljit_ins)freg_map[vn] << 16) | ((sljit_ins)freg_ebit_map[vn] << 7))
98
99 #define IMM5(imm) \
100 (COPY_BITS(imm, 2, 12, 3) | (((sljit_ins)imm & 0x3) << 6))
101 #define IMM12(imm) \
102 (COPY_BITS(imm, 11, 26, 1) | COPY_BITS(imm, 8, 12, 3) | ((sljit_ins)imm & 0xff))
103
104 /* --------------------------------------------------------------------- */
105 /* Instrucion forms */
106 /* --------------------------------------------------------------------- */
107
108 /* dot '.' changed to _
109 I immediate form (possibly followed by number of immediate bits). */
110 #define ADCI 0xf1400000
111 #define ADCS 0x4140
112 #define ADC_W 0xeb400000
113 #define ADD 0x4400
114 #define ADDS 0x1800
115 #define ADDSI3 0x1c00
116 #define ADDSI8 0x3000
117 #define ADDWI 0xf2000000
118 #define ADD_SP 0x4485
119 #define ADD_SP_I 0xb000
120 #define ADD_W 0xeb000000
121 #define ADD_WI 0xf1000000
122 #define ANDI 0xf0000000
123 #define ANDS 0x4000
124 #define AND_W 0xea000000
125 #define ASRS 0x4100
126 #define ASRSI 0x1000
127 #define ASR_W 0xfa40f000
128 #define ASR_WI 0xea4f0020
129 #define BCC 0xd000
130 #define BICI 0xf0200000
131 #define BKPT 0xbe00
132 #define BLX 0x4780
133 #define BX 0x4700
134 #define CLZ 0xfab0f080
135 #define CMNI_W 0xf1100f00
136 #define CMP 0x4280
137 #define CMPI 0x2800
138 #define CMPI_W 0xf1b00f00
139 #define CMP_X 0x4500
140 #define CMP_W 0xebb00f00
141 #define DMB_SY 0xf3bf8f5f
142 #define EORI 0xf0800000
143 #define EORS 0x4040
144 #define EOR_W 0xea800000
145 #define IT 0xbf00
146 #define LDR 0xf8d00000
147 #define LDR_SP 0x9800
148 #define LDRD 0xe9500000
149 #define LDREX 0xe8500f00
150 #define LDREXB 0xe8d00f4f
151 #define LDREXH 0xe8d00f5f
152 #define LDRI 0xf8500800
153 #define LSLS 0x4080
154 #define LSLSI 0x0000
155 #define LSL_W 0xfa00f000
156 #define LSL_WI 0xea4f0000
157 #define LSRS 0x40c0
158 #define LSRSI 0x0800
159 #define LSR_W 0xfa20f000
160 #define LSR_WI 0xea4f0010
161 #define MLA 0xfb000000
162 #define MOV 0x4600
163 #define MOVS 0x0000
164 #define MOVSI 0x2000
165 #define MOVT 0xf2c00000
166 #define MOVW 0xf2400000
167 #define MOV_W 0xea4f0000
168 #define MOV_WI 0xf04f0000
169 #define MUL 0xfb00f000
170 #define MVNS 0x43c0
171 #define MVN_W 0xea6f0000
172 #define MVN_WI 0xf06f0000
173 #define NOP 0xbf00
174 #define ORNI 0xf0600000
175 #define ORRI 0xf0400000
176 #define ORRS 0x4300
177 #define ORR_W 0xea400000
178 #define POP 0xbc00
179 #define POP_W 0xe8bd0000
180 #define PUSH 0xb400
181 #define PUSH_W 0xe92d0000
182 #define REV 0xba00
183 #define REV_W 0xfa90f080
184 #define REV16 0xba40
185 #define REV16_W 0xfa90f090
186 #define RBIT 0xfa90f0a0
187 #define RORS 0x41c0
188 #define ROR_W 0xfa60f000
189 #define ROR_WI 0xea4f0030
190 #define RSB_WI 0xf1c00000
191 #define RSBSI 0x4240
192 #define SBCI 0xf1600000
193 #define SBCS 0x4180
194 #define SBC_W 0xeb600000
195 #define SDIV 0xfb90f0f0
196 #define SMULL 0xfb800000
197 #define STR_SP 0x9000
198 #define STRD 0xe9400000
199 #define STREX 0xe8400000
200 #define STREXB 0xe8c00f40
201 #define STREXH 0xe8c00f50
202 #define SUBS 0x1a00
203 #define SUBSI3 0x1e00
204 #define SUBSI8 0x3800
205 #define SUB_W 0xeba00000
206 #define SUBWI 0xf2a00000
207 #define SUB_SP_I 0xb080
208 #define SUB_WI 0xf1a00000
209 #define SXTB 0xb240
210 #define SXTB_W 0xfa4ff080
211 #define SXTH 0xb200
212 #define SXTH_W 0xfa0ff080
213 #define TST 0x4200
214 #define TSTI 0xf0000f00
215 #define TST_W 0xea000f00
216 #define UDIV 0xfbb0f0f0
217 #define UMULL 0xfba00000
218 #define UXTB 0xb2c0
219 #define UXTB_W 0xfa5ff080
220 #define UXTH 0xb280
221 #define UXTH_W 0xfa1ff080
222 #define VABS_F32 0xeeb00ac0
223 #define VADD_F32 0xee300a00
224 #define VAND 0xef000110
225 #define VCMP_F32 0xeeb40a40
226 #define VCVT_F32_S32 0xeeb80ac0
227 #define VCVT_F32_U32 0xeeb80a40
228 #define VCVT_F64_F32 0xeeb70ac0
229 #define VCVT_S32_F32 0xeebd0ac0
230 #define VDIV_F32 0xee800a00
231 #define VDUP 0xee800b10
232 #define VDUP_s 0xffb00c00
233 #define VEOR 0xff000110
234 #define VLD1 0xf9200000
235 #define VLD1_r 0xf9a00c00
236 #define VLD1_s 0xf9a00000
237 #define VLDR_F32 0xed100a00
238 #define VMOV_F32 0xeeb00a40
239 #define VMOV 0xee000a10
240 #define VMOV2 0xec400a10
241 #define VMOV_i 0xef800010
242 #define VMOV_s 0xee000b10
243 #define VMOVN 0xffb20200
244 #define VMRS 0xeef1fa10
245 #define VMUL_F32 0xee200a00
246 #define VNEG_F32 0xeeb10a40
247 #define VORR 0xef200110
248 #define VPOP 0xecbd0b00
249 #define VPUSH 0xed2d0b00
250 #define VSHLL 0xef800a10
251 #define VSHR 0xef800010
252 #define VSRA 0xef800110
253 #define VST1 0xf9000000
254 #define VST1_s 0xf9800000
255 #define VSTR_F32 0xed000a00
256 #define VSUB_F32 0xee300a40
257 #define VTBL 0xffb00800
258
259 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
260
261 static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr, sljit_s32 is_32)
262 {
263 if (compiler->scratches == -1)
264 return 0;
265
266 if (is_32 && fr >= SLJIT_F64_SECOND(SLJIT_FR0))
267 fr -= SLJIT_F64_SECOND(0);
268
269 return (fr >= SLJIT_FR0 && fr < (SLJIT_FR0 + compiler->real_fscratches))
270 || (fr > (SLJIT_FS0 - compiler->real_fsaveds) && fr <= SLJIT_FS0)
271 || (fr >= SLJIT_TMP_FREGISTER_BASE && fr < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS));
272 }
273
274 static sljit_s32 function_check_is_vreg(struct sljit_compiler *compiler, sljit_s32 vr, sljit_s32 type)
275 {
276 sljit_s32 vr_low = vr;
277
278 if (compiler->scratches == -1)
279 return 0;
280
281 if (SLJIT_SIMD_GET_REG_SIZE(type) == 4) {
282 vr += (vr & 0x1);
283 vr_low = vr - 1;
284 }
285
286 return (vr >= SLJIT_VR0 && vr < (SLJIT_VR0 + compiler->vscratches))
287 || (vr_low > (SLJIT_VS0 - compiler->vsaveds) && vr_low <= SLJIT_VS0)
288 || (vr >= SLJIT_TMP_VREGISTER_BASE && vr < (SLJIT_TMP_VREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_VECTOR_REGISTERS));
289 }
290
291 #endif /* SLJIT_ARGUMENT_CHECKS */
292
293 static sljit_s32 push_inst16(struct sljit_compiler *compiler, sljit_ins inst)
294 {
295 sljit_u16 *ptr;
296 SLJIT_ASSERT(!(inst & 0xffff0000));
297
298 ptr = (sljit_u16*)ensure_buf(compiler, sizeof(sljit_u16));
299 FAIL_IF(!ptr);
300 *ptr = (sljit_u16)(inst);
301 compiler->size++;
302 return SLJIT_SUCCESS;
303 }
304
305 static sljit_s32 push_inst32(struct sljit_compiler *compiler, sljit_ins inst)
306 {
307 sljit_u16 *ptr = (sljit_u16*)ensure_buf(compiler, sizeof(sljit_ins));
308 FAIL_IF(!ptr);
309 *ptr++ = (sljit_u16)(inst >> 16);
310 *ptr = (sljit_u16)(inst);
311 compiler->size += 2;
312 return SLJIT_SUCCESS;
313 }
314
315 static sljit_s32 emit_imm32_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_uw imm)
316 {
317 FAIL_IF(push_inst32(compiler, MOVW | RD4(dst)
318 | COPY_BITS(imm, 12, 16, 4) | COPY_BITS(imm, 11, 26, 1) | COPY_BITS(imm, 8, 12, 3) | (imm & 0xff)));
319 return push_inst32(compiler, MOVT | RD4(dst)
320 | COPY_BITS(imm, 12 + 16, 16, 4) | COPY_BITS(imm, 11 + 16, 26, 1) | COPY_BITS(imm, 8 + 16, 12, 3) | ((imm & 0xff0000) >> 16));
321 }
322
323 /* Dst must be in bits[11-8] */
324 static void set_imm32_const(sljit_u16 *inst, sljit_ins dst, sljit_uw new_imm)
325 {
326 inst[0] = (sljit_u16)((MOVW >> 16) | COPY_BITS(new_imm, 12, 0, 4) | COPY_BITS(new_imm, 11, 10, 1));
327 inst[1] = (sljit_u16)(dst | COPY_BITS(new_imm, 8, 12, 3) | (new_imm & 0xff));
328 inst[2] = (sljit_u16)((MOVT >> 16) | COPY_BITS(new_imm, 12 + 16, 0, 4) | COPY_BITS(new_imm, 11 + 16, 10, 1));
329 inst[3] = (sljit_u16)(dst | COPY_BITS(new_imm, 8 + 16, 12, 3) | ((new_imm & 0xff0000) >> 16));
330 }
331
332 static SLJIT_INLINE void modify_imm32_const(sljit_u16 *inst, sljit_uw new_imm)
333 {
334 sljit_ins dst = inst[1] & 0x0f00;
335 SLJIT_ASSERT(((inst[0] & 0xfbf0) == (MOVW >> 16)) && ((inst[2] & 0xfbf0) == (MOVT >> 16)) && dst == (inst[3] & 0x0f00));
336 set_imm32_const(inst, dst, new_imm);
337 }
338
339 static SLJIT_INLINE sljit_u16* detect_jump_type(struct sljit_jump *jump, sljit_u16 *code_ptr, sljit_u16 *code, sljit_sw executable_offset)
340 {
341 sljit_sw diff;
342 sljit_uw target_addr;
343 sljit_uw jump_addr = (sljit_uw)code_ptr;
344 sljit_uw orig_addr = jump->addr;
345 SLJIT_UNUSED_ARG(executable_offset);
346
347 jump->addr = jump_addr;
348 if (jump->flags & SLJIT_REWRITABLE_JUMP)
349 goto exit;
350
351 if (jump->flags & JUMP_ADDR) {
352 /* Branch to ARM code is not optimized yet. */
353 if (!(jump->u.target & 0x1))
354 goto exit;
355 target_addr = jump->u.target;
356 } else {
357 SLJIT_ASSERT(jump->u.label != NULL);
358 target_addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code + jump->u.label->size, executable_offset);
359
360 if (jump->u.label->size > orig_addr)
361 jump_addr = (sljit_uw)(code + orig_addr);
362 }
363
364 diff = (sljit_sw)target_addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_addr + 4, executable_offset);
365
366 if (jump->flags & IS_COND) {
367 SLJIT_ASSERT(!(jump->flags & IS_BL));
368 /* Size of the prefix IT instruction. */
369 diff += SSIZE_OF(u16);
370 if (diff <= 0xff && diff >= -0x100) {
371 jump->flags |= PATCH_TYPE1;
372 jump->addr = (sljit_uw)(code_ptr - 1);
373 return code_ptr - 1;
374 }
375 if (diff <= 0xfffff && diff >= -0x100000) {
376 jump->flags |= PATCH_TYPE2;
377 jump->addr = (sljit_uw)(code_ptr - 1);
378 return code_ptr;
379 }
380 diff -= SSIZE_OF(u16);
381 } else if (jump->flags & IS_BL) {
382 /* Branch and link. */
383 if (diff <= 0xffffff && diff >= -0x1000000) {
384 jump->flags |= PATCH_TYPE5;
385 return code_ptr + 1;
386 }
387 goto exit;
388 } else if (diff <= 0x7ff && diff >= -0x800) {
389 jump->flags |= PATCH_TYPE3;
390 return code_ptr;
391 }
392
393 if (diff <= 0xffffff && diff >= -0x1000000) {
394 jump->flags |= PATCH_TYPE4;
395 return code_ptr + 1;
396 }
397
398 exit:
399 code_ptr[4] = code_ptr[0];
400
401 if (jump->flags & IS_COND) {
402 code_ptr[3] = code_ptr[-1];
403 jump->addr = (sljit_uw)(code_ptr - 1);
404 }
405
406 return code_ptr + 4;
407 }
408
409 static SLJIT_INLINE sljit_sw mov_addr_get_length(struct sljit_jump *jump, sljit_u16 *code_ptr, sljit_u16 *code, sljit_sw executable_offset)
410 {
411 sljit_uw addr;
412 sljit_uw jump_addr = (sljit_uw)code_ptr;
413 sljit_sw diff;
414 SLJIT_UNUSED_ARG(executable_offset);
415
416 if (jump->flags & JUMP_ADDR)
417 addr = jump->u.target;
418 else {
419 addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code + jump->u.label->size, executable_offset);
420
421 if (jump->u.label->size > jump->addr)
422 jump_addr = (sljit_uw)(code + jump->addr);
423 }
424
425 /* The pc+4 offset is represented by the 2 * SSIZE_OF(sljit_u16) below. */
426 diff = (sljit_sw)addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_addr, executable_offset);
427
428 /* Note: ADR with imm8 does not set the last bit (Thumb2 flag). */
429
430 if (diff <= 0xffd + 2 * SSIZE_OF(u16) && diff >= -0xfff + 2 * SSIZE_OF(u16)) {
431 jump->flags |= PATCH_TYPE6;
432 return 1;
433 }
434
435 return 3;
436 }
437
438 static SLJIT_INLINE void generate_jump_or_mov_addr(struct sljit_jump *jump, sljit_sw executable_offset)
439 {
440 sljit_s32 type = (jump->flags >> 4) & 0xf;
441 sljit_u16 *jump_inst = (sljit_u16*)jump->addr;
442 sljit_sw diff;
443 sljit_ins ins;
444
445 diff = (sljit_sw)((jump->flags & JUMP_ADDR) ? jump->u.target : jump->u.label->u.addr);
446
447 if (SLJIT_UNLIKELY(type == 0)) {
448 ins = (jump->flags & JUMP_MOV_ADDR) ? *jump_inst : RDN3(TMP_REG1);
449 set_imm32_const((sljit_u16*)jump->addr, ins, (sljit_uw)diff);
450 return;
451 }
452
453 if (SLJIT_UNLIKELY(type == 6)) {
454 SLJIT_ASSERT(jump->flags & JUMP_MOV_ADDR);
455 diff -= (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_inst + 2, executable_offset) & ~(sljit_sw)0x3;
456
457 SLJIT_ASSERT(diff <= 0xfff && diff >= -0xfff);
458
459 ins = ADDWI >> 16;
460 if (diff <= 0) {
461 diff = -diff;
462 ins = SUBWI >> 16;
463 }
464
465 jump_inst[1] = (sljit_u16)(jump_inst[0] | COPY_BITS(diff, 8, 12, 3) | (diff & 0xff));
466 jump_inst[0] = (sljit_u16)(ins | 0xf | COPY_BITS(diff, 11, 10, 1));
467 return;
468 }
469
470 SLJIT_ASSERT((diff & 0x1) != 0 && !(jump->flags & JUMP_MOV_ADDR));
471 diff = (diff - (sljit_sw)(jump->addr + sizeof(sljit_u32)) - executable_offset) >> 1;
472
473 switch (type) {
474 case 1:
475 /* Encoding T1 of 'B' instruction */
476 SLJIT_ASSERT(diff <= 0x7f && diff >= -0x80 && (jump->flags & IS_COND));
477 jump_inst[0] = (sljit_u16)(0xd000 | (jump->flags & 0xf00) | ((sljit_ins)diff & 0xff));
478 return;
479 case 2:
480 /* Encoding T3 of 'B' instruction */
481 SLJIT_ASSERT(diff <= 0x7ffff && diff >= -0x80000 && (jump->flags & IS_COND));
482 jump_inst[0] = (sljit_u16)(0xf000 | COPY_BITS(jump->flags, 8, 6, 4) | COPY_BITS(diff, 11, 0, 6) | COPY_BITS(diff, 19, 10, 1));
483 jump_inst[1] = (sljit_u16)(0x8000 | COPY_BITS(diff, 17, 13, 1) | COPY_BITS(diff, 18, 11, 1) | ((sljit_ins)diff & 0x7ff));
484 return;
485 case 3:
486 /* Encoding T2 of 'B' instruction */
487 SLJIT_ASSERT(diff <= 0x3ff && diff >= -0x400 && !(jump->flags & IS_COND));
488 jump_inst[0] = (sljit_u16)(0xe000 | (diff & 0x7ff));
489 return;
490 }
491
492 SLJIT_ASSERT(diff <= 0x7fffff && diff >= -0x800000);
493
494 /* Really complex instruction form for branches. Negate with sign bit. */
495 diff ^= ((diff >> 2) & 0x600000) ^ 0x600000;
496
497 jump_inst[0] = (sljit_u16)(0xf000 | COPY_BITS(diff, 11, 0, 10) | COPY_BITS(diff, 23, 10, 1));
498 jump_inst[1] = (sljit_u16)((diff & 0x7ff) | COPY_BITS(diff, 22, 13, 1) | COPY_BITS(diff, 21, 11, 1));
499
500 SLJIT_ASSERT(type == 4 || type == 5);
501
502 /* The others have a common form. */
503 if (type == 4) /* Encoding T4 of 'B' instruction */
504 jump_inst[1] |= 0x9000;
505 else /* Encoding T1 of 'BL' instruction */
506 jump_inst[1] |= 0xd000;
507 }
508
509 static void reduce_code_size(struct sljit_compiler *compiler)
510 {
511 struct sljit_label *label;
512 struct sljit_jump *jump;
513 struct sljit_const *const_;
514 SLJIT_NEXT_DEFINE_TYPES;
515 sljit_uw total_size;
516 sljit_uw size_reduce = 0;
517 sljit_sw diff;
518
519 label = compiler->labels;
520 jump = compiler->jumps;
521 const_ = compiler->consts;
522 SLJIT_NEXT_INIT_TYPES();
523
524 while (1) {
525 SLJIT_GET_NEXT_MIN();
526
527 if (next_min_addr == SLJIT_MAX_ADDRESS)
528 break;
529
530 if (next_min_addr == next_label_size) {
531 label->size -= size_reduce;
532
533 label = label->next;
534 next_label_size = SLJIT_GET_NEXT_SIZE(label);
535 }
536
537 if (next_min_addr == next_const_addr) {
538 const_->addr -= size_reduce;
539 const_ = const_->next;
540 next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
541 continue;
542 }
543
544 if (next_min_addr != next_jump_addr)
545 continue;
546
547 jump->addr -= size_reduce;
548 if (!(jump->flags & JUMP_MOV_ADDR)) {
549 total_size = JUMP_MAX_SIZE;
550
551 if (!(jump->flags & (SLJIT_REWRITABLE_JUMP | JUMP_ADDR))) {
552 /* Unit size: instruction. */
553 diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr - 2;
554 if (jump->u.label->size > jump->addr) {
555 SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
556 diff -= (sljit_sw)size_reduce;
557 }
558
559 if (jump->flags & IS_COND) {
560 diff++;
561
562 if (diff <= (0xff / SSIZE_OF(u16)) && diff >= (-0x100 / SSIZE_OF(u16)))
563 total_size = 0;
564 else if (diff <= (0xfffff / SSIZE_OF(u16)) && diff >= (-0x100000 / SSIZE_OF(u16)))
565 total_size = 1;
566 diff--;
567 } else if (!(jump->flags & IS_BL) && diff <= (0x7ff / SSIZE_OF(u16)) && diff >= (-0x800 / SSIZE_OF(u16)))
568 total_size = 1;
569
570 if (total_size == JUMP_MAX_SIZE && diff <= (0xffffff / SSIZE_OF(u16)) && diff >= (-0x1000000 / SSIZE_OF(u16)))
571 total_size = 2;
572 }
573
574 size_reduce += JUMP_MAX_SIZE - total_size;
575 } else {
576 /* Real size minus 1. Unit size: instruction. */
577 total_size = 3;
578
579 if (!(jump->flags & JUMP_ADDR)) {
580 diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
581 if (jump->u.label->size > jump->addr) {
582 SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
583 diff -= (sljit_sw)size_reduce;
584 }
585
586 if (diff <= (0xffd / SSIZE_OF(u16)) && diff >= (-0xfff / SSIZE_OF(u16)))
587 total_size = 1;
588 }
589
590 size_reduce += 3 - total_size;
591 }
592
593 jump->flags |= total_size << JUMP_SIZE_SHIFT;
594 jump = jump->next;
595 next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
596 }
597
598 compiler->size -= size_reduce;
599 }
600
601 SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler, sljit_s32 options, void *exec_allocator_data)
602 {
603 struct sljit_memory_fragment *buf;
604 sljit_u16 *code;
605 sljit_u16 *code_ptr;
606 sljit_u16 *buf_ptr;
607 sljit_u16 *buf_end;
608 sljit_uw half_count;
609 SLJIT_NEXT_DEFINE_TYPES;
610 sljit_sw addr;
611 sljit_sw executable_offset;
612
613 struct sljit_label *label;
614 struct sljit_jump *jump;
615 struct sljit_const *const_;
616
617 CHECK_ERROR_PTR();
618 CHECK_PTR(check_sljit_generate_code(compiler));
619
620 reduce_code_size(compiler);
621
622 code = (sljit_u16*)allocate_executable_memory(compiler->size * sizeof(sljit_u16), options, exec_allocator_data, &executable_offset);
623 PTR_FAIL_WITH_EXEC_IF(code);
624
625 reverse_buf(compiler);
626 buf = compiler->buf;
627
628 code_ptr = code;
629 half_count = 0;
630 label = compiler->labels;
631 jump = compiler->jumps;
632 const_ = compiler->consts;
633 SLJIT_NEXT_INIT_TYPES();
634 SLJIT_GET_NEXT_MIN();
635
636 do {
637 buf_ptr = (sljit_u16*)buf->memory;
638 buf_end = buf_ptr + (buf->used_size >> 1);
639 do {
640 *code_ptr = *buf_ptr++;
641 if (next_min_addr == half_count) {
642 SLJIT_ASSERT(!label || label->size >= half_count);
643 SLJIT_ASSERT(!jump || jump->addr >= half_count);
644 SLJIT_ASSERT(!const_ || const_->addr >= half_count);
645
646 /* These structures are ordered by their address. */
647 if (next_min_addr == next_label_size) {
648 label->u.addr = ((sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset)) | 0x1;
649 label->size = (sljit_uw)(code_ptr - code);
650 label = label->next;
651 next_label_size = SLJIT_GET_NEXT_SIZE(label);
652 }
653
654 if (next_min_addr == next_jump_addr) {
655 if (!(jump->flags & JUMP_MOV_ADDR)) {
656 half_count = half_count - 1 + (jump->flags >> JUMP_SIZE_SHIFT);
657 code_ptr = detect_jump_type(jump, code_ptr, code, executable_offset);
658 SLJIT_ASSERT((sljit_uw)code_ptr - jump->addr <
659 ((jump->flags >> JUMP_SIZE_SHIFT) + ((jump->flags & 0xf0) <= PATCH_TYPE2)) * sizeof(sljit_u16));
660 } else {
661 half_count += jump->flags >> JUMP_SIZE_SHIFT;
662 addr = (sljit_sw)code_ptr;
663 code_ptr += mov_addr_get_length(jump, code_ptr, code, executable_offset);
664 jump->addr = (sljit_uw)addr;
665 }
666
667 jump = jump->next;
668 next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
669 } else if (next_min_addr == next_const_addr) {
670 const_->addr = (sljit_uw)code_ptr;
671 const_ = const_->next;
672 next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
673 }
674
675 SLJIT_GET_NEXT_MIN();
676 }
677 code_ptr++;
678 half_count++;
679 } while (buf_ptr < buf_end);
680
681 buf = buf->next;
682 } while (buf);
683
684 if (label && label->size == half_count) {
685 label->u.addr = ((sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset)) | 0x1;
686 label->size = (sljit_uw)(code_ptr - code);
687 label = label->next;
688 }
689
690 SLJIT_ASSERT(!label);
691 SLJIT_ASSERT(!jump);
692 SLJIT_ASSERT(!const_);
693 SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size);
694
695 jump = compiler->jumps;
696 while (jump) {
697 generate_jump_or_mov_addr(jump, executable_offset);
698 jump = jump->next;
699 }
700
701 compiler->error = SLJIT_ERR_COMPILED;
702 compiler->executable_offset = executable_offset;
703 compiler->executable_size = (sljit_uw)(code_ptr - code) * sizeof(sljit_u16);
704
705 code = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
706 code_ptr = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
707
708 SLJIT_CACHE_FLUSH(code, code_ptr);
709 SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1);
710
711 /* Set thumb mode flag. */
712 return (void*)((sljit_uw)code | 0x1);
713 }
714
715 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
716 {
717 switch (feature_type) {
718 case SLJIT_HAS_FPU:
719 case SLJIT_HAS_F64_AS_F32_PAIR:
720 case SLJIT_HAS_SIMD:
721 #ifdef SLJIT_IS_FPU_AVAILABLE
722 return (SLJIT_IS_FPU_AVAILABLE) != 0;
723 #else
724 /* Available by default. */
725 return 1;
726 #endif
727
728 case SLJIT_SIMD_REGS_ARE_PAIRS:
729 case SLJIT_HAS_CLZ:
730 case SLJIT_HAS_CTZ:
731 case SLJIT_HAS_REV:
732 case SLJIT_HAS_ROT:
733 case SLJIT_HAS_CMOV:
734 case SLJIT_HAS_PREFETCH:
735 case SLJIT_HAS_COPY_F32:
736 case SLJIT_HAS_COPY_F64:
737 case SLJIT_HAS_ATOMIC:
738 case SLJIT_HAS_MEMORY_BARRIER:
739 return 1;
740
741 default:
742 return 0;
743 }
744 }
745
746 /* --------------------------------------------------------------------- */
747 /* Core code generator functions. */
748 /* --------------------------------------------------------------------- */
749
750 #define INVALID_IMM 0x80000000
751 static sljit_uw get_imm(sljit_uw imm)
752 {
753 /* Thumb immediate form. */
754 sljit_s32 counter;
755
756 if (imm <= 0xff)
757 return imm;
758
759 if ((imm & 0xffff) == (imm >> 16)) {
760 /* Some special cases. */
761 if (!(imm & 0xff00))
762 return (1 << 12) | (imm & 0xff);
763 if (!(imm & 0xff))
764 return (2 << 12) | ((imm >> 8) & 0xff);
765 if ((imm & 0xff00) == ((imm & 0xff) << 8))
766 return (3 << 12) | (imm & 0xff);
767 }
768
769 /* Assembly optimization: count leading zeroes? */
770 counter = 8;
771 if (!(imm & 0xffff0000)) {
772 counter += 16;
773 imm <<= 16;
774 }
775 if (!(imm & 0xff000000)) {
776 counter += 8;
777 imm <<= 8;
778 }
779 if (!(imm & 0xf0000000)) {
780 counter += 4;
781 imm <<= 4;
782 }
783 if (!(imm & 0xc0000000)) {
784 counter += 2;
785 imm <<= 2;
786 }
787 if (!(imm & 0x80000000)) {
788 counter += 1;
789 imm <<= 1;
790 }
791 /* Since imm >= 128, this must be true. */
792 SLJIT_ASSERT(counter <= 31);
793
794 if (imm & 0x00ffffff)
795 return INVALID_IMM; /* Cannot be encoded. */
796
797 return ((imm >> 24) & 0x7f) | COPY_BITS(counter, 4, 26, 1) | COPY_BITS(counter, 1, 12, 3) | COPY_BITS(counter, 0, 7, 1);
798 }
799
800 static sljit_s32 load_immediate(struct sljit_compiler *compiler, sljit_s32 dst, sljit_uw imm)
801 {
802 sljit_uw tmp;
803
804 /* MOVS cannot be used since it destroy flags. */
805
806 if (imm >= 0x10000) {
807 tmp = get_imm(imm);
808 if (tmp != INVALID_IMM)
809 return push_inst32(compiler, MOV_WI | RD4(dst) | tmp);
810 tmp = get_imm(~imm);
811 if (tmp != INVALID_IMM)
812 return push_inst32(compiler, MVN_WI | RD4(dst) | tmp);
813 }
814
815 /* set low 16 bits, set hi 16 bits to 0. */
816 FAIL_IF(push_inst32(compiler, MOVW | RD4(dst)
817 | COPY_BITS(imm, 12, 16, 4) | COPY_BITS(imm, 11, 26, 1) | COPY_BITS(imm, 8, 12, 3) | (imm & 0xff)));
818
819 /* set hi 16 bit if needed. */
820 if (imm >= 0x10000)
821 return push_inst32(compiler, MOVT | RD4(dst)
822 | COPY_BITS(imm, 12 + 16, 16, 4) | COPY_BITS(imm, 11 + 16, 26, 1) | COPY_BITS(imm, 8 + 16, 12, 3) | ((imm & 0xff0000) >> 16));
823 return SLJIT_SUCCESS;
824 }
825
826 #define ARG1_IMM 0x0010000
827 #define ARG2_IMM 0x0020000
828 /* SET_FLAGS must be 0x100000 as it is also the value of S bit (can be used for optimization). */
829 #define SET_FLAGS 0x0100000
830 #define UNUSED_RETURN 0x0200000
831 #define REGISTER_OP 0x0400000
832
833 static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 dst, sljit_uw arg1, sljit_uw arg2)
834 {
835 /* dst must be register
836 arg1 must be register, imm
837 arg2 must be register, imm */
838 sljit_s32 reg;
839 sljit_uw imm, imm2;
840
841 if (SLJIT_UNLIKELY((flags & (ARG1_IMM | ARG2_IMM)) == (ARG1_IMM | ARG2_IMM))) {
842 /* Both are immediates, no temporaries are used. */
843 flags &= ~ARG1_IMM;
844 FAIL_IF(load_immediate(compiler, TMP_REG1, arg1));
845 arg1 = TMP_REG1;
846 }
847
848 if (flags & (ARG1_IMM | ARG2_IMM)) {
849 reg = (sljit_s32)((flags & ARG2_IMM) ? arg1 : arg2);
850 imm = (flags & ARG2_IMM) ? arg2 : arg1;
851
852 switch (flags & 0xffff) {
853 case SLJIT_CLZ:
854 case SLJIT_CTZ:
855 case SLJIT_REV:
856 case SLJIT_REV_U16:
857 case SLJIT_REV_S16:
858 case SLJIT_REV_U32:
859 case SLJIT_REV_S32:
860 case SLJIT_MUL:
861 case SLJIT_MULADD:
862 /* No form with immediate operand. */
863 break;
864 case SLJIT_MOV:
865 SLJIT_ASSERT(!(flags & SET_FLAGS) && (flags & ARG2_IMM) && arg1 == TMP_REG2);
866 return load_immediate(compiler, dst, imm);
867 case SLJIT_ADD:
868 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
869 imm2 = NEGATE(imm);
870 if (IS_2_LO_REGS(reg, dst)) {
871 if (imm <= 0x7)
872 return push_inst16(compiler, ADDSI3 | IMM3(imm) | RD3(dst) | RN3(reg));
873 if (imm2 <= 0x7)
874 return push_inst16(compiler, SUBSI3 | IMM3(imm2) | RD3(dst) | RN3(reg));
875 if (reg == dst) {
876 if (imm <= 0xff)
877 return push_inst16(compiler, ADDSI8 | IMM8(imm) | RDN3(dst));
878 if (imm2 <= 0xff)
879 return push_inst16(compiler, SUBSI8 | IMM8(imm2) | RDN3(dst));
880 }
881 }
882 if (!(flags & SET_FLAGS)) {
883 if (imm <= 0xfff)
884 return push_inst32(compiler, ADDWI | RD4(dst) | RN4(reg) | IMM12(imm));
885 if (imm2 <= 0xfff)
886 return push_inst32(compiler, SUBWI | RD4(dst) | RN4(reg) | IMM12(imm2));
887 }
888 imm2 = get_imm(imm);
889 if (imm2 != INVALID_IMM)
890 return push_inst32(compiler, ADD_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm2);
891 imm = get_imm(NEGATE(imm));
892 if (imm != INVALID_IMM)
893 return push_inst32(compiler, SUB_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
894 break;
895 case SLJIT_ADDC:
896 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
897 imm2 = get_imm(imm);
898 if (imm2 != INVALID_IMM)
899 return push_inst32(compiler, ADCI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm2);
900 if (flags & ARG2_IMM) {
901 imm = get_imm(~imm);
902 if (imm != INVALID_IMM)
903 return push_inst32(compiler, SBCI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
904 }
905 break;
906 case SLJIT_SUB:
907 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
908 if (flags & ARG1_IMM) {
909 if (imm == 0 && IS_2_LO_REGS(reg, dst))
910 return push_inst16(compiler, RSBSI | RD3(dst) | RN3(reg));
911 imm = get_imm(imm);
912 if (imm != INVALID_IMM)
913 return push_inst32(compiler, RSB_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
914 break;
915 }
916 if (flags & UNUSED_RETURN) {
917 if (imm <= 0xff && reg_map[reg] <= 7)
918 return push_inst16(compiler, CMPI | IMM8(imm) | RDN3(reg));
919 imm2 = get_imm(imm);
920 if (imm2 != INVALID_IMM)
921 return push_inst32(compiler, CMPI_W | RN4(reg) | imm2);
922 imm = get_imm(NEGATE(imm));
923 if (imm != INVALID_IMM)
924 return push_inst32(compiler, CMNI_W | RN4(reg) | imm);
925 break;
926 }
927 imm2 = NEGATE(imm);
928 if (IS_2_LO_REGS(reg, dst)) {
929 if (imm <= 0x7)
930 return push_inst16(compiler, SUBSI3 | IMM3(imm) | RD3(dst) | RN3(reg));
931 if (imm2 <= 0x7)
932 return push_inst16(compiler, ADDSI3 | IMM3(imm2) | RD3(dst) | RN3(reg));
933 if (reg == dst) {
934 if (imm <= 0xff)
935 return push_inst16(compiler, SUBSI8 | IMM8(imm) | RDN3(dst));
936 if (imm2 <= 0xff)
937 return push_inst16(compiler, ADDSI8 | IMM8(imm2) | RDN3(dst));
938 }
939 }
940 if (!(flags & SET_FLAGS)) {
941 if (imm <= 0xfff)
942 return push_inst32(compiler, SUBWI | RD4(dst) | RN4(reg) | IMM12(imm));
943 if (imm2 <= 0xfff)
944 return push_inst32(compiler, ADDWI | RD4(dst) | RN4(reg) | IMM12(imm2));
945 }
946 imm2 = get_imm(imm);
947 if (imm2 != INVALID_IMM)
948 return push_inst32(compiler, SUB_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm2);
949 imm = get_imm(NEGATE(imm));
950 if (imm != INVALID_IMM)
951 return push_inst32(compiler, ADD_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
952 break;
953 case SLJIT_SUBC:
954 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
955 if (flags & ARG1_IMM)
956 break;
957 imm2 = get_imm(imm);
958 if (imm2 != INVALID_IMM)
959 return push_inst32(compiler, SBCI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm2);
960 imm = get_imm(~imm);
961 if (imm != INVALID_IMM)
962 return push_inst32(compiler, ADCI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
963 break;
964 case SLJIT_AND:
965 imm2 = get_imm(imm);
966 if (imm2 != INVALID_IMM)
967 return push_inst32(compiler, ((flags & UNUSED_RETURN) ? TSTI : ANDI) | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm2);
968 imm = get_imm(~imm);
969 if (imm != INVALID_IMM)
970 return push_inst32(compiler, BICI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
971 break;
972 case SLJIT_OR:
973 imm2 = get_imm(imm);
974 if (imm2 != INVALID_IMM)
975 return push_inst32(compiler, ORRI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm2);
976 imm = get_imm(~imm);
977 if (imm != INVALID_IMM)
978 return push_inst32(compiler, ORNI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
979 break;
980 case SLJIT_XOR:
981 if (imm == (sljit_uw)-1) {
982 if (IS_2_LO_REGS(dst, reg))
983 return push_inst16(compiler, MVNS | RD3(dst) | RN3(reg));
984 return push_inst32(compiler, MVN_W | (flags & SET_FLAGS) | RD4(dst) | RM4(reg));
985 }
986 imm = get_imm(imm);
987 if (imm != INVALID_IMM)
988 return push_inst32(compiler, EORI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
989 break;
990 case SLJIT_SHL:
991 case SLJIT_MSHL:
992 case SLJIT_LSHR:
993 case SLJIT_MLSHR:
994 case SLJIT_ASHR:
995 case SLJIT_MASHR:
996 case SLJIT_ROTL:
997 case SLJIT_ROTR:
998 if (flags & ARG1_IMM)
999 break;
1000 imm &= 0x1f;
1001
1002 if (imm == 0) {
1003 if (!(flags & SET_FLAGS))
1004 return push_inst16(compiler, MOV | SET_REGS44(dst, reg));
1005 if (IS_2_LO_REGS(dst, reg))
1006 return push_inst16(compiler, MOVS | RD3(dst) | RN3(reg));
1007 return push_inst32(compiler, MOV_W | SET_FLAGS | RD4(dst) | RM4(reg));
1008 }
1009
1010 switch (flags & 0xffff) {
1011 case SLJIT_SHL:
1012 case SLJIT_MSHL:
1013 if (IS_2_LO_REGS(dst, reg))
1014 return push_inst16(compiler, LSLSI | RD3(dst) | RN3(reg) | (imm << 6));
1015 return push_inst32(compiler, LSL_WI | (flags & SET_FLAGS) | RD4(dst) | RM4(reg) | IMM5(imm));
1016 case SLJIT_LSHR:
1017 case SLJIT_MLSHR:
1018 if (IS_2_LO_REGS(dst, reg))
1019 return push_inst16(compiler, LSRSI | RD3(dst) | RN3(reg) | (imm << 6));
1020 return push_inst32(compiler, LSR_WI | (flags & SET_FLAGS) | RD4(dst) | RM4(reg) | IMM5(imm));
1021 case SLJIT_ASHR:
1022 case SLJIT_MASHR:
1023 if (IS_2_LO_REGS(dst, reg))
1024 return push_inst16(compiler, ASRSI | RD3(dst) | RN3(reg) | (imm << 6));
1025 return push_inst32(compiler, ASR_WI | (flags & SET_FLAGS) | RD4(dst) | RM4(reg) | IMM5(imm));
1026 case SLJIT_ROTL:
1027 imm = (imm ^ 0x1f) + 1;
1028 /* fallthrough */
1029 default: /* SLJIT_ROTR */
1030 return push_inst32(compiler, ROR_WI | RD4(dst) | RM4(reg) | IMM5(imm));
1031 }
1032 default:
1033 SLJIT_UNREACHABLE();
1034 break;
1035 }
1036
1037 if (flags & ARG2_IMM) {
1038 imm = arg2;
1039 arg2 = (arg1 == TMP_REG1) ? TMP_REG2 : TMP_REG1;
1040 FAIL_IF(load_immediate(compiler, (sljit_s32)arg2, imm));
1041 } else {
1042 imm = arg1;
1043 arg1 = (arg2 == TMP_REG1) ? TMP_REG2 : TMP_REG1;
1044 FAIL_IF(load_immediate(compiler, (sljit_s32)arg1, imm));
1045 }
1046
1047 SLJIT_ASSERT(arg1 != arg2);
1048 }
1049
1050 /* Both arguments are registers. */
1051 switch (flags & 0xffff) {
1052 case SLJIT_MOV:
1053 case SLJIT_MOV_U32:
1054 case SLJIT_MOV_S32:
1055 case SLJIT_MOV32:
1056 case SLJIT_MOV_P:
1057 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG2);
1058 if (dst == (sljit_s32)arg2)
1059 return SLJIT_SUCCESS;
1060 return push_inst16(compiler, MOV | SET_REGS44(dst, arg2));
1061 case SLJIT_MOV_U8:
1062 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG2);
1063 if (IS_2_LO_REGS(dst, arg2))
1064 return push_inst16(compiler, UXTB | RD3(dst) | RN3(arg2));
1065 return push_inst32(compiler, UXTB_W | RD4(dst) | RM4(arg2));
1066 case SLJIT_MOV_S8:
1067 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG2);
1068 if (IS_2_LO_REGS(dst, arg2))
1069 return push_inst16(compiler, SXTB | RD3(dst) | RN3(arg2));
1070 return push_inst32(compiler, SXTB_W | RD4(dst) | RM4(arg2));
1071 case SLJIT_MOV_U16:
1072 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG2);
1073 if (IS_2_LO_REGS(dst, arg2))
1074 return push_inst16(compiler, UXTH | RD3(dst) | RN3(arg2));
1075 return push_inst32(compiler, UXTH_W | RD4(dst) | RM4(arg2));
1076 case SLJIT_MOV_S16:
1077 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG2);
1078 if (IS_2_LO_REGS(dst, arg2))
1079 return push_inst16(compiler, SXTH | RD3(dst) | RN3(arg2));
1080 return push_inst32(compiler, SXTH_W | RD4(dst) | RM4(arg2));
1081 case SLJIT_CLZ:
1082 SLJIT_ASSERT(arg1 == TMP_REG2);
1083 return push_inst32(compiler, CLZ | RN4(arg2) | RD4(dst) | RM4(arg2));
1084 case SLJIT_CTZ:
1085 SLJIT_ASSERT(arg1 == TMP_REG2);
1086 FAIL_IF(push_inst32(compiler, RBIT | RN4(arg2) | RD4(dst) | RM4(arg2)));
1087 return push_inst32(compiler, CLZ | RN4(dst) | RD4(dst) | RM4(dst));
1088 case SLJIT_REV:
1089 case SLJIT_REV_U32:
1090 case SLJIT_REV_S32:
1091 SLJIT_ASSERT(arg1 == TMP_REG2);
1092 if (IS_2_LO_REGS(dst, arg2))
1093 return push_inst16(compiler, REV | RD3(dst) | RN3(arg2));
1094 return push_inst32(compiler, REV_W | RN4(arg2) | RD4(dst) | RM4(arg2));
1095 case SLJIT_REV_U16:
1096 case SLJIT_REV_S16:
1097 SLJIT_ASSERT(arg1 == TMP_REG2);
1098
1099 if (IS_2_LO_REGS(dst, arg2))
1100 FAIL_IF(push_inst16(compiler, REV16 | RD3(dst) | RN3(arg2)));
1101 else
1102 FAIL_IF(push_inst32(compiler, REV16_W | RN4(arg2) | RD4(dst) | RM4(arg2)));
1103
1104 if (!(flags & REGISTER_OP))
1105 return SLJIT_SUCCESS;
1106
1107 flags &= 0xffff;
1108 if (reg_map[dst] <= 7)
1109 return push_inst16(compiler, (flags == SLJIT_REV_U16 ? UXTH : SXTH) | RD3(dst) | RN3(dst));
1110 return push_inst32(compiler, (flags == SLJIT_REV_U16 ? UXTH_W : SXTH_W) | RD4(dst) | RM4(dst));
1111 case SLJIT_ADD:
1112 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1113 if (IS_3_LO_REGS(dst, arg1, arg2))
1114 return push_inst16(compiler, ADDS | RD3(dst) | RN3(arg1) | RM3(arg2));
1115 if (dst == (sljit_s32)arg1 && !(flags & SET_FLAGS))
1116 return push_inst16(compiler, ADD | SET_REGS44(dst, arg2));
1117 return push_inst32(compiler, ADD_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
1118 case SLJIT_ADDC:
1119 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1120 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
1121 return push_inst16(compiler, ADCS | RD3(dst) | RN3(arg2));
1122 return push_inst32(compiler, ADC_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
1123 case SLJIT_SUB:
1124 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1125 if (flags & UNUSED_RETURN) {
1126 if (IS_2_LO_REGS(arg1, arg2))
1127 return push_inst16(compiler, CMP | RD3(arg1) | RN3(arg2));
1128 return push_inst16(compiler, CMP_X | SET_REGS44(arg1, arg2));
1129 }
1130 if (IS_3_LO_REGS(dst, arg1, arg2))
1131 return push_inst16(compiler, SUBS | RD3(dst) | RN3(arg1) | RM3(arg2));
1132 return push_inst32(compiler, SUB_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
1133 case SLJIT_SUBC:
1134 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1135 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
1136 return push_inst16(compiler, SBCS | RD3(dst) | RN3(arg2));
1137 return push_inst32(compiler, SBC_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
1138 case SLJIT_MUL:
1139 compiler->status_flags_state = 0;
1140 if (!(flags & SET_FLAGS))
1141 return push_inst32(compiler, MUL | RD4(dst) | RN4(arg1) | RM4(arg2));
1142 reg = (dst == TMP_REG2) ? TMP_REG1 : TMP_REG2;
1143 FAIL_IF(push_inst32(compiler, SMULL | RT4(dst) | RD4(reg) | RN4(arg1) | RM4(arg2)));
1144 /* cmp TMP_REG2, dst asr #31. */
1145 return push_inst32(compiler, CMP_W | RN4(reg) | 0x70e0 | RM4(dst));
1146 case SLJIT_AND:
1147 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
1148 return push_inst16(compiler, ANDS | RD3(dst) | RN3(arg2));
1149 if ((flags & UNUSED_RETURN) && IS_2_LO_REGS(arg1, arg2))
1150 return push_inst16(compiler, TST | RD3(arg1) | RN3(arg2));
1151 return push_inst32(compiler, ((flags & UNUSED_RETURN) ? TST_W : AND_W) | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
1152 case SLJIT_OR:
1153 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
1154 return push_inst16(compiler, ORRS | RD3(dst) | RN3(arg2));
1155 return push_inst32(compiler, ORR_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
1156 case SLJIT_XOR:
1157 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
1158 return push_inst16(compiler, EORS | RD3(dst) | RN3(arg2));
1159 return push_inst32(compiler, EOR_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
1160 case SLJIT_MSHL:
1161 reg = (arg2 == TMP_REG1) ? TMP_REG1 : TMP_REG2;
1162 FAIL_IF(push_inst32(compiler, ANDI | RD4(reg) | RN4(arg2) | 0x1f));
1163 arg2 = (sljit_uw)reg;
1164 /* fallthrough */
1165 case SLJIT_SHL:
1166 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
1167 return push_inst16(compiler, LSLS | RD3(dst) | RN3(arg2));
1168 return push_inst32(compiler, LSL_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
1169 case SLJIT_MLSHR:
1170 reg = (arg2 == TMP_REG1) ? TMP_REG1 : TMP_REG2;
1171 FAIL_IF(push_inst32(compiler, ANDI | RD4(reg) | RN4(arg2) | 0x1f));
1172 arg2 = (sljit_uw)reg;
1173 /* fallthrough */
1174 case SLJIT_LSHR:
1175 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
1176 return push_inst16(compiler, LSRS | RD3(dst) | RN3(arg2));
1177 return push_inst32(compiler, LSR_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
1178 case SLJIT_MASHR:
1179 reg = (arg2 == TMP_REG1) ? TMP_REG1 : TMP_REG2;
1180 FAIL_IF(push_inst32(compiler, ANDI | RD4(reg) | RN4(arg2) | 0x1f));
1181 arg2 = (sljit_uw)reg;
1182 /* fallthrough */
1183 case SLJIT_ASHR:
1184 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
1185 return push_inst16(compiler, ASRS | RD3(dst) | RN3(arg2));
1186 return push_inst32(compiler, ASR_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
1187 case SLJIT_ROTL:
1188 reg = (arg2 == TMP_REG1) ? TMP_REG1 : TMP_REG2;
1189 FAIL_IF(push_inst32(compiler, RSB_WI | RD4(reg) | RN4(arg2) | 0));
1190 arg2 = (sljit_uw)reg;
1191 /* fallthrough */
1192 case SLJIT_ROTR:
1193 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
1194 return push_inst16(compiler, RORS | RD3(dst) | RN3(arg2));
1195 return push_inst32(compiler, ROR_W | RD4(dst) | RN4(arg1) | RM4(arg2));
1196 case SLJIT_MULADD:
1197 compiler->status_flags_state = 0;
1198 return push_inst32(compiler, MLA | RD4(dst) | RN4(arg1) | RM4(arg2) | RT4(dst));
1199 }
1200
1201 SLJIT_UNREACHABLE();
1202 return SLJIT_SUCCESS;
1203 }
1204
1205 #define STORE 0x01
1206 #define SIGNED 0x02
1207
1208 #define WORD_SIZE 0x00
1209 #define BYTE_SIZE 0x04
1210 #define HALF_SIZE 0x08
1211 #define PRELOAD 0x0c
1212
1213 #define IS_WORD_SIZE(flags) (!((flags) & (BYTE_SIZE | HALF_SIZE)))
1214 #define ALIGN_CHECK(argw, imm, shift) (!((argw) & ~((imm) << (shift))))
1215
1216 /*
1217 1st letter:
1218 w = word
1219 b = byte
1220 h = half
1221
1222 2nd letter:
1223 s = signed
1224 u = unsigned
1225
1226 3rd letter:
1227 l = load
1228 s = store
1229 */
1230
1231 static const sljit_ins sljit_mem16[12] = {
1232 /* w u l */ 0x5800 /* ldr */,
1233 /* w u s */ 0x5000 /* str */,
1234 /* w s l */ 0x5800 /* ldr */,
1235 /* w s s */ 0x5000 /* str */,
1236
1237 /* b u l */ 0x5c00 /* ldrb */,
1238 /* b u s */ 0x5400 /* strb */,
1239 /* b s l */ 0x5600 /* ldrsb */,
1240 /* b s s */ 0x5400 /* strb */,
1241
1242 /* h u l */ 0x5a00 /* ldrh */,
1243 /* h u s */ 0x5200 /* strh */,
1244 /* h s l */ 0x5e00 /* ldrsh */,
1245 /* h s s */ 0x5200 /* strh */,
1246 };
1247
1248 static const sljit_ins sljit_mem16_imm5[12] = {
1249 /* w u l */ 0x6800 /* ldr imm5 */,
1250 /* w u s */ 0x6000 /* str imm5 */,
1251 /* w s l */ 0x6800 /* ldr imm5 */,
1252 /* w s s */ 0x6000 /* str imm5 */,
1253
1254 /* b u l */ 0x7800 /* ldrb imm5 */,
1255 /* b u s */ 0x7000 /* strb imm5 */,
1256 /* b s l */ 0x0000 /* not allowed */,
1257 /* b s s */ 0x7000 /* strb imm5 */,
1258
1259 /* h u l */ 0x8800 /* ldrh imm5 */,
1260 /* h u s */ 0x8000 /* strh imm5 */,
1261 /* h s l */ 0x0000 /* not allowed */,
1262 /* h s s */ 0x8000 /* strh imm5 */,
1263 };
1264
1265 #define MEM_IMM8 0xc00
1266 #define MEM_IMM12 0x800000
1267 static const sljit_ins sljit_mem32[13] = {
1268 /* w u l */ 0xf8500000 /* ldr.w */,
1269 /* w u s */ 0xf8400000 /* str.w */,
1270 /* w s l */ 0xf8500000 /* ldr.w */,
1271 /* w s s */ 0xf8400000 /* str.w */,
1272
1273 /* b u l */ 0xf8100000 /* ldrb.w */,
1274 /* b u s */ 0xf8000000 /* strb.w */,
1275 /* b s l */ 0xf9100000 /* ldrsb.w */,
1276 /* b s s */ 0xf8000000 /* strb.w */,
1277
1278 /* h u l */ 0xf8300000 /* ldrh.w */,
1279 /* h u s */ 0xf8200000 /* strsh.w */,
1280 /* h s l */ 0xf9300000 /* ldrsh.w */,
1281 /* h s s */ 0xf8200000 /* strsh.w */,
1282
1283 /* p u l */ 0xf8100000 /* pld */,
1284 };
1285
1286 /* Helper function. Dst should be reg + value, using at most 1 instruction, flags does not set. */
1287 static sljit_s32 emit_set_delta(struct sljit_compiler *compiler, sljit_s32 dst, sljit_s32 reg, sljit_sw value)
1288 {
1289 sljit_uw imm;
1290
1291 if (value >= 0) {
1292 if (value <= 0xfff)
1293 return push_inst32(compiler, ADDWI | RD4(dst) | RN4(reg) | IMM12(value));
1294 imm = get_imm((sljit_uw)value);
1295 if (imm != INVALID_IMM)
1296 return push_inst32(compiler, ADD_WI | RD4(dst) | RN4(reg) | imm);
1297 }
1298 else {
1299 value = -value;
1300 if (value <= 0xfff)
1301 return push_inst32(compiler, SUBWI | RD4(dst) | RN4(reg) | IMM12(value));
1302 imm = get_imm((sljit_uw)value);
1303 if (imm != INVALID_IMM)
1304 return push_inst32(compiler, SUB_WI | RD4(dst) | RN4(reg) | imm);
1305 }
1306 return SLJIT_ERR_UNSUPPORTED;
1307 }
1308
1309 static SLJIT_INLINE sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg,
1310 sljit_s32 arg, sljit_sw argw, sljit_s32 tmp_reg)
1311 {
1312 sljit_s32 other_r;
1313 sljit_uw imm, tmp;
1314
1315 SLJIT_ASSERT(arg & SLJIT_MEM);
1316 SLJIT_ASSERT((arg & REG_MASK) != tmp_reg || (arg == SLJIT_MEM1(tmp_reg) && argw >= -0xff && argw <= 0xfff));
1317
1318 if (SLJIT_UNLIKELY(!(arg & REG_MASK))) {
1319 imm = get_imm((sljit_uw)argw & ~(sljit_uw)0xfff);
1320 if (imm != INVALID_IMM) {
1321 FAIL_IF(push_inst32(compiler, MOV_WI | RD4(tmp_reg) | imm));
1322 return push_inst32(compiler, sljit_mem32[flags] | MEM_IMM12 | RT4(reg) | RN4(tmp_reg) | (argw & 0xfff));
1323 }
1324
1325 FAIL_IF(load_immediate(compiler, tmp_reg, (sljit_uw)argw));
1326 if (IS_2_LO_REGS(reg, tmp_reg) && sljit_mem16_imm5[flags])
1327 return push_inst16(compiler, sljit_mem16_imm5[flags] | RD3(reg) | RN3(tmp_reg));
1328 return push_inst32(compiler, sljit_mem32[flags] | MEM_IMM12 | RT4(reg) | RN4(tmp_reg));
1329 }
1330
1331 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1332 argw &= 0x3;
1333 other_r = OFFS_REG(arg);
1334 arg &= REG_MASK;
1335
1336 if (!argw && IS_3_LO_REGS(reg, arg, other_r))
1337 return push_inst16(compiler, sljit_mem16[flags] | RD3(reg) | RN3(arg) | RM3(other_r));
1338 return push_inst32(compiler, sljit_mem32[flags] | RT4(reg) | RN4(arg) | RM4(other_r) | ((sljit_ins)argw << 4));
1339 }
1340
1341 arg &= REG_MASK;
1342
1343 if (argw > 0xfff) {
1344 imm = get_imm((sljit_uw)(argw & ~0xfff));
1345 if (imm != INVALID_IMM) {
1346 push_inst32(compiler, ADD_WI | RD4(tmp_reg) | RN4(arg) | imm);
1347 arg = tmp_reg;
1348 argw = argw & 0xfff;
1349 }
1350 }
1351 else if (argw < -0xff) {
1352 tmp = (sljit_uw)((-argw + 0xfff) & ~0xfff);
1353 SLJIT_ASSERT(tmp >= (sljit_uw)-argw);
1354 imm = get_imm(tmp);
1355
1356 if (imm != INVALID_IMM) {
1357 push_inst32(compiler, SUB_WI | RD4(tmp_reg) | RN4(arg) | imm);
1358 arg = tmp_reg;
1359 argw += (sljit_sw)tmp;
1360
1361 SLJIT_ASSERT(argw >= 0 && argw <= 0xfff);
1362 }
1363 }
1364
1365 /* 16 bit instruction forms. */
1366 if (IS_2_LO_REGS(reg, arg) && sljit_mem16_imm5[flags]) {
1367 tmp = 3;
1368 if (IS_WORD_SIZE(flags)) {
1369 if (ALIGN_CHECK(argw, 0x1f, 2))
1370 tmp = 2;
1371 }
1372 else if (flags & BYTE_SIZE)
1373 {
1374 if (ALIGN_CHECK(argw, 0x1f, 0))
1375 tmp = 0;
1376 }
1377 else {
1378 SLJIT_ASSERT(flags & HALF_SIZE);
1379 if (ALIGN_CHECK(argw, 0x1f, 1))
1380 tmp = 1;
1381 }
1382
1383 if (tmp < 3)
1384 return push_inst16(compiler, sljit_mem16_imm5[flags] | RD3(reg) | RN3(arg) | ((sljit_ins)argw << (6 - tmp)));
1385 }
1386 else if (SLJIT_UNLIKELY(arg == SLJIT_SP) && IS_WORD_SIZE(flags) && ALIGN_CHECK(argw, 0xff, 2) && reg_map[reg] <= 7) {
1387 /* SP based immediate. */
1388 return push_inst16(compiler, STR_SP | (sljit_ins)((flags & STORE) ? 0 : 0x800) | RDN3(reg) | ((sljit_ins)argw >> 2));
1389 }
1390
1391 if (argw >= 0 && argw <= 0xfff)
1392 return push_inst32(compiler, sljit_mem32[flags] | MEM_IMM12 | RT4(reg) | RN4(arg) | (sljit_ins)argw);
1393 else if (argw < 0 && argw >= -0xff)
1394 return push_inst32(compiler, sljit_mem32[flags] | MEM_IMM8 | RT4(reg) | RN4(arg) | (sljit_ins)-argw);
1395
1396 SLJIT_ASSERT(arg != tmp_reg);
1397
1398 FAIL_IF(load_immediate(compiler, tmp_reg, (sljit_uw)argw));
1399 if (IS_3_LO_REGS(reg, arg, tmp_reg))
1400 return push_inst16(compiler, sljit_mem16[flags] | RD3(reg) | RN3(arg) | RM3(tmp_reg));
1401 return push_inst32(compiler, sljit_mem32[flags] | RT4(reg) | RN4(arg) | RM4(tmp_reg));
1402 }
1403
1404 #undef ALIGN_CHECK
1405 #undef IS_WORD_SIZE
1406
1407 /* --------------------------------------------------------------------- */
1408 /* Entry, exit */
1409 /* --------------------------------------------------------------------- */
1410
1411 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
1412 sljit_s32 options, sljit_s32 arg_types,
1413 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1414 {
1415 sljit_s32 fscratches;
1416 sljit_s32 fsaveds;
1417 sljit_s32 size, i, tmp, word_arg_count;
1418 sljit_s32 saved_arg_count = SLJIT_KEPT_SAVEDS_COUNT(options);
1419 sljit_uw offset;
1420 sljit_uw imm = 0;
1421 #ifdef __SOFTFP__
1422 sljit_u32 float_arg_count;
1423 #else
1424 sljit_u32 old_offset, f32_offset;
1425 sljit_u32 remap[3];
1426 sljit_u32 *remap_ptr = remap;
1427 #endif
1428
1429 CHECK_ERROR();
1430 CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, local_size));
1431 set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
1432
1433 scratches = ENTER_GET_REGS(scratches);
1434 saveds = ENTER_GET_REGS(saveds);
1435 fscratches = compiler->fscratches;
1436 fsaveds = compiler->fsaveds;
1437
1438 tmp = SLJIT_S0 - saveds;
1439 for (i = SLJIT_S0 - saved_arg_count; i > tmp; i--)
1440 imm |= (sljit_uw)1 << reg_map[i];
1441
1442 for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--)
1443 imm |= (sljit_uw)1 << reg_map[i];
1444
1445 /* At least two registers must be set for PUSH_W and one for PUSH instruction. */
1446 FAIL_IF((imm & 0xff00)
1447 ? push_inst32(compiler, PUSH_W | (1 << 14) | imm)
1448 : push_inst16(compiler, PUSH | (1 << 8) | imm));
1449
1450 /* Stack must be aligned to 8 bytes: (LR, R4) */
1451 size = GET_SAVED_REGISTERS_SIZE(scratches, saveds - saved_arg_count, 1);
1452
1453 if (fsaveds > 0 || fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG) {
1454 if ((size & SSIZE_OF(sw)) != 0) {
1455 FAIL_IF(push_inst16(compiler, SUB_SP_I | (sizeof(sljit_sw) >> 2)));
1456 size += SSIZE_OF(sw);
1457 }
1458
1459 if (fsaveds + fscratches >= SLJIT_NUMBER_OF_FLOAT_REGISTERS) {
1460 FAIL_IF(push_inst32(compiler, VPUSH | VD4(SLJIT_FS0) | ((sljit_uw)SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS << 1)));
1461 } else {
1462 if (fsaveds > 0)
1463 FAIL_IF(push_inst32(compiler, VPUSH | VD4(SLJIT_FS0) | ((sljit_uw)fsaveds << 1)));
1464 if (fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG)
1465 FAIL_IF(push_inst32(compiler, VPUSH | VD4(fscratches) | ((sljit_uw)(fscratches - (SLJIT_FIRST_SAVED_FLOAT_REG - 1)) << 1)));
1466 }
1467 }
1468
1469 local_size = ((size + local_size + 0x7) & ~0x7) - size;
1470 compiler->local_size = local_size;
1471
1472 if (options & SLJIT_ENTER_REG_ARG)
1473 arg_types = 0;
1474
1475 arg_types >>= SLJIT_ARG_SHIFT;
1476 word_arg_count = 0;
1477 saved_arg_count = 0;
1478 #ifdef __SOFTFP__
1479 SLJIT_COMPILE_ASSERT(SLJIT_FR0 == 1, float_register_index_start);
1480
1481 offset = 0;
1482 float_arg_count = 0;
1483
1484 while (arg_types) {
1485 switch (arg_types & SLJIT_ARG_MASK) {
1486 case SLJIT_ARG_TYPE_F64:
1487 if (offset & 0x7)
1488 offset += sizeof(sljit_sw);
1489
1490 if (offset < 4 * sizeof(sljit_sw))
1491 FAIL_IF(push_inst32(compiler, VMOV2 | (offset << 10) | ((offset + sizeof(sljit_sw)) << 14) | float_arg_count));
1492 else
1493 FAIL_IF(push_inst32(compiler, VLDR_F32 | 0x800100 | RN4(SLJIT_SP)
1494 | (float_arg_count << 12) | ((offset + (sljit_uw)size - 4 * sizeof(sljit_sw)) >> 2)));
1495 float_arg_count++;
1496 offset += sizeof(sljit_f64) - sizeof(sljit_sw);
1497 break;
1498 case SLJIT_ARG_TYPE_F32:
1499 if (offset < 4 * sizeof(sljit_sw))
1500 FAIL_IF(push_inst32(compiler, VMOV | (float_arg_count << 16) | (offset << 10)));
1501 else
1502 FAIL_IF(push_inst32(compiler, VLDR_F32 | 0x800000 | RN4(SLJIT_SP)
1503 | (float_arg_count << 12) | ((offset + (sljit_uw)size - 4 * sizeof(sljit_sw)) >> 2)));
1504 float_arg_count++;
1505 break;
1506 default:
1507 word_arg_count++;
1508
1509 if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
1510 tmp = SLJIT_S0 - saved_arg_count;
1511 saved_arg_count++;
1512 } else if (word_arg_count - 1 != (sljit_s32)(offset >> 2))
1513 tmp = word_arg_count;
1514 else
1515 break;
1516
1517 if (offset < 4 * sizeof(sljit_sw))
1518 FAIL_IF(push_inst16(compiler, MOV | ((sljit_ins)reg_map[tmp] & 0x7) | (((sljit_ins)reg_map[tmp] & 0x8) << 4) | (offset << 1)));
1519 else if (reg_map[tmp] <= 7)
1520 FAIL_IF(push_inst16(compiler, LDR_SP | RDN3(tmp)
1521 | ((offset + (sljit_uw)size - 4 * sizeof(sljit_sw)) >> 2)));
1522 else
1523 FAIL_IF(push_inst32(compiler, LDR | RT4(tmp) | RN4(SLJIT_SP)
1524 | ((offset + (sljit_uw)size - 4 * sizeof(sljit_sw)))));
1525 break;
1526 }
1527
1528 offset += sizeof(sljit_sw);
1529 arg_types >>= SLJIT_ARG_SHIFT;
1530 }
1531
1532 compiler->args_size = offset;
1533 #else
1534 offset = SLJIT_FR0;
1535 old_offset = SLJIT_FR0;
1536 f32_offset = 0;
1537
1538 while (arg_types) {
1539 switch (arg_types & SLJIT_ARG_MASK) {
1540 case SLJIT_ARG_TYPE_F64:
1541 if (offset != old_offset)
1542 *remap_ptr++ = VMOV_F32 | SLJIT_32 | VD4(offset) | VM4(old_offset);
1543 old_offset++;
1544 offset++;
1545 break;
1546 case SLJIT_ARG_TYPE_F32:
1547 if (f32_offset != 0) {
1548 *remap_ptr++ = VMOV_F32 | 0x20 | VD4(offset) | VM4(f32_offset);
1549 f32_offset = 0;
1550 } else {
1551 if (offset != old_offset)
1552 *remap_ptr++ = VMOV_F32 | VD4(offset) | VM4(old_offset);
1553 f32_offset = old_offset;
1554 old_offset++;
1555 }
1556 offset++;
1557 break;
1558 default:
1559 if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
1560 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(SLJIT_S0 - saved_arg_count, SLJIT_R0 + word_arg_count)));
1561 saved_arg_count++;
1562 }
1563
1564 word_arg_count++;
1565 break;
1566 }
1567 arg_types >>= SLJIT_ARG_SHIFT;
1568 }
1569
1570 SLJIT_ASSERT((sljit_uw)(remap_ptr - remap) <= sizeof(remap));
1571
1572 while (remap_ptr > remap)
1573 FAIL_IF(push_inst32(compiler, *(--remap_ptr)));
1574 #endif
1575
1576 #ifdef _WIN32
1577 if (local_size >= 4096) {
1578 imm = get_imm(4096);
1579 SLJIT_ASSERT(imm != INVALID_IMM);
1580
1581 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | imm));
1582
1583 if (local_size < 4 * 4096) {
1584 if (local_size > 2 * 4096) {
1585 if (local_size > 3 * 4096) {
1586 FAIL_IF(push_inst32(compiler, LDRI | 0x400 | RT4(TMP_REG1) | RN4(SLJIT_SP)));
1587 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | imm));
1588 }
1589
1590 FAIL_IF(push_inst32(compiler, LDRI | 0x400 | RT4(TMP_REG1) | RN4(SLJIT_SP)));
1591 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | imm));
1592 }
1593 } else {
1594 FAIL_IF(load_immediate(compiler, TMP_REG2, ((sljit_uw)local_size >> 12) - 1));
1595 FAIL_IF(push_inst32(compiler, LDRI | 0x400 | RT4(TMP_REG1) | RN4(SLJIT_SP)));
1596 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | imm));
1597 FAIL_IF(push_inst32(compiler, SUB_WI | SET_FLAGS | RD4(TMP_REG2) | RN4(TMP_REG2) | 1));
1598 FAIL_IF(push_inst16(compiler, BCC | (0x1 << 8) /* not-equal */ | (-8 & 0xff)));
1599 }
1600
1601 FAIL_IF(push_inst32(compiler, LDRI | 0x400 | RT4(TMP_REG1) | RN4(SLJIT_SP)));
1602 local_size &= 0xfff;
1603 }
1604
1605 if (local_size >= 256) {
1606 SLJIT_ASSERT(local_size < 4096);
1607
1608 if (local_size <= (127 << 2))
1609 FAIL_IF(push_inst16(compiler, SUB_SP_I | ((sljit_uw)local_size >> 2)));
1610 else
1611 FAIL_IF(emit_op_imm(compiler, SLJIT_SUB | ARG2_IMM, SLJIT_SP, SLJIT_SP, (sljit_uw)local_size));
1612
1613 FAIL_IF(push_inst32(compiler, LDRI | 0x400 | RT4(TMP_REG1) | RN4(SLJIT_SP)));
1614 } else if (local_size > 0)
1615 FAIL_IF(push_inst32(compiler, LDRI | 0x500 | RT4(TMP_REG1) | RN4(SLJIT_SP) | (sljit_uw)local_size));
1616 #else /* !_WIN32 */
1617 if (local_size > 0) {
1618 if (local_size <= (127 << 2))
1619 FAIL_IF(push_inst16(compiler, SUB_SP_I | ((sljit_uw)local_size >> 2)));
1620 else
1621 FAIL_IF(emit_op_imm(compiler, SLJIT_SUB | ARG2_IMM, SLJIT_SP, SLJIT_SP, (sljit_uw)local_size));
1622 }
1623 #endif /* _WIN32 */
1624
1625 return SLJIT_SUCCESS;
1626 }
1627
1628 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
1629 sljit_s32 options, sljit_s32 arg_types,
1630 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1631 {
1632 sljit_s32 fscratches;
1633 sljit_s32 fsaveds;
1634 sljit_s32 size;
1635
1636 CHECK_ERROR();
1637 CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, local_size));
1638 set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
1639
1640 scratches = ENTER_GET_REGS(scratches);
1641 saveds = ENTER_GET_REGS(saveds);
1642 fscratches = compiler->fscratches;
1643 fsaveds = compiler->fsaveds;
1644 size = GET_SAVED_REGISTERS_SIZE(scratches, saveds - SLJIT_KEPT_SAVEDS_COUNT(options), 1);
1645
1646 /* Doubles are saved, so alignment is unaffected. */
1647 if ((size & SSIZE_OF(sw)) != 0 && (fsaveds > 0 || fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG))
1648 size += SSIZE_OF(sw);
1649
1650 compiler->local_size = ((size + local_size + 0x7) & ~0x7) - size;
1651 return SLJIT_SUCCESS;
1652 }
1653
1654 static sljit_s32 emit_add_sp(struct sljit_compiler *compiler, sljit_uw imm)
1655 {
1656 sljit_uw imm2;
1657
1658 /* The TMP_REG1 register must keep its value. */
1659 if (imm <= (127u << 2))
1660 return push_inst16(compiler, ADD_SP_I | (imm >> 2));
1661
1662 if (imm <= 0xfff)
1663 return push_inst32(compiler, ADDWI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | IMM12(imm));
1664
1665 imm2 = get_imm(imm);
1666
1667 if (imm2 != INVALID_IMM)
1668 return push_inst32(compiler, ADD_WI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | imm2);
1669
1670 FAIL_IF(load_immediate(compiler, TMP_REG2, imm));
1671 return push_inst16(compiler, ADD_SP | RN3(TMP_REG2));
1672 }
1673
1674 static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler, sljit_s32 frame_size)
1675 {
1676 sljit_s32 local_size, fscratches, fsaveds, i, tmp;
1677 sljit_s32 restored_reg = 0;
1678 sljit_s32 lr_dst = TMP_PC;
1679 sljit_uw reg_list = 0;
1680
1681 SLJIT_ASSERT(reg_map[TMP_REG2] == 14 && frame_size <= 128);
1682
1683 local_size = compiler->local_size;
1684 fscratches = compiler->fscratches;
1685 fsaveds = compiler->fsaveds;
1686
1687 if (fsaveds > 0 || fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG) {
1688 if (local_size > 0)
1689 FAIL_IF(emit_add_sp(compiler, (sljit_uw)local_size));
1690
1691 if (fsaveds + fscratches >= SLJIT_NUMBER_OF_FLOAT_REGISTERS) {
1692 FAIL_IF(push_inst32(compiler, VPOP | VD4(SLJIT_FS0) | ((sljit_uw)SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS << 1)));
1693 } else {
1694 if (fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG)
1695 FAIL_IF(push_inst32(compiler, VPOP | VD4(fscratches) | ((sljit_uw)(fscratches - (SLJIT_FIRST_SAVED_FLOAT_REG - 1)) << 1)));
1696 if (fsaveds > 0)
1697 FAIL_IF(push_inst32(compiler, VPOP | VD4(SLJIT_FS0) | ((sljit_uw)fsaveds << 1)));
1698 }
1699
1700 local_size = GET_SAVED_REGISTERS_SIZE(compiler->scratches, compiler->saveds, 1) & 0x7;
1701 }
1702
1703 if (frame_size < 0) {
1704 lr_dst = TMP_REG2;
1705 frame_size = 0;
1706 } else if (frame_size > 0) {
1707 SLJIT_ASSERT(frame_size == 1 || (frame_size & 0x7) == 0);
1708 lr_dst = 0;
1709 frame_size &= ~0x7;
1710 }
1711
1712 tmp = SLJIT_S0 - compiler->saveds;
1713 i = SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
1714 if (tmp < i) {
1715 restored_reg = i;
1716 do {
1717 reg_list |= (sljit_uw)1 << reg_map[i];
1718 } while (--i > tmp);
1719 }
1720
1721 i = compiler->scratches;
1722 if (i >= SLJIT_FIRST_SAVED_REG) {
1723 restored_reg = i;
1724 do {
1725 reg_list |= (sljit_uw)1 << reg_map[i];
1726 } while (--i >= SLJIT_FIRST_SAVED_REG);
1727 }
1728
1729 if (lr_dst == TMP_REG2 && reg_list == 0) {
1730 reg_list |= (sljit_uw)1 << reg_map[TMP_REG2];
1731 restored_reg = TMP_REG2;
1732 lr_dst = 0;
1733 }
1734
1735 if (lr_dst == 0 && (reg_list & (reg_list - 1)) == 0) {
1736 /* The local_size does not include the saved registers. */
1737 tmp = 0;
1738 if (reg_list != 0) {
1739 tmp = 2;
1740 if (local_size <= 0xfff) {
1741 if (local_size == 0) {
1742 SLJIT_ASSERT(restored_reg != TMP_REG2);
1743 if (frame_size == 0)
1744 return push_inst32(compiler, LDRI | RT4(restored_reg) | RN4(SLJIT_SP) | 0x308);
1745 if (frame_size > 2 * SSIZE_OF(sw))
1746 return push_inst32(compiler, LDRI | RT4(restored_reg) | RN4(SLJIT_SP) | 0x100 | (sljit_ins)(frame_size - (2 * SSIZE_OF(sw))));
1747 }
1748
1749 if (reg_map[restored_reg] <= 7 && local_size <= 0x3fc)
1750 FAIL_IF(push_inst16(compiler, STR_SP | 0x800 | RDN3(restored_reg) | (sljit_ins)(local_size >> 2)));
1751 else
1752 FAIL_IF(push_inst32(compiler, LDR | RT4(restored_reg) | RN4(SLJIT_SP) | (sljit_ins)local_size));
1753 tmp = 1;
1754 } else if (frame_size == 0) {
1755 frame_size = (restored_reg == TMP_REG2) ? SSIZE_OF(sw) : 2 * SSIZE_OF(sw);
1756 tmp = 3;
1757 }
1758
1759 /* Place for the saved register. */
1760 if (restored_reg != TMP_REG2)
1761 local_size += SSIZE_OF(sw);
1762 }
1763
1764 /* Place for the lr register. */
1765 local_size += SSIZE_OF(sw);
1766
1767 if (frame_size > local_size)
1768 FAIL_IF(push_inst16(compiler, SUB_SP_I | ((sljit_ins)(frame_size - local_size) >> 2)));
1769 else if (frame_size < local_size)
1770 FAIL_IF(emit_add_sp(compiler, (sljit_uw)(local_size - frame_size)));
1771
1772 if (tmp <= 1)
1773 return SLJIT_SUCCESS;
1774
1775 if (tmp == 2) {
1776 frame_size -= SSIZE_OF(sw);
1777 if (restored_reg != TMP_REG2)
1778 frame_size -= SSIZE_OF(sw);
1779
1780 if (reg_map[restored_reg] <= 7)
1781 return push_inst16(compiler, STR_SP | 0x800 | RDN3(restored_reg) | (sljit_ins)(frame_size >> 2));
1782
1783 return push_inst32(compiler, LDR | RT4(restored_reg) | RN4(SLJIT_SP) | (sljit_ins)frame_size);
1784 }
1785
1786 tmp = (restored_reg == TMP_REG2) ? 0x304 : 0x308;
1787 return push_inst32(compiler, LDRI | RT4(restored_reg) | RN4(SLJIT_SP) | (sljit_ins)tmp);
1788 }
1789
1790 if (local_size > 0)
1791 FAIL_IF(emit_add_sp(compiler, (sljit_uw)local_size));
1792
1793 if (!(reg_list & 0xff00) && lr_dst != TMP_REG2) {
1794 if (lr_dst == TMP_PC)
1795 reg_list |= 1u << 8;
1796
1797 /* At least one register must be set for POP instruction. */
1798 SLJIT_ASSERT(reg_list != 0);
1799
1800 FAIL_IF(push_inst16(compiler, POP | reg_list));
1801 } else {
1802 if (lr_dst != 0)
1803 reg_list |= (sljit_uw)1 << reg_map[lr_dst];
1804
1805 /* At least two registers must be set for POP_W instruction. */
1806 SLJIT_ASSERT((reg_list & (reg_list - 1)) != 0);
1807
1808 FAIL_IF(push_inst32(compiler, POP_W | reg_list));
1809 }
1810
1811 if (frame_size > 0)
1812 return push_inst16(compiler, SUB_SP_I | (((sljit_ins)frame_size - sizeof(sljit_sw)) >> 2));
1813
1814 if (lr_dst != 0)
1815 return SLJIT_SUCCESS;
1816
1817 return push_inst16(compiler, ADD_SP_I | 1);
1818 }
1819
1820 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
1821 {
1822 CHECK_ERROR();
1823 CHECK(check_sljit_emit_return_void(compiler));
1824
1825 return emit_stack_frame_release(compiler, 0);
1826 }
1827
1828 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_to(struct sljit_compiler *compiler,
1829 sljit_s32 src, sljit_sw srcw)
1830 {
1831 CHECK_ERROR();
1832 CHECK(check_sljit_emit_return_to(compiler, src, srcw));
1833
1834 if (src & SLJIT_MEM) {
1835 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
1836 src = TMP_REG1;
1837 srcw = 0;
1838 } else if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
1839 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG1, src)));
1840 src = TMP_REG1;
1841 srcw = 0;
1842 }
1843
1844 FAIL_IF(emit_stack_frame_release(compiler, 1));
1845
1846 SLJIT_SKIP_CHECKS(compiler);
1847 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1848 }
1849
1850 /* --------------------------------------------------------------------- */
1851 /* Operators */
1852 /* --------------------------------------------------------------------- */
1853
1854 #if !(defined __ARM_FEATURE_IDIV) && !(defined __ARM_ARCH_EXT_IDIV__)
1855
1856 #ifdef __cplusplus
1857 extern "C" {
1858 #endif
1859
1860 #ifdef _WIN32
1861 extern unsigned long long __rt_udiv(unsigned int denominator, unsigned int numerator);
1862 extern long long __rt_sdiv(int denominator, int numerator);
1863 #elif defined(__GNUC__)
1864 extern unsigned int __aeabi_uidivmod(unsigned int numerator, int unsigned denominator);
1865 extern int __aeabi_idivmod(int numerator, int denominator);
1866 #else
1867 #error "Software divmod functions are needed"
1868 #endif
1869
1870 #ifdef __cplusplus
1871 }
1872 #endif
1873
1874 #endif /* !__ARM_FEATURE_IDIV && !__ARM_ARCH_EXT_IDIV__ */
1875
1876 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1877 {
1878 #if !(defined __ARM_FEATURE_IDIV) && !(defined __ARM_ARCH_EXT_IDIV__)
1879 sljit_uw saved_reg_list[3];
1880 sljit_uw saved_reg_count;
1881 #endif
1882
1883 CHECK_ERROR();
1884 CHECK(check_sljit_emit_op0(compiler, op));
1885
1886 op = GET_OPCODE(op);
1887 switch (op) {
1888 case SLJIT_BREAKPOINT:
1889 return push_inst16(compiler, BKPT);
1890 case SLJIT_NOP:
1891 return push_inst16(compiler, NOP);
1892 case SLJIT_LMUL_UW:
1893 case SLJIT_LMUL_SW:
1894 return push_inst32(compiler, (op == SLJIT_LMUL_UW ? UMULL : SMULL)
1895 | RD4(SLJIT_R1) | RT4(SLJIT_R0) | RN4(SLJIT_R0) | RM4(SLJIT_R1));
1896 #if (defined __ARM_FEATURE_IDIV) || (defined __ARM_ARCH_EXT_IDIV__)
1897 case SLJIT_DIVMOD_UW:
1898 case SLJIT_DIVMOD_SW:
1899 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG1, SLJIT_R0)));
1900 FAIL_IF(push_inst32(compiler, (op == SLJIT_DIVMOD_UW ? UDIV : SDIV) | RD4(SLJIT_R0) | RN4(SLJIT_R0) | RM4(SLJIT_R1)));
1901 FAIL_IF(push_inst32(compiler, MUL | RD4(SLJIT_R1) | RN4(SLJIT_R0) | RM4(SLJIT_R1)));
1902 return push_inst32(compiler, SUB_W | RD4(SLJIT_R1) | RN4(TMP_REG1) | RM4(SLJIT_R1));
1903 case SLJIT_DIV_UW:
1904 case SLJIT_DIV_SW:
1905 return push_inst32(compiler, (op == SLJIT_DIV_UW ? UDIV : SDIV) | RD4(SLJIT_R0) | RN4(SLJIT_R0) | RM4(SLJIT_R1));
1906 #else /* !__ARM_FEATURE_IDIV && !__ARM_ARCH_EXT_IDIV__ */
1907 case SLJIT_DIVMOD_UW:
1908 case SLJIT_DIVMOD_SW:
1909 case SLJIT_DIV_UW:
1910 case SLJIT_DIV_SW:
1911 SLJIT_COMPILE_ASSERT((SLJIT_DIVMOD_UW & 0x2) == 0 && SLJIT_DIV_UW - 0x2 == SLJIT_DIVMOD_UW, bad_div_opcode_assignments);
1912 SLJIT_ASSERT(reg_map[2] == 1 && reg_map[3] == 2 && reg_map[4] == 3);
1913
1914 saved_reg_count = 0;
1915 if (compiler->scratches >= 4)
1916 saved_reg_list[saved_reg_count++] = 3;
1917 if (compiler->scratches >= 3)
1918 saved_reg_list[saved_reg_count++] = 2;
1919 if (op >= SLJIT_DIV_UW)
1920 saved_reg_list[saved_reg_count++] = 1;
1921
1922 if (saved_reg_count > 0) {
1923 FAIL_IF(push_inst32(compiler, 0xf84d0d00 | (saved_reg_count >= 3 ? 16 : 8)
1924 | (saved_reg_list[0] << 12) /* str rX, [sp, #-8/-16]! */));
1925 if (saved_reg_count >= 2) {
1926 SLJIT_ASSERT(saved_reg_list[1] < 8);
1927 FAIL_IF(push_inst16(compiler, 0x9001 | (saved_reg_list[1] << 8) /* str rX, [sp, #4] */));
1928 }
1929 if (saved_reg_count >= 3) {
1930 SLJIT_ASSERT(saved_reg_list[2] < 8);
1931 FAIL_IF(push_inst16(compiler, 0x9002 | (saved_reg_list[2] << 8) /* str rX, [sp, #8] */));
1932 }
1933 }
1934
1935 #ifdef _WIN32
1936 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG1, SLJIT_R0)));
1937 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(SLJIT_R0, SLJIT_R1)));
1938 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(SLJIT_R1, TMP_REG1)));
1939 FAIL_IF(sljit_emit_ijump(compiler, SLJIT_FAST_CALL, SLJIT_IMM,
1940 ((op | 0x2) == SLJIT_DIV_UW ? SLJIT_FUNC_ADDR(__rt_udiv) : SLJIT_FUNC_ADDR(__rt_sdiv))));
1941 #elif defined(__GNUC__)
1942 FAIL_IF(sljit_emit_ijump(compiler, SLJIT_FAST_CALL, SLJIT_IMM,
1943 ((op | 0x2) == SLJIT_DIV_UW ? SLJIT_FUNC_ADDR(__aeabi_uidivmod) : SLJIT_FUNC_ADDR(__aeabi_idivmod))));
1944 #else
1945 #error "Software divmod functions are needed"
1946 #endif
1947
1948 if (saved_reg_count > 0) {
1949 if (saved_reg_count >= 3) {
1950 SLJIT_ASSERT(saved_reg_list[2] < 8);
1951 FAIL_IF(push_inst16(compiler, 0x9802 | (saved_reg_list[2] << 8) /* ldr rX, [sp, #8] */));
1952 }
1953 if (saved_reg_count >= 2) {
1954 SLJIT_ASSERT(saved_reg_list[1] < 8);
1955 FAIL_IF(push_inst16(compiler, 0x9801 | (saved_reg_list[1] << 8) /* ldr rX, [sp, #4] */));
1956 }
1957 return push_inst32(compiler, 0xf85d0b00 | (saved_reg_count >= 3 ? 16 : 8)
1958 | (saved_reg_list[0] << 12) /* ldr rX, [sp], #8/16 */);
1959 }
1960 return SLJIT_SUCCESS;
1961 #endif /* __ARM_FEATURE_IDIV || __ARM_ARCH_EXT_IDIV__ */
1962 case SLJIT_MEMORY_BARRIER:
1963 return push_inst32(compiler, DMB_SY);
1964 case SLJIT_ENDBR:
1965 case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1966 return SLJIT_SUCCESS;
1967 }
1968
1969 return SLJIT_SUCCESS;
1970 }
1971
1972 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1973 sljit_s32 dst, sljit_sw dstw,
1974 sljit_s32 src, sljit_sw srcw)
1975 {
1976 sljit_s32 dst_r, flags;
1977
1978 CHECK_ERROR();
1979 CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1980 ADJUST_LOCAL_OFFSET(dst, dstw);
1981 ADJUST_LOCAL_OFFSET(src, srcw);
1982
1983 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
1984
1985 op = GET_OPCODE(op);
1986 if (op >= SLJIT_MOV && op <= SLJIT_MOV_P) {
1987 switch (op) {
1988 case SLJIT_MOV:
1989 case SLJIT_MOV_U32:
1990 case SLJIT_MOV_S32:
1991 case SLJIT_MOV32:
1992 case SLJIT_MOV_P:
1993 flags = WORD_SIZE;
1994 break;
1995 case SLJIT_MOV_U8:
1996 flags = BYTE_SIZE;
1997 if (src == SLJIT_IMM)
1998 srcw = (sljit_u8)srcw;
1999 break;
2000 case SLJIT_MOV_S8:
2001 flags = BYTE_SIZE | SIGNED;
2002 if (src == SLJIT_IMM)
2003 srcw = (sljit_s8)srcw;
2004 break;
2005 case SLJIT_MOV_U16:
2006 flags = HALF_SIZE;
2007 if (src == SLJIT_IMM)
2008 srcw = (sljit_u16)srcw;
2009 break;
2010 case SLJIT_MOV_S16:
2011 flags = HALF_SIZE | SIGNED;
2012 if (src == SLJIT_IMM)
2013 srcw = (sljit_s16)srcw;
2014 break;
2015 default:
2016 SLJIT_UNREACHABLE();
2017 flags = 0;
2018 break;
2019 }
2020
2021 if (src == SLJIT_IMM)
2022 FAIL_IF(emit_op_imm(compiler, SLJIT_MOV | ARG2_IMM, dst_r, TMP_REG2, (sljit_uw)srcw));
2023 else if (src & SLJIT_MEM)
2024 FAIL_IF(emit_op_mem(compiler, flags, dst_r, src, srcw, TMP_REG1));
2025 else if (FAST_IS_REG(dst))
2026 return emit_op_imm(compiler, op, dst_r, TMP_REG2, (sljit_uw)src);
2027 else
2028 dst_r = src;
2029
2030 if (!(dst & SLJIT_MEM))
2031 return SLJIT_SUCCESS;
2032
2033 return emit_op_mem(compiler, flags | STORE, dst_r, dst, dstw, TMP_REG1);
2034 }
2035
2036 SLJIT_COMPILE_ASSERT(WORD_SIZE == 0, word_size_must_be_0);
2037 flags = WORD_SIZE;
2038
2039 if (op == SLJIT_REV_U16 || op == SLJIT_REV_S16) {
2040 if (!(dst & SLJIT_MEM) && (!(src & SLJIT_MEM) || op == SLJIT_REV_S16))
2041 op |= REGISTER_OP;
2042 flags |= HALF_SIZE;
2043 }
2044
2045 if (src & SLJIT_MEM) {
2046 FAIL_IF(emit_op_mem(compiler, flags, TMP_REG1, src, srcw, TMP_REG1));
2047 src = TMP_REG1;
2048 }
2049
2050 emit_op_imm(compiler, op, dst_r, TMP_REG2, (sljit_uw)src);
2051
2052 if (SLJIT_UNLIKELY(dst & SLJIT_MEM))
2053 return emit_op_mem(compiler, flags | STORE, dst_r, dst, dstw, TMP_REG1);
2054 return SLJIT_SUCCESS;
2055 }
2056
2057 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
2058 sljit_s32 dst, sljit_sw dstw,
2059 sljit_s32 src1, sljit_sw src1w,
2060 sljit_s32 src2, sljit_sw src2w)
2061 {
2062 sljit_s32 dst_reg, src2_tmp_reg, flags;
2063
2064 CHECK_ERROR();
2065 CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
2066 ADJUST_LOCAL_OFFSET(dst, dstw);
2067 ADJUST_LOCAL_OFFSET(src1, src1w);
2068 ADJUST_LOCAL_OFFSET(src2, src2w);
2069
2070 dst_reg = FAST_IS_REG(dst) ? dst : TMP_REG2;
2071 flags = HAS_FLAGS(op) ? SET_FLAGS : 0;
2072
2073 if (dst == TMP_REG1)
2074 flags |= UNUSED_RETURN;
2075
2076 if (src2 == SLJIT_IMM)
2077 flags |= ARG2_IMM;
2078 else if (src2 & SLJIT_MEM) {
2079 src2_tmp_reg = FAST_IS_REG(src1) ? TMP_REG1 : TMP_REG2;
2080 emit_op_mem(compiler, WORD_SIZE, src2_tmp_reg, src2, src2w, TMP_REG1);
2081 src2w = src2_tmp_reg;
2082 } else
2083 src2w = src2;
2084
2085 if (src1 == SLJIT_IMM)
2086 flags |= ARG1_IMM;
2087 else if (src1 & SLJIT_MEM) {
2088 emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src1, src1w, TMP_REG1);
2089 src1w = TMP_REG1;
2090 } else
2091 src1w = src1;
2092
2093 emit_op_imm(compiler, flags | GET_OPCODE(op), dst_reg, (sljit_uw)src1w, (sljit_uw)src2w);
2094
2095 if (!(dst & SLJIT_MEM))
2096 return SLJIT_SUCCESS;
2097 return emit_op_mem(compiler, WORD_SIZE | STORE, dst_reg, dst, dstw, TMP_REG1);
2098 }
2099
2100 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
2101 sljit_s32 src1, sljit_sw src1w,
2102 sljit_s32 src2, sljit_sw src2w)
2103 {
2104 CHECK_ERROR();
2105 CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
2106
2107 SLJIT_SKIP_CHECKS(compiler);
2108 return sljit_emit_op2(compiler, op, TMP_REG1, 0, src1, src1w, src2, src2w);
2109 }
2110
2111 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
2112 sljit_s32 dst_reg,
2113 sljit_s32 src1, sljit_sw src1w,
2114 sljit_s32 src2, sljit_sw src2w)
2115 {
2116 CHECK_ERROR();
2117 CHECK(check_sljit_emit_op2r(compiler, op, dst_reg, src1, src1w, src2, src2w));
2118
2119 switch (GET_OPCODE(op)) {
2120 case SLJIT_MULADD:
2121 SLJIT_SKIP_CHECKS(compiler);
2122 return sljit_emit_op2(compiler, op, dst_reg, 0, src1, src1w, src2, src2w);
2123 }
2124
2125 return SLJIT_SUCCESS;
2126 }
2127
2128 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
2129 sljit_s32 dst_reg,
2130 sljit_s32 src1_reg,
2131 sljit_s32 src2_reg,
2132 sljit_s32 src3, sljit_sw src3w)
2133 {
2134 sljit_s32 is_left;
2135
2136 CHECK_ERROR();
2137 CHECK(check_sljit_emit_shift_into(compiler, op, dst_reg, src1_reg, src2_reg, src3, src3w));
2138
2139 op = GET_OPCODE(op);
2140 is_left = (op == SLJIT_SHL || op == SLJIT_MSHL);
2141
2142 if (src1_reg == src2_reg) {
2143 SLJIT_SKIP_CHECKS(compiler);
2144 return sljit_emit_op2(compiler, is_left ? SLJIT_ROTL : SLJIT_ROTR, dst_reg, 0, src1_reg, 0, src3, src3w);
2145 }
2146
2147 ADJUST_LOCAL_OFFSET(src3, src3w);
2148
2149 if (src3 == SLJIT_IMM) {
2150 src3w &= 0x1f;
2151
2152 if (src3w == 0)
2153 return SLJIT_SUCCESS;
2154
2155 if (IS_2_LO_REGS(dst_reg, src1_reg))
2156 FAIL_IF(push_inst16(compiler, (is_left ? LSLSI : LSRSI) | RD3(dst_reg) | RN3(src1_reg) | ((sljit_ins)src3w << 6)));
2157 else
2158 FAIL_IF(push_inst32(compiler, (is_left ? LSL_WI : LSR_WI) | RD4(dst_reg) | RM4(src1_reg) | IMM5(src3w)));
2159
2160 src3w = (src3w ^ 0x1f) + 1;
2161 return push_inst32(compiler, ORR_W | RD4(dst_reg) | RN4(dst_reg) | RM4(src2_reg) | (is_left ? 0x10 : 0x0) | IMM5(src3w));
2162 }
2163
2164 if (src3 & SLJIT_MEM) {
2165 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src3, src3w, TMP_REG2));
2166 src3 = TMP_REG2;
2167 }
2168
2169 if (op == SLJIT_MSHL || op == SLJIT_MLSHR || dst_reg == src3) {
2170 FAIL_IF(push_inst32(compiler, ANDI | RD4(TMP_REG2) | RN4(src3) | 0x1f));
2171 src3 = TMP_REG2;
2172 }
2173
2174 if (dst_reg == src1_reg && IS_2_LO_REGS(dst_reg, src3))
2175 FAIL_IF(push_inst16(compiler, (is_left ? LSLS : LSRS) | RD3(dst_reg) | RN3(src3)));
2176 else
2177 FAIL_IF(push_inst32(compiler, (is_left ? LSL_W : LSR_W) | RD4(dst_reg) | RN4(src1_reg) | RM4(src3)));
2178
2179 FAIL_IF(push_inst32(compiler, (is_left ? LSR_WI : LSL_WI) | RD4(TMP_REG1) | RM4(src2_reg) | (1 << 6)));
2180 FAIL_IF(push_inst32(compiler, EORI | RD4(TMP_REG2) | RN4(src3) | 0x1f));
2181 FAIL_IF(push_inst32(compiler, (is_left ? LSR_W : LSL_W) | RD4(TMP_REG1) | RN4(TMP_REG1) | RM4(TMP_REG2)));
2182 return push_inst32(compiler, ORR_W | RD4(dst_reg) | RN4(dst_reg) | RM4(TMP_REG1));
2183 }
2184
2185 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
2186 sljit_s32 src, sljit_sw srcw)
2187 {
2188 CHECK_ERROR();
2189 CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
2190 ADJUST_LOCAL_OFFSET(src, srcw);
2191
2192 switch (op) {
2193 case SLJIT_FAST_RETURN:
2194 SLJIT_ASSERT(reg_map[TMP_REG2] == 14);
2195
2196 if (FAST_IS_REG(src))
2197 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG2, src)));
2198 else
2199 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src, srcw, TMP_REG2));
2200
2201 return push_inst16(compiler, BX | RN3(TMP_REG2));
2202 case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
2203 return SLJIT_SUCCESS;
2204 case SLJIT_PREFETCH_L1:
2205 case SLJIT_PREFETCH_L2:
2206 case SLJIT_PREFETCH_L3:
2207 case SLJIT_PREFETCH_ONCE:
2208 return emit_op_mem(compiler, PRELOAD, TMP_PC, src, srcw, TMP_REG1);
2209 }
2210
2211 return SLJIT_SUCCESS;
2212 }
2213
2214 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
2215 sljit_s32 dst, sljit_sw dstw)
2216 {
2217 sljit_s32 size, dst_r;
2218
2219 CHECK_ERROR();
2220 CHECK(check_sljit_emit_op_dst(compiler, op, dst, dstw));
2221 ADJUST_LOCAL_OFFSET(dst, dstw);
2222
2223 switch (op) {
2224 case SLJIT_FAST_ENTER:
2225 SLJIT_ASSERT(reg_map[TMP_REG2] == 14);
2226
2227 if (FAST_IS_REG(dst))
2228 return push_inst16(compiler, MOV | SET_REGS44(dst, TMP_REG2));
2229 break;
2230 case SLJIT_GET_RETURN_ADDRESS:
2231 size = GET_SAVED_REGISTERS_SIZE(compiler->scratches, compiler->saveds - SLJIT_KEPT_SAVEDS_COUNT(compiler->options), 0);
2232
2233 if (compiler->fsaveds > 0 || compiler->fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG) {
2234 /* The size of pc is not added above. */
2235 if ((size & SSIZE_OF(sw)) == 0)
2236 size += SSIZE_OF(sw);
2237
2238 size += GET_SAVED_FLOAT_REGISTERS_SIZE(compiler->fscratches, compiler->fsaveds, f64);
2239 }
2240
2241 SLJIT_ASSERT(((compiler->local_size + size + SSIZE_OF(sw)) & 0x7) == 0);
2242
2243 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
2244 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, dst_r, SLJIT_MEM1(SLJIT_SP), compiler->local_size + size, TMP_REG1));
2245 break;
2246 }
2247
2248 if (dst & SLJIT_MEM)
2249 return emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG2, dst, dstw, TMP_REG1);
2250
2251 return SLJIT_SUCCESS;
2252 }
2253
2254 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
2255 {
2256 CHECK_REG_INDEX(check_sljit_get_register_index(type, reg));
2257
2258 if (type == SLJIT_GP_REGISTER)
2259 return reg_map[reg];
2260
2261 if (type == SLJIT_FLOAT_REGISTER || type == SLJIT_SIMD_REG_64)
2262 return freg_map[reg];
2263
2264 if (type == SLJIT_SIMD_REG_128)
2265 return freg_map[reg] & ~0x1;
2266
2267 return -1;
2268 }
2269
2270 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
2271 void *instruction, sljit_u32 size)
2272 {
2273 CHECK_ERROR();
2274 CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
2275
2276 if (size == 2)
2277 return push_inst16(compiler, *(sljit_u16*)instruction);
2278 return push_inst32(compiler, *(sljit_ins*)instruction);
2279 }
2280
2281 /* --------------------------------------------------------------------- */
2282 /* Floating point operators */
2283 /* --------------------------------------------------------------------- */
2284
2285 #define FPU_LOAD (1 << 20)
2286
2287 static sljit_s32 emit_fop_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw)
2288 {
2289 sljit_uw imm;
2290 sljit_ins inst = VSTR_F32 | (flags & (SLJIT_32 | FPU_LOAD));
2291
2292 SLJIT_ASSERT(arg & SLJIT_MEM);
2293
2294 /* Fast loads and stores. */
2295 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
2296 FAIL_IF(push_inst32(compiler, ADD_W | RD4(TMP_REG1) | RN4(arg & REG_MASK) | RM4(OFFS_REG(arg)) | (((sljit_uw)argw & 0x3) << 6)));
2297 arg = SLJIT_MEM | TMP_REG1;
2298 argw = 0;
2299 }
2300
2301 if ((arg & REG_MASK) && (argw & 0x3) == 0) {
2302 if (!(argw & ~0x3fc))
2303 return push_inst32(compiler, inst | 0x800000 | RN4(arg & REG_MASK) | VD4(reg) | ((sljit_uw)argw >> 2));
2304 if (!(-argw & ~0x3fc))
2305 return push_inst32(compiler, inst | RN4(arg & REG_MASK) | VD4(reg) | ((sljit_uw)-argw >> 2));
2306 }
2307
2308 if (arg & REG_MASK) {
2309 if (emit_set_delta(compiler, TMP_REG1, arg & REG_MASK, argw) != SLJIT_ERR_UNSUPPORTED) {
2310 FAIL_IF(compiler->error);
2311 return push_inst32(compiler, inst | 0x800000 | RN4(TMP_REG1) | VD4(reg));
2312 }
2313
2314 imm = get_imm((sljit_uw)argw & ~(sljit_uw)0x3fc);
2315 if (imm != INVALID_IMM) {
2316 FAIL_IF(push_inst32(compiler, ADD_WI | RD4(TMP_REG1) | RN4(arg & REG_MASK) | imm));
2317 return push_inst32(compiler, inst | 0x800000 | RN4(TMP_REG1) | VD4(reg) | (((sljit_uw)argw & 0x3fc) >> 2));
2318 }
2319
2320 imm = get_imm((sljit_uw)-argw & ~(sljit_uw)0x3fc);
2321 if (imm != INVALID_IMM) {
2322 argw = -argw;
2323 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(TMP_REG1) | RN4(arg & REG_MASK) | imm));
2324 return push_inst32(compiler, inst | RN4(TMP_REG1) | VD4(reg) | (((sljit_uw)argw & 0x3fc) >> 2));
2325 }
2326 }
2327
2328 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)argw));
2329 if (arg & REG_MASK)
2330 FAIL_IF(push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, (arg & REG_MASK))));
2331 return push_inst32(compiler, inst | 0x800000 | RN4(TMP_REG1) | VD4(reg));
2332 }
2333
2334 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2335 sljit_s32 dst, sljit_sw dstw,
2336 sljit_s32 src, sljit_sw srcw)
2337 {
2338 op ^= SLJIT_32;
2339
2340 if (src & SLJIT_MEM) {
2341 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, TMP_FREG1, src, srcw));
2342 src = TMP_FREG1;
2343 }
2344
2345 FAIL_IF(push_inst32(compiler, VCVT_S32_F32 | (op & SLJIT_32) | VD4(TMP_FREG1) | VM4(src)));
2346
2347 if (FAST_IS_REG(dst))
2348 return push_inst32(compiler, VMOV | (1 << 20) | RT4(dst) | VN4(TMP_FREG1));
2349
2350 /* Store the integer value from a VFP register. */
2351 return emit_fop_mem(compiler, 0, TMP_FREG1, dst, dstw);
2352 }
2353
2354 static sljit_s32 sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler *compiler, sljit_ins ins,
2355 sljit_s32 dst, sljit_sw dstw,
2356 sljit_s32 src, sljit_sw srcw)
2357 {
2358 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2359
2360 if (FAST_IS_REG(src))
2361 FAIL_IF(push_inst32(compiler, VMOV | RT4(src) | VN4(TMP_FREG1)));
2362 else if (src & SLJIT_MEM) {
2363 /* Load the integer value into a VFP register. */
2364 FAIL_IF(emit_fop_mem(compiler, FPU_LOAD, TMP_FREG1, src, srcw));
2365 }
2366 else {
2367 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)srcw));
2368 FAIL_IF(push_inst32(compiler, VMOV | RT4(TMP_REG1) | VN4(TMP_FREG1)));
2369 }
2370
2371 FAIL_IF(push_inst32(compiler, ins | VD4(dst_r) | VM4(TMP_FREG1)));
2372
2373 if (dst & SLJIT_MEM)
2374 return emit_fop_mem(compiler, (ins & SLJIT_32), TMP_FREG1, dst, dstw);
2375 return SLJIT_SUCCESS;
2376 }
2377
2378 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
2379 sljit_s32 dst, sljit_sw dstw,
2380 sljit_s32 src, sljit_sw srcw)
2381 {
2382 return sljit_emit_fop1_conv_f64_from_w(compiler, VCVT_F32_S32 | (~op & SLJIT_32), dst, dstw, src, srcw);
2383 }
2384
2385 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_uw(struct sljit_compiler *compiler, sljit_s32 op,
2386 sljit_s32 dst, sljit_sw dstw,
2387 sljit_s32 src, sljit_sw srcw)
2388 {
2389 return sljit_emit_fop1_conv_f64_from_w(compiler, VCVT_F32_U32 | (~op & SLJIT_32), dst, dstw, src, srcw);
2390 }
2391
2392 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2393 sljit_s32 src1, sljit_sw src1w,
2394 sljit_s32 src2, sljit_sw src2w)
2395 {
2396 op ^= SLJIT_32;
2397
2398 if (src1 & SLJIT_MEM) {
2399 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, TMP_FREG1, src1, src1w));
2400 src1 = TMP_FREG1;
2401 }
2402
2403 if (src2 & SLJIT_MEM) {
2404 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, TMP_FREG2, src2, src2w));
2405 src2 = TMP_FREG2;
2406 }
2407
2408 FAIL_IF(push_inst32(compiler, VCMP_F32 | (op & SLJIT_32) | VD4(src1) | VM4(src2)));
2409 FAIL_IF(push_inst32(compiler, VMRS));
2410
2411 if (GET_FLAG_TYPE(op) != SLJIT_UNORDERED_OR_EQUAL)
2412 return SLJIT_SUCCESS;
2413
2414 FAIL_IF(push_inst16(compiler, IT | (0x6 << 4) | 0x8));
2415 return push_inst16(compiler, CMP /* Rm, Rn = r0 */);
2416 }
2417
2418 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2419 sljit_s32 dst, sljit_sw dstw,
2420 sljit_s32 src, sljit_sw srcw)
2421 {
2422 sljit_s32 dst_r;
2423
2424 CHECK_ERROR();
2425
2426 SLJIT_COMPILE_ASSERT((SLJIT_32 == 0x100), float_transfer_bit_error);
2427 SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
2428
2429 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2430
2431 if (GET_OPCODE(op) != SLJIT_CONV_F64_FROM_F32)
2432 op ^= SLJIT_32;
2433
2434 if (src & SLJIT_MEM) {
2435 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, dst_r, src, srcw));
2436 src = dst_r;
2437 }
2438
2439 switch (GET_OPCODE(op)) {
2440 case SLJIT_MOV_F64:
2441 if (src != dst_r) {
2442 if (!(dst & SLJIT_MEM))
2443 FAIL_IF(push_inst32(compiler, VMOV_F32 | (op & SLJIT_32) | VD4(dst_r) | VM4(src)));
2444 else
2445 dst_r = src;
2446 }
2447 break;
2448 case SLJIT_NEG_F64:
2449 FAIL_IF(push_inst32(compiler, VNEG_F32 | (op & SLJIT_32) | VD4(dst_r) | VM4(src)));
2450 break;
2451 case SLJIT_ABS_F64:
2452 FAIL_IF(push_inst32(compiler, VABS_F32 | (op & SLJIT_32) | VD4(dst_r) | VM4(src)));
2453 break;
2454 case SLJIT_CONV_F64_FROM_F32:
2455 FAIL_IF(push_inst32(compiler, VCVT_F64_F32 | (op & SLJIT_32) | VD4(dst_r) | VM4(src)));
2456 op ^= SLJIT_32;
2457 break;
2458 }
2459
2460 if (dst & SLJIT_MEM)
2461 return emit_fop_mem(compiler, (op & SLJIT_32), dst_r, dst, dstw);
2462 return SLJIT_SUCCESS;
2463 }
2464
2465 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2466 sljit_s32 dst, sljit_sw dstw,
2467 sljit_s32 src1, sljit_sw src1w,
2468 sljit_s32 src2, sljit_sw src2w)
2469 {
2470 sljit_s32 dst_r;
2471
2472 CHECK_ERROR();
2473 CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
2474 ADJUST_LOCAL_OFFSET(dst, dstw);
2475 ADJUST_LOCAL_OFFSET(src1, src1w);
2476 ADJUST_LOCAL_OFFSET(src2, src2w);
2477
2478 op ^= SLJIT_32;
2479
2480 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2481 if (src1 & SLJIT_MEM) {
2482 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, TMP_FREG1, src1, src1w));
2483 src1 = TMP_FREG1;
2484 }
2485 if (src2 & SLJIT_MEM) {
2486 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, TMP_FREG2, src2, src2w));
2487 src2 = TMP_FREG2;
2488 }
2489
2490 switch (GET_OPCODE(op)) {
2491 case SLJIT_ADD_F64:
2492 FAIL_IF(push_inst32(compiler, VADD_F32 | (op & SLJIT_32) | VD4(dst_r) | VN4(src1) | VM4(src2)));
2493 break;
2494 case SLJIT_SUB_F64:
2495 FAIL_IF(push_inst32(compiler, VSUB_F32 | (op & SLJIT_32) | VD4(dst_r) | VN4(src1) | VM4(src2)));
2496 break;
2497 case SLJIT_MUL_F64:
2498 FAIL_IF(push_inst32(compiler, VMUL_F32 | (op & SLJIT_32) | VD4(dst_r) | VN4(src1) | VM4(src2)));
2499 break;
2500 case SLJIT_DIV_F64:
2501 FAIL_IF(push_inst32(compiler, VDIV_F32 | (op & SLJIT_32) | VD4(dst_r) | VN4(src1) | VM4(src2)));
2502 break;
2503 case SLJIT_COPYSIGN_F64:
2504 FAIL_IF(push_inst32(compiler, VMOV | (1 << 20) | VN4(src2) | RT4(TMP_REG1) | ((op & SLJIT_32) ? (1 << 7) : 0)));
2505 FAIL_IF(push_inst32(compiler, VABS_F32 | (op & SLJIT_32) | VD4(dst_r) | VM4(src1)));
2506 FAIL_IF(push_inst32(compiler, CMPI_W | RN4(TMP_REG1) | 0));
2507 FAIL_IF(push_inst16(compiler, IT | (0xb << 4) | 0x8));
2508 return push_inst32(compiler, VNEG_F32 | (op & SLJIT_32) | VD4(dst_r) | VM4(dst_r));
2509 }
2510
2511 if (!(dst & SLJIT_MEM))
2512 return SLJIT_SUCCESS;
2513 return emit_fop_mem(compiler, (op & SLJIT_32), TMP_FREG1, dst, dstw);
2514 }
2515
2516 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset32(struct sljit_compiler *compiler,
2517 sljit_s32 freg, sljit_f32 value)
2518 {
2519 #if defined(__ARM_NEON) && __ARM_NEON
2520 sljit_u32 exp;
2521 sljit_ins ins;
2522 #endif /* NEON */
2523 union {
2524 sljit_u32 imm;
2525 sljit_f32 value;
2526 } u;
2527
2528 CHECK_ERROR();
2529 CHECK(check_sljit_emit_fset32(compiler, freg, value));
2530
2531 u.value = value;
2532
2533 #if defined(__ARM_NEON) && __ARM_NEON
2534 if ((u.imm << (32 - 19)) == 0) {
2535 exp = (u.imm >> (23 + 2)) & 0x3f;
2536
2537 if (exp == 0x20 || exp == 0x1f) {
2538 ins = ((u.imm >> 24) & 0x80) | ((u.imm >> 19) & 0x7f);
2539 return push_inst32(compiler, (VMOV_F32 ^ (1 << 6)) | ((ins & 0xf0) << 12) | VD4(freg) | (ins & 0xf));
2540 }
2541 }
2542 #endif /* NEON */
2543
2544 FAIL_IF(load_immediate(compiler, TMP_REG1, u.imm));
2545 return push_inst32(compiler, VMOV | VN4(freg) | RT4(TMP_REG1));
2546 }
2547
2548 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset64(struct sljit_compiler *compiler,
2549 sljit_s32 freg, sljit_f64 value)
2550 {
2551 #if defined(__ARM_NEON) && __ARM_NEON
2552 sljit_u32 exp;
2553 sljit_ins ins;
2554 #endif /* NEON */
2555 union {
2556 sljit_u32 imm[2];
2557 sljit_f64 value;
2558 } u;
2559
2560 CHECK_ERROR();
2561 CHECK(check_sljit_emit_fset64(compiler, freg, value));
2562
2563 u.value = value;
2564
2565 #if defined(__ARM_NEON) && __ARM_NEON
2566 if (u.imm[0] == 0 && (u.imm[1] << (64 - 48)) == 0) {
2567 exp = (u.imm[1] >> ((52 - 32) + 2)) & 0x1ff;
2568
2569 if (exp == 0x100 || exp == 0xff) {
2570 ins = ((u.imm[1] >> (56 - 32)) & 0x80) | ((u.imm[1] >> (48 - 32)) & 0x7f);
2571 return push_inst32(compiler, (VMOV_F32 ^ (1 << 6)) | (1 << 8) | ((ins & 0xf0) << 12) | VD4(freg) | (ins & 0xf));
2572 }
2573 }
2574 #endif /* NEON */
2575
2576 FAIL_IF(load_immediate(compiler, TMP_REG1, u.imm[0]));
2577 if (u.imm[0] == u.imm[1])
2578 return push_inst32(compiler, VMOV2 | RN4(TMP_REG1) | RT4(TMP_REG1) | VM4(freg));
2579
2580 FAIL_IF(load_immediate(compiler, TMP_REG2, u.imm[1]));
2581 return push_inst32(compiler, VMOV2 | RN4(TMP_REG2) | RT4(TMP_REG1) | VM4(freg));
2582 }
2583
2584 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fcopy(struct sljit_compiler *compiler, sljit_s32 op,
2585 sljit_s32 freg, sljit_s32 reg)
2586 {
2587 sljit_s32 reg2;
2588 sljit_ins inst;
2589
2590 CHECK_ERROR();
2591 CHECK(check_sljit_emit_fcopy(compiler, op, freg, reg));
2592
2593 if (reg & REG_PAIR_MASK) {
2594 reg2 = REG_PAIR_SECOND(reg);
2595 reg = REG_PAIR_FIRST(reg);
2596
2597 inst = VMOV2 | RN4(reg) | RT4(reg2) | VM4(freg);
2598 } else {
2599 inst = VMOV | VN4(freg) | RT4(reg);
2600
2601 if (!(op & SLJIT_32))
2602 inst |= 1 << 7;
2603 }
2604
2605 if (GET_OPCODE(op) == SLJIT_COPY_FROM_F64)
2606 inst |= 1 << 20;
2607
2608 return push_inst32(compiler, inst);
2609 }
2610
2611 /* --------------------------------------------------------------------- */
2612 /* Conditional instructions */
2613 /* --------------------------------------------------------------------- */
2614
2615 static sljit_uw get_cc(struct sljit_compiler *compiler, sljit_s32 type)
2616 {
2617 switch (type) {
2618 case SLJIT_EQUAL:
2619 case SLJIT_ATOMIC_STORED:
2620 case SLJIT_F_EQUAL:
2621 case SLJIT_ORDERED_EQUAL:
2622 case SLJIT_UNORDERED_OR_EQUAL:
2623 return 0x0;
2624
2625 case SLJIT_NOT_EQUAL:
2626 case SLJIT_ATOMIC_NOT_STORED:
2627 case SLJIT_F_NOT_EQUAL:
2628 case SLJIT_UNORDERED_OR_NOT_EQUAL:
2629 case SLJIT_ORDERED_NOT_EQUAL:
2630 return 0x1;
2631
2632 case SLJIT_CARRY:
2633 if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_ADD)
2634 return 0x2;
2635 /* fallthrough */
2636
2637 case SLJIT_LESS:
2638 return 0x3;
2639
2640 case SLJIT_NOT_CARRY:
2641 if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_ADD)
2642 return 0x3;
2643 /* fallthrough */
2644
2645 case SLJIT_GREATER_EQUAL:
2646 return 0x2;
2647
2648 case SLJIT_GREATER:
2649 case SLJIT_UNORDERED_OR_GREATER:
2650 return 0x8;
2651
2652 case SLJIT_LESS_EQUAL:
2653 case SLJIT_F_LESS_EQUAL:
2654 case SLJIT_ORDERED_LESS_EQUAL:
2655 return 0x9;
2656
2657 case SLJIT_SIG_LESS:
2658 case SLJIT_UNORDERED_OR_LESS:
2659 return 0xb;
2660
2661 case SLJIT_SIG_GREATER_EQUAL:
2662 case SLJIT_F_GREATER_EQUAL:
2663 case SLJIT_ORDERED_GREATER_EQUAL:
2664 return 0xa;
2665
2666 case SLJIT_SIG_GREATER:
2667 case SLJIT_F_GREATER:
2668 case SLJIT_ORDERED_GREATER:
2669 return 0xc;
2670
2671 case SLJIT_SIG_LESS_EQUAL:
2672 case SLJIT_UNORDERED_OR_LESS_EQUAL:
2673 return 0xd;
2674
2675 case SLJIT_OVERFLOW:
2676 if (!(compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)))
2677 return 0x1;
2678 /* fallthrough */
2679
2680 case SLJIT_UNORDERED:
2681 return 0x6;
2682
2683 case SLJIT_NOT_OVERFLOW:
2684 if (!(compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)))
2685 return 0x0;
2686 /* fallthrough */
2687
2688 case SLJIT_ORDERED:
2689 return 0x7;
2690
2691 case SLJIT_F_LESS:
2692 case SLJIT_ORDERED_LESS:
2693 return 0x4;
2694
2695 case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2696 return 0x5;
2697
2698 default: /* SLJIT_JUMP */
2699 SLJIT_UNREACHABLE();
2700 return 0xe;
2701 }
2702 }
2703
2704 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2705 {
2706 struct sljit_label *label;
2707
2708 CHECK_ERROR_PTR();
2709 CHECK_PTR(check_sljit_emit_label(compiler));
2710
2711 if (compiler->last_label && compiler->last_label->size == compiler->size)
2712 return compiler->last_label;
2713
2714 label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2715 PTR_FAIL_IF(!label);
2716 set_label(label, compiler);
2717 return label;
2718 }
2719
2720 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2721 {
2722 struct sljit_jump *jump;
2723 sljit_ins cc;
2724
2725 CHECK_ERROR_PTR();
2726 CHECK_PTR(check_sljit_emit_jump(compiler, type));
2727
2728 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2729 PTR_FAIL_IF(!jump);
2730 set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
2731 type &= 0xff;
2732
2733 if (type < SLJIT_JUMP) {
2734 jump->flags |= IS_COND;
2735 cc = get_cc(compiler, type);
2736 jump->flags |= cc << 8;
2737 PTR_FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
2738 }
2739
2740 jump->addr = compiler->size;
2741 if (type <= SLJIT_JUMP)
2742 PTR_FAIL_IF(push_inst16(compiler, BX | RN3(TMP_REG1)));
2743 else {
2744 jump->flags |= IS_BL;
2745 PTR_FAIL_IF(push_inst16(compiler, BLX | RN3(TMP_REG1)));
2746 }
2747
2748 /* Maximum number of instructions required for generating a constant. */
2749 compiler->size += JUMP_MAX_SIZE - 1;
2750 return jump;
2751 }
2752
2753 #ifdef __SOFTFP__
2754
2755 static sljit_s32 softfloat_call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types, sljit_s32 *src, sljit_u32 *extra_space)
2756 {
2757 sljit_u32 is_tail_call = *extra_space & SLJIT_CALL_RETURN;
2758 sljit_u32 offset = 0;
2759 sljit_u32 word_arg_offset = 0;
2760 sljit_u32 float_arg_count = 0;
2761 sljit_s32 types = 0;
2762 sljit_u32 src_offset = 4 * sizeof(sljit_sw);
2763 sljit_u8 offsets[4];
2764 sljit_u8 *offset_ptr = offsets;
2765
2766 if (src && FAST_IS_REG(*src))
2767 src_offset = (sljit_u32)reg_map[*src] * sizeof(sljit_sw);
2768
2769 arg_types >>= SLJIT_ARG_SHIFT;
2770
2771 while (arg_types) {
2772 types = (types << SLJIT_ARG_SHIFT) | (arg_types & SLJIT_ARG_MASK);
2773
2774 switch (arg_types & SLJIT_ARG_MASK) {
2775 case SLJIT_ARG_TYPE_F64:
2776 if (offset & 0x7)
2777 offset += sizeof(sljit_sw);
2778 *offset_ptr++ = (sljit_u8)offset;
2779 offset += sizeof(sljit_f64);
2780 float_arg_count++;
2781 break;
2782 case SLJIT_ARG_TYPE_F32:
2783 *offset_ptr++ = (sljit_u8)offset;
2784 offset += sizeof(sljit_f32);
2785 float_arg_count++;
2786 break;
2787 default:
2788 *offset_ptr++ = (sljit_u8)offset;
2789 offset += sizeof(sljit_sw);
2790 word_arg_offset += sizeof(sljit_sw);
2791 break;
2792 }
2793
2794 arg_types >>= SLJIT_ARG_SHIFT;
2795 }
2796
2797 if (offset > 4 * sizeof(sljit_sw) && (!is_tail_call || offset > compiler->args_size)) {
2798 /* Keep lr register on the stack. */
2799 if (is_tail_call)
2800 offset += sizeof(sljit_sw);
2801
2802 offset = ((offset - 4 * sizeof(sljit_sw)) + 0x7) & ~(sljit_uw)0x7;
2803
2804 *extra_space = offset;
2805
2806 if (is_tail_call)
2807 FAIL_IF(emit_stack_frame_release(compiler, (sljit_s32)offset));
2808 else
2809 FAIL_IF(push_inst16(compiler, SUB_SP_I | (offset >> 2)));
2810 } else {
2811 if (is_tail_call)
2812 FAIL_IF(emit_stack_frame_release(compiler, -1));
2813 *extra_space = 0;
2814 }
2815
2816 SLJIT_ASSERT(reg_map[TMP_REG1] == 12);
2817
2818 /* Process arguments in reversed direction. */
2819 while (types) {
2820 switch (types & SLJIT_ARG_MASK) {
2821 case SLJIT_ARG_TYPE_F64:
2822 float_arg_count--;
2823 offset = *(--offset_ptr);
2824
2825 SLJIT_ASSERT((offset & 0x7) == 0);
2826
2827 if (offset < 4 * sizeof(sljit_sw)) {
2828 if (src_offset == offset || src_offset == offset + sizeof(sljit_sw)) {
2829 FAIL_IF(push_inst16(compiler, MOV | (src_offset << 1) | 4 | (1 << 7)));
2830 *src = TMP_REG1;
2831 }
2832 FAIL_IF(push_inst32(compiler, VMOV2 | 0x100000 | (offset << 10) | ((offset + sizeof(sljit_sw)) << 14) | float_arg_count));
2833 } else
2834 FAIL_IF(push_inst32(compiler, VSTR_F32 | 0x800100 | RN4(SLJIT_SP)
2835 | (float_arg_count << 12) | ((offset - 4 * sizeof(sljit_sw)) >> 2)));
2836 break;
2837 case SLJIT_ARG_TYPE_F32:
2838 float_arg_count--;
2839 offset = *(--offset_ptr);
2840
2841 if (offset < 4 * sizeof(sljit_sw)) {
2842 if (src_offset == offset) {
2843 FAIL_IF(push_inst16(compiler, MOV | (src_offset << 1) | 4 | (1 << 7)));
2844 *src = TMP_REG1;
2845 }
2846 FAIL_IF(push_inst32(compiler, VMOV | 0x100000 | (float_arg_count << 16) | (offset << 10)));
2847 } else
2848 FAIL_IF(push_inst32(compiler, VSTR_F32 | 0x800000 | RN4(SLJIT_SP)
2849 | (float_arg_count << 12) | ((offset - 4 * sizeof(sljit_sw)) >> 2)));
2850 break;
2851 default:
2852 word_arg_offset -= sizeof(sljit_sw);
2853 offset = *(--offset_ptr);
2854
2855 SLJIT_ASSERT(offset >= word_arg_offset);
2856
2857 if (offset != word_arg_offset) {
2858 if (offset < 4 * sizeof(sljit_sw)) {
2859 if (src_offset == offset) {
2860 FAIL_IF(push_inst16(compiler, MOV | (src_offset << 1) | 4 | (1 << 7)));
2861 *src = TMP_REG1;
2862 }
2863 else if (src_offset == word_arg_offset) {
2864 *src = (sljit_s32)(1 + (offset >> 2));
2865 src_offset = offset;
2866 }
2867 FAIL_IF(push_inst16(compiler, MOV | (offset >> 2) | (word_arg_offset << 1)));
2868 } else
2869 FAIL_IF(push_inst16(compiler, STR_SP | (word_arg_offset << 6) | ((offset - 4 * sizeof(sljit_sw)) >> 2)));
2870 }
2871 break;
2872 }
2873
2874 types >>= SLJIT_ARG_SHIFT;
2875 }
2876
2877 return SLJIT_SUCCESS;
2878 }
2879
2880 static sljit_s32 softfloat_post_call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types)
2881 {
2882 if ((arg_types & SLJIT_ARG_MASK) == SLJIT_ARG_TYPE_F64)
2883 FAIL_IF(push_inst32(compiler, VMOV2 | (1 << 16) | (0 << 12) | 0));
2884 if ((arg_types & SLJIT_ARG_MASK) == SLJIT_ARG_TYPE_F32)
2885 FAIL_IF(push_inst32(compiler, VMOV | (0 << 16) | (0 << 12)));
2886
2887 return SLJIT_SUCCESS;
2888 }
2889
2890 #else
2891
2892 static sljit_s32 hardfloat_call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types)
2893 {
2894 sljit_u32 offset = SLJIT_FR0;
2895 sljit_u32 new_offset = SLJIT_FR0;
2896 sljit_u32 f32_offset = 0;
2897
2898 /* Remove return value. */
2899 arg_types >>= SLJIT_ARG_SHIFT;
2900
2901 while (arg_types) {
2902 switch (arg_types & SLJIT_ARG_MASK) {
2903 case SLJIT_ARG_TYPE_F64:
2904 if (offset != new_offset)
2905 FAIL_IF(push_inst32(compiler, VMOV_F32 | SLJIT_32 | VD4(new_offset) | VM4(offset)));
2906
2907 new_offset++;
2908 offset++;
2909 break;
2910 case SLJIT_ARG_TYPE_F32:
2911 if (f32_offset != 0) {
2912 FAIL_IF(push_inst32(compiler, VMOV_F32 | 0x400000 | VD4(f32_offset) | VM4(offset)));
2913 f32_offset = 0;
2914 } else {
2915 if (offset != new_offset)
2916 FAIL_IF(push_inst32(compiler, VMOV_F32 | 0x400000 | VD4(new_offset) | VM4(offset)));
2917 f32_offset = new_offset;
2918 new_offset++;
2919 }
2920 offset++;
2921 break;
2922 }
2923 arg_types >>= SLJIT_ARG_SHIFT;
2924 }
2925
2926 return SLJIT_SUCCESS;
2927 }
2928
2929 #endif
2930
2931 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2932 sljit_s32 arg_types)
2933 {
2934 #ifdef __SOFTFP__
2935 struct sljit_jump *jump;
2936 sljit_u32 extra_space = (sljit_u32)type;
2937 #endif
2938
2939 CHECK_ERROR_PTR();
2940 CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
2941
2942 #ifdef __SOFTFP__
2943 if ((type & 0xff) != SLJIT_CALL_REG_ARG) {
2944 PTR_FAIL_IF(softfloat_call_with_args(compiler, arg_types, NULL, &extra_space));
2945 SLJIT_ASSERT((extra_space & 0x7) == 0);
2946
2947 if ((type & SLJIT_CALL_RETURN) && extra_space == 0)
2948 type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
2949
2950 SLJIT_SKIP_CHECKS(compiler);
2951 jump = sljit_emit_jump(compiler, type);
2952 PTR_FAIL_IF(jump == NULL);
2953
2954 if (extra_space > 0) {
2955 if (type & SLJIT_CALL_RETURN)
2956 PTR_FAIL_IF(push_inst32(compiler, LDR | RT4(TMP_REG2)
2957 | RN4(SLJIT_SP) | (extra_space - sizeof(sljit_sw))));
2958
2959 PTR_FAIL_IF(push_inst16(compiler, ADD_SP_I | (extra_space >> 2)));
2960
2961 if (type & SLJIT_CALL_RETURN) {
2962 PTR_FAIL_IF(push_inst16(compiler, BX | RN3(TMP_REG2)));
2963 return jump;
2964 }
2965 }
2966
2967 SLJIT_ASSERT(!(type & SLJIT_CALL_RETURN));
2968 PTR_FAIL_IF(softfloat_post_call_with_args(compiler, arg_types));
2969 return jump;
2970 }
2971 #endif /* __SOFTFP__ */
2972
2973 if (type & SLJIT_CALL_RETURN) {
2974 /* ldmia sp!, {..., lr} */
2975 PTR_FAIL_IF(emit_stack_frame_release(compiler, -1));
2976 type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
2977 }
2978
2979 #ifndef __SOFTFP__
2980 if ((type & 0xff) != SLJIT_CALL_REG_ARG)
2981 PTR_FAIL_IF(hardfloat_call_with_args(compiler, arg_types));
2982 #endif /* !__SOFTFP__ */
2983
2984 SLJIT_SKIP_CHECKS(compiler);
2985 return sljit_emit_jump(compiler, type);
2986 }
2987
2988 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
2989 {
2990 struct sljit_jump *jump;
2991
2992 CHECK_ERROR();
2993 CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
2994 ADJUST_LOCAL_OFFSET(src, srcw);
2995
2996 SLJIT_ASSERT(reg_map[TMP_REG1] != 14);
2997
2998 if (src != SLJIT_IMM) {
2999 if (FAST_IS_REG(src)) {
3000 SLJIT_ASSERT(reg_map[src] != 14);
3001 return push_inst16(compiler, (type <= SLJIT_JUMP ? BX : BLX) | RN3(src));
3002 }
3003
3004 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, type <= SLJIT_JUMP ? TMP_PC : TMP_REG1, src, srcw, TMP_REG1));
3005 if (type >= SLJIT_FAST_CALL)
3006 return push_inst16(compiler, BLX | RN3(TMP_REG1));
3007 }
3008
3009 /* These jumps are converted to jump/call instructions when possible. */
3010 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3011 FAIL_IF(!jump);
3012 set_jump(jump, compiler, JUMP_ADDR | ((type >= SLJIT_FAST_CALL) ? IS_BL : 0));
3013 jump->u.target = (sljit_uw)srcw;
3014
3015 jump->addr = compiler->size;
3016 /* Maximum number of instructions required for generating a constant. */
3017 compiler->size += JUMP_MAX_SIZE - 1;
3018 return push_inst16(compiler, (type <= SLJIT_JUMP ? BX : BLX) | RN3(TMP_REG1));
3019 }
3020
3021 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
3022 sljit_s32 arg_types,
3023 sljit_s32 src, sljit_sw srcw)
3024 {
3025 #ifdef __SOFTFP__
3026 sljit_u32 extra_space = (sljit_u32)type;
3027 #endif
3028
3029 CHECK_ERROR();
3030 CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
3031
3032 if (src & SLJIT_MEM) {
3033 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
3034 src = TMP_REG1;
3035 }
3036
3037 if ((type & SLJIT_CALL_RETURN) && (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options)))) {
3038 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG1, src)));
3039 src = TMP_REG1;
3040 }
3041
3042 #ifdef __SOFTFP__
3043 if ((type & 0xff) != SLJIT_CALL_REG_ARG) {
3044 FAIL_IF(softfloat_call_with_args(compiler, arg_types, &src, &extra_space));
3045 SLJIT_ASSERT((extra_space & 0x7) == 0);
3046
3047 if ((type & SLJIT_CALL_RETURN) && extra_space == 0)
3048 type = SLJIT_JUMP;
3049
3050 SLJIT_SKIP_CHECKS(compiler);
3051 FAIL_IF(sljit_emit_ijump(compiler, type, src, srcw));
3052
3053 if (extra_space > 0) {
3054 if (type & SLJIT_CALL_RETURN)
3055 FAIL_IF(push_inst32(compiler, LDR | RT4(TMP_REG2)
3056 | RN4(SLJIT_SP) | (extra_space - sizeof(sljit_sw))));
3057
3058 FAIL_IF(push_inst16(compiler, ADD_SP_I | (extra_space >> 2)));
3059
3060 if (type & SLJIT_CALL_RETURN)
3061 return push_inst16(compiler, BX | RN3(TMP_REG2));
3062 }
3063
3064 SLJIT_ASSERT(!(type & SLJIT_CALL_RETURN));
3065 return softfloat_post_call_with_args(compiler, arg_types);
3066 }
3067 #endif /* __SOFTFP__ */
3068
3069 if (type & SLJIT_CALL_RETURN) {
3070 /* ldmia sp!, {..., lr} */
3071 FAIL_IF(emit_stack_frame_release(compiler, -1));
3072 type = SLJIT_JUMP;
3073 }
3074
3075 #ifndef __SOFTFP__
3076 if ((type & 0xff) != SLJIT_CALL_REG_ARG)
3077 FAIL_IF(hardfloat_call_with_args(compiler, arg_types));
3078 #endif /* !__SOFTFP__ */
3079
3080 SLJIT_SKIP_CHECKS(compiler);
3081 return sljit_emit_ijump(compiler, type, src, srcw);
3082 }
3083
3084 #ifdef __SOFTFP__
3085
3086 static SLJIT_INLINE sljit_s32 emit_fmov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3087 {
3088 if (compiler->options & SLJIT_ENTER_REG_ARG) {
3089 if (src == SLJIT_FR0)
3090 return SLJIT_SUCCESS;
3091
3092 SLJIT_SKIP_CHECKS(compiler);
3093 return sljit_emit_fop1(compiler, op, SLJIT_RETURN_FREG, 0, src, srcw);
3094 }
3095
3096 if (FAST_IS_REG(src)) {
3097 if (op & SLJIT_32)
3098 return push_inst32(compiler, VMOV | (1 << 20) | VN4(src) | RT4(SLJIT_R0));
3099 return push_inst32(compiler, VMOV2 | (1 << 20) | VM4(src) | RT4(SLJIT_R0) | RN4(SLJIT_R1));
3100 }
3101
3102 SLJIT_SKIP_CHECKS(compiler);
3103
3104 if (op & SLJIT_32)
3105 return sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, src, srcw);
3106 return sljit_emit_mem(compiler, SLJIT_MOV, SLJIT_REG_PAIR(SLJIT_R0, SLJIT_R1), src, srcw);
3107 }
3108
3109 #endif /* __SOFTFP__ */
3110
3111 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
3112 sljit_s32 dst, sljit_sw dstw,
3113 sljit_s32 type)
3114 {
3115 sljit_s32 dst_r, flags = GET_ALL_FLAGS(op);
3116 sljit_ins cc;
3117
3118 CHECK_ERROR();
3119 CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
3120 ADJUST_LOCAL_OFFSET(dst, dstw);
3121
3122 op = GET_OPCODE(op);
3123 cc = get_cc(compiler, type);
3124 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
3125
3126 if (op < SLJIT_ADD) {
3127 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | (((cc & 0x1) ^ 0x1) << 3) | 0x4));
3128 if (reg_map[dst_r] > 7) {
3129 FAIL_IF(push_inst32(compiler, MOV_WI | RD4(dst_r) | 1));
3130 FAIL_IF(push_inst32(compiler, MOV_WI | RD4(dst_r) | 0));
3131 } else {
3132 /* The movsi (immediate) instruction does not set flags in IT block. */
3133 FAIL_IF(push_inst16(compiler, MOVSI | RDN3(dst_r) | 1));
3134 FAIL_IF(push_inst16(compiler, MOVSI | RDN3(dst_r) | 0));
3135 }
3136 if (!(dst & SLJIT_MEM))
3137 return SLJIT_SUCCESS;
3138 return emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG1, dst, dstw, TMP_REG2);
3139 }
3140
3141 if (dst & SLJIT_MEM)
3142 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, dst, dstw, TMP_REG2));
3143
3144 if (op == SLJIT_AND) {
3145 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | (((cc & 0x1) ^ 0x1) << 3) | 0x4));
3146 FAIL_IF(push_inst32(compiler, ANDI | RN4(dst_r) | RD4(dst_r) | 1));
3147 FAIL_IF(push_inst32(compiler, ANDI | RN4(dst_r) | RD4(dst_r) | 0));
3148 }
3149 else {
3150 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
3151 FAIL_IF(push_inst32(compiler, ((op == SLJIT_OR) ? ORRI : EORI) | RN4(dst_r) | RD4(dst_r) | 1));
3152 }
3153
3154 if (dst & SLJIT_MEM)
3155 FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG1, dst, dstw, TMP_REG2));
3156
3157 if (!(flags & SLJIT_SET_Z))
3158 return SLJIT_SUCCESS;
3159
3160 /* The condition must always be set, even if the ORR/EORI is not executed above. */
3161 return push_inst32(compiler, MOV_W | SET_FLAGS | RD4(TMP_REG1) | RM4(dst_r));
3162 }
3163
3164 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
3165 sljit_s32 dst_reg,
3166 sljit_s32 src1, sljit_sw src1w,
3167 sljit_s32 src2_reg)
3168 {
3169 sljit_uw cc, tmp;
3170
3171 CHECK_ERROR();
3172 CHECK(check_sljit_emit_select(compiler, type, dst_reg, src1, src1w, src2_reg));
3173
3174 ADJUST_LOCAL_OFFSET(src1, src1w);
3175
3176 if (src2_reg != dst_reg && src1 == dst_reg) {
3177 src1 = src2_reg;
3178 src1w = 0;
3179 src2_reg = dst_reg;
3180 type ^= 0x1;
3181 }
3182
3183 if (src1 & SLJIT_MEM) {
3184 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, (src2_reg != dst_reg) ? dst_reg : TMP_REG1, src1, src1w, TMP_REG1));
3185
3186 if (src2_reg != dst_reg) {
3187 src1 = src2_reg;
3188 src1w = 0;
3189 type ^= 0x1;
3190 } else {
3191 src1 = TMP_REG1;
3192 src1w = 0;
3193 }
3194 } else if (dst_reg != src2_reg)
3195 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(dst_reg, src2_reg)));
3196
3197 cc = get_cc(compiler, type & ~SLJIT_32);
3198
3199 if (src1 != SLJIT_IMM) {
3200 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
3201 return push_inst16(compiler, MOV | SET_REGS44(dst_reg, src1));
3202 }
3203
3204 tmp = (sljit_uw)src1w;
3205
3206 if (tmp < 0x10000) {
3207 /* set low 16 bits, set hi 16 bits to 0. */
3208 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
3209 return push_inst32(compiler, MOVW | RD4(dst_reg)
3210 | COPY_BITS(tmp, 12, 16, 4) | COPY_BITS(tmp, 11, 26, 1) | COPY_BITS(tmp, 8, 12, 3) | (tmp & 0xff));
3211 }
3212
3213 tmp = get_imm((sljit_uw)src1w);
3214 if (tmp != INVALID_IMM) {
3215 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
3216 return push_inst32(compiler, MOV_WI | RD4(dst_reg) | tmp);
3217 }
3218
3219 tmp = get_imm(~(sljit_uw)src1w);
3220 if (tmp != INVALID_IMM) {
3221 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
3222 return push_inst32(compiler, MVN_WI | RD4(dst_reg) | tmp);
3223 }
3224
3225 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | ((cc & 0x1) << 3) | 0x4));
3226
3227 tmp = (sljit_uw)src1w;
3228 FAIL_IF(push_inst32(compiler, MOVW | RD4(dst_reg)
3229 | COPY_BITS(tmp, 12, 16, 4) | COPY_BITS(tmp, 11, 26, 1) | COPY_BITS(tmp, 8, 12, 3) | (tmp & 0xff)));
3230 return push_inst32(compiler, MOVT | RD4(dst_reg)
3231 | COPY_BITS(tmp, 12 + 16, 16, 4) | COPY_BITS(tmp, 11 + 16, 26, 1) | COPY_BITS(tmp, 8 + 16, 12, 3) | ((tmp & 0xff0000) >> 16));
3232 }
3233
3234 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
3235 sljit_s32 dst_freg,
3236 sljit_s32 src1, sljit_sw src1w,
3237 sljit_s32 src2_freg)
3238 {
3239 CHECK_ERROR();
3240 CHECK(check_sljit_emit_fselect(compiler, type, dst_freg, src1, src1w, src2_freg));
3241
3242 ADJUST_LOCAL_OFFSET(src1, src1w);
3243
3244 type ^= SLJIT_32;
3245
3246 if (dst_freg != src2_freg) {
3247 if (dst_freg == src1) {
3248 src1 = src2_freg;
3249 src1w = 0;
3250 type ^= 0x1;
3251 } else
3252 FAIL_IF(push_inst32(compiler, VMOV_F32 | (type & SLJIT_32) | VD4(dst_freg) | VM4(src2_freg)));
3253 }
3254
3255 if (src1 & SLJIT_MEM) {
3256 FAIL_IF(emit_fop_mem(compiler, (type & SLJIT_32) | FPU_LOAD, TMP_FREG2, src1, src1w));
3257 src1 = TMP_FREG2;
3258 }
3259
3260 FAIL_IF(push_inst16(compiler, IT | (get_cc(compiler, type & ~SLJIT_32) << 4) | 0x8));
3261 return push_inst32(compiler, VMOV_F32 | (type & SLJIT_32) | VD4(dst_freg) | VM4(src1));
3262 }
3263
3264 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
3265 sljit_s32 reg,
3266 sljit_s32 mem, sljit_sw memw)
3267 {
3268 sljit_s32 flags;
3269 sljit_uw imm, tmp;
3270
3271 CHECK_ERROR();
3272 CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw));
3273
3274 if (!(reg & REG_PAIR_MASK))
3275 return sljit_emit_mem_unaligned(compiler, type, reg, mem, memw);
3276
3277 if (type & (SLJIT_MEM_UNALIGNED | SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)) {
3278 if ((mem & REG_MASK) == 0) {
3279 if ((memw & 0xfff) >= (0x1000 - SSIZE_OF(sw))) {
3280 imm = get_imm((sljit_uw)((memw + 0x1000) & ~0xfff));
3281
3282 if (imm != INVALID_IMM)
3283 memw = (memw & 0xfff) - 0x1000;
3284 } else {
3285 imm = get_imm((sljit_uw)(memw & ~0xfff));
3286
3287 if (imm != INVALID_IMM)
3288 memw &= 0xfff;
3289 }
3290
3291 if (imm == INVALID_IMM) {
3292 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
3293 memw = 0;
3294 } else
3295 FAIL_IF(push_inst32(compiler, MOV_WI | RD4(TMP_REG1) | imm));
3296
3297 mem = SLJIT_MEM1(TMP_REG1);
3298 } else if (mem & OFFS_REG_MASK) {
3299 FAIL_IF(push_inst32(compiler, ADD_W | RD4(TMP_REG1) | RN4(mem & REG_MASK) | RM4(OFFS_REG(mem)) | ((sljit_uw)(memw & 0x3) << 6)));
3300 memw = 0;
3301 mem = SLJIT_MEM1(TMP_REG1);
3302 } else if (memw < -0xff) {
3303 /* Zero value can be included in the first case. */
3304 if ((-memw & 0xfff) <= SSIZE_OF(sw))
3305 tmp = (sljit_uw)((-memw + 0x7ff) & ~0x7ff);
3306 else
3307 tmp = (sljit_uw)((-memw + 0xfff) & ~0xfff);
3308
3309 SLJIT_ASSERT(tmp >= (sljit_uw)-memw);
3310 imm = get_imm(tmp);
3311
3312 if (imm != INVALID_IMM) {
3313 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(TMP_REG1) | RN4(mem & REG_MASK) | imm));
3314 memw += (sljit_sw)tmp;
3315 SLJIT_ASSERT(memw >= 0 && memw <= 0xfff - SSIZE_OF(sw));
3316 } else {
3317 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
3318 FAIL_IF(push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, mem & REG_MASK)));
3319 memw = 0;
3320 }
3321
3322 mem = SLJIT_MEM1(TMP_REG1);
3323 } else if (memw >= (0x1000 - SSIZE_OF(sw))) {
3324 if ((memw & 0xfff) >= (0x1000 - SSIZE_OF(sw))) {
3325 imm = get_imm((sljit_uw)((memw + 0x1000) & ~0xfff));
3326
3327 if (imm != INVALID_IMM)
3328 memw = (memw & 0xfff) - 0x1000;
3329 } else {
3330 imm = get_imm((sljit_uw)(memw & ~0xfff));
3331
3332 if (imm != INVALID_IMM)
3333 memw &= 0xfff;
3334 }
3335
3336 if (imm != INVALID_IMM) {
3337 SLJIT_ASSERT(memw >= -0xff && memw <= 0xfff);
3338 FAIL_IF(push_inst32(compiler, ADD_WI | RD4(TMP_REG1) | RN4(mem & REG_MASK) | imm));
3339 } else {
3340 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
3341 FAIL_IF(push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, mem & REG_MASK)));
3342 memw = 0;
3343 }
3344
3345 mem = SLJIT_MEM1(TMP_REG1);
3346 }
3347
3348 flags = WORD_SIZE;
3349
3350 SLJIT_ASSERT(memw <= 0xfff - SSIZE_OF(sw) && memw >= -0xff);
3351
3352 if (type & SLJIT_MEM_STORE) {
3353 flags |= STORE;
3354 } else if (REG_PAIR_FIRST(reg) == (mem & REG_MASK)) {
3355 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, REG_PAIR_SECOND(reg), mem, memw + SSIZE_OF(sw), TMP_REG2));
3356 return emit_op_mem(compiler, WORD_SIZE, REG_PAIR_FIRST(reg), mem, memw, TMP_REG2);
3357 }
3358
3359 FAIL_IF(emit_op_mem(compiler, flags, REG_PAIR_FIRST(reg), mem, memw, TMP_REG2));
3360 return emit_op_mem(compiler, flags, REG_PAIR_SECOND(reg), mem, memw + SSIZE_OF(sw), TMP_REG2);
3361 }
3362
3363 flags = 1 << 23;
3364
3365 if ((mem & REG_MASK) == 0) {
3366 tmp = (sljit_uw)(memw & 0x7fc);
3367 imm = get_imm((sljit_uw)((memw + (tmp <= 0x400 ? 0 : 0x400)) & ~0x3fc));
3368
3369 if (imm == INVALID_IMM) {
3370 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
3371 memw = 0;
3372 } else {
3373 FAIL_IF(push_inst32(compiler, MOV_WI | RD4(TMP_REG1) | imm));
3374 memw = (memw & 0x3fc) >> 2;
3375
3376 if (tmp > 0x400) {
3377 memw = 0x100 - memw;
3378 flags = 0;
3379 }
3380
3381 SLJIT_ASSERT(memw >= 0 && memw <= 0xff);
3382 }
3383
3384 mem = SLJIT_MEM1(TMP_REG1);
3385 } else if (mem & OFFS_REG_MASK) {
3386 FAIL_IF(push_inst32(compiler, ADD_W | RD4(TMP_REG1) | RN4(mem & REG_MASK) | RM4(OFFS_REG(mem)) | ((sljit_uw)(memw & 0x3) << 6)));
3387 memw = 0;
3388 mem = SLJIT_MEM1(TMP_REG1);
3389 } else if (memw < 0) {
3390 if ((-memw & ~0x3fc) == 0) {
3391 flags = 0;
3392 memw = -memw >> 2;
3393 } else {
3394 tmp = (sljit_uw)(-memw & 0x7fc);
3395 imm = get_imm((sljit_uw)((-memw + (tmp <= 0x400 ? 0 : 0x400)) & ~0x3fc));
3396
3397 if (imm != INVALID_IMM) {
3398 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(TMP_REG1) | RN4(mem & REG_MASK) | imm));
3399 memw = (-memw & 0x3fc) >> 2;
3400
3401 if (tmp <= 0x400)
3402 flags = 0;
3403 else
3404 memw = 0x100 - memw;
3405 } else {
3406 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
3407 FAIL_IF(push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, mem & REG_MASK)));
3408 memw = 0;
3409 }
3410
3411 mem = SLJIT_MEM1(TMP_REG1);
3412 }
3413 } else if ((memw & ~0x3fc) != 0) {
3414 tmp = (sljit_uw)(memw & 0x7fc);
3415 imm = get_imm((sljit_uw)((memw + (tmp <= 0x400 ? 0 : 0x400)) & ~0x3fc));
3416
3417 if (imm != INVALID_IMM) {
3418 FAIL_IF(push_inst32(compiler, ADD_WI | RD4(TMP_REG1) | RN4(mem & REG_MASK) | imm));
3419 memw = (memw & 0x3fc) >> 2;
3420
3421 if (tmp > 0x400) {
3422 memw = 0x100 - memw;
3423 flags = 0;
3424 }
3425 } else {
3426 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
3427 FAIL_IF(push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, mem & REG_MASK)));
3428 memw = 0;
3429 }
3430
3431 mem = SLJIT_MEM1(TMP_REG1);
3432 } else
3433 memw >>= 2;
3434
3435 SLJIT_ASSERT(memw >= 0 && memw <= 0xff);
3436 return push_inst32(compiler, ((type & SLJIT_MEM_STORE) ? STRD : LDRD) | (sljit_ins)flags | RN4(mem & REG_MASK) | RT4(REG_PAIR_FIRST(reg)) | RD4(REG_PAIR_SECOND(reg)) | (sljit_ins)memw);
3437 }
3438
3439 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
3440 sljit_s32 reg,
3441 sljit_s32 mem, sljit_sw memw)
3442 {
3443 sljit_s32 flags;
3444 sljit_ins inst;
3445
3446 CHECK_ERROR();
3447 CHECK(check_sljit_emit_mem_update(compiler, type, reg, mem, memw));
3448
3449 if ((mem & OFFS_REG_MASK) || (memw > 255 || memw < -255))
3450 return SLJIT_ERR_UNSUPPORTED;
3451
3452 if (type & SLJIT_MEM_SUPP)
3453 return SLJIT_SUCCESS;
3454
3455 switch (type & 0xff) {
3456 case SLJIT_MOV:
3457 case SLJIT_MOV_U32:
3458 case SLJIT_MOV_S32:
3459 case SLJIT_MOV32:
3460 case SLJIT_MOV_P:
3461 flags = WORD_SIZE;
3462 break;
3463 case SLJIT_MOV_U8:
3464 flags = BYTE_SIZE;
3465 break;
3466 case SLJIT_MOV_S8:
3467 flags = BYTE_SIZE | SIGNED;
3468 break;
3469 case SLJIT_MOV_U16:
3470 flags = HALF_SIZE;
3471 break;
3472 case SLJIT_MOV_S16:
3473 flags = HALF_SIZE | SIGNED;
3474 break;
3475 default:
3476 SLJIT_UNREACHABLE();
3477 flags = WORD_SIZE;
3478 break;
3479 }
3480
3481 if (type & SLJIT_MEM_STORE)
3482 flags |= STORE;
3483
3484 inst = sljit_mem32[flags] | 0x900;
3485
3486 if (!(type & SLJIT_MEM_POST))
3487 inst |= 0x400;
3488
3489 if (memw >= 0)
3490 inst |= 0x200;
3491 else
3492 memw = -memw;
3493
3494 return push_inst32(compiler, inst | RT4(reg) | RN4(mem & REG_MASK) | (sljit_ins)memw);
3495 }
3496
3497 static sljit_s32 update_mem_addr(struct sljit_compiler *compiler, sljit_s32 *mem, sljit_sw *memw, sljit_s32 max_offset)
3498 {
3499 sljit_s32 arg = *mem;
3500 sljit_sw argw = *memw;
3501 sljit_uw imm;
3502
3503 *mem = TMP_REG1;
3504
3505 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
3506 *memw = 0;
3507 return push_inst32(compiler, ADD_W | RD4(TMP_REG1) | RN4(arg & REG_MASK) | RM4(OFFS_REG(arg)) | ((sljit_uw)(argw & 0x3) << 6));
3508 }
3509
3510 arg &= REG_MASK;
3511
3512 if (arg) {
3513 if (argw <= max_offset && argw >= -0xff) {
3514 *mem = arg;
3515 return SLJIT_SUCCESS;
3516 }
3517
3518 if (argw < 0) {
3519 imm = get_imm((sljit_uw)(-argw & ~0xff));
3520
3521 if (imm) {
3522 *memw = -(-argw & 0xff);
3523 return push_inst32(compiler, SUB_WI | RD4(TMP_REG1) | RN4(arg) | imm);
3524 }
3525 } else if ((argw & 0xfff) <= max_offset) {
3526 imm = get_imm((sljit_uw)(argw & ~0xfff));
3527
3528 if (imm) {
3529 *memw = argw & 0xfff;
3530 return push_inst32(compiler, ADD_WI | RD4(TMP_REG1) | RN4(arg) | imm);
3531 }
3532 } else {
3533 imm = get_imm((sljit_uw)((argw | 0xfff) + 1));
3534
3535 if (imm) {
3536 *memw = (argw & 0xfff) - 0x1000;
3537 return push_inst32(compiler, ADD_WI | RD4(TMP_REG1) | RN4(arg) | imm);
3538 }
3539 }
3540 }
3541
3542 imm = (sljit_uw)(argw & ~0xfff);
3543
3544 if ((argw & 0xfff) > max_offset) {
3545 imm += 0x1000;
3546 *memw = (argw & 0xfff) - 0x1000;
3547 } else
3548 *memw = argw & 0xfff;
3549
3550 FAIL_IF(load_immediate(compiler, TMP_REG1, imm));
3551
3552 if (arg == 0)
3553 return SLJIT_SUCCESS;
3554
3555 return push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, arg));
3556 }
3557
3558 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem(struct sljit_compiler *compiler, sljit_s32 type,
3559 sljit_s32 freg,
3560 sljit_s32 mem, sljit_sw memw)
3561 {
3562 CHECK_ERROR();
3563 CHECK(check_sljit_emit_fmem(compiler, type, freg, mem, memw));
3564
3565 if (type & SLJIT_MEM_ALIGNED_32)
3566 return emit_fop_mem(compiler, ((type ^ SLJIT_32) & SLJIT_32) | ((type & SLJIT_MEM_STORE) ? 0 : FPU_LOAD), freg, mem, memw);
3567
3568 if (type & SLJIT_MEM_STORE) {
3569 FAIL_IF(push_inst32(compiler, VMOV | (1 << 20) | VN4(freg) | RT4(TMP_REG2)));
3570
3571 if (type & SLJIT_32)
3572 return emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG2, mem, memw, TMP_REG1);
3573
3574 FAIL_IF(update_mem_addr(compiler, &mem, &memw, 0xfff - 4));
3575 mem |= SLJIT_MEM;
3576
3577 FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG2, mem, memw, TMP_REG1));
3578 FAIL_IF(push_inst32(compiler, VMOV | (1 << 20) | VN4(freg) | 0x80 | RT4(TMP_REG2)));
3579 return emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG2, mem, memw + 4, TMP_REG1);
3580 }
3581
3582 if (type & SLJIT_32) {
3583 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, mem, memw, TMP_REG1));
3584 return push_inst32(compiler, VMOV | VN4(freg) | RT4(TMP_REG2));
3585 }
3586
3587 FAIL_IF(update_mem_addr(compiler, &mem, &memw, 0xfff - 4));
3588 mem |= SLJIT_MEM;
3589
3590 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, mem, memw, TMP_REG1));
3591 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, mem, memw + 4, TMP_REG1));
3592 return push_inst32(compiler, VMOV2 | VM4(freg) | RT4(TMP_REG2) | RN4(TMP_REG1));
3593 }
3594
3595 static sljit_s32 sljit_emit_simd_mem_offset(struct sljit_compiler *compiler, sljit_s32 *mem_ptr, sljit_sw memw)
3596 {
3597 sljit_uw imm;
3598 sljit_s32 mem = *mem_ptr;
3599
3600 if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
3601 *mem_ptr = TMP_REG1;
3602 return push_inst32(compiler, ADD_W | RD4(TMP_REG1) | RN4(mem & REG_MASK) | RM4(OFFS_REG(mem)) | ((sljit_uw)(memw & 0x3) << 6));
3603 }
3604
3605 if (SLJIT_UNLIKELY(!(mem & REG_MASK))) {
3606 *mem_ptr = TMP_REG1;
3607 return load_immediate(compiler, TMP_REG1, (sljit_uw)memw);
3608 }
3609
3610 mem &= REG_MASK;
3611
3612 if (memw == 0) {
3613 *mem_ptr = mem;
3614 return SLJIT_SUCCESS;
3615 }
3616
3617 *mem_ptr = TMP_REG1;
3618 imm = get_imm((sljit_uw)(memw < 0 ? -memw : memw));
3619
3620 if (imm != INVALID_IMM)
3621 return push_inst32(compiler, ((memw < 0) ? SUB_WI : ADD_WI) | RD4(TMP_REG1) | RN4(mem) | imm);
3622
3623 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
3624 return push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, mem));
3625 }
3626
3627 static SLJIT_INLINE sljit_s32 simd_get_quad_reg_index(sljit_s32 freg)
3628 {
3629 freg += freg & 0x1;
3630
3631 SLJIT_ASSERT((freg_map[freg] & 0x1) == (freg <= SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS));
3632
3633 if (freg <= SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS)
3634 freg--;
3635
3636 return freg;
3637 }
3638
3639 #define SLJIT_QUAD_OTHER_HALF(freg) ((((freg) & 0x1) << 1) - 1)
3640
3641 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
3642 sljit_s32 vreg,
3643 sljit_s32 srcdst, sljit_sw srcdstw)
3644 {
3645 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3646 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3647 sljit_s32 alignment = SLJIT_SIMD_GET_ELEM2_SIZE(type);
3648 sljit_ins ins;
3649
3650 CHECK_ERROR();
3651 CHECK(check_sljit_emit_simd_mov(compiler, type, vreg, srcdst, srcdstw));
3652
3653 ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
3654
3655 if (reg_size != 3 && reg_size != 4)
3656 return SLJIT_ERR_UNSUPPORTED;
3657
3658 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
3659 return SLJIT_ERR_UNSUPPORTED;
3660
3661 if (type & SLJIT_SIMD_TEST)
3662 return SLJIT_SUCCESS;
3663
3664 if (reg_size == 4)
3665 vreg = simd_get_quad_reg_index(vreg);
3666
3667 if (!(srcdst & SLJIT_MEM)) {
3668 if (reg_size == 4)
3669 srcdst = simd_get_quad_reg_index(srcdst);
3670
3671 if (type & SLJIT_SIMD_STORE)
3672 ins = VD4(srcdst) | VN4(vreg) | VM4(vreg);
3673 else
3674 ins = VD4(vreg) | VN4(srcdst) | VM4(srcdst);
3675
3676 if (reg_size == 4)
3677 ins |= (sljit_ins)1 << 6;
3678
3679 return push_inst32(compiler, VORR | ins);
3680 }
3681
3682 FAIL_IF(sljit_emit_simd_mem_offset(compiler, &srcdst, srcdstw));
3683
3684 if (elem_size > 3)
3685 elem_size = 3;
3686
3687 ins = ((type & SLJIT_SIMD_STORE) ? VST1 : VLD1) | VD4(vreg)
3688 | (sljit_ins)((reg_size == 3) ? (0x7 << 8) : (0xa << 8));
3689
3690 SLJIT_ASSERT(reg_size >= alignment);
3691
3692 if (alignment == 3)
3693 ins |= 0x10;
3694 else if (alignment >= 4)
3695 ins |= 0x20;
3696
3697 return push_inst32(compiler, ins | RN4(srcdst) | ((sljit_ins)elem_size) << 6 | 0xf);
3698 }
3699
3700 static sljit_ins simd_get_imm(sljit_s32 elem_size, sljit_uw value)
3701 {
3702 sljit_ins result;
3703
3704 if (elem_size > 1 && (sljit_u16)value == (value >> 16)) {
3705 elem_size = 1;
3706 value = (sljit_u16)value;
3707 }
3708
3709 if (elem_size == 1 && (sljit_u8)value == (value >> 8)) {
3710 elem_size = 0;
3711 value = (sljit_u8)value;
3712 }
3713
3714 switch (elem_size) {
3715 case 0:
3716 SLJIT_ASSERT(value <= 0xff);
3717 result = 0xe00;
3718 break;
3719 case 1:
3720 SLJIT_ASSERT(value <= 0xffff);
3721 result = 0;
3722
3723 while (1) {
3724 if (value <= 0xff) {
3725 result |= 0x800;
3726 break;
3727 }
3728
3729 if ((value & 0xff) == 0) {
3730 value >>= 8;
3731 result |= 0xa00;
3732 break;
3733 }
3734
3735 if (result != 0)
3736 return ~(sljit_ins)0;
3737
3738 value ^= (sljit_uw)0xffff;
3739 result = (1 << 5);
3740 }
3741 break;
3742 default:
3743 SLJIT_ASSERT(value <= 0xffffffff);
3744 result = 0;
3745
3746 while (1) {
3747 if (value <= 0xff) {
3748 result |= 0x000;
3749 break;
3750 }
3751
3752 if ((value & ~(sljit_uw)0xff00) == 0) {
3753 value >>= 8;
3754 result |= 0x200;
3755 break;
3756 }
3757
3758 if ((value & ~(sljit_uw)0xff0000) == 0) {
3759 value >>= 16;
3760 result |= 0x400;
3761 break;
3762 }
3763
3764 if ((value & ~(sljit_uw)0xff000000) == 0) {
3765 value >>= 24;
3766 result |= 0x600;
3767 break;
3768 }
3769
3770 if ((value & (sljit_uw)0xff) == 0xff && (value >> 16) == 0) {
3771 value >>= 8;
3772 result |= 0xc00;
3773 break;
3774 }
3775
3776 if ((value & (sljit_uw)0xffff) == 0xffff && (value >> 24) == 0) {
3777 value >>= 16;
3778 result |= 0xd00;
3779 break;
3780 }
3781
3782 if (result != 0)
3783 return ~(sljit_ins)0;
3784
3785 value = ~value;
3786 result = (1 << 5);
3787 }
3788 break;
3789 }
3790
3791 return ((sljit_ins)value & 0xf) | (((sljit_ins)value & 0x70) << 12) | (((sljit_ins)value & 0x80) << 21) | result;
3792 }
3793
3794 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3795 sljit_s32 vreg,
3796 sljit_s32 src, sljit_sw srcw)
3797 {
3798 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3799 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3800 sljit_ins ins, imm;
3801
3802 CHECK_ERROR();
3803 CHECK(check_sljit_emit_simd_replicate(compiler, type, vreg, src, srcw));
3804
3805 ADJUST_LOCAL_OFFSET(src, srcw);
3806
3807 if (reg_size != 3 && reg_size != 4)
3808 return SLJIT_ERR_UNSUPPORTED;
3809
3810 if ((type & SLJIT_SIMD_FLOAT) ? (elem_size < 2 || elem_size > 3) : (elem_size > 2))
3811 return SLJIT_ERR_UNSUPPORTED;
3812
3813 if (type & SLJIT_SIMD_TEST)
3814 return SLJIT_SUCCESS;
3815
3816 if (reg_size == 4)
3817 vreg = simd_get_quad_reg_index(vreg);
3818
3819 if (src == SLJIT_IMM && srcw == 0)
3820 return push_inst32(compiler, VMOV_i | ((reg_size == 4) ? (1 << 6) : 0) | VD4(vreg));
3821
3822 if (SLJIT_UNLIKELY(elem_size == 3)) {
3823 SLJIT_ASSERT(type & SLJIT_SIMD_FLOAT);
3824
3825 if (src & SLJIT_MEM) {
3826 FAIL_IF(emit_fop_mem(compiler, FPU_LOAD | SLJIT_32, vreg, src, srcw));
3827 src = vreg;
3828 } else if (vreg != src)
3829 FAIL_IF(push_inst32(compiler, VORR | VD4(vreg) | VN4(src) | VM4(src)));
3830
3831 vreg += SLJIT_QUAD_OTHER_HALF(vreg);
3832
3833 if (vreg != src)
3834 return push_inst32(compiler, VORR | VD4(vreg) | VN4(src) | VM4(src));
3835 return SLJIT_SUCCESS;
3836 }
3837
3838 if (src & SLJIT_MEM) {
3839 FAIL_IF(sljit_emit_simd_mem_offset(compiler, &src, srcw));
3840
3841 ins = (sljit_ins)(elem_size << 6);
3842
3843 if (reg_size == 4)
3844 ins |= 1 << 5;
3845
3846 return push_inst32(compiler, VLD1_r | ins | VD4(vreg) | RN4(src) | 0xf);
3847 }
3848
3849 if (type & SLJIT_SIMD_FLOAT) {
3850 SLJIT_ASSERT(elem_size == 2);
3851 ins = ((sljit_ins)freg_ebit_map[src] << (16 + 2 + 1)) | ((sljit_ins)1 << (16 + 2));
3852
3853 if (reg_size == 4)
3854 ins |= (sljit_ins)1 << 6;
3855
3856 return push_inst32(compiler, VDUP_s | ins | VD4(vreg) | (sljit_ins)freg_map[src]);
3857 }
3858
3859 if (src == SLJIT_IMM) {
3860 if (elem_size < 2)
3861 srcw &= ((sljit_sw)1 << (((sljit_sw)1 << elem_size) << 3)) - 1;
3862
3863 imm = simd_get_imm(elem_size, (sljit_uw)srcw);
3864
3865 if (imm != ~(sljit_ins)0) {
3866 if (reg_size == 4)
3867 imm |= (sljit_ins)1 << 6;
3868
3869 return push_inst32(compiler, VMOV_i | imm | VD4(vreg));
3870 }
3871
3872 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)srcw));
3873 src = TMP_REG1;
3874 }
3875
3876 switch (elem_size) {
3877 case 0:
3878 ins = 1 << 22;
3879 break;
3880 case 1:
3881 ins = 1 << 5;
3882 break;
3883 default:
3884 ins = 0;
3885 break;
3886 }
3887
3888 if (reg_size == 4)
3889 ins |= (sljit_ins)1 << 21;
3890
3891 return push_inst32(compiler, VDUP | ins | VN4(vreg) | RT4(src));
3892 }
3893
3894 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
3895 sljit_s32 vreg, sljit_s32 lane_index,
3896 sljit_s32 srcdst, sljit_sw srcdstw)
3897 {
3898 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3899 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3900 sljit_ins ins;
3901
3902 CHECK_ERROR();
3903 CHECK(check_sljit_emit_simd_lane_mov(compiler, type, vreg, lane_index, srcdst, srcdstw));
3904
3905 ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
3906
3907 if (reg_size != 3 && reg_size != 4)
3908 return SLJIT_ERR_UNSUPPORTED;
3909
3910 if ((type & SLJIT_SIMD_FLOAT) ? (elem_size < 2 || elem_size > 3) : (elem_size > 2))
3911 return SLJIT_ERR_UNSUPPORTED;
3912
3913 if (type & SLJIT_SIMD_TEST)
3914 return SLJIT_SUCCESS;
3915
3916 if (reg_size == 4)
3917 vreg = simd_get_quad_reg_index(vreg);
3918
3919 if (type & SLJIT_SIMD_LANE_ZERO) {
3920 ins = (reg_size == 3) ? 0 : ((sljit_ins)1 << 6);
3921
3922 if (type & SLJIT_SIMD_FLOAT) {
3923 if (elem_size == 3 && !(srcdst & SLJIT_MEM)) {
3924 if (lane_index == 1)
3925 vreg += SLJIT_QUAD_OTHER_HALF(vreg);
3926
3927 if (srcdst != vreg)
3928 FAIL_IF(push_inst32(compiler, VORR | VD4(vreg) | VN4(srcdst) | VM4(srcdst)));
3929
3930 vreg += SLJIT_QUAD_OTHER_HALF(vreg);
3931 return push_inst32(compiler, VMOV_i | VD4(vreg));
3932 }
3933
3934 if (srcdst == vreg || (elem_size == 3 && srcdst == (vreg + SLJIT_QUAD_OTHER_HALF(vreg)))) {
3935 FAIL_IF(push_inst32(compiler, VORR | ins | VD4(TMP_FREG2) | VN4(vreg) | VM4(vreg)));
3936 srcdst = TMP_FREG2;
3937 srcdstw = 0;
3938 }
3939 }
3940
3941 FAIL_IF(push_inst32(compiler, VMOV_i | ins | VD4(vreg)));
3942 }
3943
3944 if (reg_size == 4 && lane_index >= (0x8 >> elem_size)) {
3945 lane_index -= (0x8 >> elem_size);
3946 vreg += SLJIT_QUAD_OTHER_HALF(vreg);
3947 }
3948
3949 if (srcdst & SLJIT_MEM) {
3950 if (elem_size == 3)
3951 return emit_fop_mem(compiler, ((type & SLJIT_SIMD_STORE) ? 0 : FPU_LOAD) | SLJIT_32, vreg, srcdst, srcdstw);
3952
3953 FAIL_IF(sljit_emit_simd_mem_offset(compiler, &srcdst, srcdstw));
3954
3955 lane_index = lane_index << elem_size;
3956 ins = (sljit_ins)((elem_size << 10) | (lane_index << 5));
3957 return push_inst32(compiler, ((type & SLJIT_SIMD_STORE) ? VST1_s : VLD1_s) | ins | VD4(vreg) | RN4(srcdst) | 0xf);
3958 }
3959
3960 if (type & SLJIT_SIMD_FLOAT) {
3961 if (elem_size == 3) {
3962 if (type & SLJIT_SIMD_STORE)
3963 return push_inst32(compiler, VORR | VD4(srcdst) | VN4(vreg) | VM4(vreg));
3964 return push_inst32(compiler, VMOV_F32 | SLJIT_32 | VD4(vreg) | VM4(srcdst));
3965 }
3966
3967 if (type & SLJIT_SIMD_STORE) {
3968 if (freg_ebit_map[vreg] == 0) {
3969 if (lane_index == 1)
3970 vreg = SLJIT_F64_SECOND(vreg);
3971
3972 return push_inst32(compiler, VMOV_F32 | VD4(srcdst) | VM4(vreg));
3973 }
3974
3975 FAIL_IF(push_inst32(compiler, VMOV_s | (1 << 20) | ((sljit_ins)lane_index << 21) | VN4(vreg) | RT4(TMP_REG1)));
3976 return push_inst32(compiler, VMOV | VN4(srcdst) | RT4(TMP_REG1));
3977 }
3978
3979 FAIL_IF(push_inst32(compiler, VMOV | (1 << 20) | VN4(srcdst) | RT4(TMP_REG1)));
3980 return push_inst32(compiler, VMOV_s | ((sljit_ins)lane_index << 21) | VN4(vreg) | RT4(TMP_REG1));
3981 }
3982
3983 if (srcdst == SLJIT_IMM) {
3984 if (elem_size < 2)
3985 srcdstw &= ((sljit_sw)1 << (((sljit_sw)1 << elem_size) << 3)) - 1;
3986
3987 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)srcdstw));
3988 srcdst = TMP_REG1;
3989 }
3990
3991 if (elem_size == 0)
3992 ins = 0x400000;
3993 else if (elem_size == 1)
3994 ins = 0x20;
3995 else
3996 ins = 0;
3997
3998 lane_index = lane_index << elem_size;
3999 ins |= (sljit_ins)(((lane_index & 0x4) << 19) | ((lane_index & 0x3) << 5));
4000
4001 if (type & SLJIT_SIMD_STORE) {
4002 ins |= (1 << 20);
4003
4004 if (elem_size < 2 && !(type & SLJIT_SIMD_LANE_SIGNED))
4005 ins |= (1 << 23);
4006 }
4007
4008 return push_inst32(compiler, VMOV_s | ins | VN4(vreg) | RT4(srcdst));
4009 }
4010
4011 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
4012 sljit_s32 vreg,
4013 sljit_s32 src, sljit_s32 src_lane_index)
4014 {
4015 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4016 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4017 sljit_ins ins;
4018
4019 CHECK_ERROR();
4020 CHECK(check_sljit_emit_simd_lane_replicate(compiler, type, vreg, src, src_lane_index));
4021
4022 if (reg_size != 3 && reg_size != 4)
4023 return SLJIT_ERR_UNSUPPORTED;
4024
4025 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
4026 return SLJIT_ERR_UNSUPPORTED;
4027
4028 if (type & SLJIT_SIMD_TEST)
4029 return SLJIT_SUCCESS;
4030
4031 if (reg_size == 4) {
4032 vreg = simd_get_quad_reg_index(vreg);
4033 src = simd_get_quad_reg_index(src);
4034
4035 if (src_lane_index >= (0x8 >> elem_size)) {
4036 src_lane_index -= (0x8 >> elem_size);
4037 src += SLJIT_QUAD_OTHER_HALF(src);
4038 }
4039 }
4040
4041 if (elem_size == 3) {
4042 if (vreg != src)
4043 FAIL_IF(push_inst32(compiler, VORR | VD4(vreg) | VN4(src) | VM4(src)));
4044
4045 vreg += SLJIT_QUAD_OTHER_HALF(vreg);
4046
4047 if (vreg != src)
4048 return push_inst32(compiler, VORR | VD4(vreg) | VN4(src) | VM4(src));
4049 return SLJIT_SUCCESS;
4050 }
4051
4052 ins = ((((sljit_ins)src_lane_index << 1) | 1) << (16 + elem_size));
4053
4054 if (reg_size == 4)
4055 ins |= (sljit_ins)1 << 6;
4056
4057 return push_inst32(compiler, VDUP_s | ins | VD4(vreg) | VM4(src));
4058 }
4059
4060 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
4061 sljit_s32 vreg,
4062 sljit_s32 src, sljit_sw srcw)
4063 {
4064 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4065 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4066 sljit_s32 elem2_size = SLJIT_SIMD_GET_ELEM2_SIZE(type);
4067 sljit_s32 dst_reg;
4068
4069 CHECK_ERROR();
4070 CHECK(check_sljit_emit_simd_extend(compiler, type, vreg, src, srcw));
4071
4072 ADJUST_LOCAL_OFFSET(src, srcw);
4073
4074 if (reg_size != 3 && reg_size != 4)
4075 return SLJIT_ERR_UNSUPPORTED;
4076
4077 if ((type & SLJIT_SIMD_FLOAT) && (elem_size != 2 || elem2_size != 3))
4078 return SLJIT_ERR_UNSUPPORTED;
4079
4080 if (type & SLJIT_SIMD_TEST)
4081 return SLJIT_SUCCESS;
4082
4083 if (reg_size == 4)
4084 vreg = simd_get_quad_reg_index(vreg);
4085
4086 if (src & SLJIT_MEM) {
4087 FAIL_IF(sljit_emit_simd_mem_offset(compiler, &src, srcw));
4088 if (reg_size == 4 && elem2_size - elem_size == 1)
4089 FAIL_IF(push_inst32(compiler, VLD1 | (0x7 << 8) | VD4(vreg) | RN4(src) | 0xf));
4090 else
4091 FAIL_IF(push_inst32(compiler, VLD1_s | (sljit_ins)((reg_size - elem2_size + elem_size) << 10) | VD4(vreg) | RN4(src) | 0xf));
4092 src = vreg;
4093 } else if (reg_size == 4)
4094 src = simd_get_quad_reg_index(src);
4095
4096 if (!(type & SLJIT_SIMD_FLOAT)) {
4097 dst_reg = (reg_size == 4) ? vreg : TMP_FREG2;
4098
4099 do {
4100 FAIL_IF(push_inst32(compiler, VSHLL | ((type & SLJIT_SIMD_EXTEND_SIGNED) ? 0 : (1 << 28))
4101 | ((sljit_ins)1 << (19 + elem_size)) | VD4(dst_reg) | VM4(src)));
4102 src = dst_reg;
4103 } while (++elem_size < elem2_size);
4104
4105 if (dst_reg == TMP_FREG2)
4106 return push_inst32(compiler, VORR | VD4(vreg) | VN4(TMP_FREG2) | VM4(TMP_FREG2));
4107 return SLJIT_SUCCESS;
4108 }
4109
4110 /* No SIMD variant, must use VFP instead. */
4111 SLJIT_ASSERT(reg_size == 4);
4112
4113 if (vreg == src) {
4114 vreg += SLJIT_QUAD_OTHER_HALF(vreg);
4115 FAIL_IF(push_inst32(compiler, VCVT_F64_F32 | VD4(vreg) | VM4(src) | 0x20));
4116 vreg += SLJIT_QUAD_OTHER_HALF(vreg);
4117 return push_inst32(compiler, VCVT_F64_F32 | VD4(vreg) | VM4(src));
4118 }
4119
4120 FAIL_IF(push_inst32(compiler, VCVT_F64_F32 | VD4(vreg) | VM4(src)));
4121 vreg += SLJIT_QUAD_OTHER_HALF(vreg);
4122 return push_inst32(compiler, VCVT_F64_F32 | VD4(vreg) | VM4(src) | 0x20);
4123 }
4124
4125 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
4126 sljit_s32 vreg,
4127 sljit_s32 dst, sljit_sw dstw)
4128 {
4129 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4130 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4131 sljit_ins ins, imms;
4132 sljit_s32 dst_r;
4133
4134 CHECK_ERROR();
4135 CHECK(check_sljit_emit_simd_sign(compiler, type, vreg, dst, dstw));
4136
4137 ADJUST_LOCAL_OFFSET(dst, dstw);
4138
4139 if (reg_size != 3 && reg_size != 4)
4140 return SLJIT_ERR_UNSUPPORTED;
4141
4142 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
4143 return SLJIT_ERR_UNSUPPORTED;
4144
4145 if (type & SLJIT_SIMD_TEST)
4146 return SLJIT_SUCCESS;
4147
4148 switch (elem_size) {
4149 case 0:
4150 imms = 0x243219;
4151 ins = VSHR | (1 << 28) | (0x9 << 16);
4152 break;
4153 case 1:
4154 imms = (reg_size == 4) ? 0x243219 : 0x2231;
4155 ins = VSHR | (1 << 28) | (0x11 << 16);
4156 break;
4157 case 2:
4158 imms = (reg_size == 4) ? 0x2231 : 0x21;
4159 ins = VSHR | (1 << 28) | (0x21 << 16);
4160 break;
4161 default:
4162 imms = 0x21;
4163 ins = VSHR | (1 << 28) | (0x1 << 16) | (1 << 7);
4164 break;
4165 }
4166
4167 if (reg_size == 4) {
4168 vreg = simd_get_quad_reg_index(vreg);
4169 ins |= (sljit_ins)1 << 6;
4170 }
4171
4172 SLJIT_ASSERT((freg_map[TMP_FREG2] & 0x1) == 0);
4173 FAIL_IF(push_inst32(compiler, ins | VD4(TMP_FREG2) | VM4(vreg)));
4174
4175 if (reg_size == 4 && elem_size > 0)
4176 FAIL_IF(push_inst32(compiler, VMOVN | ((sljit_ins)(elem_size - 1) << 18) | VD4(TMP_FREG2) | VM4(TMP_FREG2)));
4177
4178 ins = (reg_size == 4 && elem_size == 0) ? (1 << 6) : 0;
4179
4180 while (imms >= 0x100) {
4181 FAIL_IF(push_inst32(compiler, VSRA | (1 << 28) | ins | ((imms & 0xff) << 16) | VD4(TMP_FREG2) | VM4(TMP_FREG2)));
4182 imms >>= 8;
4183 }
4184
4185 FAIL_IF(push_inst32(compiler, VSRA | (1 << 28) | ins | (1 << 7) | (imms << 16) | VD4(TMP_FREG2) | VM4(TMP_FREG2)));
4186
4187 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
4188 FAIL_IF(push_inst32(compiler, VMOV_s | (1 << 20) | (1 << 23) | (0x2 << 21) | RT4(dst_r) | VN4(TMP_FREG2)));
4189
4190 if (reg_size == 4 && elem_size == 0) {
4191 SLJIT_ASSERT(freg_map[TMP_FREG2] + 1 == freg_map[TMP_FREG1]);
4192 FAIL_IF(push_inst32(compiler, VMOV_s | (1 << 20) | (1 << 23) | (0x2 << 21) | RT4(TMP_REG2)| VN4(TMP_FREG1)));
4193 FAIL_IF(push_inst32(compiler, ORR_W | RD4(dst_r) | RN4(dst_r) | RM4(TMP_REG2) | (0x2 << 12)));
4194 }
4195
4196 if (dst_r == TMP_REG1)
4197 return emit_op_mem(compiler, STORE | WORD_SIZE, TMP_REG1, dst, dstw, TMP_REG2);
4198
4199 return SLJIT_SUCCESS;
4200 }
4201
4202 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
4203 sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
4204 {
4205 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
4206 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
4207 sljit_s32 alignment;
4208 sljit_ins ins = 0, load_ins;
4209
4210 CHECK_ERROR();
4211 CHECK(check_sljit_emit_simd_op2(compiler, type, dst_vreg, src1_vreg, src2, src2w));
4212 ADJUST_LOCAL_OFFSET(src2, src2w);
4213
4214 if (reg_size != 3 && reg_size != 4)
4215 return SLJIT_ERR_UNSUPPORTED;
4216
4217 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
4218 return SLJIT_ERR_UNSUPPORTED;
4219
4220 if (type & SLJIT_SIMD_TEST)
4221 return SLJIT_SUCCESS;
4222
4223 switch (SLJIT_SIMD_GET_OPCODE(type)) {
4224 case SLJIT_SIMD_OP2_AND:
4225 ins = VAND;
4226 break;
4227 case SLJIT_SIMD_OP2_OR:
4228 ins = VORR;
4229 break;
4230 case SLJIT_SIMD_OP2_XOR:
4231 ins = VEOR;
4232 break;
4233 case SLJIT_SIMD_OP2_SHUFFLE:
4234 ins = VTBL;
4235 break;
4236 }
4237
4238 if (src2 & SLJIT_MEM) {
4239 if (elem_size > 3)
4240 elem_size = 3;
4241
4242 load_ins = VLD1 | (sljit_ins)((reg_size == 3) ? (0x7 << 8) : (0xa << 8));
4243 alignment = SLJIT_SIMD_GET_ELEM2_SIZE(type);
4244
4245 SLJIT_ASSERT(reg_size >= alignment);
4246
4247 if (alignment == 3)
4248 load_ins |= 0x10;
4249 else if (alignment >= 4)
4250 load_ins |= 0x20;
4251
4252 FAIL_IF(sljit_emit_simd_mem_offset(compiler, &src2, src2w));
4253 FAIL_IF(push_inst32(compiler, load_ins | VD4(TMP_FREG2) | RN4(src2) | ((sljit_ins)elem_size) << 6 | 0xf));
4254 src2 = TMP_FREG2;
4255 }
4256
4257 if (reg_size == 4) {
4258 dst_vreg = simd_get_quad_reg_index(dst_vreg);
4259 src1_vreg = simd_get_quad_reg_index(src1_vreg);
4260 src2 = simd_get_quad_reg_index(src2);
4261
4262 if (SLJIT_SIMD_GET_OPCODE(type) == SLJIT_SIMD_OP2_SHUFFLE) {
4263 ins |= (sljit_ins)1 << 8;
4264
4265 FAIL_IF(push_inst32(compiler, ins | VD4(dst_vreg != src1_vreg ? dst_vreg : TMP_FREG2) | VN4(src1_vreg) | VM4(src2)));
4266 src2 += SLJIT_QUAD_OTHER_HALF(src2);
4267 FAIL_IF(push_inst32(compiler, ins | VD4(dst_vreg + SLJIT_QUAD_OTHER_HALF(dst_vreg)) | VN4(src1_vreg) | VM4(src2)));
4268
4269 if (dst_vreg == src1_vreg)
4270 return push_inst32(compiler, VORR | VD4(dst_vreg) | VN4(TMP_FREG2) | VM4(TMP_FREG2));
4271 return SLJIT_SUCCESS;
4272 }
4273
4274 ins |= (sljit_ins)1 << 6;
4275 }
4276
4277 return push_inst32(compiler, ins | VD4(dst_vreg) | VN4(src1_vreg) | VM4(src2));
4278 }
4279
4280 #undef FPU_LOAD
4281
4282 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
4283 sljit_s32 dst_reg,
4284 sljit_s32 mem_reg)
4285 {
4286 sljit_ins ins;
4287
4288 CHECK_ERROR();
4289 CHECK(check_sljit_emit_atomic_load(compiler, op, dst_reg, mem_reg));
4290
4291 if (op & SLJIT_ATOMIC_USE_CAS)
4292 return SLJIT_ERR_UNSUPPORTED;
4293
4294 switch (GET_OPCODE(op)) {
4295 case SLJIT_MOV_S8:
4296 case SLJIT_MOV_S16:
4297 case SLJIT_MOV_S32:
4298 return SLJIT_ERR_UNSUPPORTED;
4299
4300 case SLJIT_MOV_U8:
4301 ins = LDREXB;
4302 break;
4303 case SLJIT_MOV_U16:
4304 ins = LDREXH;
4305 break;
4306 default:
4307 ins = LDREX;
4308 break;
4309 }
4310
4311 if (op & SLJIT_ATOMIC_TEST)
4312 return SLJIT_SUCCESS;
4313
4314 return push_inst32(compiler, ins | RN4(mem_reg) | RT4(dst_reg));
4315 }
4316
4317 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
4318 sljit_s32 src_reg,
4319 sljit_s32 mem_reg,
4320 sljit_s32 temp_reg)
4321 {
4322 sljit_ins ins;
4323
4324 /* temp_reg == mem_reg is undefined so use another temp register */
4325 SLJIT_UNUSED_ARG(temp_reg);
4326
4327 CHECK_ERROR();
4328 CHECK(check_sljit_emit_atomic_store(compiler, op, src_reg, mem_reg, temp_reg));
4329
4330 if (op & SLJIT_ATOMIC_USE_CAS)
4331 return SLJIT_ERR_UNSUPPORTED;
4332
4333 switch (GET_OPCODE(op)) {
4334 case SLJIT_MOV_S8:
4335 case SLJIT_MOV_S16:
4336 case SLJIT_MOV_S32:
4337 return SLJIT_ERR_UNSUPPORTED;
4338
4339 case SLJIT_MOV_U8:
4340 ins = STREXB | RM4(TMP_REG1);
4341 break;
4342 case SLJIT_MOV_U16:
4343 ins = STREXH | RM4(TMP_REG1);
4344 break;
4345 default:
4346 ins = STREX | RD4(TMP_REG1);
4347 break;
4348 }
4349
4350 if (op & SLJIT_ATOMIC_TEST)
4351 return SLJIT_SUCCESS;
4352
4353 FAIL_IF(push_inst32(compiler, ins | RN4(mem_reg) | RT4(src_reg)));
4354 if (op & SLJIT_SET_ATOMIC_STORED)
4355 return push_inst32(compiler, CMPI_W | RN4(TMP_REG1));
4356
4357 return SLJIT_SUCCESS;
4358 }
4359
4360 SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
4361 {
4362 struct sljit_const *const_;
4363 sljit_s32 dst_r;
4364
4365 CHECK_ERROR_PTR();
4366 CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value));
4367 ADJUST_LOCAL_OFFSET(dst, dstw);
4368
4369 const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
4370 PTR_FAIL_IF(!const_);
4371 set_const(const_, compiler);
4372
4373 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
4374 PTR_FAIL_IF(emit_imm32_const(compiler, dst_r, (sljit_uw)init_value));
4375
4376 if (dst & SLJIT_MEM)
4377 PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2));
4378 return const_;
4379 }
4380
4381 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_mov_addr(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
4382 {
4383 struct sljit_jump *jump;
4384 sljit_s32 dst_r;
4385
4386 CHECK_ERROR_PTR();
4387 CHECK_PTR(check_sljit_emit_mov_addr(compiler, dst, dstw));
4388 ADJUST_LOCAL_OFFSET(dst, dstw);
4389
4390 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
4391 PTR_FAIL_IF(!jump);
4392 set_mov_addr(jump, compiler, 0);
4393
4394 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
4395 PTR_FAIL_IF(push_inst16(compiler, RDN3(dst_r)));
4396 compiler->size += 3;
4397
4398 if (dst & SLJIT_MEM)
4399 PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2));
4400 return jump;
4401 }
4402
4403 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset)
4404 {
4405 sljit_u16 *inst = (sljit_u16*)addr;
4406 SLJIT_UNUSED_ARG(executable_offset);
4407
4408 SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 0);
4409 modify_imm32_const(inst, new_target);
4410 SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 1);
4411 inst = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
4412 SLJIT_CACHE_FLUSH(inst, inst + 4);
4413 }
4414
4415 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset)
4416 {
4417 sljit_set_jump_addr(addr, (sljit_uw)new_constant, executable_offset);
4418 }