Debian Codegraph @

src/3rdparty/pcre2/deps/sljit/sljit_src/sljitNativeARM_64.c

raw
1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #ifdef __ARM_FEATURE_ATOMICS
28 #define ARM_ATOMIC_INFO " (LSE)"
29 #else
30 #define ARM_ATOMIC_INFO ""
31 #endif
32
33 SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
34 {
35 return "ARM-64" ARM_ATOMIC_INFO SLJIT_CPUINFO;
36 }
37
38 /* Length of an instruction word */
39 typedef sljit_u32 sljit_ins;
40
41 #define TMP_ZERO (0)
42
43 #define TMP_REG1 (SLJIT_NUMBER_OF_REGISTERS + 2)
44 #define TMP_REG2 (SLJIT_NUMBER_OF_REGISTERS + 3)
45 #define TMP_LR (SLJIT_NUMBER_OF_REGISTERS + 4)
46 #define TMP_FP (SLJIT_NUMBER_OF_REGISTERS + 5)
47
48 #define TMP_FREG1 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
49 #define TMP_FREG2 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2)
50
51 /* r18 - platform register, currently not used */
52 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 8] = {
53 31, 0, 1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14, 15, 16, 17, 8, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 31, 9, 10, 30, 29
54 };
55
56 static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = {
57 0, 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 15, 14, 13, 12, 11, 10, 9, 8, 30, 31
58 };
59
60 #define W_OP ((sljit_ins)1 << 31)
61 #define RD(rd) ((sljit_ins)reg_map[rd])
62 #define RT(rt) ((sljit_ins)reg_map[rt])
63 #define RN(rn) ((sljit_ins)reg_map[rn] << 5)
64 #define RT2(rt2) ((sljit_ins)reg_map[rt2] << 10)
65 #define RM(rm) ((sljit_ins)reg_map[rm] << 16)
66 #define VD(vd) ((sljit_ins)freg_map[vd])
67 #define VT(vt) ((sljit_ins)freg_map[vt])
68 #define VT2(vt) ((sljit_ins)freg_map[vt] << 10)
69 #define VN(vn) ((sljit_ins)freg_map[vn] << 5)
70 #define VM(vm) ((sljit_ins)freg_map[vm] << 16)
71
72 /* --------------------------------------------------------------------- */
73 /* Instruction forms */
74 /* --------------------------------------------------------------------- */
75
76 #define ADC 0x9a000000
77 #define ADD 0x8b000000
78 #define ADDE 0x8b200000
79 #define ADDI 0x91000000
80 #define ADR 0x10000000
81 #define ADRP 0x90000000
82 #define AND 0x8a000000
83 #define ANDI 0x92000000
84 #define AND_v 0x0e201c00
85 #define ASRV 0x9ac02800
86 #define B 0x14000000
87 #define B_CC 0x54000000
88 #define BL 0x94000000
89 #define BLR 0xd63f0000
90 #define BR 0xd61f0000
91 #define BRK 0xd4200000
92 #define CAS 0xc8a07c00
93 #define CASB 0x08a07c00
94 #define CASH 0x48a07c00
95 #define CBZ 0xb4000000
96 #define CCMPI 0xfa400800
97 #define CLZ 0xdac01000
98 #define CSEL 0x9a800000
99 #define CSINC 0x9a800400
100 #define CSINV 0xda800000
101 #define DMB_SY 0xd5033fbf
102 #define DUP_e 0x0e000400
103 #define DUP_g 0x0e000c00
104 #define EOR 0xca000000
105 #define EOR_v 0x2e201c00
106 #define EORI 0xd2000000
107 #define EXTR 0x93c00000
108 #define FABS 0x1e60c000
109 #define FADD 0x1e602800
110 #define FCMP 0x1e602000
111 #define FCSEL 0x1e600c00
112 #define FCVT 0x1e224000
113 #define FCVTL 0x0e217800
114 #define FCVTZS 0x9e780000
115 #define FDIV 0x1e601800
116 #define FMOV 0x1e604000
117 #define FMOV_R 0x9e660000
118 #define FMOV_I 0x1e601000
119 #define FMUL 0x1e600800
120 #define FNEG 0x1e614000
121 #define FSUB 0x1e603800
122 #define INS 0x4e001c00
123 #define INS_e 0x6e000400
124 #define LD1 0x0c407000
125 #define LD1_s 0x0d400000
126 #define LD1R 0x0d40c000
127 #define LDRI 0xf9400000
128 #define LDRI_F64 0xfd400000
129 #define LDRI_POST 0xf8400400
130 #define LDP 0xa9400000
131 #define LDP_F64 0x6d400000
132 #define LDP_POST 0xa8c00000
133 #define LDR_PRE 0xf8400c00
134 #define LDXR 0xc85f7c00
135 #define LDXRB 0x085f7c00
136 #define LDXRH 0x485f7c00
137 #define LSLV 0x9ac02000
138 #define LSRV 0x9ac02400
139 #define MADD 0x9b000000
140 #define MOVI 0x0f000400
141 #define MOVK 0xf2800000
142 #define MOVN 0x92800000
143 #define MOVZ 0xd2800000
144 #define NOP 0xd503201f
145 #define ORN 0xaa200000
146 #define ORR 0xaa000000
147 #define ORR_v 0x0ea01c00
148 #define ORRI 0xb2000000
149 #define RBIT 0xdac00000
150 #define RET 0xd65f0000
151 #define REV 0xdac00c00
152 #define REV16 0xdac00400
153 #define RORV 0x9ac02c00
154 #define SBC 0xda000000
155 #define SBFM 0x93400000
156 #define SCVTF 0x9e620000
157 #define SDIV 0x9ac00c00
158 #define SMADDL 0x9b200000
159 #define SMOV 0x0e002c00
160 #define SMULH 0x9b403c00
161 #define SSHLL 0x0f00a400
162 #define ST1 0x0c007000
163 #define ST1_s 0x0d000000
164 #define STP 0xa9000000
165 #define STP_F64 0x6d000000
166 #define STP_PRE 0xa9800000
167 #define STRB 0x38206800
168 #define STRBI 0x39000000
169 #define STRI 0xf9000000
170 #define STRI_F64 0xfd000000
171 #define STR_FI 0x3d000000
172 #define STR_FR 0x3c206800
173 #define STUR_FI 0x3c000000
174 #define STURBI 0x38000000
175 #define STXR 0xc8007c00
176 #define STXRB 0x8007c00
177 #define STXRH 0x48007c00
178 #define SUB 0xcb000000
179 #define SUBI 0xd1000000
180 #define SUBS 0xeb000000
181 #define TBZ 0x36000000
182 #define TBL_v 0x0e000000
183 #define UBFM 0xd3400000
184 #define UCVTF 0x9e630000
185 #define UDIV 0x9ac00800
186 #define UMOV 0x0e003c00
187 #define UMULH 0x9bc03c00
188 #define USHLL 0x2f00a400
189 #define USHR 0x2f000400
190 #define USRA 0x2f001400
191 #define XTN 0x0e212800
192
193 #define CSET (CSINC | RM(TMP_ZERO) | RN(TMP_ZERO))
194 #define LDR (STRI | (1 << 22))
195 #define LDRB (STRBI | (1 << 22))
196 #define LDRH (LDRB | (1 << 30))
197 #define MOV (ORR | RN(TMP_ZERO))
198
199 static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins)
200 {
201 sljit_ins *ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
202 FAIL_IF(!ptr);
203 *ptr = ins;
204 compiler->size++;
205 return SLJIT_SUCCESS;
206 }
207
208 static SLJIT_INLINE sljit_s32 emit_imm64_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_uw imm)
209 {
210 FAIL_IF(push_inst(compiler, MOVZ | RD(dst) | ((sljit_ins)(imm & 0xffff) << 5)));
211 FAIL_IF(push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)(imm >> 16) & 0xffff) << 5) | (1 << 21)));
212 FAIL_IF(push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)(imm >> 32) & 0xffff) << 5) | (2 << 21)));
213 return push_inst(compiler, MOVK | RD(dst) | ((sljit_ins)(imm >> 48) << 5) | (3 << 21));
214 }
215
216 static SLJIT_INLINE sljit_ins* detect_jump_type(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code, sljit_sw executable_offset)
217 {
218 sljit_sw diff;
219 sljit_uw target_addr;
220 sljit_uw jump_addr = (sljit_uw)code_ptr;
221 sljit_uw orig_addr = jump->addr;
222 SLJIT_UNUSED_ARG(executable_offset);
223
224 jump->addr = jump_addr;
225 if (jump->flags & SLJIT_REWRITABLE_JUMP)
226 goto exit;
227
228 if (jump->flags & JUMP_ADDR)
229 target_addr = jump->u.target;
230 else {
231 SLJIT_ASSERT(jump->u.label != NULL);
232 target_addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code + jump->u.label->size, executable_offset);
233
234 if (jump->u.label->size > orig_addr)
235 jump_addr = (sljit_uw)(code + orig_addr);
236 }
237
238 diff = (sljit_sw)target_addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_addr, executable_offset);
239
240 if (jump->flags & IS_COND) {
241 diff += SSIZE_OF(ins);
242 if (diff <= 0xfffff && diff >= -0x100000) {
243 *(--code_ptr) ^= (jump->flags & IS_CBZ) ? (0x1 << 24) : 0x1;
244 jump->flags |= PATCH_COND;
245 jump->addr -= sizeof(sljit_ins);
246 return code_ptr;
247 }
248 diff -= SSIZE_OF(ins);
249 }
250
251 if (diff <= 0x7ffffff && diff >= -0x8000000) {
252 if (jump->flags & IS_COND)
253 code_ptr[-1] -= (4 << 5);
254 jump->flags |= PATCH_B;
255 return code_ptr;
256 }
257
258 if (target_addr < 0x100000000l) {
259 if (jump->flags & IS_COND)
260 code_ptr[-1] -= (2 << 5);
261 code_ptr[2] = code_ptr[0];
262 return code_ptr + 2;
263 }
264
265 if (diff <= 0xfffff000l && diff >= -0x100000000l) {
266 if (jump->flags & IS_COND)
267 code_ptr[-1] -= (2 << 5);
268 jump->flags |= PATCH_B32;
269 code_ptr[2] = code_ptr[0];
270 return code_ptr + 2;
271 }
272
273 if (target_addr < 0x1000000000000l) {
274 if (jump->flags & IS_COND)
275 code_ptr[-1] -= (1 << 5);
276 jump->flags |= PATCH_ABS48;
277 code_ptr[3] = code_ptr[0];
278 return code_ptr + 3;
279 }
280
281 exit:
282 jump->flags |= PATCH_ABS64;
283 code_ptr[4] = code_ptr[0];
284 return code_ptr + 4;
285 }
286
287 static SLJIT_INLINE sljit_sw mov_addr_get_length(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code, sljit_sw executable_offset)
288 {
289 sljit_uw addr;
290 sljit_uw jump_addr = (sljit_uw)code_ptr;
291 sljit_sw diff;
292 SLJIT_UNUSED_ARG(executable_offset);
293
294 SLJIT_ASSERT(jump->flags < ((sljit_uw)4 << JUMP_SIZE_SHIFT));
295 if (jump->flags & JUMP_ADDR)
296 addr = jump->u.target;
297 else {
298 addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code + jump->u.label->size, executable_offset);
299
300 if (jump->u.label->size > jump->addr)
301 jump_addr = (sljit_uw)(code + jump->addr);
302 }
303
304 diff = (sljit_sw)addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(jump_addr, executable_offset);
305
306 if (diff <= 0xfffff && diff >= -0x100000) {
307 jump->flags |= PATCH_B;
308 return 0;
309 }
310
311 if (diff <= 0xfffff000l && diff >= -0x100000000l) {
312 SLJIT_ASSERT(jump->flags >= ((sljit_uw)1 << JUMP_SIZE_SHIFT));
313 jump->flags |= PATCH_B32;
314 return 1;
315 }
316
317 if (addr < 0x100000000l) {
318 SLJIT_ASSERT(jump->flags >= ((sljit_uw)1 << JUMP_SIZE_SHIFT));
319 return 1;
320 }
321
322 if (addr < 0x1000000000000l) {
323 SLJIT_ASSERT(jump->flags >= ((sljit_uw)2 << JUMP_SIZE_SHIFT));
324 jump->flags |= PATCH_ABS48;
325 return 2;
326 }
327
328 SLJIT_ASSERT(jump->flags >= ((sljit_uw)3 << JUMP_SIZE_SHIFT));
329 jump->flags |= PATCH_ABS64;
330 return 3;
331 }
332
333 static SLJIT_INLINE void generate_jump_or_mov_addr(struct sljit_jump *jump, sljit_sw executable_offset)
334 {
335 sljit_sw addr = (sljit_sw)((jump->flags & JUMP_ADDR) ? jump->u.target : jump->u.label->u.addr);
336 sljit_ins* buf_ptr = (sljit_ins*)jump->addr;
337 sljit_u32 dst;
338 SLJIT_UNUSED_ARG(executable_offset);
339
340 if (!(jump->flags & JUMP_MOV_ADDR)) {
341 if (jump->flags & PATCH_COND) {
342 addr = (addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset)) >> 2;
343 SLJIT_ASSERT(addr <= 0x3ffff && addr >= -0x40000);
344 buf_ptr[0] = (buf_ptr[0] & ~(sljit_ins)0xffffe0) | (sljit_ins)((addr & 0x7ffff) << 5);
345 return;
346 }
347
348 if (jump->flags & PATCH_B) {
349 addr = (addr - (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset)) >> 2;
350 SLJIT_ASSERT(addr <= 0x1ffffff && addr >= -0x2000000);
351 buf_ptr[0] = ((jump->flags & IS_BL) ? BL : B) | (sljit_ins)(addr & 0x3ffffff);
352 return;
353 }
354
355 dst = (buf_ptr[0] >> 5) & 0x1f;
356
357 if (jump->flags & PATCH_B32) {
358 addr -= (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset) & ~(sljit_sw)0xfff;
359 SLJIT_ASSERT(addr <= 0xfffff000l && addr >= -0x100000000l);
360 buf_ptr[0] = ADRP | (((sljit_ins)(addr >> 12) & 0x3) << 29) | (((sljit_ins)(addr >> 14) & 0x7ffff) << 5) | dst;
361 buf_ptr[1] = ADDI | dst | (dst << 5) | ((sljit_ins)(addr & 0xfff) << 10);
362 return;
363 }
364 } else {
365 dst = *buf_ptr;
366
367 if (jump->flags & PATCH_B) {
368 addr -= (sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset);
369 SLJIT_ASSERT(addr <= 0xfffff && addr >= -0x100000);
370 buf_ptr[0] = ADR | (((sljit_ins)addr & 0x3) << 29) | (((sljit_ins)(addr >> 2) & 0x7ffff) << 5) | dst;
371 return;
372 }
373
374 if (jump->flags & PATCH_B32) {
375 addr -= ((sljit_sw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset)) & ~(sljit_sw)0xfff;
376 SLJIT_ASSERT(addr <= 0xffffffffl && addr >= -0x100000000l);
377 buf_ptr[0] = ADRP | (((sljit_ins)(addr >> 12) & 0x3) << 29) | (((sljit_ins)(addr >> 14) & 0x7ffff) << 5) | dst;
378 buf_ptr[1] = ADDI | dst | (dst << 5) | ((sljit_ins)(addr & 0xfff) << 10);
379 return;
380 }
381 }
382
383 SLJIT_ASSERT((jump->flags & (PATCH_ABS48 | PATCH_ABS64)) || (sljit_uw)addr <= (sljit_uw)0xffffffff);
384 SLJIT_ASSERT((jump->flags & PATCH_ABS64) || (sljit_uw)addr <= (sljit_uw)0xffffffffffff);
385
386 buf_ptr[0] = MOVZ | (((sljit_ins)addr & 0xffff) << 5) | dst;
387 buf_ptr[1] = MOVK | (((sljit_ins)(addr >> 16) & 0xffff) << 5) | (1 << 21) | dst;
388 if (jump->flags & (PATCH_ABS48 | PATCH_ABS64))
389 buf_ptr[2] = MOVK | (((sljit_ins)(addr >> 32) & 0xffff) << 5) | (2 << 21) | dst;
390
391 if (jump->flags & PATCH_ABS64)
392 buf_ptr[3] = MOVK | ((sljit_ins)((sljit_uw)addr >> 48) << 5) | (3 << 21) | dst;
393 }
394
395 static SLJIT_INLINE sljit_ins *process_extended_label(sljit_ins *code_ptr, struct sljit_extended_label *ext_label)
396 {
397 SLJIT_ASSERT(ext_label->label.u.index == SLJIT_LABEL_ALIGNED);
398 return (sljit_ins*)((sljit_uw)code_ptr & ~(ext_label->data));
399 }
400
401 static void reduce_code_size(struct sljit_compiler *compiler)
402 {
403 struct sljit_label *label;
404 struct sljit_jump *jump;
405 struct sljit_const *const_;
406 SLJIT_NEXT_DEFINE_TYPES;
407 sljit_uw total_size;
408 sljit_uw size_reduce = 0;
409 sljit_sw diff;
410
411 label = compiler->labels;
412 jump = compiler->jumps;
413 const_ = compiler->consts;
414 SLJIT_NEXT_INIT_TYPES();
415
416 while (1) {
417 SLJIT_GET_NEXT_MIN();
418
419 if (next_min_addr == SLJIT_MAX_ADDRESS)
420 break;
421
422 if (next_min_addr == next_label_size) {
423 label->size -= size_reduce;
424
425 label = label->next;
426 next_label_size = SLJIT_GET_NEXT_SIZE(label);
427 }
428
429 if (next_min_addr == next_const_addr) {
430 const_->addr -= size_reduce;
431 const_ = const_->next;
432 next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
433 continue;
434 }
435
436 if (next_min_addr != next_jump_addr)
437 continue;
438
439 jump->addr -= size_reduce;
440 if (!(jump->flags & JUMP_MOV_ADDR)) {
441 total_size = JUMP_MAX_SIZE;
442
443 if (!(jump->flags & SLJIT_REWRITABLE_JUMP)) {
444 if (jump->flags & JUMP_ADDR) {
445 if (jump->u.target < 0x100000000l)
446 total_size = 3;
447 else if (jump->u.target < 0x1000000000000l)
448 total_size = 4;
449 } else {
450 /* Unit size: instruction. */
451 diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
452 if (jump->u.label->size > jump->addr) {
453 SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
454 diff -= (sljit_sw)size_reduce;
455 }
456
457 if ((jump->flags & IS_COND) && (diff + 1) <= (0xfffff / SSIZE_OF(ins)) && (diff + 1) >= (-0x100000 / SSIZE_OF(ins)))
458 total_size = 0;
459 else if (diff <= (0x7ffffff / SSIZE_OF(ins)) && diff >= (-0x8000000 / SSIZE_OF(ins)))
460 total_size = 1;
461 else if (diff <= (0xfffff000l / SSIZE_OF(ins)) && diff >= (-0x100000000l / SSIZE_OF(ins)))
462 total_size = 3;
463 }
464 }
465
466 size_reduce += JUMP_MAX_SIZE - total_size;
467 } else {
468 /* Real size minus 1. Unit size: instruction. */
469 total_size = 3;
470
471 if (!(jump->flags & JUMP_ADDR)) {
472 diff = (sljit_sw)jump->u.label->size - (sljit_sw)jump->addr;
473 if (jump->u.label->size > jump->addr) {
474 SLJIT_ASSERT(jump->u.label->size - size_reduce >= jump->addr);
475 diff -= (sljit_sw)size_reduce;
476 }
477
478 if (diff <= (0xfffff / SSIZE_OF(ins)) && diff >= (-0x100000 / SSIZE_OF(ins)))
479 total_size = 0;
480 else if (diff <= (0xfffff000l / SSIZE_OF(ins)) && diff >= (-0x100000000l / SSIZE_OF(ins)))
481 total_size = 1;
482 } else if (jump->u.target < 0x100000000l)
483 total_size = 1;
484 else if (jump->u.target < 0x1000000000000l)
485 total_size = 2;
486
487 size_reduce += 3 - total_size;
488 }
489
490 jump->flags |= total_size << JUMP_SIZE_SHIFT;
491 jump = jump->next;
492 next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
493 }
494
495 compiler->size -= size_reduce;
496 }
497
498 SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler, sljit_s32 options, void *exec_allocator_data)
499 {
500 struct sljit_memory_fragment *buf;
501 sljit_ins *code;
502 sljit_ins *code_ptr;
503 sljit_ins *buf_ptr;
504 sljit_ins *buf_end;
505 sljit_uw word_count;
506 SLJIT_NEXT_DEFINE_TYPES;
507 sljit_sw executable_offset;
508 sljit_sw addr;
509
510 struct sljit_label *label;
511 struct sljit_jump *jump;
512 struct sljit_const *const_;
513
514 CHECK_ERROR_PTR();
515 CHECK_PTR(check_sljit_generate_code(compiler, options));
516
517 reduce_code_size(compiler);
518
519 code = (sljit_ins*)allocate_executable_memory(compiler->size * sizeof(sljit_ins), options, exec_allocator_data, &executable_offset);
520 PTR_FAIL_WITH_EXEC_IF(code);
521
522 reverse_buf(compiler);
523 buf = compiler->buf;
524
525 code_ptr = code;
526 word_count = 0;
527 label = compiler->labels;
528 jump = compiler->jumps;
529 const_ = compiler->consts;
530 SLJIT_NEXT_INIT_TYPES();
531 SLJIT_GET_NEXT_MIN();
532
533 do {
534 buf_ptr = (sljit_ins*)buf->memory;
535 buf_end = buf_ptr + (buf->used_size >> 2);
536 do {
537 *code_ptr = *buf_ptr++;
538 if (next_min_addr == word_count) {
539 SLJIT_ASSERT(!label || label->size >= word_count);
540 SLJIT_ASSERT(!jump || jump->addr >= word_count);
541 SLJIT_ASSERT(!const_ || const_->addr >= word_count);
542
543 /* These structures are ordered by their address. */
544 if (next_min_addr == next_label_size) {
545 if (label->u.index >= SLJIT_LABEL_ALIGNED) {
546 code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);
547 *code_ptr = buf_ptr[-1];
548 }
549
550 label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
551 label->size = (sljit_uw)(code_ptr - code);
552 label = label->next;
553 next_label_size = SLJIT_GET_NEXT_SIZE(label);
554 }
555
556 if (next_min_addr == next_jump_addr) {
557 if (!(jump->flags & JUMP_MOV_ADDR)) {
558 word_count = word_count - 1 + (jump->flags >> JUMP_SIZE_SHIFT);
559 code_ptr = detect_jump_type(jump, code_ptr, code, executable_offset);
560 SLJIT_ASSERT((jump->flags & PATCH_COND) || ((sljit_uw)code_ptr - jump->addr < (jump->flags >> JUMP_SIZE_SHIFT) * sizeof(sljit_ins)));
561 } else {
562 word_count += jump->flags >> JUMP_SIZE_SHIFT;
563 addr = (sljit_sw)code_ptr;
564 code_ptr += mov_addr_get_length(jump, code_ptr, code, executable_offset);
565 jump->addr = (sljit_uw)addr;
566 }
567
568 jump = jump->next;
569 next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump);
570 } else if (next_min_addr == next_const_addr) {
571 const_->addr = (sljit_uw)code_ptr;
572 const_ = const_->next;
573 next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
574 }
575
576 SLJIT_GET_NEXT_MIN();
577 }
578 code_ptr++;
579 word_count++;
580 } while (buf_ptr < buf_end);
581
582 buf = buf->next;
583 } while (buf);
584
585 if (label && label->size == word_count) {
586 if (label->u.index >= SLJIT_LABEL_ALIGNED)
587 code_ptr = process_extended_label(code_ptr, (struct sljit_extended_label*)label);
588
589 label->u.addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
590 label->size = (sljit_uw)(code_ptr - code);
591 label = label->next;
592 }
593
594 SLJIT_ASSERT(!label);
595 SLJIT_ASSERT(!jump);
596 SLJIT_ASSERT(!const_);
597 SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size);
598
599 jump = compiler->jumps;
600 while (jump) {
601 generate_jump_or_mov_addr(jump, executable_offset);
602 jump = jump->next;
603 }
604
605 compiler->error = SLJIT_ERR_COMPILED;
606 compiler->executable_offset = executable_offset;
607 compiler->executable_size = (sljit_uw)(code_ptr - code) * sizeof(sljit_ins);
608
609 code = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
610 code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
611
612 SLJIT_CACHE_FLUSH(code, code_ptr);
613 SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1);
614 return code;
615 }
616
617 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
618 {
619 switch (feature_type) {
620 case SLJIT_HAS_FPU:
621 case SLJIT_HAS_SIMD:
622 #ifdef SLJIT_IS_FPU_AVAILABLE
623 return (SLJIT_IS_FPU_AVAILABLE) != 0;
624 #else
625 /* Available by default. */
626 return 1;
627 #endif
628
629 case SLJIT_HAS_CLZ:
630 case SLJIT_HAS_CTZ:
631 case SLJIT_HAS_REV:
632 case SLJIT_HAS_ROT:
633 case SLJIT_HAS_CMOV:
634 case SLJIT_HAS_PREFETCH:
635 case SLJIT_HAS_COPY_F32:
636 case SLJIT_HAS_COPY_F64:
637 case SLJIT_HAS_ATOMIC:
638 case SLJIT_HAS_MEMORY_BARRIER:
639 return 1;
640
641 default:
642 return 0;
643 }
644 }
645
646 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
647 {
648 switch (type) {
649 case SLJIT_UNORDERED_OR_EQUAL:
650 case SLJIT_ORDERED_NOT_EQUAL:
651 return 2;
652 }
653
654 return 0;
655 }
656
657 /* --------------------------------------------------------------------- */
658 /* Core code generator functions. */
659 /* --------------------------------------------------------------------- */
660
661 #define COUNT_TRAILING_ZERO(value, result) \
662 result = 0; \
663 if (!(value & 0xffffffff)) { \
664 result += 32; \
665 value >>= 32; \
666 } \
667 if (!(value & 0xffff)) { \
668 result += 16; \
669 value >>= 16; \
670 } \
671 if (!(value & 0xff)) { \
672 result += 8; \
673 value >>= 8; \
674 } \
675 if (!(value & 0xf)) { \
676 result += 4; \
677 value >>= 4; \
678 } \
679 if (!(value & 0x3)) { \
680 result += 2; \
681 value >>= 2; \
682 } \
683 if (!(value & 0x1)) { \
684 result += 1; \
685 value >>= 1; \
686 }
687
688 #define LOGICAL_IMM_CHECK (sljit_ins)0x100
689
690 static sljit_ins logical_imm(sljit_sw imm, sljit_u32 len)
691 {
692 sljit_s32 negated;
693 sljit_u32 ones, right;
694 sljit_uw mask, uimm;
695 sljit_ins ins;
696
697 if (len & LOGICAL_IMM_CHECK) {
698 len &= ~LOGICAL_IMM_CHECK;
699 if (len == 32 && (imm == 0 || imm == -1))
700 return 0;
701 if (len == 16 && ((sljit_s32)imm == 0 || (sljit_s32)imm == -1))
702 return 0;
703 }
704
705 SLJIT_ASSERT((len == 32 && imm != 0 && imm != -1)
706 || (len == 16 && (sljit_s32)imm != 0 && (sljit_s32)imm != -1));
707
708 uimm = (sljit_uw)imm;
709 while (1) {
710 if (len <= 0) {
711 SLJIT_UNREACHABLE();
712 return 0;
713 }
714
715 mask = ((sljit_uw)1 << len) - 1;
716 if ((uimm & mask) != ((uimm >> len) & mask))
717 break;
718 len >>= 1;
719 }
720
721 len <<= 1;
722
723 negated = 0;
724 if (uimm & 0x1) {
725 negated = 1;
726 uimm = ~uimm;
727 }
728
729 if (len < 64)
730 uimm &= ((sljit_uw)1 << len) - 1;
731
732 /* Unsigned right shift. */
733 COUNT_TRAILING_ZERO(uimm, right);
734
735 /* Signed shift. We also know that the highest bit is set. */
736 imm = (sljit_sw)~uimm;
737 SLJIT_ASSERT(imm < 0);
738
739 COUNT_TRAILING_ZERO(imm, ones);
740
741 if (~imm)
742 return 0;
743
744 if (len == 64)
745 ins = 1 << 22;
746 else
747 ins = (0x3f - ((len << 1) - 1)) << 10;
748
749 if (negated)
750 return ins | ((len - ones - 1) << 10) | ((len - ones - right) << 16);
751
752 return ins | ((ones - 1) << 10) | ((len - right) << 16);
753 }
754
755 #undef COUNT_TRAILING_ZERO
756
757 static sljit_s32 load_immediate(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw simm)
758 {
759 sljit_uw imm = (sljit_uw)simm;
760 sljit_u32 i, zeros, ones, first;
761 sljit_ins bitmask;
762
763 /* Handling simple immediates first. */
764 if (imm <= 0xffff)
765 return push_inst(compiler, MOVZ | RD(dst) | ((sljit_ins)imm << 5));
766
767 if (simm < 0 && simm >= -0x10000)
768 return push_inst(compiler, MOVN | RD(dst) | (((sljit_ins)~imm & 0xffff) << 5));
769
770 if (imm <= 0xffffffffl) {
771 if ((imm & 0xffff) == 0)
772 return push_inst(compiler, MOVZ | RD(dst) | ((sljit_ins)(imm >> 16) << 5) | (1 << 21));
773 if ((imm & 0xffff0000l) == 0xffff0000)
774 return push_inst(compiler, (MOVN ^ W_OP) | RD(dst) | (((sljit_ins)~imm & 0xffff) << 5));
775 if ((imm & 0xffff) == 0xffff)
776 return push_inst(compiler, (MOVN ^ W_OP) | RD(dst) | (((sljit_ins)~imm & 0xffff0000u) >> (16 - 5)) | (1 << 21));
777
778 bitmask = logical_imm(simm, 16);
779 if (bitmask != 0)
780 return push_inst(compiler, (ORRI ^ W_OP) | RD(dst) | RN(TMP_ZERO) | bitmask);
781
782 FAIL_IF(push_inst(compiler, MOVZ | RD(dst) | (((sljit_ins)imm & 0xffff) << 5)));
783 return push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)imm & 0xffff0000u) >> (16 - 5)) | (1 << 21));
784 }
785
786 bitmask = logical_imm(simm, 32);
787 if (bitmask != 0)
788 return push_inst(compiler, ORRI | RD(dst) | RN(TMP_ZERO) | bitmask);
789
790 if (simm < 0 && simm >= -0x100000000l) {
791 if ((imm & 0xffff) == 0xffff)
792 return push_inst(compiler, MOVN | RD(dst) | (((sljit_ins)~imm & 0xffff0000u) >> (16 - 5)) | (1 << 21));
793
794 FAIL_IF(push_inst(compiler, MOVN | RD(dst) | (((sljit_ins)~imm & 0xffff) << 5)));
795 return push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)imm & 0xffff0000u) >> (16 - 5)) | (1 << 21));
796 }
797
798 /* A large amount of number can be constructed from ORR and MOVx, but computing them is costly. */
799
800 zeros = 0;
801 ones = 0;
802 for (i = 4; i > 0; i--) {
803 if ((simm & 0xffff) == 0)
804 zeros++;
805 if ((simm & 0xffff) == 0xffff)
806 ones++;
807 simm >>= 16;
808 }
809
810 simm = (sljit_sw)imm;
811 first = 1;
812 if (ones > zeros) {
813 simm = ~simm;
814 for (i = 0; i < 4; i++) {
815 if (!(simm & 0xffff)) {
816 simm >>= 16;
817 continue;
818 }
819 if (first) {
820 first = 0;
821 FAIL_IF(push_inst(compiler, MOVN | RD(dst) | (((sljit_ins)simm & 0xffff) << 5) | (i << 21)));
822 }
823 else
824 FAIL_IF(push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)~simm & 0xffff) << 5) | (i << 21)));
825 simm >>= 16;
826 }
827 return SLJIT_SUCCESS;
828 }
829
830 for (i = 0; i < 4; i++) {
831 if (!(simm & 0xffff)) {
832 simm >>= 16;
833 continue;
834 }
835 if (first) {
836 first = 0;
837 FAIL_IF(push_inst(compiler, MOVZ | RD(dst) | (((sljit_ins)simm & 0xffff) << 5) | (i << 21)));
838 }
839 else
840 FAIL_IF(push_inst(compiler, MOVK | RD(dst) | (((sljit_ins)simm & 0xffff) << 5) | (i << 21)));
841 simm >>= 16;
842 }
843 return SLJIT_SUCCESS;
844 }
845
846 #define ARG1_IMM 0x0010000
847 #define ARG2_IMM 0x0020000
848 #define INT_OP 0x0040000
849 #define SET_FLAGS 0x0080000
850 #define UNUSED_RETURN 0x0100000
851
852 #define CHECK_FLAGS(flag_bits) \
853 if (flags & SET_FLAGS) { \
854 inv_bits |= flag_bits; \
855 if (flags & UNUSED_RETURN) \
856 dst = TMP_ZERO; \
857 }
858
859 static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 dst, sljit_sw arg1, sljit_sw arg2)
860 {
861 /* dst must be register, TMP_REG1
862 arg1 must be register, TMP_REG1, imm
863 arg2 must be register, TMP_REG2, imm */
864 sljit_ins inv_bits = (flags & INT_OP) ? W_OP : 0;
865 sljit_ins inst_bits;
866 sljit_s32 op = (flags & 0xffff);
867 sljit_s32 reg;
868 sljit_sw imm, nimm;
869
870 if (SLJIT_UNLIKELY((flags & (ARG1_IMM | ARG2_IMM)) == (ARG1_IMM | ARG2_IMM))) {
871 /* Both are immediates. */
872 flags &= ~ARG1_IMM;
873 if (arg1 == 0 && op != SLJIT_ADD && op != SLJIT_SUB)
874 arg1 = TMP_ZERO;
875 else {
876 FAIL_IF(load_immediate(compiler, TMP_REG1, arg1));
877 arg1 = TMP_REG1;
878 }
879 }
880
881 if (flags & (ARG1_IMM | ARG2_IMM)) {
882 reg = (sljit_s32)((flags & ARG2_IMM) ? arg1 : arg2);
883 imm = (flags & ARG2_IMM) ? arg2 : arg1;
884
885 switch (op) {
886 case SLJIT_CLZ:
887 case SLJIT_CTZ:
888 case SLJIT_REV:
889 case SLJIT_REV_U16:
890 case SLJIT_REV_S16:
891 case SLJIT_REV_U32:
892 case SLJIT_REV_S32:
893 case SLJIT_ADDC:
894 case SLJIT_SUBC:
895 case SLJIT_MUL:
896 case SLJIT_MULADD:
897 /* No form with immediate operand (except imm 0, which
898 is represented by a ZERO register). */
899 break;
900 case SLJIT_MOV:
901 SLJIT_ASSERT(!(flags & SET_FLAGS) && (flags & ARG2_IMM) && arg1 == TMP_REG1);
902 return load_immediate(compiler, dst, imm);
903 case SLJIT_SUB:
904 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
905 if (flags & ARG1_IMM)
906 break;
907 imm = -imm;
908 SLJIT_FALLTHROUGH
909 case SLJIT_ADD:
910 if (op != SLJIT_SUB)
911 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
912
913 if (imm == 0) {
914 CHECK_FLAGS(1 << 29);
915 return push_inst(compiler, ((op == SLJIT_ADD ? ADDI : SUBI) ^ inv_bits) | RD(dst) | RN(reg));
916 }
917 if (imm > 0 && imm <= 0xfff) {
918 CHECK_FLAGS(1 << 29);
919 return push_inst(compiler, (ADDI ^ inv_bits) | RD(dst) | RN(reg) | ((sljit_ins)imm << 10));
920 }
921 nimm = -imm;
922 if (nimm > 0 && nimm <= 0xfff) {
923 CHECK_FLAGS(1 << 29);
924 return push_inst(compiler, (SUBI ^ inv_bits) | RD(dst) | RN(reg) | ((sljit_ins)nimm << 10));
925 }
926 if (imm > 0 && imm <= 0xffffff && !(imm & 0xfff)) {
927 CHECK_FLAGS(1 << 29);
928 return push_inst(compiler, (ADDI ^ inv_bits) | RD(dst) | RN(reg) | (((sljit_ins)imm >> 12) << 10) | (1 << 22));
929 }
930 if (nimm > 0 && nimm <= 0xffffff && !(nimm & 0xfff)) {
931 CHECK_FLAGS(1 << 29);
932 return push_inst(compiler, (SUBI ^ inv_bits) | RD(dst) | RN(reg) | (((sljit_ins)nimm >> 12) << 10) | (1 << 22));
933 }
934 if (imm > 0 && imm <= 0xffffff && !(flags & SET_FLAGS)) {
935 FAIL_IF(push_inst(compiler, (ADDI ^ inv_bits) | RD(dst) | RN(reg) | (((sljit_ins)imm >> 12) << 10) | (1 << 22)));
936 return push_inst(compiler, (ADDI ^ inv_bits) | RD(dst) | RN(dst) | (((sljit_ins)imm & 0xfff) << 10));
937 }
938 if (nimm > 0 && nimm <= 0xffffff && !(flags & SET_FLAGS)) {
939 FAIL_IF(push_inst(compiler, (SUBI ^ inv_bits) | RD(dst) | RN(reg) | (((sljit_ins)nimm >> 12) << 10) | (1 << 22)));
940 return push_inst(compiler, (SUBI ^ inv_bits) | RD(dst) | RN(dst) | (((sljit_ins)nimm & 0xfff) << 10));
941 }
942 break;
943 case SLJIT_AND:
944 inst_bits = logical_imm(imm, LOGICAL_IMM_CHECK | ((flags & INT_OP) ? 16 : 32));
945 if (!inst_bits)
946 break;
947 CHECK_FLAGS(3 << 29);
948 return push_inst(compiler, (ANDI ^ inv_bits) | RD(dst) | RN(reg) | inst_bits);
949 case SLJIT_XOR:
950 if (imm == -1) {
951 FAIL_IF(push_inst(compiler, (ORN ^ inv_bits) | RD(dst) | RN(TMP_ZERO) | RM(reg)));
952 goto set_flags;
953 }
954 SLJIT_FALLTHROUGH
955 case SLJIT_OR:
956 inst_bits = logical_imm(imm, LOGICAL_IMM_CHECK | ((flags & INT_OP) ? 16 : 32));
957 if (!inst_bits)
958 break;
959 if (op == SLJIT_OR)
960 inst_bits |= ORRI;
961 else
962 inst_bits |= EORI;
963 FAIL_IF(push_inst(compiler, (inst_bits ^ inv_bits) | RD(dst) | RN(reg)));
964 goto set_flags;
965 case SLJIT_SHL:
966 case SLJIT_MSHL:
967 if (flags & ARG1_IMM)
968 break;
969
970 if (flags & INT_OP) {
971 imm &= 0x1f;
972 inst_bits = (((sljit_ins)-imm & 0x1f) << 16) | ((31 - (sljit_ins)imm) << 10);
973 } else {
974 imm &= 0x3f;
975 inst_bits = ((sljit_ins)1 << 22) | (((sljit_ins)-imm & 0x3f) << 16) | ((63 - (sljit_ins)imm) << 10);
976 }
977
978 inv_bits |= inv_bits >> 9;
979 FAIL_IF(push_inst(compiler, (UBFM ^ inv_bits) | RD(dst) | RN(arg1) | inst_bits));
980 goto set_flags;
981 case SLJIT_LSHR:
982 case SLJIT_MLSHR:
983 case SLJIT_ASHR:
984 case SLJIT_MASHR:
985 if (flags & ARG1_IMM)
986 break;
987
988 inv_bits |= inv_bits >> 9;
989 if (op >= SLJIT_ASHR)
990 inv_bits |= 1 << 30;
991
992 if (flags & INT_OP) {
993 imm &= 0x1f;
994 inst_bits = ((sljit_ins)imm << 16) | (31 << 10);
995 } else {
996 imm &= 0x3f;
997 inst_bits = ((sljit_ins)1 << 22) | ((sljit_ins)imm << 16) | (63 << 10);
998 }
999
1000 FAIL_IF(push_inst(compiler, (UBFM ^ inv_bits) | RD(dst) | RN(arg1) | inst_bits));
1001 goto set_flags;
1002 case SLJIT_ROTL:
1003 case SLJIT_ROTR:
1004 if (flags & ARG1_IMM)
1005 break;
1006
1007 if (op == SLJIT_ROTL)
1008 imm = -imm;
1009
1010 imm &= (flags & INT_OP) ? 0x1f : 0x3f;
1011 return push_inst(compiler, (EXTR ^ (inv_bits | (inv_bits >> 9))) | RD(dst) | RN(arg1) | RM(arg1) | ((sljit_ins)imm << 10));
1012 default:
1013 SLJIT_UNREACHABLE();
1014 break;
1015 }
1016
1017 if (flags & ARG2_IMM) {
1018 if (arg2 == 0)
1019 arg2 = TMP_ZERO;
1020 else {
1021 FAIL_IF(load_immediate(compiler, TMP_REG2, arg2));
1022 arg2 = TMP_REG2;
1023 }
1024 }
1025 else {
1026 if (arg1 == 0)
1027 arg1 = TMP_ZERO;
1028 else {
1029 FAIL_IF(load_immediate(compiler, TMP_REG1, arg1));
1030 arg1 = TMP_REG1;
1031 }
1032 }
1033 }
1034
1035 /* Both arguments are registers. */
1036 switch (op) {
1037 case SLJIT_MOV:
1038 case SLJIT_MOV_P:
1039 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1040 if (dst == arg2)
1041 return SLJIT_SUCCESS;
1042 return push_inst(compiler, MOV | RD(dst) | RM(arg2));
1043 case SLJIT_MOV_U8:
1044 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1045 inv_bits |= inv_bits >> 9;
1046 return push_inst(compiler, (UBFM ^ inv_bits) | RD(dst) | RN(arg2) | (7 << 10));
1047 case SLJIT_MOV_S8:
1048 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1049 inv_bits |= inv_bits >> 9;
1050 return push_inst(compiler, (SBFM ^ inv_bits) | RD(dst) | RN(arg2) | (7 << 10));
1051 case SLJIT_MOV_U16:
1052 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1053 inv_bits |= inv_bits >> 9;
1054 return push_inst(compiler, (UBFM ^ inv_bits) | RD(dst) | RN(arg2) | (15 << 10));
1055 case SLJIT_MOV_S16:
1056 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1057 inv_bits |= inv_bits >> 9;
1058 return push_inst(compiler, (SBFM ^ inv_bits) | RD(dst) | RN(arg2) | (15 << 10));
1059 case SLJIT_MOV32:
1060 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1061 if (dst == arg2)
1062 return SLJIT_SUCCESS;
1063 SLJIT_FALLTHROUGH
1064 case SLJIT_MOV_U32:
1065 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1066 return push_inst(compiler, (MOV ^ W_OP) | RD(dst) | RM(arg2));
1067 case SLJIT_MOV_S32:
1068 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG1);
1069 return push_inst(compiler, SBFM | (1 << 22) | RD(dst) | RN(arg2) | (31 << 10));
1070 case SLJIT_CLZ:
1071 SLJIT_ASSERT(arg1 == TMP_REG1);
1072 return push_inst(compiler, (CLZ ^ inv_bits) | RD(dst) | RN(arg2));
1073 case SLJIT_CTZ:
1074 SLJIT_ASSERT(arg1 == TMP_REG1);
1075 FAIL_IF(push_inst(compiler, (RBIT ^ inv_bits) | RD(dst) | RN(arg2)));
1076 return push_inst(compiler, (CLZ ^ inv_bits) | RD(dst) | RN(dst));
1077 case SLJIT_REV:
1078 SLJIT_ASSERT(arg1 == TMP_REG1);
1079 inv_bits |= inv_bits >> 21;
1080 return push_inst(compiler, (REV ^ inv_bits) | RD(dst) | RN(arg2));
1081 case SLJIT_REV_U16:
1082 case SLJIT_REV_S16:
1083 SLJIT_ASSERT(arg1 == TMP_REG1 && dst != TMP_REG2);
1084 FAIL_IF(push_inst(compiler, (REV16 ^ (sljit_ins)0x80000000) | RD(dst) | RN(arg2)));
1085 if (dst == TMP_REG1 || (arg2 == TMP_REG2 && op == SLJIT_REV_U16))
1086 return SLJIT_SUCCESS;
1087 inv_bits |= inv_bits >> 9;
1088 return push_inst(compiler, ((op == SLJIT_REV_U16 ? UBFM : SBFM) ^ inv_bits) | RD(dst) | RN(dst) | (15 << 10));
1089 case SLJIT_REV_U32:
1090 case SLJIT_REV_S32:
1091 SLJIT_ASSERT(arg1 == TMP_REG1 && dst != TMP_REG2);
1092 FAIL_IF(push_inst(compiler, (REV ^ (sljit_ins)0x80000400) | RD(dst) | RN(arg2)));
1093 if (op == SLJIT_REV_U32 || dst == TMP_REG1)
1094 return SLJIT_SUCCESS;
1095 return push_inst(compiler, SBFM | (1 << 22) | RD(dst) | RN(dst) | (31 << 10));
1096 case SLJIT_ADD:
1097 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1098 CHECK_FLAGS(1 << 29);
1099 return push_inst(compiler, (ADD ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1100 case SLJIT_ADDC:
1101 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1102 CHECK_FLAGS(1 << 29);
1103 return push_inst(compiler, (ADC ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1104 case SLJIT_SUB:
1105 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1106 CHECK_FLAGS(1 << 29);
1107 return push_inst(compiler, (SUB ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1108 case SLJIT_SUBC:
1109 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1110 CHECK_FLAGS(1 << 29);
1111 return push_inst(compiler, (SBC ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1112 case SLJIT_MUL:
1113 compiler->status_flags_state = 0;
1114 if (!(flags & SET_FLAGS))
1115 return push_inst(compiler, (MADD ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2) | RT2(TMP_ZERO));
1116 if (flags & INT_OP) {
1117 FAIL_IF(push_inst(compiler, SMADDL | RD(dst) | RN(arg1) | RM(arg2) | (31 << 10)));
1118 FAIL_IF(push_inst(compiler, ADD | RD(TMP_LR) | RN(TMP_ZERO) | RM(dst) | (2 << 22) | (31 << 10)));
1119 return push_inst(compiler, SUBS | RD(TMP_ZERO) | RN(TMP_LR) | RM(dst) | (2 << 22) | (63 << 10));
1120 }
1121 FAIL_IF(push_inst(compiler, SMULH | RD(TMP_LR) | RN(arg1) | RM(arg2)));
1122 FAIL_IF(push_inst(compiler, MADD | RD(dst) | RN(arg1) | RM(arg2) | RT2(TMP_ZERO)));
1123 return push_inst(compiler, SUBS | RD(TMP_ZERO) | RN(TMP_LR) | RM(dst) | (2 << 22) | (63 << 10));
1124 case SLJIT_AND:
1125 CHECK_FLAGS(3 << 29);
1126 return push_inst(compiler, (AND ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1127 case SLJIT_OR:
1128 FAIL_IF(push_inst(compiler, (ORR ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2)));
1129 break; /* Set flags. */
1130 case SLJIT_XOR:
1131 FAIL_IF(push_inst(compiler, (EOR ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2)));
1132 break; /* Set flags. */
1133 case SLJIT_SHL:
1134 case SLJIT_MSHL:
1135 FAIL_IF(push_inst(compiler, (LSLV ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2)));
1136 break; /* Set flags. */
1137 case SLJIT_LSHR:
1138 case SLJIT_MLSHR:
1139 FAIL_IF(push_inst(compiler, (LSRV ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2)));
1140 break; /* Set flags. */
1141 case SLJIT_ASHR:
1142 case SLJIT_MASHR:
1143 FAIL_IF(push_inst(compiler, (ASRV ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2)));
1144 break; /* Set flags. */
1145 case SLJIT_ROTL:
1146 FAIL_IF(push_inst(compiler, (SUB ^ inv_bits) | RD(TMP_REG2) | RN(TMP_ZERO) | RM(arg2)));
1147 arg2 = TMP_REG2;
1148 SLJIT_FALLTHROUGH
1149 case SLJIT_ROTR:
1150 return push_inst(compiler, (RORV ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2));
1151 case SLJIT_MULADD:
1152 compiler->status_flags_state = 0;
1153 return push_inst(compiler, (MADD ^ inv_bits) | RD(dst) | RN(arg1) | RM(arg2) | RT2(dst));
1154 default:
1155 SLJIT_UNREACHABLE();
1156 return SLJIT_SUCCESS;
1157 }
1158
1159 set_flags:
1160 if (flags & SET_FLAGS)
1161 return push_inst(compiler, (SUBS ^ inv_bits) | RD(TMP_ZERO) | RN(dst) | RM(TMP_ZERO));
1162 return SLJIT_SUCCESS;
1163 }
1164
1165 #define STORE 0x10
1166 #define SIGNED 0x20
1167
1168 #define BYTE_SIZE 0x0
1169 #define HALF_SIZE 0x1
1170 #define INT_SIZE 0x2
1171 #define WORD_SIZE 0x3
1172
1173 #define MEM_SIZE_SHIFT(flags) ((sljit_ins)(flags) & 0x3)
1174
1175 static sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg,
1176 sljit_s32 arg, sljit_sw argw, sljit_s32 tmp_reg)
1177 {
1178 sljit_u32 shift = MEM_SIZE_SHIFT(flags);
1179 sljit_u32 type = (shift << 30);
1180
1181 if (!(flags & STORE))
1182 type |= (flags & SIGNED) ? 0x00800000 : 0x00400000;
1183
1184 SLJIT_ASSERT(arg & SLJIT_MEM);
1185
1186 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1187 argw &= 0x3;
1188
1189 if (argw == 0 || argw == shift)
1190 return push_inst(compiler, STRB | type | RT(reg)
1191 | RN(arg & REG_MASK) | RM(OFFS_REG(arg)) | (argw ? (1 << 12) : 0));
1192
1193 FAIL_IF(push_inst(compiler, ADD | RD(tmp_reg) | RN(arg & REG_MASK) | RM(OFFS_REG(arg)) | ((sljit_ins)argw << 10)));
1194 return push_inst(compiler, STRBI | type | RT(reg) | RN(tmp_reg));
1195 }
1196
1197 arg &= REG_MASK;
1198
1199 if (!arg) {
1200 FAIL_IF(load_immediate(compiler, tmp_reg, argw & ~(0xfff << shift)));
1201
1202 argw = (argw >> shift) & 0xfff;
1203
1204 return push_inst(compiler, STRBI | type | RT(reg) | RN(tmp_reg) | ((sljit_ins)argw << 10));
1205 }
1206
1207 if ((argw & ((1 << shift) - 1)) == 0) {
1208 if (argw >= 0) {
1209 if ((argw >> shift) <= 0xfff)
1210 return push_inst(compiler, STRBI | type | RT(reg) | RN(arg) | ((sljit_ins)argw << (10 - shift)));
1211
1212 if (argw <= 0xffffff) {
1213 FAIL_IF(push_inst(compiler, ADDI | (1 << 22) | RD(tmp_reg) | RN(arg) | (((sljit_ins)argw >> 12) << 10)));
1214
1215 argw = ((argw & 0xfff) >> shift);
1216 return push_inst(compiler, STRBI | type | RT(reg) | RN(tmp_reg) | ((sljit_ins)argw << 10));
1217 }
1218 } else if (argw < -256 && argw >= -0xfff000) {
1219 FAIL_IF(push_inst(compiler, SUBI | (1 << 22) | RD(tmp_reg) | RN(arg) | (((sljit_ins)(-argw + 0xfff) >> 12) << 10)));
1220 argw = ((0x1000 + argw) & 0xfff) >> shift;
1221 return push_inst(compiler, STRBI | type | RT(reg) | RN(tmp_reg) | ((sljit_ins)argw << 10));
1222 }
1223 }
1224
1225 if (argw <= 0xff && argw >= -0x100)
1226 return push_inst(compiler, STURBI | type | RT(reg) | RN(arg) | (((sljit_ins)argw & 0x1ff) << 12));
1227
1228 if (((argw + 0x100) & 0xfff) <= 0x1ff && argw <= 0xfff0ff && argw >= -0xfff100) {
1229 if (argw >= 0) {
1230 if (argw & 0x100)
1231 argw += 0x1000;
1232
1233 FAIL_IF(push_inst(compiler, ADDI | (1 << 22) | RD(tmp_reg) | RN(arg) | (((sljit_ins)argw >> 12) << 10)));
1234 return push_inst(compiler, STURBI | type | RT(reg) | RN(tmp_reg) | (((sljit_ins)argw & 0x1ff) << 12));
1235 } else {
1236 if (!(argw & 0x100))
1237 argw -= 0x1000;
1238
1239 FAIL_IF(push_inst(compiler, SUBI | (1 << 22) | RD(tmp_reg) | RN(arg) | (((sljit_ins)-argw >> 12) << 10)));
1240 return push_inst(compiler, STURBI | type | RT(reg) | RN(tmp_reg) | (((sljit_ins)argw & 0x1ff) << 12));
1241 }
1242 }
1243
1244 FAIL_IF(load_immediate(compiler, tmp_reg, argw));
1245
1246 return push_inst(compiler, STRB | type | RT(reg) | RN(arg) | RM(tmp_reg));
1247 }
1248
1249 /* --------------------------------------------------------------------- */
1250 /* Entry, exit */
1251 /* --------------------------------------------------------------------- */
1252
1253 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
1254 sljit_s32 options, sljit_s32 arg_types,
1255 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1256 {
1257 sljit_s32 fscratches;
1258 sljit_s32 fsaveds;
1259 sljit_s32 prev, fprev, saved_regs_size, i, tmp;
1260 sljit_s32 saved_arg_count = SLJIT_KEPT_SAVEDS_COUNT(options);
1261 sljit_ins offs;
1262
1263 CHECK_ERROR();
1264 CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, local_size));
1265 set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
1266
1267 scratches = ENTER_GET_REGS(scratches);
1268 saveds = ENTER_GET_REGS(saveds);
1269 fscratches = compiler->fscratches;
1270 fsaveds = compiler->fsaveds;
1271
1272 saved_regs_size = GET_SAVED_REGISTERS_SIZE(scratches, saveds - saved_arg_count, 2);
1273 saved_regs_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
1274
1275 local_size = (local_size + saved_regs_size + 0xf) & ~0xf;
1276 compiler->local_size = local_size;
1277
1278 if (local_size <= 512) {
1279 FAIL_IF(push_inst(compiler, STP_PRE | RT(TMP_FP) | RT2(TMP_LR)
1280 | RN(SLJIT_SP) | (sljit_ins)((-(local_size >> 3) & 0x7f) << 15)));
1281 offs = (sljit_ins)(local_size - 2 * SSIZE_OF(sw)) << (15 - 3);
1282 local_size = 0;
1283 } else {
1284 saved_regs_size = ((saved_regs_size - 2 * SSIZE_OF(sw)) + 0xf) & ~0xf;
1285
1286 FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | ((sljit_ins)saved_regs_size << 10)));
1287 offs = (sljit_ins)(saved_regs_size - 2 * SSIZE_OF(sw)) << (15 - 3);
1288 local_size -= saved_regs_size;
1289 SLJIT_ASSERT(local_size > 0);
1290 }
1291
1292 prev = -1;
1293
1294 tmp = SLJIT_S0 - saveds;
1295 for (i = SLJIT_S0 - saved_arg_count; i > tmp; i--) {
1296 if (prev == -1) {
1297 prev = i;
1298 continue;
1299 }
1300 FAIL_IF(push_inst(compiler, STP | RT(prev) | RT2(i) | RN(SLJIT_SP) | offs));
1301 offs -= (sljit_ins)2 << 15;
1302 prev = -1;
1303 }
1304
1305 for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
1306 if (prev == -1) {
1307 prev = i;
1308 continue;
1309 }
1310 FAIL_IF(push_inst(compiler, STP | RT(prev) | RT2(i) | RN(SLJIT_SP) | offs));
1311 offs -= (sljit_ins)2 << 15;
1312 prev = -1;
1313 }
1314
1315 fprev = -1;
1316
1317 tmp = SLJIT_FS0 - fsaveds;
1318 for (i = SLJIT_FS0; i > tmp; i--) {
1319 if (fprev == -1) {
1320 fprev = i;
1321 continue;
1322 }
1323 FAIL_IF(push_inst(compiler, STP_F64 | VT(fprev) | VT2(i) | RN(SLJIT_SP) | offs));
1324 offs -= (sljit_ins)2 << 15;
1325 fprev = -1;
1326 }
1327
1328 for (i = fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
1329 if (fprev == -1) {
1330 fprev = i;
1331 continue;
1332 }
1333 FAIL_IF(push_inst(compiler, STP_F64 | VT(fprev) | VT2(i) | RN(SLJIT_SP) | offs));
1334 offs -= (sljit_ins)2 << 15;
1335 fprev = -1;
1336 }
1337
1338 if (fprev != -1)
1339 FAIL_IF(push_inst(compiler, STRI_F64 | VT(fprev) | RN(SLJIT_SP) | (offs >> 5) | (1 << 10)));
1340
1341 if (prev != -1)
1342 FAIL_IF(push_inst(compiler, STRI | RT(prev) | RN(SLJIT_SP) | (offs >> 5) | ((fprev == -1) ? (1 << 10) : 0)));
1343
1344
1345 #ifdef _WIN32
1346 if (local_size > 4096)
1347 FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | (1 << 10) | (1 << 22)));
1348 #endif /* _WIN32 */
1349
1350 if (!(options & SLJIT_ENTER_REG_ARG)) {
1351 arg_types >>= SLJIT_ARG_SHIFT;
1352 saved_arg_count = 0;
1353 tmp = SLJIT_R0;
1354
1355 while (arg_types) {
1356 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64) {
1357 if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
1358 FAIL_IF(push_inst(compiler, MOV | RD(SLJIT_S0 - saved_arg_count) | RM(tmp)));
1359 saved_arg_count++;
1360 }
1361 tmp++;
1362 }
1363 arg_types >>= SLJIT_ARG_SHIFT;
1364 }
1365 }
1366
1367 #ifdef _WIN32
1368 if (local_size > 4096) {
1369 if (local_size < 4 * 4096) {
1370 /* No need for a loop. */
1371
1372 if (local_size >= 2 * 4096) {
1373 if (local_size >= 3 * 4096) {
1374 FAIL_IF(push_inst(compiler, LDRI | RT(TMP_ZERO) | RN(SLJIT_SP)));
1375 FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | (1 << 10) | (1 << 22)));
1376 }
1377
1378 FAIL_IF(push_inst(compiler, LDRI | RT(TMP_ZERO) | RN(SLJIT_SP)));
1379 FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | (1 << 10) | (1 << 22)));
1380 }
1381 }
1382 else {
1383 FAIL_IF(push_inst(compiler, MOVZ | RD(TMP_REG1) | ((((sljit_ins)local_size >> 12) - 1) << 5)));
1384 FAIL_IF(push_inst(compiler, LDRI | RT(TMP_ZERO) | RN(SLJIT_SP)));
1385 FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | (1 << 10) | (1 << 22)));
1386 FAIL_IF(push_inst(compiler, SUBI | (1 << 29) | RD(TMP_REG1) | RN(TMP_REG1) | (1 << 10)));
1387 FAIL_IF(push_inst(compiler, B_CC | ((((sljit_ins) -3) & 0x7ffff) << 5) | 0x1 /* not-equal */));
1388 }
1389
1390 local_size &= 0xfff;
1391
1392 if (local_size > 0)
1393 FAIL_IF(push_inst(compiler, LDRI | RT(TMP_ZERO) | RN(SLJIT_SP)));
1394 else
1395 FAIL_IF(push_inst(compiler, STP | RT(TMP_FP) | RT2(TMP_LR) | RN(SLJIT_SP)));
1396 }
1397
1398 if (local_size > 0) {
1399 if (local_size <= 512)
1400 FAIL_IF(push_inst(compiler, STP_PRE | RT(TMP_FP) | RT2(TMP_LR)
1401 | RN(SLJIT_SP) | (sljit_ins)((-(local_size >> 3) & 0x7f) << 15)));
1402 else {
1403 if (local_size >= 4096)
1404 local_size = (1 << (22 - 10));
1405
1406 FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | ((sljit_ins)local_size << 10)));
1407 FAIL_IF(push_inst(compiler, STP | RT(TMP_FP) | RT2(TMP_LR) | RN(SLJIT_SP)));
1408 }
1409 }
1410
1411 #else /* !_WIN32 */
1412
1413 /* The local_size does not include saved registers size. */
1414 if (local_size != 0) {
1415 if (local_size > 0xfff) {
1416 FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | (((sljit_ins)local_size >> 12) << 10) | (1 << 22)));
1417 local_size &= 0xfff;
1418 }
1419
1420 if (local_size > 512 || local_size == 0) {
1421 if (local_size != 0)
1422 FAIL_IF(push_inst(compiler, SUBI | RD(SLJIT_SP) | RN(SLJIT_SP) | ((sljit_ins)local_size << 10)));
1423
1424 FAIL_IF(push_inst(compiler, STP | RT(TMP_FP) | RT2(TMP_LR) | RN(SLJIT_SP)));
1425 } else
1426 FAIL_IF(push_inst(compiler, STP_PRE | RT(TMP_FP) | RT2(TMP_LR)
1427 | RN(SLJIT_SP) | (sljit_ins)((-(local_size >> 3) & 0x7f) << 15)));
1428 }
1429
1430 #endif /* _WIN32 */
1431
1432 return push_inst(compiler, ADDI | RD(TMP_FP) | RN(SLJIT_SP) | (0 << 10));
1433 }
1434
1435 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
1436 sljit_s32 options, sljit_s32 arg_types,
1437 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1438 {
1439 sljit_s32 fscratches;
1440 sljit_s32 fsaveds;
1441 sljit_s32 saved_regs_size;
1442
1443 CHECK_ERROR();
1444 CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, local_size));
1445 set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
1446
1447 scratches = ENTER_GET_REGS(scratches);
1448 saveds = ENTER_GET_REGS(saveds);
1449 fscratches = compiler->fscratches;
1450 fsaveds = compiler->fsaveds;
1451 saved_regs_size = GET_SAVED_REGISTERS_SIZE(scratches, saveds - SLJIT_KEPT_SAVEDS_COUNT(options), 2);
1452 saved_regs_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
1453
1454 compiler->local_size = (local_size + saved_regs_size + 0xf) & ~0xf;
1455 return SLJIT_SUCCESS;
1456 }
1457
1458 static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler, sljit_s32 is_return_to)
1459 {
1460 sljit_s32 local_size, prev, fprev, i, tmp;
1461 sljit_ins offs;
1462
1463 local_size = compiler->local_size;
1464
1465 if (!is_return_to) {
1466 if (local_size > 512 && local_size <= 512 + 496) {
1467 FAIL_IF(push_inst(compiler, LDP_POST | RT(TMP_FP) | RT2(TMP_LR)
1468 | RN(SLJIT_SP) | ((sljit_ins)(local_size - 512) << (15 - 3))));
1469 local_size = 512;
1470 } else
1471 FAIL_IF(push_inst(compiler, LDP | RT(TMP_FP) | RT2(TMP_LR) | RN(SLJIT_SP)));
1472 } else {
1473 if (local_size > 512 && local_size <= 512 + 248) {
1474 FAIL_IF(push_inst(compiler, LDRI_POST | RT(TMP_FP) | RN(SLJIT_SP) | ((sljit_ins)(local_size - 512) << 12)));
1475 local_size = 512;
1476 } else
1477 FAIL_IF(push_inst(compiler, LDRI | RT(TMP_FP) | RN(SLJIT_SP) | 0));
1478 }
1479
1480 if (local_size > 512) {
1481 local_size -= 512;
1482 if (local_size > 0xfff) {
1483 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RN(SLJIT_SP)
1484 | (((sljit_ins)local_size >> 12) << 10) | (1 << 22)));
1485 local_size &= 0xfff;
1486 }
1487
1488 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RN(SLJIT_SP) | ((sljit_ins)local_size << 10)));
1489 local_size = 512;
1490 }
1491
1492 offs = (sljit_ins)(local_size - 2 * SSIZE_OF(sw)) << (15 - 3);
1493 prev = -1;
1494
1495 tmp = SLJIT_S0 - compiler->saveds;
1496 for (i = SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options); i > tmp; i--) {
1497 if (prev == -1) {
1498 prev = i;
1499 continue;
1500 }
1501 FAIL_IF(push_inst(compiler, LDP | RT(prev) | RT2(i) | RN(SLJIT_SP) | offs));
1502 offs -= (sljit_ins)2 << 15;
1503 prev = -1;
1504 }
1505
1506 for (i = compiler->scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
1507 if (prev == -1) {
1508 prev = i;
1509 continue;
1510 }
1511 FAIL_IF(push_inst(compiler, LDP | RT(prev) | RT2(i) | RN(SLJIT_SP) | offs));
1512 offs -= (sljit_ins)2 << 15;
1513 prev = -1;
1514 }
1515
1516 fprev = -1;
1517
1518 tmp = SLJIT_FS0 - compiler->fsaveds;
1519 for (i = SLJIT_FS0; i > tmp; i--) {
1520 if (fprev == -1) {
1521 fprev = i;
1522 continue;
1523 }
1524 FAIL_IF(push_inst(compiler, LDP_F64 | VT(fprev) | VT2(i) | RN(SLJIT_SP) | offs));
1525 offs -= (sljit_ins)2 << 15;
1526 fprev = -1;
1527 }
1528
1529 for (i = compiler->fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
1530 if (fprev == -1) {
1531 fprev = i;
1532 continue;
1533 }
1534 FAIL_IF(push_inst(compiler, LDP_F64 | VT(fprev) | VT2(i) | RN(SLJIT_SP) | offs));
1535 offs -= (sljit_ins)2 << 15;
1536 fprev = -1;
1537 }
1538
1539 if (fprev != -1)
1540 FAIL_IF(push_inst(compiler, LDRI_F64 | VT(fprev) | RN(SLJIT_SP) | (offs >> 5) | (1 << 10)));
1541
1542 if (prev != -1)
1543 FAIL_IF(push_inst(compiler, LDRI | RT(prev) | RN(SLJIT_SP) | (offs >> 5) | ((fprev == -1) ? (1 << 10) : 0)));
1544
1545 /* This and the next call/jump instruction can be executed parallelly. */
1546 return push_inst(compiler, ADDI | RD(SLJIT_SP) | RN(SLJIT_SP) | (sljit_ins)(local_size << 10));
1547 }
1548
1549 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
1550 {
1551 CHECK_ERROR();
1552 CHECK(check_sljit_emit_return_void(compiler));
1553
1554 FAIL_IF(emit_stack_frame_release(compiler, 0));
1555
1556 return push_inst(compiler, RET | RN(TMP_LR));
1557 }
1558
1559 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_to(struct sljit_compiler *compiler,
1560 sljit_s32 src, sljit_sw srcw)
1561 {
1562 CHECK_ERROR();
1563 CHECK(check_sljit_emit_return_to(compiler, src, srcw));
1564
1565 if (src & SLJIT_MEM) {
1566 ADJUST_LOCAL_OFFSET(src, srcw);
1567 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
1568 src = TMP_REG1;
1569 srcw = 0;
1570 } else if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
1571 FAIL_IF(push_inst(compiler, MOV | RD(TMP_REG1) | RM(src)));
1572 src = TMP_REG1;
1573 srcw = 0;
1574 }
1575
1576 FAIL_IF(emit_stack_frame_release(compiler, 1));
1577
1578 SLJIT_SKIP_CHECKS(compiler);
1579 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1580 }
1581
1582 /* --------------------------------------------------------------------- */
1583 /* Operators */
1584 /* --------------------------------------------------------------------- */
1585
1586 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1587 {
1588 sljit_ins inv_bits = (op & SLJIT_32) ? W_OP : 0;
1589
1590 CHECK_ERROR();
1591 CHECK(check_sljit_emit_op0(compiler, op));
1592
1593 op = GET_OPCODE(op);
1594 switch (op) {
1595 case SLJIT_BREAKPOINT:
1596 return push_inst(compiler, BRK | (0xf000 << 5));
1597 case SLJIT_NOP:
1598 return push_inst(compiler, NOP);
1599 case SLJIT_LMUL_UW:
1600 case SLJIT_LMUL_SW:
1601 FAIL_IF(push_inst(compiler, MOV | RD(TMP_REG1) | RM(SLJIT_R0)));
1602 FAIL_IF(push_inst(compiler, MADD | RD(SLJIT_R0) | RN(SLJIT_R0) | RM(SLJIT_R1) | RT2(TMP_ZERO)));
1603 return push_inst(compiler, (op == SLJIT_LMUL_UW ? UMULH : SMULH) | RD(SLJIT_R1) | RN(TMP_REG1) | RM(SLJIT_R1));
1604 case SLJIT_DIVMOD_UW:
1605 case SLJIT_DIVMOD_SW:
1606 FAIL_IF(push_inst(compiler, (MOV ^ inv_bits) | RD(TMP_REG1) | RM(SLJIT_R0)));
1607 FAIL_IF(push_inst(compiler, ((op == SLJIT_DIVMOD_UW ? UDIV : SDIV) ^ inv_bits) | RD(SLJIT_R0) | RN(SLJIT_R0) | RM(SLJIT_R1)));
1608 FAIL_IF(push_inst(compiler, (MADD ^ inv_bits) | RD(SLJIT_R1) | RN(SLJIT_R0) | RM(SLJIT_R1) | RT2(TMP_ZERO)));
1609 return push_inst(compiler, (SUB ^ inv_bits) | RD(SLJIT_R1) | RN(TMP_REG1) | RM(SLJIT_R1));
1610 case SLJIT_DIV_UW:
1611 case SLJIT_DIV_SW:
1612 return push_inst(compiler, ((op == SLJIT_DIV_UW ? UDIV : SDIV) ^ inv_bits) | RD(SLJIT_R0) | RN(SLJIT_R0) | RM(SLJIT_R1));
1613 case SLJIT_MEMORY_BARRIER:
1614 return push_inst(compiler, DMB_SY);
1615 case SLJIT_ENDBR:
1616 case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1617 return SLJIT_SUCCESS;
1618 }
1619
1620 return SLJIT_SUCCESS;
1621 }
1622
1623 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1624 sljit_s32 dst, sljit_sw dstw,
1625 sljit_s32 src, sljit_sw srcw)
1626 {
1627 sljit_s32 dst_r, flags, mem_flags;
1628 sljit_s32 op_flags = GET_ALL_FLAGS(op);
1629
1630 CHECK_ERROR();
1631 CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1632 ADJUST_LOCAL_OFFSET(dst, dstw);
1633 ADJUST_LOCAL_OFFSET(src, srcw);
1634
1635 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1636
1637 op = GET_OPCODE(op);
1638 if (op >= SLJIT_MOV && op <= SLJIT_MOV_P) {
1639 /* Both operands are registers. */
1640 if (FAST_IS_REG(dst) && FAST_IS_REG(src))
1641 return emit_op_imm(compiler, op | ((op_flags & SLJIT_32) ? INT_OP : 0), dst_r, TMP_REG1, src);
1642
1643 switch (op) {
1644 case SLJIT_MOV:
1645 case SLJIT_MOV_P:
1646 mem_flags = WORD_SIZE;
1647 break;
1648 case SLJIT_MOV_U8:
1649 mem_flags = BYTE_SIZE;
1650 if (src == SLJIT_IMM)
1651 srcw = (sljit_u8)srcw;
1652 break;
1653 case SLJIT_MOV_S8:
1654 mem_flags = BYTE_SIZE | SIGNED;
1655 if (src == SLJIT_IMM)
1656 srcw = (sljit_s8)srcw;
1657 break;
1658 case SLJIT_MOV_U16:
1659 mem_flags = HALF_SIZE;
1660 if (src == SLJIT_IMM)
1661 srcw = (sljit_u16)srcw;
1662 break;
1663 case SLJIT_MOV_S16:
1664 mem_flags = HALF_SIZE | SIGNED;
1665 if (src == SLJIT_IMM)
1666 srcw = (sljit_s16)srcw;
1667 break;
1668 case SLJIT_MOV_U32:
1669 mem_flags = INT_SIZE;
1670 if (src == SLJIT_IMM)
1671 srcw = (sljit_u32)srcw;
1672 break;
1673 case SLJIT_MOV_S32:
1674 case SLJIT_MOV32:
1675 mem_flags = INT_SIZE | SIGNED;
1676 if (src == SLJIT_IMM)
1677 srcw = (sljit_s32)srcw;
1678 break;
1679 default:
1680 SLJIT_UNREACHABLE();
1681 mem_flags = 0;
1682 break;
1683 }
1684
1685 if (src == SLJIT_IMM)
1686 FAIL_IF(emit_op_imm(compiler, SLJIT_MOV | ARG2_IMM, dst_r, TMP_REG1, srcw));
1687 else if (!(src & SLJIT_MEM))
1688 dst_r = src;
1689 else
1690 FAIL_IF(emit_op_mem(compiler, mem_flags, dst_r, src, srcw, TMP_REG2));
1691
1692 if (dst & SLJIT_MEM)
1693 return emit_op_mem(compiler, mem_flags | STORE, dst_r, dst, dstw, TMP_REG2);
1694 return SLJIT_SUCCESS;
1695 }
1696
1697 flags = HAS_FLAGS(op_flags) ? SET_FLAGS : 0;
1698
1699 switch (op) {
1700 case SLJIT_REV_U16:
1701 case SLJIT_REV_S16:
1702 mem_flags = HALF_SIZE;
1703 break;
1704 case SLJIT_REV_U32:
1705 case SLJIT_REV_S32:
1706 mem_flags = INT_SIZE;
1707 break;
1708 default:
1709 mem_flags = WORD_SIZE;
1710
1711 if (op_flags & SLJIT_32) {
1712 flags |= INT_OP;
1713 mem_flags = INT_SIZE;
1714 }
1715 break;
1716 }
1717
1718 if (src & SLJIT_MEM) {
1719 FAIL_IF(emit_op_mem(compiler, mem_flags, TMP_REG2, src, srcw, TMP_REG2));
1720 src = TMP_REG2;
1721 }
1722
1723 emit_op_imm(compiler, flags | op, dst_r, TMP_REG1, src);
1724
1725 if (SLJIT_UNLIKELY(dst & SLJIT_MEM))
1726 return emit_op_mem(compiler, mem_flags | STORE, dst_r, dst, dstw, TMP_REG2);
1727 return SLJIT_SUCCESS;
1728 }
1729
1730 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
1731 sljit_s32 dst, sljit_sw dstw,
1732 sljit_s32 src1, sljit_sw src1w,
1733 sljit_s32 src2, sljit_sw src2w)
1734 {
1735 sljit_s32 dst_r, flags, mem_flags;
1736
1737 CHECK_ERROR();
1738 CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
1739 ADJUST_LOCAL_OFFSET(dst, dstw);
1740 ADJUST_LOCAL_OFFSET(src1, src1w);
1741 ADJUST_LOCAL_OFFSET(src2, src2w);
1742
1743 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1744 flags = HAS_FLAGS(op) ? SET_FLAGS : 0;
1745 mem_flags = WORD_SIZE;
1746
1747 if (op & SLJIT_32) {
1748 flags |= INT_OP;
1749 mem_flags = INT_SIZE;
1750 }
1751
1752 if (dst == TMP_REG2)
1753 flags |= UNUSED_RETURN;
1754
1755 if (src1 & SLJIT_MEM) {
1756 FAIL_IF(emit_op_mem(compiler, mem_flags, TMP_REG1, src1, src1w, TMP_REG1));
1757 src1 = TMP_REG1;
1758 }
1759
1760 if (src2 & SLJIT_MEM) {
1761 FAIL_IF(emit_op_mem(compiler, mem_flags, TMP_REG2, src2, src2w, TMP_REG2));
1762 src2 = TMP_REG2;
1763 }
1764
1765 if (src1 == SLJIT_IMM)
1766 flags |= ARG1_IMM;
1767 else
1768 src1w = src1;
1769
1770 if (src2 == SLJIT_IMM)
1771 flags |= ARG2_IMM;
1772 else
1773 src2w = src2;
1774
1775 emit_op_imm(compiler, flags | GET_OPCODE(op), dst_r, src1w, src2w);
1776
1777 if (dst & SLJIT_MEM)
1778 return emit_op_mem(compiler, mem_flags | STORE, dst_r, dst, dstw, TMP_REG2);
1779 return SLJIT_SUCCESS;
1780 }
1781
1782 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
1783 sljit_s32 src1, sljit_sw src1w,
1784 sljit_s32 src2, sljit_sw src2w)
1785 {
1786 CHECK_ERROR();
1787 CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
1788
1789 SLJIT_SKIP_CHECKS(compiler);
1790 return sljit_emit_op2(compiler, op, TMP_REG2, 0, src1, src1w, src2, src2w);
1791 }
1792
1793 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
1794 sljit_s32 dst_reg,
1795 sljit_s32 src1, sljit_sw src1w,
1796 sljit_s32 src2, sljit_sw src2w)
1797 {
1798 CHECK_ERROR();
1799 CHECK(check_sljit_emit_op2r(compiler, op, dst_reg, src1, src1w, src2, src2w));
1800
1801 switch (GET_OPCODE(op)) {
1802 case SLJIT_MULADD:
1803 SLJIT_SKIP_CHECKS(compiler);
1804 return sljit_emit_op2(compiler, op, dst_reg, 0, src1, src1w, src2, src2w);
1805 }
1806
1807 return SLJIT_SUCCESS;
1808 }
1809
1810 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
1811 sljit_s32 dst_reg,
1812 sljit_s32 src1_reg,
1813 sljit_s32 src2_reg,
1814 sljit_s32 src3, sljit_sw src3w)
1815 {
1816 sljit_ins inv_bits, imm;
1817 sljit_s32 is_left;
1818 sljit_sw mask;
1819
1820 CHECK_ERROR();
1821 CHECK(check_sljit_emit_shift_into(compiler, op, dst_reg, src1_reg, src2_reg, src3, src3w));
1822
1823 is_left = (GET_OPCODE(op) == SLJIT_SHL || GET_OPCODE(op) == SLJIT_MSHL);
1824
1825 if (src1_reg == src2_reg) {
1826 SLJIT_SKIP_CHECKS(compiler);
1827 return sljit_emit_op2(compiler, (is_left ? SLJIT_ROTL : SLJIT_ROTR) | (op & SLJIT_32), dst_reg, 0, src1_reg, 0, src3, src3w);
1828 }
1829
1830 ADJUST_LOCAL_OFFSET(src3, src3w);
1831
1832 inv_bits = (op & SLJIT_32) ? W_OP : 0;
1833
1834 if (src3 == SLJIT_IMM) {
1835 mask = inv_bits ? 0x1f : 0x3f;
1836 src3w &= mask;
1837
1838 if (src3w == 0)
1839 return SLJIT_SUCCESS;
1840
1841 if (is_left)
1842 src3w = (src3w ^ mask) + 1;
1843
1844 return push_inst(compiler, (EXTR ^ (inv_bits | (inv_bits >> 9))) | RD(dst_reg)
1845 | RN(is_left ? src1_reg : src2_reg) | RM(is_left ? src2_reg : src1_reg) | ((sljit_ins)src3w << 10));
1846 }
1847
1848 if (src3 & SLJIT_MEM) {
1849 FAIL_IF(emit_op_mem(compiler, inv_bits ? INT_SIZE : WORD_SIZE, TMP_REG2, src3, src3w, TMP_REG2));
1850 src3 = TMP_REG2;
1851 } else if (dst_reg == src3) {
1852 FAIL_IF(push_inst(compiler, MOV | RD(TMP_REG2) | RM(src3)));
1853 src3 = TMP_REG2;
1854 }
1855
1856 FAIL_IF(push_inst(compiler, ((is_left ? LSLV : LSRV) ^ inv_bits) | RD(dst_reg) | RN(src1_reg) | RM(src3)));
1857
1858 if (!(op & SLJIT_SHIFT_INTO_NON_ZERO)) {
1859 /* Shift left/right by 1. */
1860 if (is_left)
1861 imm = (sljit_ins)(inv_bits ? ((1 << 16) | (31 << 10)) : ((1 << 16) | (63 << 10) | (1 << 22)));
1862 else
1863 imm = (sljit_ins)(inv_bits ? ((31 << 16) | (30 << 10)) : ((63 << 16) | (62 << 10) | (1 << 22)));
1864
1865 FAIL_IF(push_inst(compiler, (UBFM ^ (inv_bits | (inv_bits >> 9))) | RD(TMP_REG1) | RN(src2_reg) | imm));
1866
1867 /* Set imm to mask. */
1868 imm = (sljit_ins)(inv_bits ? (4 << 10) : ((5 << 10) | (1 << 22)));
1869 FAIL_IF(push_inst(compiler, (EORI ^ inv_bits) | RD(TMP_REG2) | RN(src3) | imm));
1870
1871 src2_reg = TMP_REG1;
1872 } else
1873 FAIL_IF(push_inst(compiler, (SUB ^ inv_bits) | RD(TMP_REG2) | RN(TMP_ZERO) | RM(src3)));
1874
1875 FAIL_IF(push_inst(compiler, ((is_left ? LSRV : LSLV) ^ inv_bits) | RD(TMP_REG1) | RN(src2_reg) | RM(TMP_REG2)));
1876 return push_inst(compiler, (ORR ^ inv_bits) | RD(dst_reg) | RN(dst_reg) | RM(TMP_REG1));
1877 }
1878
1879 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2_shift(struct sljit_compiler *compiler, sljit_s32 op,
1880 sljit_s32 dst, sljit_sw dstw,
1881 sljit_s32 src1, sljit_sw src1w,
1882 sljit_s32 src2, sljit_sw src2w,
1883 sljit_sw shift_arg)
1884 {
1885 sljit_s32 dst_r, tmp_r;
1886
1887 CHECK_ERROR();
1888 CHECK(check_sljit_emit_op2_shift(compiler, op, dst, dstw, src1, src1w, src2, src2w, shift_arg));
1889 ADJUST_LOCAL_OFFSET(dst, dstw);
1890 ADJUST_LOCAL_OFFSET(src1, src1w);
1891 ADJUST_LOCAL_OFFSET(src2, src2w);
1892
1893 shift_arg &= 0x3f;
1894
1895 if (src2 == SLJIT_IMM) {
1896 src2w = src2w << shift_arg;
1897 shift_arg = 0;
1898 }
1899
1900 if (shift_arg == 0) {
1901 SLJIT_SKIP_CHECKS(compiler);
1902 return sljit_emit_op2(compiler, GET_OPCODE(op), dst, dstw, src1, src1w, src2, src2w);
1903 }
1904
1905 if (src1 == SLJIT_IMM) {
1906 FAIL_IF(load_immediate(compiler, TMP_REG2, src1w));
1907 src1 = TMP_REG2;
1908 } else if (src1 & SLJIT_MEM) {
1909 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src1, src1w, TMP_REG2));
1910 src1 = TMP_REG2;
1911 }
1912
1913 if (src2 & SLJIT_MEM) {
1914 tmp_r = (src1 == TMP_REG2) ? TMP_REG1 : TMP_REG2;
1915 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, tmp_r, src2, src2w, tmp_r));
1916 src2 = tmp_r;
1917 }
1918
1919 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1920 FAIL_IF(push_inst(compiler, ADD | RD(dst_r) | RN(src1) | RM(src2) | ((sljit_ins)shift_arg << 10)));
1921
1922 if (dst & SLJIT_MEM)
1923 return emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2);
1924 return SLJIT_SUCCESS;
1925 }
1926
1927 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
1928 sljit_s32 src, sljit_sw srcw)
1929 {
1930 CHECK_ERROR();
1931 CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
1932 ADJUST_LOCAL_OFFSET(src, srcw);
1933
1934 switch (op) {
1935 case SLJIT_FAST_RETURN:
1936 if (FAST_IS_REG(src)) {
1937 if (src != TMP_LR)
1938 FAIL_IF(push_inst(compiler, MOV | RD(TMP_LR) | RM(src)));
1939 } else
1940 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_LR, src, srcw, TMP_REG1));
1941
1942 return push_inst(compiler, RET | RN(TMP_LR));
1943 case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
1944 return SLJIT_SUCCESS;
1945 case SLJIT_PREFETCH_L1:
1946 case SLJIT_PREFETCH_L2:
1947 case SLJIT_PREFETCH_L3:
1948 case SLJIT_PREFETCH_ONCE:
1949 SLJIT_ASSERT(reg_map[1] == 0 && reg_map[3] == 2 && reg_map[5] == 4);
1950
1951 /* The reg_map[op] should provide the appropriate constant. */
1952 if (op == SLJIT_PREFETCH_L1)
1953 op = 1;
1954 else if (op == SLJIT_PREFETCH_L2)
1955 op = 3;
1956 else if (op == SLJIT_PREFETCH_L3)
1957 op = 5;
1958 else
1959 op = 2;
1960
1961 /* Signed word sized load is the prefetch instruction. */
1962 return emit_op_mem(compiler, WORD_SIZE | SIGNED, op, src, srcw, TMP_REG1);
1963 }
1964
1965 return SLJIT_SUCCESS;
1966 }
1967
1968 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
1969 sljit_s32 dst, sljit_sw dstw)
1970 {
1971 sljit_s32 dst_r = TMP_LR;
1972
1973 CHECK_ERROR();
1974 CHECK(check_sljit_emit_op_dst(compiler, op, dst, dstw));
1975 ADJUST_LOCAL_OFFSET(dst, dstw);
1976
1977 switch (op) {
1978 case SLJIT_FAST_ENTER:
1979 if (FAST_IS_REG(dst)) {
1980 if (dst == TMP_LR)
1981 return SLJIT_SUCCESS;
1982 return push_inst(compiler, MOV | RD(dst) | RM(TMP_LR));
1983 }
1984 break;
1985 case SLJIT_GET_RETURN_ADDRESS:
1986 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1987 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, dst_r, SLJIT_MEM1(SLJIT_SP), 0x8, TMP_REG2));
1988 break;
1989 }
1990
1991 if (dst & SLJIT_MEM)
1992 return emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2);
1993
1994 return SLJIT_SUCCESS;
1995 }
1996
1997 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
1998 {
1999 CHECK_REG_INDEX(check_sljit_get_register_index(type, reg));
2000
2001 if (type == SLJIT_GP_REGISTER)
2002 return reg_map[reg];
2003
2004 if (type != SLJIT_FLOAT_REGISTER && type != SLJIT_SIMD_REG_64 && type != SLJIT_SIMD_REG_128)
2005 return -1;
2006
2007 return freg_map[reg];
2008 }
2009
2010 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
2011 void *instruction, sljit_u32 size)
2012 {
2013 SLJIT_UNUSED_ARG(size);
2014 CHECK_ERROR();
2015 CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
2016
2017 return push_inst(compiler, *(sljit_ins*)instruction);
2018 }
2019
2020 /* --------------------------------------------------------------------- */
2021 /* Floating point operators */
2022 /* --------------------------------------------------------------------- */
2023
2024 static sljit_s32 emit_fop_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw)
2025 {
2026 sljit_u32 shift = MEM_SIZE_SHIFT(flags);
2027 sljit_ins type = (shift << 30);
2028
2029 SLJIT_ASSERT(arg & SLJIT_MEM);
2030
2031 if (!(flags & STORE))
2032 type |= 0x00400000;
2033
2034 if (arg & OFFS_REG_MASK) {
2035 argw &= 3;
2036 if (argw == 0 || argw == shift)
2037 return push_inst(compiler, STR_FR | type | VT(reg)
2038 | RN(arg & REG_MASK) | RM(OFFS_REG(arg)) | (argw ? (1 << 12) : 0));
2039
2040 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG2) | RN(arg & REG_MASK) | RM(OFFS_REG(arg)) | ((sljit_ins)argw << 10)));
2041 return push_inst(compiler, STR_FI | type | VT(reg) | RN(TMP_REG2));
2042 }
2043
2044 arg &= REG_MASK;
2045
2046 if (!arg) {
2047 FAIL_IF(load_immediate(compiler, TMP_REG2, argw & ~(0xfff << shift)));
2048
2049 argw = (argw >> shift) & 0xfff;
2050
2051 return push_inst(compiler, STR_FI | type | VT(reg) | RN(TMP_REG2) | ((sljit_ins)argw << 10));
2052 }
2053
2054 if (argw >= 0 && (argw & ((1 << shift) - 1)) == 0) {
2055 if ((argw >> shift) <= 0xfff)
2056 return push_inst(compiler, STR_FI | type | VT(reg) | RN(arg) | ((sljit_ins)argw << (10 - shift)));
2057
2058 if (argw <= 0xffffff) {
2059 FAIL_IF(push_inst(compiler, ADDI | (1 << 22) | RD(TMP_REG2) | RN(arg) | (((sljit_ins)argw >> 12) << 10)));
2060
2061 argw = ((argw & 0xfff) >> shift);
2062 return push_inst(compiler, STR_FI | type | VT(reg) | RN(TMP_REG2) | ((sljit_ins)argw << 10));
2063 }
2064 }
2065
2066 if (argw <= 255 && argw >= -256)
2067 return push_inst(compiler, STUR_FI | type | VT(reg) | RN(arg) | (((sljit_ins)argw & 0x1ff) << 12));
2068
2069 FAIL_IF(load_immediate(compiler, TMP_REG2, argw));
2070 return push_inst(compiler, STR_FR | type | VT(reg) | RN(arg) | RM(TMP_REG2));
2071 }
2072
2073 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2074 sljit_s32 dst, sljit_sw dstw,
2075 sljit_s32 src, sljit_sw srcw)
2076 {
2077 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2078 sljit_ins inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2079
2080 if (GET_OPCODE(op) == SLJIT_CONV_S32_FROM_F64)
2081 inv_bits |= W_OP;
2082
2083 if (src & SLJIT_MEM) {
2084 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_32) ? INT_SIZE : WORD_SIZE, TMP_FREG1, src, srcw));
2085 src = TMP_FREG1;
2086 }
2087
2088 FAIL_IF(push_inst(compiler, (FCVTZS ^ inv_bits) | RD(dst_r) | VN(src)));
2089
2090 if (dst & SLJIT_MEM)
2091 return emit_op_mem(compiler, ((GET_OPCODE(op) == SLJIT_CONV_S32_FROM_F64) ? INT_SIZE : WORD_SIZE) | STORE, TMP_REG1, dst, dstw, TMP_REG2);
2092 return SLJIT_SUCCESS;
2093 }
2094
2095 static sljit_s32 sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler *compiler, sljit_ins ins,
2096 sljit_s32 dst, sljit_sw dstw,
2097 sljit_s32 src, sljit_sw srcw)
2098 {
2099 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2100
2101 if (src & SLJIT_MEM) {
2102 emit_op_mem(compiler, (ins & W_OP) ? WORD_SIZE : INT_SIZE, TMP_REG1, src, srcw, TMP_REG1);
2103 src = TMP_REG1;
2104 } else if (src == SLJIT_IMM) {
2105 FAIL_IF(load_immediate(compiler, TMP_REG1, srcw));
2106 src = TMP_REG1;
2107 }
2108
2109 FAIL_IF(push_inst(compiler, ins | VD(dst_r) | RN(src)));
2110
2111 if (dst & SLJIT_MEM)
2112 return emit_fop_mem(compiler, ((ins & (1 << 22)) ? WORD_SIZE : INT_SIZE) | STORE, TMP_FREG1, dst, dstw);
2113 return SLJIT_SUCCESS;
2114 }
2115
2116 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
2117 sljit_s32 dst, sljit_sw dstw,
2118 sljit_s32 src, sljit_sw srcw)
2119 {
2120 sljit_ins inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2121
2122 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_S32) {
2123 inv_bits |= W_OP;
2124
2125 if (src == SLJIT_IMM)
2126 srcw = (sljit_s32)srcw;
2127 }
2128
2129 return sljit_emit_fop1_conv_f64_from_w(compiler, SCVTF ^ inv_bits, dst, dstw, src, srcw);
2130 }
2131
2132 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_uw(struct sljit_compiler *compiler, sljit_s32 op,
2133 sljit_s32 dst, sljit_sw dstw,
2134 sljit_s32 src, sljit_sw srcw)
2135 {
2136 sljit_ins inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2137
2138 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_U32) {
2139 inv_bits |= W_OP;
2140
2141 if (src == SLJIT_IMM)
2142 srcw = (sljit_u32)srcw;
2143 }
2144
2145 return sljit_emit_fop1_conv_f64_from_w(compiler, UCVTF ^ inv_bits, dst, dstw, src, srcw);
2146 }
2147
2148 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2149 sljit_s32 src1, sljit_sw src1w,
2150 sljit_s32 src2, sljit_sw src2w)
2151 {
2152 sljit_s32 mem_flags = (op & SLJIT_32) ? INT_SIZE : WORD_SIZE;
2153 sljit_ins inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2154
2155 if (src1 & SLJIT_MEM) {
2156 FAIL_IF(emit_fop_mem(compiler, mem_flags, TMP_FREG1, src1, src1w));
2157 src1 = TMP_FREG1;
2158 }
2159
2160 if (src2 & SLJIT_MEM) {
2161 FAIL_IF(emit_fop_mem(compiler, mem_flags, TMP_FREG2, src2, src2w));
2162 src2 = TMP_FREG2;
2163 }
2164
2165 FAIL_IF(push_inst(compiler, (FCMP ^ inv_bits) | VN(src1) | VM(src2)));
2166
2167 if (GET_FLAG_TYPE(op) != SLJIT_UNORDERED_OR_EQUAL)
2168 return SLJIT_SUCCESS;
2169
2170 FAIL_IF(push_inst(compiler, CSINC | (0x0 << 12) | RD(TMP_REG1) | RN(TMP_ZERO) | RM(TMP_ZERO)));
2171 return push_inst(compiler, CCMPI | (0x0 << 16) | (0x7 << 12) | RN(TMP_REG1) | 0x4);
2172 }
2173
2174 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2175 sljit_s32 dst, sljit_sw dstw,
2176 sljit_s32 src, sljit_sw srcw)
2177 {
2178 sljit_s32 dst_r, mem_flags = (op & SLJIT_32) ? INT_SIZE : WORD_SIZE;
2179 sljit_ins inv_bits;
2180
2181 CHECK_ERROR();
2182
2183 SLJIT_COMPILE_ASSERT((INT_SIZE ^ 0x1) == WORD_SIZE, must_be_one_bit_difference);
2184 SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
2185
2186 inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2187 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2188
2189 if (src & SLJIT_MEM) {
2190 FAIL_IF(emit_fop_mem(compiler, (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32) ? (mem_flags ^ 0x1) : mem_flags, dst_r, src, srcw));
2191 src = dst_r;
2192 }
2193
2194 switch (GET_OPCODE(op)) {
2195 case SLJIT_MOV_F64:
2196 if (src != dst_r) {
2197 if (!(dst & SLJIT_MEM))
2198 FAIL_IF(push_inst(compiler, (FMOV ^ inv_bits) | VD(dst_r) | VN(src)));
2199 else
2200 dst_r = src;
2201 }
2202 break;
2203 case SLJIT_NEG_F64:
2204 FAIL_IF(push_inst(compiler, (FNEG ^ inv_bits) | VD(dst_r) | VN(src)));
2205 break;
2206 case SLJIT_ABS_F64:
2207 FAIL_IF(push_inst(compiler, (FABS ^ inv_bits) | VD(dst_r) | VN(src)));
2208 break;
2209 case SLJIT_CONV_F64_FROM_F32:
2210 FAIL_IF(push_inst(compiler, FCVT | (sljit_ins)((op & SLJIT_32) ? (1 << 22) : (1 << 15)) | VD(dst_r) | VN(src)));
2211 break;
2212 }
2213
2214 if (dst & SLJIT_MEM)
2215 return emit_fop_mem(compiler, mem_flags | STORE, dst_r, dst, dstw);
2216 return SLJIT_SUCCESS;
2217 }
2218
2219 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2220 sljit_s32 dst, sljit_sw dstw,
2221 sljit_s32 src1, sljit_sw src1w,
2222 sljit_s32 src2, sljit_sw src2w)
2223 {
2224 sljit_s32 dst_r, mem_flags = (op & SLJIT_32) ? INT_SIZE : WORD_SIZE;
2225 sljit_ins inv_bits = (op & SLJIT_32) ? (1 << 22) : 0;
2226
2227 CHECK_ERROR();
2228 CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
2229 ADJUST_LOCAL_OFFSET(dst, dstw);
2230 ADJUST_LOCAL_OFFSET(src1, src1w);
2231 ADJUST_LOCAL_OFFSET(src2, src2w);
2232
2233 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2234 if (src1 & SLJIT_MEM) {
2235 FAIL_IF(emit_fop_mem(compiler, mem_flags, TMP_FREG1, src1, src1w));
2236 src1 = TMP_FREG1;
2237 }
2238 if (src2 & SLJIT_MEM) {
2239 FAIL_IF(emit_fop_mem(compiler, mem_flags, TMP_FREG2, src2, src2w));
2240 src2 = TMP_FREG2;
2241 }
2242
2243 switch (GET_OPCODE(op)) {
2244 case SLJIT_ADD_F64:
2245 FAIL_IF(push_inst(compiler, (FADD ^ inv_bits) | VD(dst_r) | VN(src1) | VM(src2)));
2246 break;
2247 case SLJIT_SUB_F64:
2248 FAIL_IF(push_inst(compiler, (FSUB ^ inv_bits) | VD(dst_r) | VN(src1) | VM(src2)));
2249 break;
2250 case SLJIT_MUL_F64:
2251 FAIL_IF(push_inst(compiler, (FMUL ^ inv_bits) | VD(dst_r) | VN(src1) | VM(src2)));
2252 break;
2253 case SLJIT_DIV_F64:
2254 FAIL_IF(push_inst(compiler, (FDIV ^ inv_bits) | VD(dst_r) | VN(src1) | VM(src2)));
2255 break;
2256 case SLJIT_COPYSIGN_F64:
2257 FAIL_IF(push_inst(compiler, (FMOV_R ^ ((op & SLJIT_32) ? (W_OP | (1 << 22)) : 0)) | VN(src2) | RD(TMP_REG1)));
2258 FAIL_IF(push_inst(compiler, (FABS ^ inv_bits) | VD(dst_r) | VN(src1)));
2259 FAIL_IF(push_inst(compiler, TBZ | ((op & SLJIT_32) ? 0 : ((sljit_ins)1 << 31)) | (0x1f << 19) | (2 << 5) | RT(TMP_REG1)));
2260 return push_inst(compiler, (FNEG ^ inv_bits) | VD(dst_r) | VN(dst_r));
2261 }
2262
2263 if (!(dst & SLJIT_MEM))
2264 return SLJIT_SUCCESS;
2265 return emit_fop_mem(compiler, mem_flags | STORE, TMP_FREG1, dst, dstw);
2266 }
2267
2268 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset32(struct sljit_compiler *compiler,
2269 sljit_s32 freg, sljit_f32 value)
2270 {
2271 sljit_u32 exp;
2272 union {
2273 sljit_u32 imm;
2274 sljit_f32 value;
2275 } u;
2276
2277 CHECK_ERROR();
2278 CHECK(check_sljit_emit_fset32(compiler, freg, value));
2279
2280 u.value = value;
2281
2282 if (u.imm == 0)
2283 return push_inst(compiler, (FMOV_R ^ (W_OP | (1 << 22))) | RN(TMP_ZERO) | VD(freg) | (1 << 16));
2284
2285 if ((u.imm << (32 - 19)) == 0) {
2286 exp = (u.imm >> (23 + 2)) & 0x3f;
2287
2288 if (exp == 0x20 || exp == 0x1f)
2289 return push_inst(compiler, (FMOV_I ^ (1 << 22)) | (sljit_ins)((((u.imm >> 24) & 0x80) | ((u.imm >> 19) & 0x7f)) << 13) | VD(freg));
2290 }
2291
2292 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_s32)u.imm));
2293 return push_inst(compiler, (FMOV_R ^ (W_OP | (1 << 22))) | RN(TMP_REG1) | VD(freg) | (1 << 16));
2294 }
2295
2296 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset64(struct sljit_compiler *compiler,
2297 sljit_s32 freg, sljit_f64 value)
2298 {
2299 sljit_uw exp;
2300 union {
2301 sljit_uw imm;
2302 sljit_f64 value;
2303 } u;
2304
2305 CHECK_ERROR();
2306 CHECK(check_sljit_emit_fset64(compiler, freg, value));
2307
2308 u.value = value;
2309
2310 if (u.imm == 0)
2311 return push_inst(compiler, FMOV_R | RN(TMP_ZERO) | VD(freg) | (sljit_ins)1 << 16);
2312
2313 if ((u.imm << (64 - 48)) == 0) {
2314 exp = (u.imm >> (52 + 2)) & 0x1ff;
2315
2316 if (exp == 0x100 || exp == 0xff)
2317 return push_inst(compiler, FMOV_I | (sljit_ins)((((u.imm >> 56) & 0x80) | ((u.imm >> 48) & 0x7f)) << 13) | VD(freg));
2318 }
2319
2320 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_sw)u.imm));
2321 return push_inst(compiler, FMOV_R | RN(TMP_REG1) | VD(freg) | (1 << 16));
2322 }
2323
2324 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fcopy(struct sljit_compiler *compiler, sljit_s32 op,
2325 sljit_s32 freg, sljit_s32 reg)
2326 {
2327 sljit_ins inst;
2328
2329 CHECK_ERROR();
2330 CHECK(check_sljit_emit_fcopy(compiler, op, freg, reg));
2331
2332 if (GET_OPCODE(op) == SLJIT_COPY_TO_F64)
2333 inst = FMOV_R | RN(reg) | VD(freg) | (1 << 16);
2334 else
2335 inst = FMOV_R | VN(freg) | RD(reg);
2336
2337 if (op & SLJIT_32)
2338 inst ^= W_OP | (1 << 22);
2339
2340 return push_inst(compiler, inst);
2341 }
2342
2343 /* --------------------------------------------------------------------- */
2344 /* Conditional instructions */
2345 /* --------------------------------------------------------------------- */
2346
2347 static sljit_ins get_cc(struct sljit_compiler *compiler, sljit_s32 type)
2348 {
2349 switch (type) {
2350 case SLJIT_EQUAL:
2351 case SLJIT_ATOMIC_STORED:
2352 case SLJIT_F_EQUAL:
2353 case SLJIT_ORDERED_EQUAL:
2354 case SLJIT_UNORDERED_OR_EQUAL:
2355 return 0x1;
2356
2357 case SLJIT_NOT_EQUAL:
2358 case SLJIT_ATOMIC_NOT_STORED:
2359 case SLJIT_F_NOT_EQUAL:
2360 case SLJIT_UNORDERED_OR_NOT_EQUAL:
2361 case SLJIT_ORDERED_NOT_EQUAL:
2362 return 0x0;
2363
2364 case SLJIT_CARRY:
2365 if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_ADD)
2366 return 0x3;
2367 SLJIT_FALLTHROUGH
2368
2369 case SLJIT_LESS:
2370 return 0x2;
2371
2372 case SLJIT_NOT_CARRY:
2373 if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_ADD)
2374 return 0x2;
2375 SLJIT_FALLTHROUGH
2376
2377 case SLJIT_GREATER_EQUAL:
2378 return 0x3;
2379
2380 case SLJIT_GREATER:
2381 case SLJIT_UNORDERED_OR_GREATER:
2382 return 0x9;
2383
2384 case SLJIT_LESS_EQUAL:
2385 case SLJIT_F_LESS_EQUAL:
2386 case SLJIT_ORDERED_LESS_EQUAL:
2387 return 0x8;
2388
2389 case SLJIT_SIG_LESS:
2390 case SLJIT_UNORDERED_OR_LESS:
2391 return 0xa;
2392
2393 case SLJIT_SIG_GREATER_EQUAL:
2394 case SLJIT_F_GREATER_EQUAL:
2395 case SLJIT_ORDERED_GREATER_EQUAL:
2396 return 0xb;
2397
2398 case SLJIT_SIG_GREATER:
2399 case SLJIT_F_GREATER:
2400 case SLJIT_ORDERED_GREATER:
2401 return 0xd;
2402
2403 case SLJIT_SIG_LESS_EQUAL:
2404 case SLJIT_UNORDERED_OR_LESS_EQUAL:
2405 return 0xc;
2406
2407 case SLJIT_OVERFLOW:
2408 if (!(compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)))
2409 return 0x0;
2410 SLJIT_FALLTHROUGH
2411
2412 case SLJIT_UNORDERED:
2413 return 0x7;
2414
2415 case SLJIT_NOT_OVERFLOW:
2416 if (!(compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)))
2417 return 0x1;
2418 SLJIT_FALLTHROUGH
2419
2420 case SLJIT_ORDERED:
2421 return 0x6;
2422
2423 case SLJIT_F_LESS:
2424 case SLJIT_ORDERED_LESS:
2425 return 0x5;
2426
2427 case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2428 return 0x4;
2429
2430 default:
2431 SLJIT_UNREACHABLE();
2432 return 0xe;
2433 }
2434 }
2435
2436 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2437 {
2438 struct sljit_label *label;
2439
2440 CHECK_ERROR_PTR();
2441 CHECK_PTR(check_sljit_emit_label(compiler));
2442
2443 if (compiler->last_label && compiler->last_label->size == compiler->size)
2444 return compiler->last_label;
2445
2446 label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2447 PTR_FAIL_IF(!label);
2448 set_label(label, compiler);
2449 return label;
2450 }
2451
2452 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_aligned_label(struct sljit_compiler *compiler,
2453 sljit_s32 alignment, struct sljit_read_only_buffer *buffers)
2454 {
2455 sljit_uw mask, i;
2456 struct sljit_label *label;
2457 struct sljit_label *next_label;
2458 struct sljit_extended_label *ext_label;
2459
2460 CHECK_ERROR_PTR();
2461 CHECK_PTR(check_sljit_emit_aligned_label(compiler, alignment, buffers));
2462
2463 sljit_reset_read_only_buffers(buffers);
2464
2465 if (alignment <= SLJIT_LABEL_ALIGN_4) {
2466 SLJIT_SKIP_CHECKS(compiler);
2467 label = sljit_emit_label(compiler);
2468 PTR_FAIL_IF(!label);
2469 } else {
2470 /* The used space is filled with NOPs. */
2471 mask = ((sljit_uw)1 << alignment) - sizeof(sljit_ins);
2472
2473 for (i = (mask >> 2); i != 0; i--)
2474 PTR_FAIL_IF(push_inst(compiler, NOP));
2475
2476 ext_label = (struct sljit_extended_label*)ensure_abuf(compiler, sizeof(struct sljit_extended_label));
2477 PTR_FAIL_IF(!ext_label);
2478 set_extended_label(ext_label, compiler, SLJIT_LABEL_ALIGNED, mask);
2479 label = &ext_label->label;
2480 }
2481
2482 if (buffers == NULL)
2483 return label;
2484
2485 next_label = label;
2486
2487 while (1) {
2488 buffers->u.label = next_label;
2489
2490 for (i = (buffers->size + 3) >> 2; i > 0; i--)
2491 PTR_FAIL_IF(push_inst(compiler, NOP));
2492
2493 buffers = buffers->next;
2494
2495 if (buffers == NULL)
2496 break;
2497
2498 SLJIT_SKIP_CHECKS(compiler);
2499 next_label = sljit_emit_label(compiler);
2500 PTR_FAIL_IF(!next_label);
2501 }
2502
2503 return label;
2504 }
2505
2506 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2507 {
2508 struct sljit_jump *jump;
2509
2510 CHECK_ERROR_PTR();
2511 CHECK_PTR(check_sljit_emit_jump(compiler, type));
2512
2513 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2514 PTR_FAIL_IF(!jump);
2515 set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
2516 type &= 0xff;
2517
2518 if (type < SLJIT_JUMP) {
2519 jump->flags |= IS_COND;
2520 PTR_FAIL_IF(push_inst(compiler, B_CC | (6 << 5) | get_cc(compiler, type)));
2521 } else if (type >= SLJIT_FAST_CALL)
2522 jump->flags |= IS_BL;
2523
2524 jump->addr = compiler->size;
2525 PTR_FAIL_IF(push_inst(compiler, ((type >= SLJIT_FAST_CALL) ? BLR : BR) | RN(TMP_REG2)));
2526
2527 /* Maximum number of instructions required for generating a constant. */
2528 compiler->size += JUMP_MAX_SIZE - 1;
2529 return jump;
2530 }
2531
2532 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2533 sljit_s32 arg_types)
2534 {
2535 SLJIT_UNUSED_ARG(arg_types);
2536 CHECK_ERROR_PTR();
2537 CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
2538
2539 if (type & SLJIT_CALL_RETURN) {
2540 PTR_FAIL_IF(emit_stack_frame_release(compiler, 0));
2541 type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
2542 }
2543
2544 SLJIT_SKIP_CHECKS(compiler);
2545 return sljit_emit_jump(compiler, type);
2546 }
2547
2548 static SLJIT_INLINE struct sljit_jump* emit_cmp_to0(struct sljit_compiler *compiler, sljit_s32 type,
2549 sljit_s32 src, sljit_sw srcw)
2550 {
2551 struct sljit_jump *jump;
2552 sljit_ins inv_bits = (type & SLJIT_32) ? W_OP : 0;
2553
2554 SLJIT_ASSERT((type & 0xff) == SLJIT_EQUAL || (type & 0xff) == SLJIT_NOT_EQUAL);
2555 ADJUST_LOCAL_OFFSET(src, srcw);
2556
2557 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2558 PTR_FAIL_IF(!jump);
2559 set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
2560 jump->flags |= IS_CBZ | IS_COND;
2561
2562 if (src & SLJIT_MEM) {
2563 PTR_FAIL_IF(emit_op_mem(compiler, inv_bits ? INT_SIZE : WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
2564 src = TMP_REG1;
2565 }
2566 else if (src == SLJIT_IMM) {
2567 PTR_FAIL_IF(load_immediate(compiler, TMP_REG1, srcw));
2568 src = TMP_REG1;
2569 }
2570
2571 SLJIT_ASSERT(FAST_IS_REG(src));
2572
2573 if ((type & 0xff) == SLJIT_EQUAL)
2574 inv_bits |= 1 << 24;
2575
2576 PTR_FAIL_IF(push_inst(compiler, (CBZ ^ inv_bits) | (6 << 5) | RT(src)));
2577 jump->addr = compiler->size;
2578 PTR_FAIL_IF(push_inst(compiler, BR | RN(TMP_REG2)));
2579
2580 /* Maximum number of instructions required for generating a constant. */
2581 compiler->size += JUMP_MAX_SIZE - 1;
2582 return jump;
2583 }
2584
2585 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
2586 {
2587 struct sljit_jump *jump;
2588
2589 CHECK_ERROR();
2590 CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
2591
2592 if (src != SLJIT_IMM) {
2593 if (src & SLJIT_MEM) {
2594 ADJUST_LOCAL_OFFSET(src, srcw);
2595 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src, srcw, TMP_REG2));
2596 src = TMP_REG2;
2597 }
2598 return push_inst(compiler, ((type >= SLJIT_FAST_CALL) ? BLR : BR) | RN(src));
2599 }
2600
2601 /* These jumps are converted to jump/call instructions when possible. */
2602 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2603 FAIL_IF(!jump);
2604 set_jump(jump, compiler, JUMP_ADDR | ((type >= SLJIT_FAST_CALL) ? IS_BL : 0));
2605 jump->u.target = (sljit_uw)srcw;
2606
2607 jump->addr = compiler->size;
2608 /* Maximum number of instructions required for generating a constant. */
2609 compiler->size += JUMP_MAX_SIZE - 1;
2610 return push_inst(compiler, ((type >= SLJIT_FAST_CALL) ? BLR : BR) | RN(TMP_REG2));
2611 }
2612
2613 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
2614 sljit_s32 arg_types,
2615 sljit_s32 src, sljit_sw srcw)
2616 {
2617 SLJIT_UNUSED_ARG(arg_types);
2618 CHECK_ERROR();
2619 CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
2620
2621 if (src & SLJIT_MEM) {
2622 ADJUST_LOCAL_OFFSET(src, srcw);
2623 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
2624 src = TMP_REG1;
2625 }
2626
2627 if (type & SLJIT_CALL_RETURN) {
2628 if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
2629 FAIL_IF(push_inst(compiler, MOV | RD(TMP_REG1) | RM(src)));
2630 src = TMP_REG1;
2631 }
2632
2633 FAIL_IF(emit_stack_frame_release(compiler, 0));
2634 type = SLJIT_JUMP;
2635 }
2636
2637 SLJIT_SKIP_CHECKS(compiler);
2638 return sljit_emit_ijump(compiler, type, src, srcw);
2639 }
2640
2641 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2642 sljit_s32 dst, sljit_sw dstw,
2643 sljit_s32 type)
2644 {
2645 sljit_s32 dst_r, src_r, flags, mem_flags;
2646 sljit_ins cc;
2647
2648 CHECK_ERROR();
2649 CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
2650 ADJUST_LOCAL_OFFSET(dst, dstw);
2651
2652 cc = get_cc(compiler, type);
2653 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2654
2655 if (GET_OPCODE(op) < SLJIT_ADD) {
2656 FAIL_IF(push_inst(compiler, CSINC | (cc << 12) | RD(dst_r) | RN(TMP_ZERO) | RM(TMP_ZERO)));
2657
2658 if (dst & SLJIT_MEM) {
2659 mem_flags = (GET_OPCODE(op) == SLJIT_MOV ? WORD_SIZE : INT_SIZE) | STORE;
2660 return emit_op_mem(compiler, mem_flags, TMP_REG1, dst, dstw, TMP_REG2);
2661 }
2662
2663 return SLJIT_SUCCESS;
2664 }
2665
2666 flags = HAS_FLAGS(op) ? SET_FLAGS : 0;
2667 mem_flags = WORD_SIZE;
2668
2669 if (op & SLJIT_32) {
2670 flags |= INT_OP;
2671 mem_flags = INT_SIZE;
2672 }
2673
2674 src_r = dst;
2675
2676 if (dst & SLJIT_MEM) {
2677 FAIL_IF(emit_op_mem(compiler, mem_flags, TMP_REG1, dst, dstw, TMP_REG1));
2678 src_r = TMP_REG1;
2679 }
2680
2681 FAIL_IF(push_inst(compiler, CSINC | (cc << 12) | RD(TMP_REG2) | RN(TMP_ZERO) | RM(TMP_ZERO)));
2682 emit_op_imm(compiler, flags | GET_OPCODE(op), dst_r, src_r, TMP_REG2);
2683
2684 if (dst & SLJIT_MEM)
2685 return emit_op_mem(compiler, mem_flags | STORE, TMP_REG1, dst, dstw, TMP_REG2);
2686 return SLJIT_SUCCESS;
2687 }
2688
2689 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
2690 sljit_s32 dst_reg,
2691 sljit_s32 src1, sljit_sw src1w,
2692 sljit_s32 src2_reg)
2693 {
2694 sljit_ins inv_bits = (type & SLJIT_32) ? W_OP : 0;
2695 sljit_ins op = CSEL;
2696 sljit_ins cmp = 0;
2697 sljit_ins cc;
2698
2699 CHECK_ERROR();
2700 CHECK(check_sljit_emit_select(compiler, type, dst_reg, src1, src1w, src2_reg));
2701
2702 ADJUST_LOCAL_OFFSET(src1, src1w);
2703
2704 if (src1 == SLJIT_IMM) {
2705 if (type & SLJIT_32)
2706 src1w = (sljit_s32)src1w;
2707
2708 if (src1w <= 1 && src1w >= -1) {
2709 src1 = TMP_ZERO;
2710 cmp = (SUBI ^ inv_bits) | (1 << 29) | RD(TMP_ZERO);
2711
2712 if (src1w == 1) {
2713 op = CSINC;
2714 cmp = (SUBI ^ inv_bits) | (1 << 29) | RD(TMP_ZERO) | (1 << 10);
2715 } else if (src1w == -1) {
2716 op = CSINV;
2717 cmp = (ADDI ^ inv_bits) | (1 << 29) | RD(TMP_ZERO) | (1 << 10);
2718 }
2719
2720 src1w = 0;
2721 } else {
2722 FAIL_IF(load_immediate(compiler, TMP_REG2, src1w));
2723 src1 = TMP_REG2;
2724 }
2725 } else if (src1 & SLJIT_MEM) {
2726 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src1, src1w, TMP_REG2));
2727 src1 = TMP_REG2;
2728 }
2729
2730 if (type & SLJIT_COMPARE_SELECT) {
2731 type ^= 0x1;
2732 if (cmp == 0)
2733 cmp = (SUB ^ inv_bits) | (1 << 29) | RD(TMP_ZERO) | RM(src1);
2734 FAIL_IF(push_inst(compiler, cmp | RN(src2_reg)));
2735 }
2736
2737 cc = get_cc(compiler, type & ~(SLJIT_32 | SLJIT_COMPARE_SELECT));
2738 return push_inst(compiler, (op ^ inv_bits) | (cc << 12) | RD(dst_reg) | RN(src2_reg) | RM(src1));
2739 }
2740
2741 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
2742 sljit_s32 dst_freg,
2743 sljit_s32 src1, sljit_sw src1w,
2744 sljit_s32 src2_freg)
2745 {
2746 sljit_ins inv_bits = (type & SLJIT_32) ? (1 << 22) : 0;
2747 sljit_ins cc;
2748
2749 CHECK_ERROR();
2750 CHECK(check_sljit_emit_fselect(compiler, type, dst_freg, src1, src1w, src2_freg));
2751
2752 ADJUST_LOCAL_OFFSET(src1, src1w);
2753
2754 if (src1 & SLJIT_MEM) {
2755 FAIL_IF(emit_fop_mem(compiler, (type & SLJIT_32) ? INT_SIZE : WORD_SIZE, TMP_FREG2, src1, src1w));
2756 src1 = TMP_FREG2;
2757 }
2758
2759 cc = get_cc(compiler, type & ~SLJIT_32);
2760 return push_inst(compiler, (FCSEL ^ inv_bits) | (cc << 12) | VD(dst_freg) | VN(src2_freg) | VM(src1));
2761 }
2762
2763 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
2764 sljit_s32 reg,
2765 sljit_s32 mem, sljit_sw memw)
2766 {
2767 sljit_u32 inst;
2768
2769 CHECK_ERROR();
2770 CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw));
2771
2772 if (!(reg & REG_PAIR_MASK))
2773 return sljit_emit_mem_unaligned(compiler, type, reg, mem, memw);
2774
2775 ADJUST_LOCAL_OFFSET(mem, memw);
2776
2777 if (!(mem & REG_MASK)) {
2778 FAIL_IF(load_immediate(compiler, TMP_REG1, memw & ~0x1f8));
2779
2780 mem = SLJIT_MEM1(TMP_REG1);
2781 memw &= 0x1f8;
2782 } else if (mem & OFFS_REG_MASK) {
2783 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG1) | RN(mem & REG_MASK) | RM(OFFS_REG(mem)) | ((sljit_ins)(memw & 0x3) << 10)));
2784
2785 mem = SLJIT_MEM1(TMP_REG1);
2786 memw = 0;
2787 } else if ((memw & 0x7) != 0 || memw > 0x1f8 || memw < -0x200) {
2788 inst = ADDI;
2789
2790 if (memw < 0) {
2791 /* Remains negative for integer min. */
2792 memw = -memw;
2793 inst = SUBI;
2794 } else if ((memw & 0x7) == 0 && memw <= 0x7ff0) {
2795 if (!(type & SLJIT_MEM_STORE) && (mem & REG_MASK) == REG_PAIR_FIRST(reg)) {
2796 FAIL_IF(push_inst(compiler, LDRI | RD(REG_PAIR_SECOND(reg)) | RN(mem & REG_MASK) | ((sljit_ins)memw << 7)));
2797 return push_inst(compiler, LDRI | RD(REG_PAIR_FIRST(reg)) | RN(mem & REG_MASK) | ((sljit_ins)(memw + 0x8) << 7));
2798 }
2799
2800 inst = (type & SLJIT_MEM_STORE) ? STRI : LDRI;
2801
2802 FAIL_IF(push_inst(compiler, inst | RD(REG_PAIR_FIRST(reg)) | RN(mem & REG_MASK) | ((sljit_ins)memw << 7)));
2803 return push_inst(compiler, inst | RD(REG_PAIR_SECOND(reg)) | RN(mem & REG_MASK) | ((sljit_ins)(memw + 0x8) << 7));
2804 }
2805
2806 if ((sljit_uw)memw <= 0xfff) {
2807 FAIL_IF(push_inst(compiler, inst | RD(TMP_REG1) | RN(mem & REG_MASK) | ((sljit_ins)memw << 10)));
2808 memw = 0;
2809 } else if ((sljit_uw)memw <= 0xffffff) {
2810 FAIL_IF(push_inst(compiler, inst | (1 << 22) | RD(TMP_REG1) | RN(mem & REG_MASK) | (((sljit_ins)memw >> 12) << 10)));
2811
2812 if ((memw & 0xe07) != 0) {
2813 FAIL_IF(push_inst(compiler, inst | RD(TMP_REG1) | RN(TMP_REG1) | (((sljit_ins)memw & 0xfff) << 10)));
2814 memw = 0;
2815 } else {
2816 memw &= 0xfff;
2817 }
2818 } else {
2819 FAIL_IF(load_immediate(compiler, TMP_REG1, memw));
2820 FAIL_IF(push_inst(compiler, (inst == ADDI ? ADD : SUB) | RD(TMP_REG1) | RN(mem & REG_MASK) | RM(TMP_REG1)));
2821 memw = 0;
2822 }
2823
2824 mem = SLJIT_MEM1(TMP_REG1);
2825
2826 if (inst == SUBI)
2827 memw = -memw;
2828 }
2829
2830 SLJIT_ASSERT((memw & 0x7) == 0 && memw <= 0x1f8 && memw >= -0x200);
2831 return push_inst(compiler, ((type & SLJIT_MEM_STORE) ? STP : LDP) | RT(REG_PAIR_FIRST(reg)) | RT2(REG_PAIR_SECOND(reg)) | RN(mem & REG_MASK) | (sljit_ins)((memw & 0x3f8) << 12));
2832 }
2833
2834 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
2835 sljit_s32 reg,
2836 sljit_s32 mem, sljit_sw memw)
2837 {
2838 sljit_u32 sign = 0, inst;
2839
2840 CHECK_ERROR();
2841 CHECK(check_sljit_emit_mem_update(compiler, type, reg, mem, memw));
2842
2843 if ((mem & OFFS_REG_MASK) || (memw > 255 || memw < -256))
2844 return SLJIT_ERR_UNSUPPORTED;
2845
2846 if (type & SLJIT_MEM_SUPP)
2847 return SLJIT_SUCCESS;
2848
2849 switch (type & 0xff) {
2850 case SLJIT_MOV:
2851 case SLJIT_MOV_P:
2852 inst = STURBI | (MEM_SIZE_SHIFT(WORD_SIZE) << 30) | 0x400;
2853 break;
2854 case SLJIT_MOV_S8:
2855 sign = 1;
2856 SLJIT_FALLTHROUGH
2857 case SLJIT_MOV_U8:
2858 inst = STURBI | (MEM_SIZE_SHIFT(BYTE_SIZE) << 30) | 0x400;
2859 break;
2860 case SLJIT_MOV_S16:
2861 sign = 1;
2862 SLJIT_FALLTHROUGH
2863 case SLJIT_MOV_U16:
2864 inst = STURBI | (MEM_SIZE_SHIFT(HALF_SIZE) << 30) | 0x400;
2865 break;
2866 case SLJIT_MOV_S32:
2867 sign = 1;
2868 SLJIT_FALLTHROUGH
2869 case SLJIT_MOV_U32:
2870 case SLJIT_MOV32:
2871 inst = STURBI | (MEM_SIZE_SHIFT(INT_SIZE) << 30) | 0x400;
2872 break;
2873 default:
2874 SLJIT_UNREACHABLE();
2875 inst = STURBI | (MEM_SIZE_SHIFT(WORD_SIZE) << 30) | 0x400;
2876 break;
2877 }
2878
2879 if (!(type & SLJIT_MEM_STORE))
2880 inst |= sign ? 0x00800000 : 0x00400000;
2881
2882 if (!(type & SLJIT_MEM_POST))
2883 inst |= 0x800;
2884
2885 return push_inst(compiler, inst | RT(reg) | RN(mem & REG_MASK) | (sljit_ins)((memw & 0x1ff) << 12));
2886 }
2887
2888 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
2889 sljit_s32 freg,
2890 sljit_s32 mem, sljit_sw memw)
2891 {
2892 sljit_u32 inst;
2893
2894 CHECK_ERROR();
2895 CHECK(check_sljit_emit_fmem_update(compiler, type, freg, mem, memw));
2896
2897 if ((mem & OFFS_REG_MASK) || (memw > 255 || memw < -256))
2898 return SLJIT_ERR_UNSUPPORTED;
2899
2900 if (type & SLJIT_MEM_SUPP)
2901 return SLJIT_SUCCESS;
2902
2903 inst = STUR_FI | 0x80000400;
2904
2905 if (!(type & SLJIT_32))
2906 inst |= 0x40000000;
2907
2908 if (!(type & SLJIT_MEM_STORE))
2909 inst |= 0x00400000;
2910
2911 if (!(type & SLJIT_MEM_POST))
2912 inst |= 0x800;
2913
2914 return push_inst(compiler, inst | VT(freg) | RN(mem & REG_MASK) | (sljit_ins)((memw & 0x1ff) << 12));
2915 }
2916
2917 static sljit_s32 sljit_emit_simd_mem_offset(struct sljit_compiler *compiler, sljit_s32 *mem_ptr, sljit_sw memw)
2918 {
2919 sljit_ins ins;
2920 sljit_s32 mem = *mem_ptr;
2921
2922 if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
2923 *mem_ptr = TMP_REG2;
2924 return push_inst(compiler, ADD | RD(TMP_REG2) | RN(mem & REG_MASK) | RM(OFFS_REG(mem)) | ((sljit_ins)(memw & 0x3) << 10));
2925 }
2926
2927 if (!(mem & REG_MASK)) {
2928 *mem_ptr = TMP_REG2;
2929 return load_immediate(compiler, TMP_REG2, memw);
2930 }
2931
2932 mem &= REG_MASK;
2933
2934 if (memw == 0) {
2935 *mem_ptr = mem;
2936 return SLJIT_SUCCESS;
2937 }
2938
2939 *mem_ptr = TMP_REG2;
2940
2941 if (memw < -0xffffff || memw > 0xffffff) {
2942 FAIL_IF(load_immediate(compiler, TMP_REG2, memw));
2943 return push_inst(compiler, ADD | RD(TMP_REG2) | RN(TMP_REG2) | RM(mem));
2944 }
2945
2946 ins = ADDI;
2947
2948 if (memw < 0) {
2949 memw = -memw;
2950 ins = SUBI;
2951 }
2952
2953 if (memw > 0xfff) {
2954 FAIL_IF(push_inst(compiler, ins | (1 << 22) | RD(TMP_REG2) | RN(mem) | ((sljit_ins)(memw >> 12) << 10)));
2955
2956 memw &= 0xfff;
2957 if (memw == 0)
2958 return SLJIT_SUCCESS;
2959
2960 mem = TMP_REG2;
2961 }
2962
2963 return push_inst(compiler, ins | RD(TMP_REG2) | RN(mem) | ((sljit_ins)memw << 10));
2964 }
2965
2966 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
2967 sljit_s32 vreg,
2968 sljit_s32 srcdst, sljit_sw srcdstw)
2969 {
2970 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
2971 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
2972 sljit_ins ins;
2973
2974 CHECK_ERROR();
2975 CHECK(check_sljit_emit_simd_mov(compiler, type, vreg, srcdst, srcdstw));
2976
2977 ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
2978
2979 if (reg_size != 3 && reg_size != 4)
2980 return SLJIT_ERR_UNSUPPORTED;
2981
2982 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
2983 return SLJIT_ERR_UNSUPPORTED;
2984
2985 if (type & SLJIT_SIMD_TEST)
2986 return SLJIT_SUCCESS;
2987
2988 if (!(srcdst & SLJIT_MEM)) {
2989 if (type & SLJIT_SIMD_STORE)
2990 ins = VD(srcdst) | VN(vreg) | VM(vreg);
2991 else
2992 ins = VD(vreg) | VN(srcdst) | VM(srcdst);
2993
2994 if (reg_size == 4)
2995 ins |= (1 << 30);
2996
2997 return push_inst(compiler, ORR_v | ins);
2998 }
2999
3000 FAIL_IF(sljit_emit_simd_mem_offset(compiler, &srcdst, srcdstw));
3001
3002 if (elem_size > 3)
3003 elem_size = 3;
3004
3005 ins = (type & SLJIT_SIMD_STORE) ? ST1 : LD1;
3006
3007 if (reg_size == 4)
3008 ins |= (1 << 30);
3009
3010 return push_inst(compiler, ins | ((sljit_ins)elem_size << 10) | RN(srcdst) | VT(vreg));
3011 }
3012
3013 static sljit_ins simd_get_imm(sljit_s32 elem_size, sljit_uw value)
3014 {
3015 sljit_ins result;
3016
3017 if (elem_size > 2 && (sljit_u32)value == (value >> 32)) {
3018 elem_size = 2;
3019 value = (sljit_u32)value;
3020 }
3021
3022 if (elem_size == 2 && (sljit_u16)value == (value >> 16)) {
3023 elem_size = 1;
3024 value = (sljit_u16)value;
3025 }
3026
3027 if (elem_size == 1 && (sljit_u8)value == (value >> 8)) {
3028 elem_size = 0;
3029 value = (sljit_u8)value;
3030 }
3031
3032 switch (elem_size) {
3033 case 0:
3034 SLJIT_ASSERT(value <= 0xff);
3035 result = 0xe000;
3036 break;
3037 case 1:
3038 SLJIT_ASSERT(value <= 0xffff);
3039 result = 0;
3040
3041 while (1) {
3042 if (value <= 0xff) {
3043 result |= 0x8000;
3044 break;
3045 }
3046
3047 if ((value & 0xff) == 0) {
3048 value >>= 8;
3049 result |= 0xa000;
3050 break;
3051 }
3052
3053 if (result != 0)
3054 return ~(sljit_ins)0;
3055
3056 value ^= (sljit_uw)0xffff;
3057 result = (1 << 29);
3058 }
3059 break;
3060 case 2:
3061 SLJIT_ASSERT(value <= 0xffffffff);
3062 result = 0;
3063
3064 while (1) {
3065 if (value <= 0xff) {
3066 result |= 0x0000;
3067 break;
3068 }
3069
3070 if ((value & ~(sljit_uw)0xff00) == 0) {
3071 value >>= 8;
3072 result |= 0x2000;
3073 break;
3074 }
3075
3076 if ((value & ~(sljit_uw)0xff0000) == 0) {
3077 value >>= 16;
3078 result |= 0x4000;
3079 break;
3080 }
3081
3082 if ((value & ~(sljit_uw)0xff000000) == 0) {
3083 value >>= 24;
3084 result |= 0x6000;
3085 break;
3086 }
3087
3088 if ((value & (sljit_uw)0xff) == 0xff && (value >> 16) == 0) {
3089 value >>= 8;
3090 result |= 0xc000;
3091 break;
3092 }
3093
3094 if ((value & (sljit_uw)0xffff) == 0xffff && (value >> 24) == 0) {
3095 value >>= 16;
3096 result |= 0xd000;
3097 break;
3098 }
3099
3100 if (result != 0)
3101 return ~(sljit_ins)0;
3102
3103 value ^= (sljit_uw)0xffffffff;
3104 result = (1 << 29);
3105 }
3106 break;
3107 default:
3108 return ~(sljit_ins)0;
3109 }
3110
3111 return (((sljit_ins)value & 0x1f) << 5) | (((sljit_ins)value & 0xe0) << 11) | result;
3112 }
3113
3114 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3115 sljit_s32 vreg,
3116 sljit_s32 src, sljit_sw srcw)
3117 {
3118 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3119 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3120 sljit_ins ins, imm;
3121
3122 CHECK_ERROR();
3123 CHECK(check_sljit_emit_simd_replicate(compiler, type, vreg, src, srcw));
3124
3125 ADJUST_LOCAL_OFFSET(src, srcw);
3126
3127 if (reg_size != 3 && reg_size != 4)
3128 return SLJIT_ERR_UNSUPPORTED;
3129
3130 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
3131 return SLJIT_ERR_UNSUPPORTED;
3132
3133 if (type & SLJIT_SIMD_TEST)
3134 return SLJIT_SUCCESS;
3135
3136 if (src & SLJIT_MEM) {
3137 FAIL_IF(sljit_emit_simd_mem_offset(compiler, &src, srcw));
3138
3139 ins = (sljit_ins)elem_size << 10;
3140
3141 if (reg_size == 4)
3142 ins |= (sljit_ins)1 << 30;
3143
3144 return push_inst(compiler, LD1R | ins | RN(src) | VT(vreg));
3145 }
3146
3147 ins = (sljit_ins)1 << (16 + elem_size);
3148
3149 if (reg_size == 4)
3150 ins |= (sljit_ins)1 << 30;
3151
3152 if (type & SLJIT_SIMD_FLOAT) {
3153 if (src == SLJIT_IMM)
3154 return push_inst(compiler, MOVI | (ins & ((sljit_ins)1 << 30)) | VD(vreg));
3155
3156 return push_inst(compiler, DUP_e | ins | VD(vreg) | VN(src));
3157 }
3158
3159 if (src == SLJIT_IMM) {
3160 if (elem_size < 3)
3161 srcw &= ((sljit_sw)1 << (((sljit_sw)1 << elem_size) << 3)) - 1;
3162
3163 imm = simd_get_imm(elem_size, (sljit_uw)srcw);
3164
3165 if (imm != ~(sljit_ins)0) {
3166 imm |= ins & ((sljit_ins)1 << 30);
3167
3168 return push_inst(compiler, MOVI | imm | VD(vreg));
3169 }
3170
3171 FAIL_IF(load_immediate(compiler, TMP_REG2, srcw));
3172 src = TMP_REG2;
3173 }
3174
3175 return push_inst(compiler, DUP_g | ins | VD(vreg) | RN(src));
3176 }
3177
3178 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
3179 sljit_s32 vreg, sljit_s32 lane_index,
3180 sljit_s32 srcdst, sljit_sw srcdstw)
3181 {
3182 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3183 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3184 sljit_ins ins;
3185
3186 CHECK_ERROR();
3187 CHECK(check_sljit_emit_simd_lane_mov(compiler, type, vreg, lane_index, srcdst, srcdstw));
3188
3189 ADJUST_LOCAL_OFFSET(srcdst, srcdstw);
3190
3191 if (reg_size != 3 && reg_size != 4)
3192 return SLJIT_ERR_UNSUPPORTED;
3193
3194 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
3195 return SLJIT_ERR_UNSUPPORTED;
3196
3197 if (type & SLJIT_SIMD_TEST)
3198 return SLJIT_SUCCESS;
3199
3200 if (type & SLJIT_SIMD_LANE_ZERO) {
3201 ins = (reg_size == 3) ? 0 : ((sljit_ins)1 << 30);
3202
3203 if ((type & SLJIT_SIMD_FLOAT) && vreg == srcdst) {
3204 FAIL_IF(push_inst(compiler, ORR_v | ins | VD(TMP_FREG1) | VN(vreg) | VM(vreg)));
3205 srcdst = TMP_FREG1;
3206 srcdstw = 0;
3207 }
3208
3209 FAIL_IF(push_inst(compiler, MOVI | ins | VD(vreg)));
3210 }
3211
3212 if (srcdst & SLJIT_MEM) {
3213 FAIL_IF(sljit_emit_simd_mem_offset(compiler, &srcdst, srcdstw));
3214
3215 if (elem_size == 3)
3216 ins = 0x8400;
3217 else if (elem_size == 0)
3218 ins = 0;
3219 else
3220 ins = (sljit_ins)0x2000 << elem_size;
3221
3222 lane_index = lane_index << elem_size;
3223 ins |= (sljit_ins)(((lane_index & 0x8) << 27) | ((lane_index & 0x7) << 10));
3224
3225 return push_inst(compiler, ((type & SLJIT_SIMD_STORE) ? ST1_s : LD1_s) | ins | RN(srcdst) | VT(vreg));
3226 }
3227
3228 if (type & SLJIT_SIMD_FLOAT) {
3229 if (type & SLJIT_SIMD_STORE)
3230 ins = INS_e | ((sljit_ins)1 << (16 + elem_size)) | ((sljit_ins)lane_index << (11 + elem_size)) | VD(srcdst) | VN(vreg);
3231 else
3232 ins = INS_e | ((((sljit_ins)lane_index << 1) | 1) << (16 + elem_size)) | VD(vreg) | VN(srcdst);
3233
3234 return push_inst(compiler, ins);
3235 }
3236
3237 if (srcdst == SLJIT_IMM) {
3238 if (elem_size < 3)
3239 srcdstw &= ((sljit_sw)1 << (((sljit_sw)1 << elem_size) << 3)) - 1;
3240
3241 FAIL_IF(load_immediate(compiler, TMP_REG2, srcdstw));
3242 srcdst = TMP_REG2;
3243 }
3244
3245 if (type & SLJIT_SIMD_STORE) {
3246 ins = RD(srcdst) | VN(vreg);
3247
3248 if ((type & SLJIT_SIMD_LANE_SIGNED) && (elem_size < 2 || (elem_size == 2 && !(type & SLJIT_32)))) {
3249 ins |= SMOV;
3250
3251 if (!(type & SLJIT_32))
3252 ins |= (sljit_ins)1 << 30;
3253 } else
3254 ins |= UMOV;
3255 } else
3256 ins = INS | VD(vreg) | RN(srcdst);
3257
3258 if (elem_size == 3)
3259 ins |= (sljit_ins)1 << 30;
3260
3261 return push_inst(compiler, ins | ((((sljit_ins)lane_index << 1) | 1) << (16 + elem_size)));
3262 }
3263
3264 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3265 sljit_s32 vreg,
3266 sljit_s32 src, sljit_s32 src_lane_index)
3267 {
3268 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3269 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3270 sljit_ins ins;
3271
3272 CHECK_ERROR();
3273 CHECK(check_sljit_emit_simd_lane_replicate(compiler, type, vreg, src, src_lane_index));
3274
3275 if (reg_size != 3 && reg_size != 4)
3276 return SLJIT_ERR_UNSUPPORTED;
3277
3278 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
3279 return SLJIT_ERR_UNSUPPORTED;
3280
3281 if (type & SLJIT_SIMD_TEST)
3282 return SLJIT_SUCCESS;
3283
3284 ins = (((sljit_ins)src_lane_index << 1) | 1) << (16 + elem_size);
3285
3286 if (reg_size == 4)
3287 ins |= (sljit_ins)1 << 30;
3288
3289 return push_inst(compiler, DUP_e | ins | VD(vreg) | VN(src));
3290 }
3291
3292 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
3293 sljit_s32 vreg,
3294 sljit_s32 src, sljit_sw srcw)
3295 {
3296 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3297 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3298 sljit_s32 elem2_size = SLJIT_SIMD_GET_ELEM2_SIZE(type);
3299
3300 CHECK_ERROR();
3301 CHECK(check_sljit_emit_simd_extend(compiler, type, vreg, src, srcw));
3302
3303 ADJUST_LOCAL_OFFSET(src, srcw);
3304
3305 if (reg_size != 3 && reg_size != 4)
3306 return SLJIT_ERR_UNSUPPORTED;
3307
3308 if ((type & SLJIT_SIMD_FLOAT) && (elem_size != 2 || elem2_size != 3))
3309 return SLJIT_ERR_UNSUPPORTED;
3310
3311 if (type & SLJIT_SIMD_TEST)
3312 return SLJIT_SUCCESS;
3313
3314 if (src & SLJIT_MEM) {
3315 FAIL_IF(sljit_emit_simd_mem_offset(compiler, &src, srcw));
3316
3317 if (reg_size == 4 && elem2_size - elem_size == 1)
3318 FAIL_IF(push_inst(compiler, LD1 | ((sljit_ins)elem_size << 10) | RN(src) | VT(vreg)));
3319 else
3320 FAIL_IF(push_inst(compiler, LD1_s | ((sljit_ins)0x2000 << (reg_size - elem2_size + elem_size)) | RN(src) | VT(vreg)));
3321 src = vreg;
3322 }
3323
3324 if (type & SLJIT_SIMD_FLOAT) {
3325 SLJIT_ASSERT(reg_size == 4);
3326 return push_inst(compiler, FCVTL | (1 << 22) | VD(vreg) | VN(src));
3327 }
3328
3329 do {
3330 FAIL_IF(push_inst(compiler, ((type & SLJIT_SIMD_EXTEND_SIGNED) ? SSHLL : USHLL)
3331 | ((sljit_ins)1 << (19 + elem_size)) | VD(vreg) | VN(src)));
3332 src = vreg;
3333 } while (++elem_size < elem2_size);
3334
3335 return SLJIT_SUCCESS;
3336 }
3337
3338 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
3339 sljit_s32 vreg,
3340 sljit_s32 dst, sljit_sw dstw)
3341 {
3342 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3343 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3344 sljit_ins ins, imms;
3345 sljit_s32 dst_r;
3346
3347 CHECK_ERROR();
3348 CHECK(check_sljit_emit_simd_sign(compiler, type, vreg, dst, dstw));
3349
3350 ADJUST_LOCAL_OFFSET(dst, dstw);
3351
3352 if (reg_size != 3 && reg_size != 4)
3353 return SLJIT_ERR_UNSUPPORTED;
3354
3355 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
3356 return SLJIT_ERR_UNSUPPORTED;
3357
3358 if (type & SLJIT_SIMD_TEST)
3359 return SLJIT_SUCCESS;
3360
3361 switch (elem_size) {
3362 case 0:
3363 imms = 0x643219;
3364 ins = USHR | (0x9 << 16);
3365 break;
3366 case 1:
3367 imms = (reg_size == 4) ? 0x643219 : 0x6231;
3368 ins = USHR | (0x11 << 16);
3369 break;
3370 case 2:
3371 imms = (reg_size == 4) ? 0x6231 : 0x61;
3372 ins = USHR | (0x21 << 16);
3373 break;
3374 default:
3375 imms = 0x61;
3376 ins = USHR | (0x41 << 16);
3377 break;
3378 }
3379
3380 if (reg_size == 4)
3381 ins |= (1 << 30);
3382
3383 FAIL_IF(push_inst(compiler, ins | VD(TMP_FREG1) | VN(vreg)));
3384
3385 if (reg_size == 4 && elem_size > 0)
3386 FAIL_IF(push_inst(compiler, XTN | ((sljit_ins)(elem_size - 1) << 22) | VD(TMP_FREG1) | VN(TMP_FREG1)));
3387
3388 if (imms >= 0x100) {
3389 ins = (reg_size == 4 && elem_size == 0) ? (1 << 30) : 0;
3390
3391 do {
3392 FAIL_IF(push_inst(compiler, USRA | ins | ((imms & 0xff) << 16) | VD(TMP_FREG1) | VN(TMP_FREG1)));
3393 imms >>= 8;
3394 } while (imms >= 0x100);
3395 }
3396
3397 FAIL_IF(push_inst(compiler, USRA | (1 << 30) | (imms << 16) | VD(TMP_FREG1) | VN(TMP_FREG1)));
3398
3399 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
3400 ins = (0x1 << 16);
3401
3402 if (reg_size == 4 && elem_size == 0) {
3403 FAIL_IF(push_inst(compiler, INS_e | (0x3 << 16) | (0x8 << 11) | VD(TMP_FREG1) | VN(TMP_FREG1)));
3404 ins = (0x2 << 16);
3405 }
3406
3407 FAIL_IF(push_inst(compiler, UMOV | ins | RD(dst_r) | VN(TMP_FREG1)));
3408
3409 if (dst_r == TMP_REG2)
3410 return emit_op_mem(compiler, STORE | ((type & SLJIT_32) ? INT_SIZE : WORD_SIZE), TMP_REG2, dst, dstw, TMP_REG1);
3411
3412 return SLJIT_SUCCESS;
3413 }
3414
3415 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
3416 sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
3417 {
3418 sljit_s32 reg_size = SLJIT_SIMD_GET_REG_SIZE(type);
3419 sljit_s32 elem_size = SLJIT_SIMD_GET_ELEM_SIZE(type);
3420 sljit_ins ins = 0;
3421
3422 CHECK_ERROR();
3423 CHECK(check_sljit_emit_simd_op2(compiler, type, dst_vreg, src1_vreg, src2, src2w));
3424 ADJUST_LOCAL_OFFSET(src2, src2w);
3425
3426 if (reg_size != 3 && reg_size != 4)
3427 return SLJIT_ERR_UNSUPPORTED;
3428
3429 if ((type & SLJIT_SIMD_FLOAT) && (elem_size < 2 || elem_size > 3))
3430 return SLJIT_ERR_UNSUPPORTED;
3431
3432 if (type & SLJIT_SIMD_TEST)
3433 return SLJIT_SUCCESS;
3434
3435 switch (SLJIT_SIMD_GET_OPCODE(type)) {
3436 case SLJIT_SIMD_OP2_AND:
3437 ins = AND_v;
3438 break;
3439 case SLJIT_SIMD_OP2_OR:
3440 ins = ORR_v;
3441 break;
3442 case SLJIT_SIMD_OP2_XOR:
3443 ins = EOR_v;
3444 break;
3445 case SLJIT_SIMD_OP2_SHUFFLE:
3446 ins = TBL_v;
3447 break;
3448 }
3449
3450 if (src2 & SLJIT_MEM) {
3451 if (elem_size > 3)
3452 elem_size = 3;
3453
3454 FAIL_IF(sljit_emit_simd_mem_offset(compiler, &src2, src2w));
3455 push_inst(compiler, LD1 | (reg_size == 4 ? (1 << 30) : 0) | ((sljit_ins)elem_size << 10) | RN(src2) | VT(TMP_FREG1));
3456 src2 = TMP_FREG1;
3457 }
3458
3459 if (reg_size == 4)
3460 ins |= (sljit_ins)1 << 30;
3461
3462 return push_inst(compiler, ins | VD(dst_vreg) | VN(src1_vreg) | VM(src2));
3463 }
3464
3465 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
3466 sljit_s32 dst_reg,
3467 sljit_s32 mem_reg)
3468 {
3469 sljit_ins ins;
3470
3471 CHECK_ERROR();
3472 CHECK(check_sljit_emit_atomic_load(compiler, op, dst_reg, mem_reg));
3473
3474 #ifndef __ARM_FEATURE_ATOMICS
3475 if (op & SLJIT_ATOMIC_USE_CAS)
3476 return SLJIT_ERR_UNSUPPORTED;
3477 #endif /* ARM_FEATURE_ATOMICS */
3478
3479 switch (GET_OPCODE(op)) {
3480 case SLJIT_MOV_S8:
3481 case SLJIT_MOV_S16:
3482 case SLJIT_MOV_S32:
3483 return SLJIT_ERR_UNSUPPORTED;
3484
3485 case SLJIT_MOV32:
3486 case SLJIT_MOV_U32:
3487 #ifdef __ARM_FEATURE_ATOMICS
3488 if (!(op & SLJIT_ATOMIC_USE_LS))
3489 ins = LDR ^ (1 << 30);
3490 else
3491 #endif /* ARM_FEATURE_ATOMICS */
3492 ins = LDXR ^ (1 << 30);
3493 break;
3494 case SLJIT_MOV_U8:
3495 #ifdef __ARM_FEATURE_ATOMICS
3496 if (!(op & SLJIT_ATOMIC_USE_LS))
3497 ins = LDRB;
3498 else
3499 #endif /* ARM_FEATURE_ATOMICS */
3500 ins = LDXRB;
3501 break;
3502 case SLJIT_MOV_U16:
3503 #ifdef __ARM_FEATURE_ATOMICS
3504 if (!(op & SLJIT_ATOMIC_USE_LS))
3505 ins = LDRH;
3506 else
3507 #endif /* ARM_FEATURE_ATOMICS */
3508 ins = LDXRH;
3509 break;
3510 default:
3511 #ifdef __ARM_FEATURE_ATOMICS
3512 if (!(op & SLJIT_ATOMIC_USE_LS))
3513 ins = LDR;
3514 else
3515 #endif /* ARM_FEATURE_ATOMICS */
3516 ins = LDXR;
3517 break;
3518 }
3519
3520 if (op & SLJIT_ATOMIC_TEST)
3521 return SLJIT_SUCCESS;
3522
3523 return push_inst(compiler, ins | RN(mem_reg) | RT(dst_reg));
3524 }
3525
3526 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
3527 sljit_s32 src_reg,
3528 sljit_s32 mem_reg,
3529 sljit_s32 temp_reg)
3530 {
3531 sljit_ins ins;
3532 sljit_ins cmp = 0;
3533 SLJIT_UNUSED_ARG(temp_reg); /* !__ARM_FEATURE_ATOMICS */
3534
3535 CHECK_ERROR();
3536 CHECK(check_sljit_emit_atomic_store(compiler, op, src_reg, mem_reg, temp_reg));
3537
3538 #ifdef __ARM_FEATURE_ATOMICS
3539 if (!(op & SLJIT_ATOMIC_USE_LS)) {
3540 if (op & SLJIT_SET_ATOMIC_STORED)
3541 cmp = (SUBS ^ W_OP) | RD(TMP_ZERO);
3542
3543 switch (GET_OPCODE(op)) {
3544 case SLJIT_MOV_S8:
3545 case SLJIT_MOV_S16:
3546 case SLJIT_MOV_S32:
3547 return SLJIT_ERR_UNSUPPORTED;
3548
3549 case SLJIT_MOV32:
3550 case SLJIT_MOV_U32:
3551 ins = CAS ^ (1 << 30);
3552 break;
3553 case SLJIT_MOV_U16:
3554 ins = CASH;
3555 break;
3556 case SLJIT_MOV_U8:
3557 ins = CASB;
3558 break;
3559 default:
3560 ins = CAS;
3561 if (cmp)
3562 cmp ^= W_OP;
3563 break;
3564 }
3565
3566 if (op & SLJIT_ATOMIC_TEST)
3567 return SLJIT_SUCCESS;
3568
3569 if (cmp)
3570 FAIL_IF(push_inst(compiler, ((MOV ^ W_OP) ^ (cmp & W_OP)) | RM(temp_reg) | RD(TMP_REG2)));
3571
3572 FAIL_IF(push_inst(compiler, ins | RM(temp_reg) | RN(mem_reg) | RD(src_reg)));
3573 if (!cmp)
3574 return SLJIT_SUCCESS;
3575
3576 return push_inst(compiler, cmp | RM(TMP_REG2) | RN(temp_reg));
3577 }
3578 #else /* !__ARM_FEATURE_ATOMICS */
3579 if (op & SLJIT_ATOMIC_USE_CAS)
3580 return SLJIT_ERR_UNSUPPORTED;
3581 #endif /* __ARM_FEATURE_ATOMICS */
3582
3583 if (op & SLJIT_SET_ATOMIC_STORED)
3584 cmp = (SUBI ^ W_OP) | (1 << 29);
3585
3586 switch (GET_OPCODE(op)) {
3587 case SLJIT_MOV_S8:
3588 case SLJIT_MOV_S16:
3589 case SLJIT_MOV_S32:
3590 return SLJIT_ERR_UNSUPPORTED;
3591
3592 case SLJIT_MOV32:
3593 case SLJIT_MOV_U32:
3594 ins = STXR ^ (1 << 30);
3595 break;
3596 case SLJIT_MOV_U8:
3597 ins = STXRB;
3598 break;
3599 case SLJIT_MOV_U16:
3600 ins = STXRH;
3601 break;
3602 default:
3603 ins = STXR;
3604 break;
3605 }
3606
3607 if (op & SLJIT_ATOMIC_TEST)
3608 return SLJIT_SUCCESS;
3609
3610 FAIL_IF(push_inst(compiler, ins | RM(TMP_REG2) | RN(mem_reg) | RT(src_reg)));
3611 if (!cmp)
3612 return SLJIT_SUCCESS;
3613 return push_inst(compiler, cmp | RD(TMP_ZERO) | RN(TMP_REG2));
3614 }
3615
3616 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
3617 {
3618 sljit_s32 dst_reg;
3619 sljit_ins ins;
3620
3621 CHECK_ERROR();
3622 CHECK(check_sljit_get_local_base(compiler, dst, dstw, offset));
3623 ADJUST_LOCAL_OFFSET(dst, dstw);
3624 ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_SP), offset);
3625
3626 dst_reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
3627
3628 /* Not all instruction forms support accessing SP register. */
3629 if (offset <= 0xffffff && offset >= -0xffffff) {
3630 ins = ADDI;
3631 if (offset < 0) {
3632 offset = -offset;
3633 ins = SUBI;
3634 }
3635
3636 if (offset <= 0xfff)
3637 FAIL_IF(push_inst(compiler, ins | RD(dst_reg) | RN(SLJIT_SP) | (sljit_ins)(offset << 10)));
3638 else {
3639 FAIL_IF(push_inst(compiler, ins | RD(dst_reg) | RN(SLJIT_SP) | (sljit_ins)((offset & 0xfff000) >> (12 - 10)) | (1 << 22)));
3640
3641 offset &= 0xfff;
3642 if (offset != 0)
3643 FAIL_IF(push_inst(compiler, ins | RD(dst_reg) | RN(dst_reg) | (sljit_ins)(offset << 10)));
3644 }
3645 }
3646 else {
3647 FAIL_IF(load_immediate (compiler, dst_reg, offset));
3648 /* Add extended register form. */
3649 FAIL_IF(push_inst(compiler, ADDE | (0x3 << 13) | RD(dst_reg) | RN(SLJIT_SP) | RM(dst_reg)));
3650 }
3651
3652 if (SLJIT_UNLIKELY(dst & SLJIT_MEM))
3653 return emit_op_mem(compiler, WORD_SIZE | STORE, dst_reg, dst, dstw, TMP_REG1);
3654 return SLJIT_SUCCESS;
3655 }
3656
3657 #define SLJIT_EMIT_CONST_U8(c) \
3658 (((c) & 0x100) != 0 ? (MOVN | (sljit_ins)((~(c) & 0xff) << 5)) : (MOVZ | (sljit_ins)(((c) & 0xff) << 5)))
3659 #define SLJIT_EMIT_CONST_S32(c) \
3660 (((c) < 0) ? (MOVN | (sljit_ins)((~(c) & 0xffff) << 5)) : (MOVZ | (sljit_ins)(((c) & 0xffff) << 5)))
3661
3662 SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 op,
3663 sljit_s32 dst, sljit_sw dstw,
3664 sljit_sw init_value)
3665 {
3666 struct sljit_const *const_;
3667 sljit_s32 dst_r;
3668 sljit_s32 mem_flags = WORD_SIZE | STORE;
3669
3670 CHECK_ERROR_PTR();
3671 CHECK_PTR(check_sljit_emit_const(compiler, op, dst, dstw, init_value));
3672 ADJUST_LOCAL_OFFSET(dst, dstw);
3673
3674 const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
3675 PTR_FAIL_IF(!const_);
3676 set_const(const_, compiler);
3677
3678 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
3679
3680 switch (GET_OPCODE(op)) {
3681 case SLJIT_MOV_U8:
3682 PTR_FAIL_IF(push_inst(compiler, SLJIT_EMIT_CONST_U8(init_value) | RD(dst_r)));
3683 mem_flags = BYTE_SIZE | STORE;
3684 break;
3685
3686 case SLJIT_MOV32:
3687 case SLJIT_MOV_S32:
3688 if (GET_OPCODE(op) == SLJIT_MOV32) {
3689 init_value = (sljit_u32)init_value;
3690 mem_flags = INT_SIZE | STORE;
3691 } else
3692 init_value = (sljit_s32)init_value;
3693
3694 PTR_FAIL_IF(push_inst(compiler, SLJIT_EMIT_CONST_S32(init_value) | RD(dst_r)));
3695 PTR_FAIL_IF(push_inst(compiler, MOVK | (1 << 21) | (sljit_ins)((init_value >> 11) & 0x1fffe0) | RD(dst_r)));
3696 break;
3697
3698 default:
3699 PTR_FAIL_IF(emit_imm64_const(compiler, dst_r, (sljit_uw)init_value));
3700 break;
3701 }
3702
3703 if (dst & SLJIT_MEM)
3704 PTR_FAIL_IF(emit_op_mem(compiler, mem_flags, dst_r, dst, dstw, TMP_REG2));
3705 return const_;
3706 }
3707
3708 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_op_addr(struct sljit_compiler *compiler, sljit_s32 op,
3709 sljit_s32 dst, sljit_sw dstw)
3710 {
3711 struct sljit_jump *jump;
3712 sljit_s32 dst_r, target_r;
3713 SLJIT_UNUSED_ARG(op);
3714
3715 CHECK_ERROR_PTR();
3716 CHECK_PTR(check_sljit_emit_op_addr(compiler, op, dst, dstw));
3717 ADJUST_LOCAL_OFFSET(dst, dstw);
3718
3719 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
3720
3721 if (op != SLJIT_ADD_ABS_ADDR)
3722 target_r = dst_r;
3723 else {
3724 target_r = TMP_REG2;
3725
3726 if (dst & SLJIT_MEM)
3727 PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, dst, dstw, TMP_REG2));
3728 }
3729
3730 PTR_FAIL_IF(push_inst(compiler, RD(target_r)));
3731
3732 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
3733 PTR_FAIL_IF(!jump);
3734 set_mov_addr(jump, compiler, 1);
3735
3736 compiler->size += 3;
3737
3738 if (op == SLJIT_ADD_ABS_ADDR)
3739 PTR_FAIL_IF(push_inst(compiler, ADD | RD(dst_r) | RN(dst_r) | RM(TMP_REG2)));
3740
3741 if (dst & SLJIT_MEM)
3742 PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2));
3743
3744 return jump;
3745 }
3746
3747 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset)
3748 {
3749 sljit_ins* inst = (sljit_ins*)addr;
3750 sljit_u32 dst;
3751 SLJIT_UNUSED_ARG(executable_offset);
3752
3753 SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 0);
3754
3755 dst = inst[0] & 0x1f;
3756 SLJIT_ASSERT((inst[0] & 0xffe00000) == MOVZ && (inst[1] & 0xffe00000) == (MOVK | (1 << 21)));
3757 inst[0] = MOVZ | dst | (((sljit_u32)new_target & 0xffff) << 5);
3758 inst[1] = MOVK | dst | (((sljit_u32)(new_target >> 16) & 0xffff) << 5) | (1 << 21);
3759 inst[2] = MOVK | dst | (((sljit_u32)(new_target >> 32) & 0xffff) << 5) | (2 << 21);
3760 inst[3] = MOVK | dst | ((sljit_u32)(new_target >> 48) << 5) | (3 << 21);
3761
3762 SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 1);
3763 inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
3764 SLJIT_CACHE_FLUSH(inst, inst + 4);
3765 }
3766
3767 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_s32 op, sljit_sw new_constant, sljit_sw executable_offset)
3768 {
3769 sljit_ins* inst;
3770
3771 switch (GET_OPCODE(op)) {
3772 case SLJIT_MOV_U8:
3773 inst = (sljit_ins*)addr;
3774 SLJIT_ASSERT((inst[0] & 0xffe00000) == MOVZ || (inst[0] & 0xffe00000) == MOVN);
3775
3776 SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 0);
3777 inst[0] = SLJIT_EMIT_CONST_U8(new_constant) | (inst[0] & 0x1f);
3778 SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1);
3779 inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
3780 SLJIT_CACHE_FLUSH(inst, inst + 1);
3781 return;
3782
3783 case SLJIT_MOV32:
3784 case SLJIT_MOV_S32:
3785 if (GET_OPCODE(op) == SLJIT_MOV32)
3786 new_constant = (sljit_u32)new_constant;
3787 else
3788 new_constant = (sljit_s32)new_constant;
3789
3790 inst = (sljit_ins*)addr;
3791 SLJIT_ASSERT(((inst[0] & 0xffe00000) == MOVZ || (inst[0] & 0xffe00000) == MOVN) && (inst[1] & 0xffe00000) == (MOVK | (1 << 21)));
3792
3793 SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0);
3794 inst[0] = SLJIT_EMIT_CONST_S32(new_constant) | (inst[0] & 0x1f);
3795 inst[1] = MOVK | (1 << 21) | (sljit_ins)((new_constant >> 11) & 0x1fffe0) | (inst[1] & 0x1f);
3796 SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1);
3797 inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
3798 SLJIT_CACHE_FLUSH(inst, inst + 2);
3799 return;
3800
3801 default:
3802 sljit_set_jump_addr(addr, (sljit_uw)new_constant, executable_offset);
3803 return;
3804 }
3805 }