Debian Codegraph @

src/3rdparty/pcre2/deps/sljit/sljit_src/sljitLir.c

raw
1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include "sljitLir.h"
28
29 #ifdef _WIN32
30
31 #include <windows.h>
32
33 #endif /* _WIN32 */
34
35 #if !(defined SLJIT_STD_MACROS_DEFINED && SLJIT_STD_MACROS_DEFINED)
36
37 /* These libraries are needed for the macros below. */
38 #include <stdlib.h>
39 #include <string.h>
40
41 #endif /* SLJIT_STD_MACROS_DEFINED */
42
43 #define CHECK_ERROR() \
44 do { \
45 if (SLJIT_UNLIKELY(compiler->error)) \
46 return compiler->error; \
47 } while (0)
48
49 #define CHECK_ERROR_PTR() \
50 do { \
51 if (SLJIT_UNLIKELY(compiler->error)) \
52 return NULL; \
53 } while (0)
54
55 #define FAIL_IF(expr) \
56 do { \
57 if (SLJIT_UNLIKELY(expr)) \
58 return compiler->error; \
59 } while (0)
60
61 #define PTR_FAIL_IF(expr) \
62 do { \
63 if (SLJIT_UNLIKELY(expr)) \
64 return NULL; \
65 } while (0)
66
67 #define FAIL_IF_NULL(ptr) \
68 do { \
69 if (SLJIT_UNLIKELY(!(ptr))) { \
70 compiler->error = SLJIT_ERR_ALLOC_FAILED; \
71 return SLJIT_ERR_ALLOC_FAILED; \
72 } \
73 } while (0)
74
75 #define PTR_FAIL_IF_NULL(ptr) \
76 do { \
77 if (SLJIT_UNLIKELY(!(ptr))) { \
78 compiler->error = SLJIT_ERR_ALLOC_FAILED; \
79 return NULL; \
80 } \
81 } while (0)
82
83 #define PTR_FAIL_WITH_EXEC_IF(ptr) \
84 do { \
85 if (SLJIT_UNLIKELY(!(ptr))) { \
86 compiler->error = SLJIT_ERR_EX_ALLOC_FAILED; \
87 return NULL; \
88 } \
89 } while (0)
90
91 #if !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)
92
93 #define SSIZE_OF(type) ((sljit_s32)sizeof(sljit_ ## type))
94
95 #define VARIABLE_FLAG_SHIFT (10)
96 /* All variable flags are even. */
97 #define VARIABLE_FLAG_MASK (0x3e << VARIABLE_FLAG_SHIFT)
98 #define GET_FLAG_TYPE(op) ((op) >> VARIABLE_FLAG_SHIFT)
99 #define GET_FLAG_TYPE_MASK(op) (((op) >> VARIABLE_FLAG_SHIFT) & 0x3e)
100
101 #define GET_OPCODE(op) \
102 ((op) & 0xff)
103
104 #define HAS_FLAGS(op) \
105 ((op) & (SLJIT_SET_Z | VARIABLE_FLAG_MASK))
106
107 #define GET_ALL_FLAGS(op) \
108 ((op) & (SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK))
109
110 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
111 #define TYPE_CAST_NEEDED(op) \
112 ((op) >= SLJIT_MOV_U8 && (op) <= SLJIT_MOV_S32)
113 #else /* !SLJIT_64BIT_ARCHITECTURE */
114 #define TYPE_CAST_NEEDED(op) \
115 ((op) >= SLJIT_MOV_U8 && (op) <= SLJIT_MOV_S16)
116 #endif /* SLJIT_64BIT_ARCHITECTURE */
117
118 #define BUF_SIZE 4096
119
120 #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
121 #define ABUF_SIZE 2048
122 #else /* !SLJIT_32BIT_ARCHITECTURE */
123 #define ABUF_SIZE 4096
124 #endif /* SLJIT_32BIT_ARCHITECTURE */
125
126 /* Parameter parsing. */
127 #define REG_MASK 0x7f
128 #define OFFS_REG(reg) (((reg) >> 8) & REG_MASK)
129 #define OFFS_REG_MASK (REG_MASK << 8)
130 #define TO_OFFS_REG(reg) ((reg) << 8)
131 #define FAST_IS_REG(reg) ((reg) < REG_MASK)
132
133 /* Mask for argument types. */
134 #define SLJIT_ARG_MASK 0x7
135 #define SLJIT_ARG_FULL_MASK (SLJIT_ARG_MASK | SLJIT_ARG_TYPE_SCRATCH_REG)
136
137 /* Mask for register pairs. */
138 #define REG_PAIR_MASK 0x7f00
139 #define REG_PAIR_FIRST(reg) ((reg) & 0x7f)
140 #define REG_PAIR_SECOND(reg) ((reg) >> 8)
141
142 /* Mask for sljit_emit_enter. */
143 #define ENTER_GET_REGS(regs) ((regs) & 0xff)
144 #define ENTER_GET_FLOAT_REGS(regs) (((regs) >> 8) & 0xff)
145 #define ENTER_GET_VECTOR_REGS(regs) (((regs) >> 16) & 0xff)
146 #define SLJIT_KEPT_SAVEDS_COUNT(options) ((options) & 0x3)
147
148 /* Getters for simd operations, which returns with log2(size). */
149 #define SLJIT_SIMD_GET_OPCODE(type) ((type) & 0xff)
150 #define SLJIT_SIMD_GET_REG_SIZE(type) (((type) >> 12) & 0x3f)
151 #define SLJIT_SIMD_GET_ELEM_SIZE(type) (((type) >> 18) & 0x3f)
152 #define SLJIT_SIMD_GET_ELEM2_SIZE(type) (((type) >> 24) & 0x3f)
153
154 #define SLJIT_SIMD_CHECK_REG(type) (((type) & 0x3f000) >= SLJIT_SIMD_REG_64 && ((type) & 0x3f000) <= SLJIT_SIMD_REG_512)
155 #define SLJIT_SIMD_TYPE_MASK(m) ((sljit_s32)0xff000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))
156 #define SLJIT_SIMD_TYPE_MASK2(m) ((sljit_s32)0xc0000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))
157
158 /* Jump flags. */
159 #define JUMP_ADDR 0x1
160 #define JUMP_MOV_ADDR 0x2
161 /* SLJIT_REWRITABLE_JUMP is 0x1000. */
162
163 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
164 # define PATCH_MB 0x04
165 # define PATCH_MW 0x08
166 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
167 # define PATCH_MD 0x10
168 # define MOV_ADDR_HI 0x20
169 # define JUMP_MAX_SIZE ((sljit_uw)(10 + 3))
170 # define CJUMP_MAX_SIZE ((sljit_uw)(2 + 10 + 3))
171 #else /* !SLJIT_CONFIG_X86_64 */
172 # define JUMP_MAX_SIZE ((sljit_uw)5)
173 # define CJUMP_MAX_SIZE ((sljit_uw)6)
174 #endif /* SLJIT_CONFIG_X86_64 */
175 # define TYPE_SHIFT 13
176 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
177 /* Bits 7..12 is for debug jump size, SLJIT_REWRITABLE_JUMP is 0x1000 */
178 # define JUMP_SIZE_SHIFT 7
179 #endif /* SLJIT_DEBUG */
180 #endif /* SLJIT_CONFIG_X86 */
181
182 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6) || (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
183 # define IS_BL 0x04
184 # define PATCH_B 0x08
185 #endif /* SLJIT_CONFIG_ARM_V6 || SLJIT_CONFIG_ARM_V7 */
186
187 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
188 # define CPOOL_SIZE 512
189 #endif /* SLJIT_CONFIG_ARM_V6 */
190
191 #if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
192 # define JUMP_SIZE_SHIFT 26
193 # define JUMP_MAX_SIZE ((sljit_uw)3)
194 #endif /* SLJIT_CONFIG_ARM_V7 */
195
196 #if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
197 # define IS_COND 0x04
198 # define IS_BL 0x08
199 /* conditional + imm8 */
200 # define PATCH_TYPE1 0x10
201 /* conditional + imm20 */
202 # define PATCH_TYPE2 0x20
203 /* imm11 */
204 # define PATCH_TYPE3 0x30
205 /* imm24 */
206 # define PATCH_TYPE4 0x40
207 /* BL + imm24 */
208 # define PATCH_TYPE5 0x50
209 /* addwi/subwi */
210 # define PATCH_TYPE6 0x60
211 /* 0xf00 cc code for branches */
212 # define JUMP_SIZE_SHIFT 26
213 # define JUMP_MAX_SIZE ((sljit_uw)5)
214 #endif /* SLJIT_CONFIG_ARM_THUMB2 */
215
216 #if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
217 # define IS_COND 0x004
218 # define IS_CBZ 0x008
219 # define IS_BL 0x010
220 # define PATCH_COND 0x020
221 # define PATCH_B 0x040
222 # define PATCH_B32 0x080
223 # define PATCH_ABS48 0x100
224 # define PATCH_ABS64 0x200
225 # define JUMP_SIZE_SHIFT 58
226 # define JUMP_MAX_SIZE ((sljit_uw)5)
227 #endif /* SLJIT_CONFIG_ARM_64 */
228
229 #if (defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
230 # define IS_COND 0x004
231 # define IS_CALL 0x008
232 # define PATCH_B 0x010
233 # define PATCH_ABS_B 0x020
234 #if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
235 # define PATCH_ABS32 0x040
236 # define PATCH_ABS48 0x080
237 # define JUMP_SIZE_SHIFT 58
238 # define JUMP_MAX_SIZE ((sljit_uw)7)
239 #else /* !SLJIT_CONFIG_PPC_64 */
240 # define JUMP_SIZE_SHIFT 26
241 # define JUMP_MAX_SIZE ((sljit_uw)4)
242 #endif /* SLJIT_CONFIG_PPC_64 */
243 #endif /* SLJIT_CONFIG_PPC */
244
245 #if (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
246 # define IS_MOVABLE 0x004
247 # define IS_JAL 0x008
248 # define IS_CALL 0x010
249 # define IS_BIT26_COND 0x020
250 # define IS_BIT16_COND 0x040
251 # define IS_BIT23_COND 0x080
252
253 # define IS_COND (IS_BIT26_COND | IS_BIT16_COND | IS_BIT23_COND)
254
255 # define PATCH_B 0x100
256 # define PATCH_J 0x200
257
258 #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64)
259 # define PATCH_ABS32 0x400
260 # define PATCH_ABS48 0x800
261 #endif /* SLJIT_CONFIG_MIPS_64 */
262
263 /* instruction types */
264 # define MOVABLE_INS 0
265 /* 1 - 31 last destination register */
266 /* no destination (i.e: store) */
267 # define UNMOVABLE_INS 32
268 /* FPU status register */
269 # define FCSR_FCC 33
270 #endif /* SLJIT_CONFIG_MIPS */
271
272 #if (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
273 # define IS_COND 0x004
274 # define IS_CALL 0x008
275
276 # define PATCH_B 0x010
277 # define PATCH_J 0x020
278
279 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
280 # define PATCH_REL32 0x040
281 # define PATCH_ABS32 0x080
282 # define PATCH_ABS44 0x100
283 # define PATCH_ABS52 0x200
284 # define JUMP_SIZE_SHIFT 58
285 # define JUMP_MAX_SIZE ((sljit_uw)6)
286 #else /* !SLJIT_CONFIG_RISCV_64 */
287 # define JUMP_SIZE_SHIFT 26
288 # define JUMP_MAX_SIZE ((sljit_uw)2)
289 #endif /* SLJIT_CONFIG_RISCV_64 */
290 #endif /* SLJIT_CONFIG_RISCV */
291
292 #if (defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
293 # define IS_COND 0x004
294 # define IS_CALL 0x008
295
296 # define PATCH_B 0x010
297 # define PATCH_J 0x020
298
299 # define PATCH_REL32 0x040
300 # define PATCH_ABS32 0x080
301 # define PATCH_ABS52 0x100
302 # define JUMP_SIZE_SHIFT 58
303 # define JUMP_MAX_SIZE ((sljit_uw)4)
304
305 #endif /* SLJIT_CONFIG_LOONGARCH */
306 /* Stack management. */
307
308 #define GET_SAVED_REGISTERS_SIZE(scratches, saveds, extra) \
309 (((scratches < SLJIT_NUMBER_OF_SCRATCH_REGISTERS ? 0 : (scratches - SLJIT_NUMBER_OF_SCRATCH_REGISTERS)) + \
310 (saveds) + (sljit_s32)(extra)) * (sljit_s32)sizeof(sljit_sw))
311
312 #define GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, type) \
313 (((fscratches < SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS ? 0 : (fscratches - SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS)) + \
314 (fsaveds)) * SSIZE_OF(type))
315
316 #define ADJUST_LOCAL_OFFSET(p, i) \
317 if ((p) == (SLJIT_MEM1(SLJIT_SP))) \
318 (i) += SLJIT_LOCALS_OFFSET;
319
320 #endif /* !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED) */
321
322 /* Utils can still be used even if SLJIT_CONFIG_UNSUPPORTED is set. */
323 #include "sljitUtils.c"
324
325 #if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
326 #define SLJIT_CODE_TO_PTR(code) ((void*)((sljit_up)(code) & ~(sljit_up)0x1))
327 #elif (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
328 #define SLJIT_CODE_TO_PTR(code) ((void*)(*(sljit_up*)code))
329 #else /* !SLJIT_CONFIG_ARM_THUMB2 && !SLJIT_INDIRECT_CALL */
330 #define SLJIT_CODE_TO_PTR(code) ((void*)(code))
331 #endif /* SLJIT_CONFIG_ARM_THUMB2 || SLJIT_INDIRECT_CALL */
332
333 #if !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)
334
335 #if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR)
336
337 #if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR)
338
339 #if defined(__NetBSD__)
340 #include "allocator_src/sljitProtExecAllocatorNetBSD.c"
341 #else /* !__NetBSD__ */
342 #include "allocator_src/sljitProtExecAllocatorPosix.c"
343 #endif /* __NetBSD__ */
344
345 #elif (defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR)
346
347 #if defined(_WIN32)
348 #include "allocator_src/sljitWXExecAllocatorWindows.c"
349 #else /* !_WIN32 */
350 #include "allocator_src/sljitWXExecAllocatorPosix.c"
351 #endif /* _WIN32 */
352
353 #else /* !SLJIT_PROT_EXECUTABLAE_ALLOCATOR && !SLJIT_WX_EXECUTABLE_ALLOCATOR */
354
355 #if defined(_WIN32)
356 #include "allocator_src/sljitExecAllocatorWindows.c"
357 #elif defined(__APPLE__)
358 #include "allocator_src/sljitExecAllocatorApple.c"
359 #elif defined(__FreeBSD__)
360 #include "allocator_src/sljitExecAllocatorFreeBSD.c"
361 #else /* !_WIN32 && !__APPLE__ && !__FreeBSD__ */
362 #include "allocator_src/sljitExecAllocatorPosix.c"
363 #endif /* _WIN32 */
364
365 #endif /* SLJIT_PROT_EXECUTABLE_ALLOCATOR */
366
367 #else /* !SLJIT_EXECUTABLE_ALLOCATOR */
368
369 #ifndef SLJIT_UPDATE_WX_FLAGS
370 #define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec)
371 #endif /* SLJIT_UPDATE_WX_FLAGS */
372
373 #endif /* SLJIT_EXECUTABLE_ALLOCATOR */
374
375 #if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR)
376 #define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr) + (exec_offset))
377 #else /* !SLJIT_PROT_EXECUTABLE_ALLOCATOR */
378 #define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr))
379 #endif /* SLJIT_PROT_EXECUTABLE_ALLOCATOR */
380
381 /* Argument checking features. */
382
383 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
384
385 /* Returns with error when an invalid argument is passed. */
386
387 #define CHECK_ARGUMENT(x) \
388 do { \
389 if (SLJIT_UNLIKELY(!(x))) \
390 return 1; \
391 } while (0)
392
393 #define CHECK_RETURN_TYPE sljit_s32
394 #define CHECK_RETURN_OK return 0
395
396 #define CHECK(x) \
397 do { \
398 if (SLJIT_UNLIKELY(x)) { \
399 compiler->error = SLJIT_ERR_BAD_ARGUMENT; \
400 return SLJIT_ERR_BAD_ARGUMENT; \
401 } \
402 } while (0)
403
404 #define CHECK_PTR(x) \
405 do { \
406 if (SLJIT_UNLIKELY(x)) { \
407 compiler->error = SLJIT_ERR_BAD_ARGUMENT; \
408 return NULL; \
409 } \
410 } while (0)
411
412 #define CHECK_REG_INDEX(x) \
413 do { \
414 if (SLJIT_UNLIKELY(x)) { \
415 return -2; \
416 } \
417 } while (0)
418
419 #elif (defined SLJIT_DEBUG && SLJIT_DEBUG)
420
421 /* Assertion failure occures if an invalid argument is passed. */
422 #undef SLJIT_ARGUMENT_CHECKS
423 #define SLJIT_ARGUMENT_CHECKS 1
424
425 #define CHECK_ARGUMENT(x) SLJIT_ASSERT(x)
426 #define CHECK_RETURN_TYPE void
427 #define CHECK_RETURN_OK return
428 #define CHECK(x) x
429 #define CHECK_PTR(x) x
430 #define CHECK_REG_INDEX(x) x
431
432 #elif (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
433
434 /* Arguments are not checked. */
435 #define CHECK_RETURN_TYPE void
436 #define CHECK_RETURN_OK return
437 #define CHECK(x) x
438 #define CHECK_PTR(x) x
439 #define CHECK_REG_INDEX(x) x
440
441 #else /* !SLJIT_ARGUMENT_CHECKS && !SLJIT_DEBUG && !SLJIT_VERBOSE */
442
443 /* Arguments are not checked. */
444 #define CHECK(x)
445 #define CHECK_PTR(x)
446 #define CHECK_REG_INDEX(x)
447
448 #endif /* SLJIT_ARGUMENT_CHECKS */
449
450 /* --------------------------------------------------------------------- */
451 /* Public functions */
452 /* --------------------------------------------------------------------- */
453
454 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
455 #define SLJIT_NEEDS_COMPILER_INIT 1
456 static sljit_s32 compiler_initialized = 0;
457 /* A thread safe initialization. */
458 static void init_compiler(void);
459 #endif /* SLJIT_CONFIG_X86 */
460
461 SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data)
462 {
463 struct sljit_compiler *compiler = (struct sljit_compiler*)SLJIT_MALLOC(sizeof(struct sljit_compiler), allocator_data);
464 if (!compiler)
465 return NULL;
466 SLJIT_ZEROMEM(compiler, sizeof(struct sljit_compiler));
467
468 SLJIT_COMPILE_ASSERT(
469 sizeof(sljit_s8) == 1 && sizeof(sljit_u8) == 1
470 && sizeof(sljit_s16) == 2 && sizeof(sljit_u16) == 2
471 && sizeof(sljit_s32) == 4 && sizeof(sljit_u32) == 4
472 && (sizeof(sljit_up) == 4 || sizeof(sljit_up) == 8)
473 && sizeof(sljit_up) <= sizeof(sljit_sw)
474 && sizeof(sljit_up) == sizeof(sljit_sp)
475 && (sizeof(sljit_sw) == 4 || sizeof(sljit_sw) == 8)
476 && (sizeof(sljit_uw) == sizeof(sljit_sw)),
477 invalid_integer_types);
478 SLJIT_COMPILE_ASSERT(SLJIT_REWRITABLE_JUMP != SLJIT_32,
479 rewritable_jump_and_single_op_must_not_be_the_same);
480 SLJIT_COMPILE_ASSERT(!(SLJIT_EQUAL & 0x1) && !(SLJIT_LESS & 0x1) && !(SLJIT_F_EQUAL & 0x1) && !(SLJIT_JUMP & 0x1),
481 conditional_flags_must_be_even_numbers);
482
483 /* Only the non-zero members must be set. */
484 compiler->error = SLJIT_SUCCESS;
485
486 compiler->allocator_data = allocator_data;
487 compiler->buf = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, allocator_data);
488 compiler->abuf = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, allocator_data);
489
490 if (!compiler->buf || !compiler->abuf) {
491 if (compiler->buf)
492 SLJIT_FREE(compiler->buf, allocator_data);
493 if (compiler->abuf)
494 SLJIT_FREE(compiler->abuf, allocator_data);
495 SLJIT_FREE(compiler, allocator_data);
496 return NULL;
497 }
498
499 compiler->buf->next = NULL;
500 compiler->buf->used_size = 0;
501 compiler->abuf->next = NULL;
502 compiler->abuf->used_size = 0;
503
504 compiler->scratches = -1;
505 compiler->saveds = -1;
506 compiler->fscratches = -1;
507 compiler->fsaveds = -1;
508 #if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS) \
509 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
510 || (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
511 compiler->vscratches = -1;
512 compiler->vsaveds = -1;
513 #endif /* SLJIT_SEPARATE_VECTOR_REGISTERS || SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
514 compiler->local_size = -1;
515
516 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
517 compiler->args_size = -1;
518 #endif /* SLJIT_CONFIG_X86_32 */
519
520 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
521 compiler->cpool = (sljit_uw*)SLJIT_MALLOC(CPOOL_SIZE * sizeof(sljit_uw)
522 + CPOOL_SIZE * sizeof(sljit_u8), allocator_data);
523 if (!compiler->cpool) {
524 SLJIT_FREE(compiler->buf, allocator_data);
525 SLJIT_FREE(compiler->abuf, allocator_data);
526 SLJIT_FREE(compiler, allocator_data);
527 return NULL;
528 }
529 compiler->cpool_unique = (sljit_u8*)(compiler->cpool + CPOOL_SIZE);
530 compiler->cpool_diff = 0xffffffff;
531 #endif /* SLJIT_CONFIG_ARM_V6 */
532
533 #if (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
534 compiler->delay_slot = UNMOVABLE_INS;
535 #endif /* SLJIT_CONFIG_MIPS */
536
537 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
538 || (defined SLJIT_DEBUG && SLJIT_DEBUG)
539 SLJIT_ASSERT(compiler->last_flags == 0 && compiler->logical_local_size == 0);
540 compiler->last_return = -1;
541 #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_DEBUG */
542
543 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
544 || (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
545 #if !(defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
546 compiler->real_fscratches = -1;
547 compiler->real_fsaveds = -1;
548 #endif /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
549 SLJIT_ASSERT(compiler->skip_checks == 0);
550 #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
551
552 #if (defined SLJIT_NEEDS_COMPILER_INIT && SLJIT_NEEDS_COMPILER_INIT)
553 if (!compiler_initialized) {
554 init_compiler();
555 compiler_initialized = 1;
556 }
557 #endif /* SLJIT_NEEDS_COMPILER_INIT */
558
559 return compiler;
560 }
561
562 SLJIT_API_FUNC_ATTRIBUTE void sljit_free_compiler(struct sljit_compiler *compiler)
563 {
564 struct sljit_memory_fragment *buf;
565 struct sljit_memory_fragment *curr;
566 void *allocator_data = compiler->allocator_data;
567 SLJIT_UNUSED_ARG(allocator_data);
568
569 buf = compiler->buf;
570 while (buf) {
571 curr = buf;
572 buf = buf->next;
573 SLJIT_FREE(curr, allocator_data);
574 }
575
576 buf = compiler->abuf;
577 while (buf) {
578 curr = buf;
579 buf = buf->next;
580 SLJIT_FREE(curr, allocator_data);
581 }
582
583 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
584 SLJIT_FREE(compiler->cpool, allocator_data);
585 #endif /* SLJIT_CONFIG_ARM_V6 */
586 SLJIT_FREE(compiler, allocator_data);
587 }
588
589 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_compiler_memory_error(struct sljit_compiler *compiler)
590 {
591 if (compiler->error == SLJIT_SUCCESS)
592 compiler->error = SLJIT_ERR_ALLOC_FAILED;
593 }
594
595 SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data)
596 {
597 SLJIT_UNUSED_ARG(exec_allocator_data);
598
599 SLJIT_FREE_EXEC(SLJIT_CODE_TO_PTR(code), exec_allocator_data);
600 }
601
602 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_label(struct sljit_jump *jump, struct sljit_label* label)
603 {
604 if (SLJIT_LIKELY(!!jump) && SLJIT_LIKELY(!!label)) {
605 jump->flags &= (sljit_uw)~JUMP_ADDR;
606 jump->u.label = label;
607 }
608 }
609
610 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_target(struct sljit_jump *jump, sljit_uw target)
611 {
612 if (SLJIT_LIKELY(!!jump)) {
613 jump->flags |= JUMP_ADDR;
614 jump->u.target = target;
615 }
616 }
617
618 #define SLJIT_CURRENT_FLAGS_ALL \
619 (SLJIT_CURRENT_FLAGS_32 | SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB | SLJIT_CURRENT_FLAGS_COMPARE)
620
621 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_current_flags(struct sljit_compiler *compiler, sljit_s32 current_flags)
622 {
623 SLJIT_UNUSED_ARG(compiler);
624 SLJIT_UNUSED_ARG(current_flags);
625
626 #if (defined SLJIT_HAS_STATUS_FLAGS_STATE && SLJIT_HAS_STATUS_FLAGS_STATE)
627 compiler->status_flags_state = current_flags;
628 #endif /* SLJIT_HAS_STATUS_FLAGS_STATE */
629
630 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
631 compiler->last_flags = 0;
632 if ((current_flags & ~(VARIABLE_FLAG_MASK | SLJIT_SET_Z | SLJIT_CURRENT_FLAGS_ALL)) == 0) {
633 compiler->last_flags = GET_FLAG_TYPE(current_flags) | (current_flags & (SLJIT_32 | SLJIT_SET_Z));
634 }
635 #endif /* SLJIT_ARGUMENT_CHECKS */
636 }
637
638 /* --------------------------------------------------------------------- */
639 /* Private functions */
640 /* --------------------------------------------------------------------- */
641
642 static void* ensure_buf(struct sljit_compiler *compiler, sljit_uw size)
643 {
644 sljit_u8 *ret;
645 struct sljit_memory_fragment *new_frag;
646
647 SLJIT_ASSERT(size <= 256);
648 if (compiler->buf->used_size + size <= (BUF_SIZE - (sljit_uw)SLJIT_OFFSETOF(struct sljit_memory_fragment, memory))) {
649 ret = compiler->buf->memory + compiler->buf->used_size;
650 compiler->buf->used_size += size;
651 return ret;
652 }
653 new_frag = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, compiler->allocator_data);
654 PTR_FAIL_IF_NULL(new_frag);
655 new_frag->next = compiler->buf;
656 compiler->buf = new_frag;
657 new_frag->used_size = size;
658 return new_frag->memory;
659 }
660
661 static void* ensure_abuf(struct sljit_compiler *compiler, sljit_uw size)
662 {
663 sljit_u8 *ret;
664 struct sljit_memory_fragment *new_frag;
665
666 SLJIT_ASSERT(size <= 256);
667 if (compiler->abuf->used_size + size <= (ABUF_SIZE - (sljit_uw)SLJIT_OFFSETOF(struct sljit_memory_fragment, memory))) {
668 ret = compiler->abuf->memory + compiler->abuf->used_size;
669 compiler->abuf->used_size += size;
670 return ret;
671 }
672 new_frag = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, compiler->allocator_data);
673 PTR_FAIL_IF_NULL(new_frag);
674 new_frag->next = compiler->abuf;
675 compiler->abuf = new_frag;
676 new_frag->used_size = size;
677 return new_frag->memory;
678 }
679
680 SLJIT_API_FUNC_ATTRIBUTE void* sljit_alloc_memory(struct sljit_compiler *compiler, sljit_s32 size)
681 {
682 CHECK_ERROR_PTR();
683
684 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
685 if (size <= 0 || size > 128)
686 return NULL;
687 size = (size + 7) & ~7;
688 #else /* !SLJIT_64BIT_ARCHITECTURE */
689 if (size <= 0 || size > 64)
690 return NULL;
691 size = (size + 3) & ~3;
692 #endif /* SLJIT_64BIT_ARCHITECTURE */
693 return ensure_abuf(compiler, (sljit_uw)size);
694 }
695
696 static SLJIT_INLINE void reverse_buf(struct sljit_compiler *compiler)
697 {
698 struct sljit_memory_fragment *buf = compiler->buf;
699 struct sljit_memory_fragment *prev = NULL;
700 struct sljit_memory_fragment *tmp;
701
702 do {
703 tmp = buf->next;
704 buf->next = prev;
705 prev = buf;
706 buf = tmp;
707 } while (buf != NULL);
708
709 compiler->buf = prev;
710 }
711
712 static SLJIT_INLINE void* allocate_executable_memory(sljit_uw size, sljit_s32 options,
713 void *exec_allocator_data, sljit_sw *executable_offset)
714 {
715 void *code;
716 struct sljit_generate_code_buffer *buffer;
717
718 if (SLJIT_LIKELY(!(options & SLJIT_GENERATE_CODE_BUFFER))) {
719 code = SLJIT_MALLOC_EXEC(size, exec_allocator_data);
720 *executable_offset = SLJIT_EXEC_OFFSET(code);
721 return code;
722 }
723
724 buffer = (struct sljit_generate_code_buffer*)exec_allocator_data;
725
726 if (size <= buffer->size) {
727 *executable_offset = buffer->executable_offset;
728 return buffer->buffer;
729 }
730
731 return NULL;
732 }
733
734 #define SLJIT_MAX_ADDRESS ~(sljit_uw)0
735
736 #define SLJIT_GET_NEXT_SIZE(ptr) (ptr != NULL) ? ((ptr)->size) : SLJIT_MAX_ADDRESS
737 #define SLJIT_GET_NEXT_ADDRESS(ptr) (ptr != NULL) ? ((ptr)->addr) : SLJIT_MAX_ADDRESS
738
739 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
740
741 #define SLJIT_NEXT_DEFINE_TYPES \
742 sljit_uw next_label_size; \
743 sljit_uw next_jump_addr; \
744 sljit_uw next_const_addr; \
745 sljit_uw next_min_addr
746
747 #define SLJIT_NEXT_INIT_TYPES() \
748 next_label_size = SLJIT_GET_NEXT_SIZE(label); \
749 next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump); \
750 next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
751
752 #define SLJIT_GET_NEXT_MIN() \
753 next_min_addr = sljit_get_next_min(next_label_size, next_jump_addr, next_const_addr);
754
755 static SLJIT_INLINE sljit_uw sljit_get_next_min(sljit_uw next_label_size,
756 sljit_uw next_jump_addr, sljit_uw next_const_addr)
757 {
758 sljit_uw result = next_jump_addr;
759
760 SLJIT_ASSERT(result == SLJIT_MAX_ADDRESS || result != next_const_addr);
761
762 if (next_const_addr < result)
763 result = next_const_addr;
764
765 if (next_label_size < result)
766 result = next_label_size;
767
768 return result;
769 }
770
771 #endif /* !SLJIT_CONFIG_X86 */
772
773 #if !(defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
774
775 static void update_float_register_count(struct sljit_compiler *compiler, sljit_s32 scratches, sljit_s32 saveds)
776 {
777 sljit_s32 vscratches = ENTER_GET_VECTOR_REGS(scratches);
778 sljit_s32 vsaveds = ENTER_GET_VECTOR_REGS(saveds);
779
780 if (compiler->fscratches < vscratches)
781 compiler->fscratches = vscratches;
782
783 if (compiler->fsaveds < vsaveds)
784 compiler->fsaveds = vsaveds;
785
786 if (compiler->fsaveds + compiler->fscratches > SLJIT_NUMBER_OF_FLOAT_REGISTERS)
787 compiler->fscratches = SLJIT_NUMBER_OF_FLOAT_REGISTERS - compiler->fsaveds;
788 }
789
790 #endif /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
791
792 static SLJIT_INLINE void set_emit_enter(struct sljit_compiler *compiler,
793 sljit_s32 options, sljit_s32 args,
794 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
795 {
796 SLJIT_UNUSED_ARG(args);
797 SLJIT_UNUSED_ARG(local_size);
798
799 compiler->options = options;
800 compiler->scratches = ENTER_GET_REGS(scratches);
801 compiler->saveds = ENTER_GET_REGS(saveds);
802 /* These members may be copied to real_* members below. */
803 compiler->fscratches = ENTER_GET_FLOAT_REGS(scratches);
804 compiler->fsaveds = ENTER_GET_FLOAT_REGS(saveds);
805 #if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
806 compiler->vscratches = ENTER_GET_VECTOR_REGS(scratches);
807 compiler->vsaveds = ENTER_GET_VECTOR_REGS(saveds);
808 #else /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
809 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
810 || (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
811 compiler->real_fscratches = compiler->fscratches;
812 compiler->real_fsaveds = compiler->fsaveds;
813 compiler->vscratches = ENTER_GET_VECTOR_REGS(scratches);
814 compiler->vsaveds = ENTER_GET_VECTOR_REGS(saveds);
815 #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
816 update_float_register_count(compiler, scratches, saveds);
817 #endif /* SLJIT_SEPARATE_VECTOR_REGISTERS */
818 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
819 compiler->last_return = args & SLJIT_ARG_MASK;
820 compiler->logical_local_size = local_size;
821 #endif /* SLJIT_ARGUMENT_CHECKS */
822 }
823
824 static SLJIT_INLINE void set_label(struct sljit_label *label, struct sljit_compiler *compiler)
825 {
826 label->next = NULL;
827 label->u.index = compiler->label_count++;
828 label->size = compiler->size;
829 if (compiler->last_label != NULL)
830 compiler->last_label->next = label;
831 else
832 compiler->labels = label;
833 compiler->last_label = label;
834 }
835
836 static SLJIT_INLINE void set_jump(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_u32 flags)
837 {
838 jump->next = NULL;
839 jump->flags = flags;
840 jump->u.label = NULL;
841 if (compiler->last_jump != NULL)
842 compiler->last_jump->next = jump;
843 else
844 compiler->jumps = jump;
845 compiler->last_jump = jump;
846 }
847
848 static SLJIT_INLINE void set_mov_addr(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_uw offset)
849 {
850 jump->next = NULL;
851 jump->addr = compiler->size - offset;
852 jump->flags = JUMP_MOV_ADDR;
853 jump->u.label = NULL;
854 if (compiler->last_jump != NULL)
855 compiler->last_jump->next = jump;
856 else
857 compiler->jumps = jump;
858 compiler->last_jump = jump;
859 }
860
861 static SLJIT_INLINE void set_const(struct sljit_const *const_, struct sljit_compiler *compiler)
862 {
863 const_->next = NULL;
864 const_->addr = compiler->size;
865 if (compiler->last_const != NULL)
866 compiler->last_const->next = const_;
867 else
868 compiler->consts = const_;
869 compiler->last_const = const_;
870 }
871
872 #define ADDRESSING_DEPENDS_ON(exp, reg) \
873 (((exp) & SLJIT_MEM) && (((exp) & REG_MASK) == reg || OFFS_REG(exp) == reg))
874
875 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
876
877 static sljit_s32 function_check_arguments(sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds, sljit_s32 fscratches)
878 {
879 sljit_s32 word_arg_count, scratch_arg_end, saved_arg_count, float_arg_count, curr_type;
880
881 curr_type = (arg_types & SLJIT_ARG_FULL_MASK);
882
883 if (curr_type >= SLJIT_ARG_TYPE_F64) {
884 if (curr_type > SLJIT_ARG_TYPE_F32 || fscratches == 0)
885 return 0;
886 } else if (curr_type >= SLJIT_ARG_TYPE_W) {
887 if (scratches == 0)
888 return 0;
889 }
890
891 arg_types >>= SLJIT_ARG_SHIFT;
892
893 word_arg_count = 0;
894 scratch_arg_end = 0;
895 saved_arg_count = 0;
896 float_arg_count = 0;
897 while (arg_types != 0) {
898 if (word_arg_count + float_arg_count >= 4)
899 return 0;
900
901 curr_type = (arg_types & SLJIT_ARG_MASK);
902
903 if (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) {
904 if (saveds == -1 || curr_type < SLJIT_ARG_TYPE_W || curr_type > SLJIT_ARG_TYPE_P)
905 return 0;
906
907 word_arg_count++;
908 scratch_arg_end = word_arg_count;
909 } else {
910 if (curr_type < SLJIT_ARG_TYPE_W || curr_type > SLJIT_ARG_TYPE_F32)
911 return 0;
912
913 if (curr_type < SLJIT_ARG_TYPE_F64) {
914 word_arg_count++;
915 saved_arg_count++;
916 } else
917 float_arg_count++;
918 }
919
920 arg_types >>= SLJIT_ARG_SHIFT;
921 }
922
923 if (saveds == -1)
924 return (word_arg_count <= scratches && float_arg_count <= fscratches);
925
926 return (saved_arg_count <= saveds && scratch_arg_end <= scratches && float_arg_count <= fscratches);
927 }
928
929 #define FUNCTION_CHECK_IS_REG(r) \
930 (((r) >= SLJIT_R0 && (r) < (SLJIT_R0 + compiler->scratches)) \
931 || ((r) > (SLJIT_S0 - compiler->saveds) && (r) <= SLJIT_S0) \
932 || ((r) >= SLJIT_TMP_REGISTER_BASE && (r) < (SLJIT_TMP_REGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_REGISTERS)))
933
934 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
935 #define CHECK_IF_VIRTUAL_REGISTER(p) ((p) <= SLJIT_S3 && (p) >= SLJIT_S8)
936 #else /* !SLJIT_CONFIG_X86_32 */
937 #define CHECK_IF_VIRTUAL_REGISTER(p) 0
938 #endif /* SLJIT_CONFIG_X86_32 */
939
940 static sljit_s32 function_check_src_mem(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
941 {
942 if (compiler->scratches == -1)
943 return 0;
944
945 if (!(p & SLJIT_MEM))
946 return 0;
947
948 if (p == SLJIT_MEM1(SLJIT_SP))
949 return (i >= 0 && i < compiler->logical_local_size);
950
951 if (!(!(p & REG_MASK) || FUNCTION_CHECK_IS_REG(p & REG_MASK)))
952 return 0;
953
954 if (CHECK_IF_VIRTUAL_REGISTER(p & REG_MASK))
955 return 0;
956
957 if (p & OFFS_REG_MASK) {
958 if (!(p & REG_MASK))
959 return 0;
960
961 if (!(FUNCTION_CHECK_IS_REG(OFFS_REG(p))))
962 return 0;
963
964 if (CHECK_IF_VIRTUAL_REGISTER(OFFS_REG(p)))
965 return 0;
966
967 if ((i & ~0x3) != 0)
968 return 0;
969 }
970
971 return (p & ~(SLJIT_MEM | REG_MASK | OFFS_REG_MASK)) == 0;
972 }
973
974 #define FUNCTION_CHECK_SRC_MEM(p, i) \
975 CHECK_ARGUMENT(function_check_src_mem(compiler, p, i));
976
977 static sljit_s32 function_check_src(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
978 {
979 if (compiler->scratches == -1)
980 return 0;
981
982 if (FUNCTION_CHECK_IS_REG(p))
983 return (i == 0);
984
985 if (p == SLJIT_IMM)
986 return 1;
987
988 return function_check_src_mem(compiler, p, i);
989 }
990
991 #define FUNCTION_CHECK_SRC(p, i) \
992 CHECK_ARGUMENT(function_check_src(compiler, p, i));
993
994 static sljit_s32 function_check_dst(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
995 {
996 if (compiler->scratches == -1)
997 return 0;
998
999 if (FUNCTION_CHECK_IS_REG(p))
1000 return (i == 0);
1001
1002 return function_check_src_mem(compiler, p, i);
1003 }
1004
1005 #define FUNCTION_CHECK_DST(p, i) \
1006 CHECK_ARGUMENT(function_check_dst(compiler, p, i));
1007
1008 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
1009 || (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
1010
1011 #define FUNCTION_CHECK_IS_FREG(fr, is_32) \
1012 function_check_is_freg(compiler, (fr), (is_32))
1013
1014 static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr, sljit_s32 is_32);
1015
1016 #define FUNCTION_FCHECK(p, i, is_32) \
1017 CHECK_ARGUMENT(function_fcheck(compiler, (p), (i), (is_32)));
1018
1019 static sljit_s32 function_fcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i, sljit_s32 is_32)
1020 {
1021 if (compiler->scratches == -1)
1022 return 0;
1023
1024 if (FUNCTION_CHECK_IS_FREG(p, is_32))
1025 return (i == 0);
1026
1027 return function_check_src_mem(compiler, p, i);
1028 }
1029
1030 static sljit_s32 function_check_is_vreg(struct sljit_compiler *compiler, sljit_s32 vr, sljit_s32 type);
1031
1032 #define FUNCTION_CHECK_IS_VREG(vr, type) \
1033 function_check_is_vreg(compiler, (vr), (type))
1034
1035 #define FUNCTION_VCHECK(p, i, type) \
1036 CHECK_ARGUMENT(function_vcheck(compiler, (p), (i), (type)))
1037
1038 static sljit_s32 function_vcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i, sljit_s32 type)
1039 {
1040 if (compiler->scratches == -1)
1041 return 0;
1042
1043 if (FUNCTION_CHECK_IS_VREG(p, type))
1044 return (i == 0);
1045
1046 return function_check_src_mem(compiler, p, i);
1047 }
1048
1049 #else /* !SLJIT_CONFIG_ARM_32 && !SLJIT_CONFIG_MIPS_32 */
1050
1051 #define FUNCTION_CHECK_IS_FREG(fr, is_32) \
1052 function_check_is_freg(compiler, (fr))
1053
1054 static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr)
1055 {
1056 sljit_s32 fscratches, fsaveds;
1057
1058 if (compiler->scratches == -1)
1059 return 0;
1060
1061 #if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
1062 fscratches = compiler->fscratches;
1063 fsaveds = compiler->fsaveds;
1064 #else /* SLJIT_SEPARATE_VECTOR_REGISTERS */
1065 fscratches = compiler->real_fscratches;
1066 fsaveds = compiler->real_fsaveds;
1067 #endif /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
1068
1069 return (fr >= SLJIT_FR0 && fr < (SLJIT_FR0 + fscratches))
1070 || (fr > (SLJIT_FS0 - fsaveds) && fr <= SLJIT_FS0)
1071 || (fr >= SLJIT_TMP_FREGISTER_BASE && fr < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS));
1072 }
1073
1074 #define FUNCTION_FCHECK(p, i, is_32) \
1075 CHECK_ARGUMENT(function_fcheck(compiler, (p), (i)));
1076
1077 static sljit_s32 function_fcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1078 {
1079 if (compiler->scratches == -1)
1080 return 0;
1081
1082 if (function_check_is_freg(compiler, p))
1083 return (i == 0);
1084
1085 return function_check_src_mem(compiler, p, i);
1086 }
1087
1088 #define FUNCTION_CHECK_IS_VREG(vr, type) \
1089 function_check_is_vreg(compiler, (vr))
1090
1091 static sljit_s32 function_check_is_vreg(struct sljit_compiler *compiler, sljit_s32 vr)
1092 {
1093 if (compiler->scratches == -1)
1094 return 0;
1095
1096 return (vr >= SLJIT_VR0 && vr < (SLJIT_VR0 + compiler->vscratches))
1097 || (vr > (SLJIT_VS0 - compiler->vsaveds) && vr <= SLJIT_VS0)
1098 || (vr >= SLJIT_TMP_VREGISTER_BASE && vr < (SLJIT_TMP_VREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_VECTOR_REGISTERS));
1099 }
1100
1101 #define FUNCTION_VCHECK(p, i, type) \
1102 CHECK_ARGUMENT(function_vcheck(compiler, (p), (i)))
1103
1104 static sljit_s32 function_vcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1105 {
1106 if (compiler->scratches == -1)
1107 return 0;
1108
1109 if (function_check_is_vreg(compiler, p))
1110 return (i == 0);
1111
1112 return function_check_src_mem(compiler, p, i);
1113 }
1114
1115 #endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1116
1117 #endif /* SLJIT_ARGUMENT_CHECKS */
1118
1119 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1120
1121 SLJIT_API_FUNC_ATTRIBUTE void sljit_compiler_verbose(struct sljit_compiler *compiler, FILE* verbose)
1122 {
1123 compiler->verbose = verbose;
1124 }
1125
1126 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
1127 #ifdef _WIN64
1128 #ifdef __GNUC__
1129 # define SLJIT_PRINT_D "ll"
1130 #else /* !__GNUC__ */
1131 # define SLJIT_PRINT_D "I64"
1132 #endif /* __GNUC__ */
1133 #else /* !_WIN64 */
1134 # define SLJIT_PRINT_D "l"
1135 #endif /* _WIN64 */
1136 #else /* !SLJIT_64BIT_ARCHITECTURE */
1137 # define SLJIT_PRINT_D ""
1138 #endif /* SLJIT_64BIT_ARCHITECTURE */
1139
1140 static void sljit_verbose_reg(struct sljit_compiler *compiler, sljit_s32 r)
1141 {
1142 if (r < (SLJIT_R0 + compiler->scratches))
1143 fprintf(compiler->verbose, "r%d", r - SLJIT_R0);
1144 else if (r < SLJIT_SP)
1145 fprintf(compiler->verbose, "s%d", SLJIT_NUMBER_OF_REGISTERS - r);
1146 else if (r == SLJIT_SP)
1147 fprintf(compiler->verbose, "sp");
1148 else
1149 fprintf(compiler->verbose, "t%d", r - SLJIT_TMP_REGISTER_BASE);
1150 }
1151
1152 static void sljit_verbose_freg(struct sljit_compiler *compiler, sljit_s32 r)
1153 {
1154 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
1155 || (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
1156 if (r >= SLJIT_F64_SECOND(SLJIT_FR0)) {
1157 fprintf(compiler->verbose, "^");
1158 r -= SLJIT_F64_SECOND(0);
1159 }
1160 #endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1161
1162 if (r < (SLJIT_FR0 + compiler->fscratches))
1163 fprintf(compiler->verbose, "fr%d", r - SLJIT_FR0);
1164 else if (r < SLJIT_TMP_FREGISTER_BASE)
1165 fprintf(compiler->verbose, "fs%d", SLJIT_NUMBER_OF_FLOAT_REGISTERS - r);
1166 else
1167 fprintf(compiler->verbose, "ft%d", r - SLJIT_TMP_FREGISTER_BASE);
1168 }
1169
1170 static void sljit_verbose_vreg(struct sljit_compiler *compiler, sljit_s32 r)
1171 {
1172 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
1173 || (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
1174 if (r >= SLJIT_F64_SECOND(SLJIT_VR0)) {
1175 fprintf(compiler->verbose, "^");
1176 r -= SLJIT_F64_SECOND(0);
1177 }
1178 #endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1179
1180 if (r < (SLJIT_VR0 + compiler->vscratches))
1181 fprintf(compiler->verbose, "vr%d", r - SLJIT_VR0);
1182 else if (r < SLJIT_TMP_VREGISTER_BASE)
1183 fprintf(compiler->verbose, "vs%d", SLJIT_NUMBER_OF_VECTOR_REGISTERS - r);
1184 else
1185 fprintf(compiler->verbose, "vt%d", r - SLJIT_TMP_VREGISTER_BASE);
1186 }
1187
1188 static void sljit_verbose_mem(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1189 {
1190 if (!(p & REG_MASK)) {
1191 fprintf(compiler->verbose, "[%" SLJIT_PRINT_D "d]", i);
1192 return;
1193 }
1194
1195 fputc('[', compiler->verbose);
1196 sljit_verbose_reg(compiler, (p) & REG_MASK);
1197 if (p & OFFS_REG_MASK) {
1198 fprintf(compiler->verbose, " + ");
1199 sljit_verbose_reg(compiler, OFFS_REG(p));
1200 if (i)
1201 fprintf(compiler->verbose, " * %d", 1 << (i));
1202 } else if (i)
1203 fprintf(compiler->verbose, " + %" SLJIT_PRINT_D "d", (i));
1204 fputc(']', compiler->verbose);
1205 }
1206
1207 static void sljit_verbose_param(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1208 {
1209 if (p == SLJIT_IMM)
1210 fprintf(compiler->verbose, "#%" SLJIT_PRINT_D "d", i);
1211 else if (p & SLJIT_MEM)
1212 sljit_verbose_mem(compiler, p, i);
1213 else
1214 sljit_verbose_reg(compiler, p);
1215 }
1216
1217 static void sljit_verbose_fparam(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1218 {
1219 if (p & SLJIT_MEM)
1220 sljit_verbose_mem(compiler, p, i);
1221 else
1222 sljit_verbose_freg(compiler, p);
1223 }
1224
1225 static void sljit_verbose_vparam(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1226 {
1227 if (p & SLJIT_MEM)
1228 sljit_verbose_mem(compiler, p, i);
1229 else
1230 sljit_verbose_vreg(compiler, p);
1231 }
1232
1233 static const char* op0_names[] = {
1234 "breakpoint", "nop", "lmul.uw", "lmul.sw",
1235 "divmod.u", "divmod.s", "div.u", "div.s",
1236 "memory_barrier", "endbr", "skip_frames_before_return"
1237 };
1238
1239 static const char* op1_names[] = {
1240 "mov", "mov", "mov", "mov",
1241 "mov", "mov", "mov", "mov",
1242 "mov", "clz", "ctz", "rev",
1243 "rev", "rev", "rev", "rev"
1244 };
1245
1246 static const char* op1_types[] = {
1247 "", ".u8", ".s8", ".u16",
1248 ".s16", ".u32", ".s32", "32",
1249 ".p", "", "", "",
1250 ".u16", ".s16", ".u32", ".s32"
1251 };
1252
1253 static const char* op2_names[] = {
1254 "add", "addc", "sub", "subc",
1255 "mul", "and", "or", "xor",
1256 "shl", "mshl", "lshr", "mlshr",
1257 "ashr", "mashr", "rotl", "rotr"
1258 };
1259
1260 static const char* op2r_names[] = {
1261 "muladd"
1262 };
1263
1264 static const char* op_src_dst_names[] = {
1265 "fast_return", "skip_frames_before_fast_return",
1266 "prefetch_l1", "prefetch_l2",
1267 "prefetch_l3", "prefetch_once",
1268 "fast_enter", "get_return_address"
1269 };
1270
1271 static const char* fop1_names[] = {
1272 "mov", "conv", "conv", "conv",
1273 "conv", "conv", "conv", "conv",
1274 "cmp", "neg", "abs",
1275 };
1276
1277 static const char* fop1_conv_types[] = {
1278 "sw", "s32", "sw", "s32",
1279 "uw", "u32"
1280 };
1281
1282 static const char* fop2_names[] = {
1283 "add", "sub", "mul", "div"
1284 };
1285
1286 static const char* fop2r_names[] = {
1287 "copysign"
1288 };
1289
1290 static const char* simd_op2_names[] = {
1291 "and", "or", "xor", "shuffle"
1292 };
1293
1294 static const char* jump_names[] = {
1295 "equal", "not_equal",
1296 "less", "greater_equal",
1297 "greater", "less_equal",
1298 "sig_less", "sig_greater_equal",
1299 "sig_greater", "sig_less_equal",
1300 "overflow", "not_overflow",
1301 "carry", "not_carry",
1302 "atomic_stored", "atomic_not_stored",
1303 "f_equal", "f_not_equal",
1304 "f_less", "f_greater_equal",
1305 "f_greater", "f_less_equal",
1306 "unordered", "ordered",
1307 "ordered_equal", "unordered_or_not_equal",
1308 "ordered_less", "unordered_or_greater_equal",
1309 "ordered_greater", "unordered_or_less_equal",
1310 "unordered_or_equal", "ordered_not_equal",
1311 "unordered_or_less", "ordered_greater_equal",
1312 "unordered_or_greater", "ordered_less_equal",
1313 "jump", "fast_call",
1314 "call", "call_reg_arg"
1315 };
1316
1317 static const char* call_arg_names[] = {
1318 "void", "w", "32", "p", "f64", "f32"
1319 };
1320
1321 #endif /* SLJIT_VERBOSE */
1322
1323 /* --------------------------------------------------------------------- */
1324 /* Arch dependent */
1325 /* --------------------------------------------------------------------- */
1326
1327 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
1328 || (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1329
1330 #define SLJIT_SKIP_CHECKS(compiler) (compiler)->skip_checks = 1
1331 #define SLJIT_CHECK_OPCODE(op, flags) ((op) & ~(SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK | (flags)))
1332
1333 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_generate_code(struct sljit_compiler *compiler)
1334 {
1335 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1336 struct sljit_jump *jump;
1337 #endif /* SLJIT_ARGUMENT_CHECKS */
1338
1339 SLJIT_UNUSED_ARG(compiler);
1340
1341 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1342 CHECK_ARGUMENT(compiler->size > 0);
1343 jump = compiler->jumps;
1344 while (jump) {
1345 /* All jumps have target. */
1346 CHECK_ARGUMENT((jump->flags & JUMP_ADDR) || jump->u.label != NULL);
1347 jump = jump->next;
1348 }
1349 #endif /* SLJIT_ARGUMENT_CHECKS */
1350 CHECK_RETURN_OK;
1351 }
1352
1353 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1354 #define SLJIT_ENTER_CPU_SPECIFIC_OPTIONS (SLJIT_ENTER_USE_VEX)
1355 #else /* !SLJIT_CONFIG_X86 */
1356 #define SLJIT_ENTER_CPU_SPECIFIC_OPTIONS (0)
1357 #endif /* !SLJIT_CONFIG_X86 */
1358
1359 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_enter(struct sljit_compiler *compiler,
1360 sljit_s32 options, sljit_s32 arg_types,
1361 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1362 {
1363 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1364 sljit_s32 real_scratches = ENTER_GET_REGS(scratches);
1365 sljit_s32 real_saveds = ENTER_GET_REGS(saveds);
1366 sljit_s32 real_fscratches = ENTER_GET_FLOAT_REGS(scratches);
1367 sljit_s32 real_fsaveds = ENTER_GET_FLOAT_REGS(saveds);
1368 sljit_s32 real_vscratches = ENTER_GET_VECTOR_REGS(scratches);
1369 sljit_s32 real_vsaveds = ENTER_GET_VECTOR_REGS(saveds);
1370 #endif /* SLJIT_ARGUMENT_CHECKS */
1371 SLJIT_UNUSED_ARG(compiler);
1372
1373 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1374 if (options & SLJIT_ENTER_REG_ARG) {
1375 CHECK_ARGUMENT(!(options & ~(0x3 | SLJIT_ENTER_REG_ARG | SLJIT_ENTER_CPU_SPECIFIC_OPTIONS)));
1376 } else {
1377 CHECK_ARGUMENT((options & ~SLJIT_ENTER_CPU_SPECIFIC_OPTIONS) == 0);
1378 }
1379 CHECK_ARGUMENT(SLJIT_KEPT_SAVEDS_COUNT(options) <= 3 && SLJIT_KEPT_SAVEDS_COUNT(options) <= saveds);
1380 CHECK_ARGUMENT((scratches & ~0xffffff) == 0 && (saveds & ~0xffffff) == 0);
1381 CHECK_ARGUMENT(real_scratches >= 0 && real_scratches <= SLJIT_NUMBER_OF_REGISTERS);
1382 CHECK_ARGUMENT(real_saveds >= 0 && real_saveds <= SLJIT_NUMBER_OF_SAVED_REGISTERS);
1383 CHECK_ARGUMENT(real_scratches + real_saveds <= SLJIT_NUMBER_OF_REGISTERS);
1384 CHECK_ARGUMENT(real_fscratches >= 0 && real_fscratches <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1385 CHECK_ARGUMENT(real_fsaveds >= 0 && real_fsaveds <= SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS);
1386 CHECK_ARGUMENT(real_fscratches + real_fsaveds <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1387 CHECK_ARGUMENT(real_vscratches >= 0 && real_vscratches <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1388 CHECK_ARGUMENT(real_vsaveds >= 0 && real_vsaveds <= SLJIT_NUMBER_OF_SAVED_VECTOR_REGISTERS);
1389 CHECK_ARGUMENT(real_vscratches + real_vsaveds <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1390 CHECK_ARGUMENT(local_size >= 0 && local_size <= SLJIT_MAX_LOCAL_SIZE);
1391 CHECK_ARGUMENT((arg_types & SLJIT_ARG_FULL_MASK) <= SLJIT_ARG_TYPE_F32);
1392 CHECK_ARGUMENT(function_check_arguments(arg_types, real_scratches,
1393 (options & SLJIT_ENTER_REG_ARG) ? 0 : real_saveds, real_fscratches));
1394
1395 compiler->last_flags = 0;
1396 #endif /* SLJIT_ARGUMENT_CHECKS */
1397 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1398 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1399 fprintf(compiler->verbose, " enter ret[%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
1400
1401 arg_types >>= SLJIT_ARG_SHIFT;
1402 if (arg_types) {
1403 fprintf(compiler->verbose, "], args[");
1404 do {
1405 fprintf(compiler->verbose, "%s%s", call_arg_names[arg_types & SLJIT_ARG_MASK],
1406 (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) ? "_r" : "");
1407 arg_types >>= SLJIT_ARG_SHIFT;
1408 if (arg_types)
1409 fprintf(compiler->verbose, ",");
1410 } while (arg_types);
1411 }
1412
1413 fprintf(compiler->verbose, "],");
1414
1415 if (options & SLJIT_ENTER_REG_ARG) {
1416 if (SLJIT_KEPT_SAVEDS_COUNT(options) > 0)
1417 fprintf(compiler->verbose, " opt:reg_arg(%d),", SLJIT_KEPT_SAVEDS_COUNT(options));
1418 else
1419 fprintf(compiler->verbose, " opt:reg_arg,");
1420 }
1421
1422 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1423 if (options & SLJIT_ENTER_USE_VEX) {
1424 fprintf(compiler->verbose, " opt:use_vex,");
1425 }
1426 #endif /* !SLJIT_CONFIG_X86 */
1427
1428 fprintf(compiler->verbose, " scratches:%d, saveds:%d, fscratches:%d, fsaveds:%d, vscratches:%d, vsaveds:%d, local_size:%d\n",
1429 ENTER_GET_REGS(scratches), ENTER_GET_REGS(saveds), ENTER_GET_FLOAT_REGS(scratches), ENTER_GET_FLOAT_REGS(saveds),
1430 ENTER_GET_VECTOR_REGS(scratches), ENTER_GET_VECTOR_REGS(saveds), local_size);
1431 }
1432 #endif /* SLJIT_VERBOSE */
1433 CHECK_RETURN_OK;
1434 }
1435
1436 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_set_context(struct sljit_compiler *compiler,
1437 sljit_s32 options, sljit_s32 arg_types,
1438 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1439 {
1440 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1441 sljit_s32 real_scratches = ENTER_GET_REGS(scratches);
1442 sljit_s32 real_saveds = ENTER_GET_REGS(saveds);
1443 sljit_s32 real_fscratches = ENTER_GET_FLOAT_REGS(scratches);
1444 sljit_s32 real_fsaveds = ENTER_GET_FLOAT_REGS(saveds);
1445 sljit_s32 real_vscratches = ENTER_GET_VECTOR_REGS(scratches);
1446 sljit_s32 real_vsaveds = ENTER_GET_VECTOR_REGS(saveds);
1447 #endif /* SLJIT_ARGUMENT_CHECKS */
1448 SLJIT_UNUSED_ARG(compiler);
1449
1450 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1451 if (options & SLJIT_ENTER_REG_ARG) {
1452 CHECK_ARGUMENT(!(options & ~(0x3 | SLJIT_ENTER_REG_ARG | SLJIT_ENTER_CPU_SPECIFIC_OPTIONS)));
1453 } else {
1454 CHECK_ARGUMENT((options & ~SLJIT_ENTER_CPU_SPECIFIC_OPTIONS) == 0);
1455 }
1456 CHECK_ARGUMENT(SLJIT_KEPT_SAVEDS_COUNT(options) <= 3 && SLJIT_KEPT_SAVEDS_COUNT(options) <= saveds);
1457 CHECK_ARGUMENT((scratches & ~0xffffff) == 0 && (saveds & ~0xffffff) == 0);
1458 CHECK_ARGUMENT(real_scratches >= 0 && real_scratches <= SLJIT_NUMBER_OF_REGISTERS);
1459 CHECK_ARGUMENT(real_saveds >= 0 && real_saveds <= SLJIT_NUMBER_OF_SAVED_REGISTERS);
1460 CHECK_ARGUMENT(real_scratches + real_saveds <= SLJIT_NUMBER_OF_REGISTERS);
1461 CHECK_ARGUMENT(real_fscratches >= 0 && real_fscratches <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1462 CHECK_ARGUMENT(real_fsaveds >= 0 && real_fsaveds <= SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS);
1463 CHECK_ARGUMENT(real_fscratches + real_fsaveds <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1464 CHECK_ARGUMENT(real_vscratches >= 0 && real_vscratches <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1465 CHECK_ARGUMENT(real_vsaveds >= 0 && real_vsaveds <= SLJIT_NUMBER_OF_SAVED_VECTOR_REGISTERS);
1466 CHECK_ARGUMENT(real_vscratches + real_vsaveds <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1467 CHECK_ARGUMENT(local_size >= 0 && local_size <= SLJIT_MAX_LOCAL_SIZE);
1468 CHECK_ARGUMENT((arg_types & SLJIT_ARG_FULL_MASK) < SLJIT_ARG_TYPE_F64);
1469 CHECK_ARGUMENT(function_check_arguments(arg_types, real_scratches,
1470 (options & SLJIT_ENTER_REG_ARG) ? 0 : real_saveds, real_fscratches));
1471
1472 compiler->last_flags = 0;
1473 #endif /* SLJIT_ARGUMENT_CHECKS */
1474 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1475 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1476 fprintf(compiler->verbose, " set_context ret[%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
1477
1478 arg_types >>= SLJIT_ARG_SHIFT;
1479 if (arg_types) {
1480 fprintf(compiler->verbose, "], args[");
1481 do {
1482 fprintf(compiler->verbose, "%s%s", call_arg_names[arg_types & SLJIT_ARG_MASK],
1483 (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) ? "_r" : "");
1484 arg_types >>= SLJIT_ARG_SHIFT;
1485 if (arg_types)
1486 fprintf(compiler->verbose, ",");
1487 } while (arg_types);
1488 }
1489
1490 fprintf(compiler->verbose, "],");
1491
1492 if (options & SLJIT_ENTER_REG_ARG) {
1493 if (SLJIT_KEPT_SAVEDS_COUNT(options) > 0)
1494 fprintf(compiler->verbose, " opt:reg_arg(%d),", SLJIT_KEPT_SAVEDS_COUNT(options));
1495 else
1496 fprintf(compiler->verbose, " opt:reg_arg,");
1497 }
1498
1499 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1500 if (options & SLJIT_ENTER_USE_VEX) {
1501 fprintf(compiler->verbose, " opt:use_vex,");
1502 }
1503 #endif /* !SLJIT_CONFIG_X86 */
1504
1505 fprintf(compiler->verbose, " scratches:%d, saveds:%d, fscratches:%d, fsaveds:%d, vscratches:%d, vsaveds:%d, local_size:%d\n",
1506 ENTER_GET_REGS(scratches), ENTER_GET_REGS(saveds), ENTER_GET_FLOAT_REGS(scratches), ENTER_GET_FLOAT_REGS(saveds),
1507 ENTER_GET_VECTOR_REGS(scratches), ENTER_GET_VECTOR_REGS(saveds), local_size);
1508 }
1509 #endif /* SLJIT_VERBOSE */
1510 CHECK_RETURN_OK;
1511 }
1512
1513 #undef SLJIT_ENTER_CPU_SPECIFIC_OPTIONS
1514
1515 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return_void(struct sljit_compiler *compiler)
1516 {
1517 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1518 compiler->skip_checks = 0;
1519 CHECK_RETURN_OK;
1520 }
1521
1522 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1523 CHECK_ARGUMENT(compiler->last_return == SLJIT_ARG_TYPE_RET_VOID);
1524 #endif /* SLJIT_ARGUMENT_CHECKS */
1525
1526 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1527 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1528 fprintf(compiler->verbose, " return_void\n");
1529 }
1530 #endif /* SLJIT_VERBOSE */
1531 CHECK_RETURN_OK;
1532 }
1533
1534 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
1535 {
1536 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1537 CHECK_ARGUMENT(compiler->scratches >= 0);
1538
1539 switch (compiler->last_return) {
1540 case SLJIT_ARG_TYPE_W:
1541 CHECK_ARGUMENT(op >= SLJIT_MOV && op <= SLJIT_MOV_S32);
1542 break;
1543 case SLJIT_ARG_TYPE_32:
1544 CHECK_ARGUMENT(op == SLJIT_MOV32 || (op >= SLJIT_MOV32_U8 && op <= SLJIT_MOV32_S16));
1545 break;
1546 case SLJIT_ARG_TYPE_P:
1547 CHECK_ARGUMENT(op == SLJIT_MOV_P);
1548 break;
1549 case SLJIT_ARG_TYPE_F64:
1550 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1551 CHECK_ARGUMENT(op == SLJIT_MOV_F64);
1552 break;
1553 case SLJIT_ARG_TYPE_F32:
1554 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1555 CHECK_ARGUMENT(op == SLJIT_MOV_F32);
1556 break;
1557 default:
1558 /* Context not initialized, void, etc. */
1559 CHECK_ARGUMENT(0);
1560 break;
1561 }
1562
1563 if (SLJIT_CHECK_OPCODE(op, 0) < SLJIT_MOV_F64) {
1564 FUNCTION_CHECK_SRC(src, srcw);
1565 } else {
1566 FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
1567 }
1568 compiler->last_flags = 0;
1569 #endif /* SLJIT_ARGUMENT_CHECKS */
1570 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1571 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1572 if (GET_OPCODE(op) < SLJIT_MOV_F64) {
1573 fprintf(compiler->verbose, " return%s%s ", !(op & SLJIT_32) ? "" : "32",
1574 op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1575 sljit_verbose_param(compiler, src, srcw);
1576 } else {
1577 fprintf(compiler->verbose, " return%s ", !(op & SLJIT_32) ? ".f64" : ".f32");
1578 sljit_verbose_fparam(compiler, src, srcw);
1579 }
1580 fprintf(compiler->verbose, "\n");
1581 }
1582 #endif /* SLJIT_VERBOSE */
1583 CHECK_RETURN_OK;
1584 }
1585
1586 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return_to(struct sljit_compiler *compiler,
1587 sljit_s32 src, sljit_sw srcw)
1588 {
1589 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1590 FUNCTION_CHECK_SRC(src, srcw);
1591 #endif /* SLJIT_ARGUMENT_CHECKS */
1592 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1593 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1594 fprintf(compiler->verbose, " return_to ");
1595 sljit_verbose_param(compiler, src, srcw);
1596 fprintf(compiler->verbose, "\n");
1597 }
1598 #endif /* SLJIT_VERBOSE */
1599 CHECK_RETURN_OK;
1600 }
1601
1602 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1603 {
1604 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1605 CHECK_ARGUMENT((op >= SLJIT_BREAKPOINT && op <= SLJIT_LMUL_SW)
1606 || ((op & ~SLJIT_32) >= SLJIT_DIVMOD_UW && (op & ~SLJIT_32) <= SLJIT_DIV_SW)
1607 || (op >= SLJIT_MEMORY_BARRIER && op <= SLJIT_SKIP_FRAMES_BEFORE_RETURN));
1608 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) < SLJIT_LMUL_UW || SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_MEMORY_BARRIER || compiler->scratches >= 2);
1609 if ((SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_LMUL_UW && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_DIV_SW) || op == SLJIT_SKIP_FRAMES_BEFORE_RETURN)
1610 compiler->last_flags = 0;
1611 #endif /* SLJIT_ARGUMENT_CHECKS */
1612 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1613 if (SLJIT_UNLIKELY(!!compiler->verbose))
1614 {
1615 fprintf(compiler->verbose, " %s", op0_names[GET_OPCODE(op) - SLJIT_OP0_BASE]);
1616 if (GET_OPCODE(op) >= SLJIT_DIVMOD_UW && GET_OPCODE(op) <= SLJIT_DIV_SW) {
1617 fprintf(compiler->verbose, (op & SLJIT_32) ? "32" : "w");
1618 }
1619 fprintf(compiler->verbose, "\n");
1620 }
1621 #endif /* SLJIT_VERBOSE */
1622 CHECK_RETURN_OK;
1623 }
1624
1625 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1626 sljit_s32 dst, sljit_sw dstw,
1627 sljit_s32 src, sljit_sw srcw)
1628 {
1629 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1630 compiler->skip_checks = 0;
1631 CHECK_RETURN_OK;
1632 }
1633
1634 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1635 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_MOV && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_REV_S32);
1636
1637 switch (GET_OPCODE(op)) {
1638 case SLJIT_MOV:
1639 case SLJIT_MOV_U32:
1640 case SLJIT_MOV_S32:
1641 case SLJIT_MOV32:
1642 case SLJIT_MOV_P:
1643 case SLJIT_REV_U32:
1644 case SLJIT_REV_S32:
1645 /* Nothing allowed */
1646 CHECK_ARGUMENT(!(op & (SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
1647 break;
1648 default:
1649 /* Only SLJIT_32 is allowed. */
1650 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
1651 break;
1652 }
1653
1654 FUNCTION_CHECK_DST(dst, dstw);
1655 FUNCTION_CHECK_SRC(src, srcw);
1656 #endif /* SLJIT_ARGUMENT_CHECKS */
1657 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1658 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1659 fprintf(compiler->verbose, " %s%s%s ", op1_names[GET_OPCODE(op) - SLJIT_OP1_BASE],
1660 !(op & SLJIT_32) ? "" : "32", op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1661
1662 sljit_verbose_param(compiler, dst, dstw);
1663 fprintf(compiler->verbose, ", ");
1664 sljit_verbose_param(compiler, src, srcw);
1665 fprintf(compiler->verbose, "\n");
1666 }
1667 #endif /* SLJIT_VERBOSE */
1668 CHECK_RETURN_OK;
1669 }
1670
1671 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
1672 sljit_s32 dst_reg,
1673 sljit_s32 mem_reg)
1674 {
1675 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1676 compiler->skip_checks = 0;
1677 CHECK_RETURN_OK;
1678 }
1679
1680 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1681 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_ATOMIC));
1682 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | SLJIT_SET_Z | VARIABLE_FLAG_MASK) >= SLJIT_MOV
1683 && SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | SLJIT_SET_Z | VARIABLE_FLAG_MASK) <= SLJIT_MOV_P);
1684 CHECK_ARGUMENT((op & (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS)) != (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS));
1685
1686 /* All arguments must be valid registers. */
1687 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1688 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(mem_reg) && !CHECK_IF_VIRTUAL_REGISTER(mem_reg));
1689
1690 if (GET_OPCODE(op) < SLJIT_MOV_U8 || GET_OPCODE(op) > SLJIT_MOV_S16) {
1691 /* Nothing allowed. */
1692 CHECK_ARGUMENT(!(op & SLJIT_32));
1693 }
1694
1695 compiler->last_flags = 0;
1696 #endif /* SLJIT_ARGUMENT_CHECKS */
1697 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1698 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1699 if (op & SLJIT_ATOMIC_TEST)
1700 CHECK_RETURN_OK;
1701 if (sljit_emit_atomic_load(compiler, op | SLJIT_ATOMIC_TEST, dst_reg, mem_reg)) {
1702 fprintf(compiler->verbose, " # atomic_load: unsupported form, no instructions are emitted\n");
1703 CHECK_RETURN_OK;
1704 }
1705
1706 fprintf(compiler->verbose, " atomic_load");
1707 if (op & SLJIT_ATOMIC_USE_CAS)
1708 fprintf(compiler->verbose, "_cas");
1709 if (op & SLJIT_ATOMIC_USE_LS)
1710 fprintf(compiler->verbose, "_ls");
1711
1712 fprintf(compiler->verbose, "%s%s ", !(op & SLJIT_32) ? "" : "32",
1713 op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1714 sljit_verbose_reg(compiler, dst_reg);
1715 fprintf(compiler->verbose, ", [");
1716 sljit_verbose_reg(compiler, mem_reg);
1717 fprintf(compiler->verbose, "]\n");
1718 }
1719 #endif /* SLJIT_VERBOSE */
1720 CHECK_RETURN_OK;
1721 }
1722
1723 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
1724 sljit_s32 src_reg,
1725 sljit_s32 mem_reg,
1726 sljit_s32 temp_reg)
1727 {
1728 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1729 compiler->skip_checks = 0;
1730 CHECK_RETURN_OK;
1731 }
1732
1733 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1734 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_ATOMIC));
1735 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | SLJIT_SET_Z) >= SLJIT_MOV
1736 && SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | SLJIT_SET_Z) <= SLJIT_MOV_P);
1737 CHECK_ARGUMENT((op & (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS)) != (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS));
1738
1739 /* All arguments must be valid registers. */
1740 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src_reg));
1741 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(mem_reg) && !CHECK_IF_VIRTUAL_REGISTER(mem_reg));
1742 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(temp_reg) && (src_reg != temp_reg || (op & SLJIT_ATOMIC_USE_LS)));
1743
1744 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK) || GET_FLAG_TYPE_MASK(op) == SLJIT_ATOMIC_STORED);
1745
1746 if (GET_OPCODE(op) < SLJIT_MOV_U8 || GET_OPCODE(op) > SLJIT_MOV_S16) {
1747 /* Nothing allowed. */
1748 CHECK_ARGUMENT(!(op & SLJIT_32));
1749 }
1750
1751 compiler->last_flags = GET_FLAG_TYPE_MASK(op) | (op & SLJIT_32);
1752 #endif /* SLJIT_ARGUMENT_CHECKS */
1753 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1754 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1755 if (op & SLJIT_ATOMIC_TEST)
1756 CHECK_RETURN_OK;
1757 if (sljit_emit_atomic_store(compiler, op | SLJIT_ATOMIC_TEST, src_reg, mem_reg, temp_reg)) {
1758 fprintf(compiler->verbose, " # atomic_store: unsupported form, no instructions are emitted\n");
1759 CHECK_RETURN_OK;
1760 }
1761
1762 fprintf(compiler->verbose, " atomic_store");
1763 if (op & SLJIT_ATOMIC_USE_CAS)
1764 fprintf(compiler->verbose, "_cas");
1765 if (op & SLJIT_ATOMIC_USE_LS)
1766 fprintf(compiler->verbose, "_ls");
1767
1768 fprintf(compiler->verbose, "%s%s%s ", !(op & SLJIT_32) ? "" : "32",
1769 op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE], !(op & VARIABLE_FLAG_MASK) ? "" : ".stored");
1770 sljit_verbose_reg(compiler, src_reg);
1771 fprintf(compiler->verbose, ", [");
1772 sljit_verbose_reg(compiler, mem_reg);
1773 fprintf(compiler->verbose, "], ");
1774 sljit_verbose_reg(compiler, temp_reg);
1775 fprintf(compiler->verbose, "\n");
1776 }
1777 #endif /* SLJIT_VERBOSE */
1778 CHECK_RETURN_OK;
1779 }
1780
1781 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 unset,
1782 sljit_s32 dst, sljit_sw dstw,
1783 sljit_s32 src1, sljit_sw src1w,
1784 sljit_s32 src2, sljit_sw src2w)
1785 {
1786 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1787 compiler->skip_checks = 0;
1788 CHECK_RETURN_OK;
1789 }
1790
1791 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1792 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_ADD && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_ROTR);
1793
1794 switch (GET_OPCODE(op)) {
1795 case SLJIT_AND:
1796 case SLJIT_OR:
1797 case SLJIT_XOR:
1798 case SLJIT_SHL:
1799 case SLJIT_MSHL:
1800 case SLJIT_LSHR:
1801 case SLJIT_MLSHR:
1802 case SLJIT_ASHR:
1803 case SLJIT_MASHR:
1804 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK));
1805 break;
1806 case SLJIT_MUL:
1807 CHECK_ARGUMENT(!(op & SLJIT_SET_Z));
1808 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK)
1809 || GET_FLAG_TYPE(op) == SLJIT_OVERFLOW);
1810 break;
1811 case SLJIT_ADD:
1812 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK)
1813 || GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY)
1814 || GET_FLAG_TYPE(op) == SLJIT_OVERFLOW);
1815 break;
1816 case SLJIT_SUB:
1817 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK)
1818 || (GET_FLAG_TYPE(op) >= SLJIT_LESS && GET_FLAG_TYPE(op) <= SLJIT_OVERFLOW)
1819 || GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY));
1820 break;
1821 case SLJIT_ADDC:
1822 case SLJIT_SUBC:
1823 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK)
1824 || GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY));
1825 CHECK_ARGUMENT((compiler->last_flags & 0xff) == GET_FLAG_TYPE(SLJIT_SET_CARRY));
1826 CHECK_ARGUMENT((op & SLJIT_32) == (compiler->last_flags & SLJIT_32));
1827 break;
1828 case SLJIT_ROTL:
1829 case SLJIT_ROTR:
1830 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
1831 break;
1832 default:
1833 SLJIT_UNREACHABLE();
1834 break;
1835 }
1836
1837 if (unset) {
1838 CHECK_ARGUMENT(HAS_FLAGS(op));
1839 } else {
1840 FUNCTION_CHECK_DST(dst, dstw);
1841 }
1842 FUNCTION_CHECK_SRC(src1, src1w);
1843 FUNCTION_CHECK_SRC(src2, src2w);
1844 compiler->last_flags = GET_FLAG_TYPE(op) | (op & (SLJIT_32 | SLJIT_SET_Z));
1845 #endif /* SLJIT_ARGUMENT_CHECKS */
1846 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1847 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1848 fprintf(compiler->verbose, " %s%s%s%s%s ", op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE], !(op & SLJIT_32) ? "" : "32",
1849 !(op & SLJIT_SET_Z) ? "" : ".z", !(op & VARIABLE_FLAG_MASK) ? "" : ".",
1850 !(op & VARIABLE_FLAG_MASK) ? "" : jump_names[GET_FLAG_TYPE(op)]);
1851 if (unset)
1852 fprintf(compiler->verbose, "unset");
1853 else
1854 sljit_verbose_param(compiler, dst, dstw);
1855 fprintf(compiler->verbose, ", ");
1856 sljit_verbose_param(compiler, src1, src1w);
1857 fprintf(compiler->verbose, ", ");
1858 sljit_verbose_param(compiler, src2, src2w);
1859 fprintf(compiler->verbose, "\n");
1860 }
1861 #endif /* SLJIT_VERBOSE */
1862 CHECK_RETURN_OK;
1863 }
1864
1865 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
1866 sljit_s32 dst_reg,
1867 sljit_s32 src1, sljit_sw src1w,
1868 sljit_s32 src2, sljit_sw src2w)
1869 {
1870 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1871 CHECK_ARGUMENT((op | SLJIT_32) == SLJIT_MULADD32);
1872 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1873 FUNCTION_CHECK_SRC(src1, src1w);
1874 FUNCTION_CHECK_SRC(src2, src2w);
1875 compiler->last_flags = 0;
1876 #endif /* SLJIT_ARGUMENT_CHECKS */
1877 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1878 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1879 fprintf(compiler->verbose, " %s%s ", op2r_names[GET_OPCODE(op) - SLJIT_OP2R_BASE], !(op & SLJIT_32) ? "" : "32");
1880
1881 sljit_verbose_reg(compiler, dst_reg);
1882 fprintf(compiler->verbose, ", ");
1883 sljit_verbose_param(compiler, src1, src1w);
1884 fprintf(compiler->verbose, ", ");
1885 sljit_verbose_param(compiler, src2, src2w);
1886 fprintf(compiler->verbose, "\n");
1887 }
1888 #endif /* SLJIT_VERBOSE */
1889 CHECK_RETURN_OK;
1890 }
1891
1892 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
1893 sljit_s32 dst_reg,
1894 sljit_s32 src1_reg,
1895 sljit_s32 src2_reg,
1896 sljit_s32 src3, sljit_sw src3w)
1897 {
1898 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1899 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) == SLJIT_SHL || SLJIT_CHECK_OPCODE(op, 0) == SLJIT_LSHR
1900 || SLJIT_CHECK_OPCODE(op, 0) == SLJIT_MSHL || SLJIT_CHECK_OPCODE(op, 0) == SLJIT_MLSHR);
1901 CHECK_ARGUMENT((op & ~(0xff | SLJIT_32 | SLJIT_SHIFT_INTO_NON_ZERO)) == 0);
1902 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1903 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src1_reg));
1904 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src2_reg));
1905 FUNCTION_CHECK_SRC(src3, src3w);
1906 CHECK_ARGUMENT(dst_reg != src2_reg);
1907 #endif /* SLJIT_ARGUMENT_CHECKS */
1908 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1909 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1910 fprintf(compiler->verbose, " %s%s.into%s ", op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE], !(op & SLJIT_32) ? "" : "32",
1911 (op & SLJIT_SHIFT_INTO_NON_ZERO) ? ".nz" : "");
1912
1913 sljit_verbose_reg(compiler, dst_reg);
1914 fprintf(compiler->verbose, ", ");
1915 sljit_verbose_reg(compiler, src1_reg);
1916 fprintf(compiler->verbose, ", ");
1917 sljit_verbose_reg(compiler, src2_reg);
1918 fprintf(compiler->verbose, ", ");
1919 sljit_verbose_param(compiler, src3, src3w);
1920 fprintf(compiler->verbose, "\n");
1921 }
1922 #endif /* SLJIT_VERBOSE */
1923 CHECK_RETURN_OK;
1924 }
1925
1926 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
1927 sljit_s32 src, sljit_sw srcw)
1928 {
1929 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1930 CHECK_ARGUMENT(op >= SLJIT_FAST_RETURN && op <= SLJIT_PREFETCH_ONCE);
1931 FUNCTION_CHECK_SRC(src, srcw);
1932
1933 if (op == SLJIT_FAST_RETURN || op == SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN) {
1934 CHECK_ARGUMENT(src != SLJIT_IMM);
1935 compiler->last_flags = 0;
1936 } else if (op >= SLJIT_PREFETCH_L1 && op <= SLJIT_PREFETCH_ONCE) {
1937 CHECK_ARGUMENT(src & SLJIT_MEM);
1938 }
1939 #endif /* SLJIT_ARGUMENT_CHECKS */
1940 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1941 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1942 fprintf(compiler->verbose, " %s ", op_src_dst_names[op - SLJIT_OP_SRC_DST_BASE]);
1943 sljit_verbose_param(compiler, src, srcw);
1944 fprintf(compiler->verbose, "\n");
1945 }
1946 #endif /* SLJIT_VERBOSE */
1947 CHECK_RETURN_OK;
1948 }
1949
1950 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
1951 sljit_s32 dst, sljit_sw dstw)
1952 {
1953 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1954 CHECK_ARGUMENT(op >= SLJIT_FAST_ENTER && op <= SLJIT_GET_RETURN_ADDRESS);
1955 FUNCTION_CHECK_DST(dst, dstw);
1956
1957 if (op == SLJIT_FAST_ENTER)
1958 compiler->last_flags = 0;
1959 #endif /* SLJIT_ARGUMENT_CHECKS */
1960 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1961 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1962 fprintf(compiler->verbose, " %s ", op_src_dst_names[op - SLJIT_OP_SRC_DST_BASE]);
1963 sljit_verbose_param(compiler, dst, dstw);
1964 fprintf(compiler->verbose, "\n");
1965 }
1966 #endif /* SLJIT_VERBOSE */
1967 CHECK_RETURN_OK;
1968 }
1969
1970 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
1971 {
1972 SLJIT_UNUSED_ARG(type);
1973 SLJIT_UNUSED_ARG(reg);
1974 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1975 if (type == SLJIT_GP_REGISTER) {
1976 CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_REGISTERS)
1977 || (reg >= SLJIT_TMP_REGISTER_BASE && reg < (SLJIT_TMP_REGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_REGISTERS)));
1978 }
1979 #if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
1980 else if (((type >> 12) == 0 || ((type >> 12) >= 3 && (type >> 12) <= 6))) {
1981 CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_VECTOR_REGISTERS)
1982 || (reg >= SLJIT_TMP_VREGISTER_BASE && reg < (SLJIT_TMP_VREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_VECTOR_REGISTERS)));
1983 }
1984 #endif /* SLJIT_SEPARATE_VECTOR_REGISTERS */
1985 else {
1986 CHECK_ARGUMENT(type == SLJIT_FLOAT_REGISTER || ((type >> 12) == 0 || ((type >> 12) >= 3 && (type >> 12) <= 6) || (type & (3 << 12)) || (type & (4 << 12)) || (type & (5 << 12)) || (type & (6 << 12))));
1987 CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_FLOAT_REGISTERS)
1988 || (reg >= SLJIT_TMP_FREGISTER_BASE && reg < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS)));
1989 }
1990 #endif /* SLJIT_ARGUMENT_CHECKS */
1991 CHECK_RETURN_OK;
1992 }
1993
1994 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_custom(struct sljit_compiler *compiler,
1995 void *instruction, sljit_u32 size)
1996 {
1997 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1998 sljit_u32 i;
1999 #endif /* SLJIT_VERBOSE */
2000
2001 SLJIT_UNUSED_ARG(compiler);
2002
2003 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2004 CHECK_ARGUMENT(instruction);
2005
2006 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
2007 CHECK_ARGUMENT(size > 0 && size < 16);
2008 #elif (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
2009 CHECK_ARGUMENT((size == 2 && (((sljit_sw)instruction) & 0x1) == 0)
2010 || (size == 4 && (((sljit_sw)instruction) & 0x3) == 0));
2011 #elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
2012 CHECK_ARGUMENT(size == 2 || size == 4 || size == 6);
2013 #else /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM_THUMB2 && !SLJIT_CONFIG_S390X */
2014 CHECK_ARGUMENT(size == 4 && (((sljit_sw)instruction) & 0x3) == 0);
2015 #endif /* SLJIT_CONFIG_X86 */
2016
2017 compiler->last_flags = 0;
2018 #endif /* SLJIT_ARGUMENT_CHECKS */
2019 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2020 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2021 fprintf(compiler->verbose, " op_custom");
2022 for (i = 0; i < size; i++)
2023 fprintf(compiler->verbose, " 0x%x", ((sljit_u8*)instruction)[i]);
2024 fprintf(compiler->verbose, "\n");
2025 }
2026 #endif /* SLJIT_VERBOSE */
2027 CHECK_RETURN_OK;
2028 }
2029
2030 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2031 sljit_s32 dst, sljit_sw dstw,
2032 sljit_s32 src, sljit_sw srcw)
2033 {
2034 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2035 compiler->skip_checks = 0;
2036 CHECK_RETURN_OK;
2037 }
2038
2039 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2040 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2041 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_MOV_F64 && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_ABS_F64);
2042 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
2043 FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
2044 FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
2045 #endif /* SLJIT_ARGUMENT_CHECKS */
2046 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2047 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2048 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32)
2049 fprintf(compiler->verbose, " %s%s ", fop1_names[SLJIT_CONV_F64_FROM_F32 - SLJIT_FOP1_BASE],
2050 (op & SLJIT_32) ? ".f32.from.f64" : ".f64.from.f32");
2051 else
2052 fprintf(compiler->verbose, " %s%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
2053 (op & SLJIT_32) ? ".f32" : ".f64");
2054
2055 sljit_verbose_fparam(compiler, dst, dstw);
2056 fprintf(compiler->verbose, ", ");
2057 sljit_verbose_fparam(compiler, src, srcw);
2058 fprintf(compiler->verbose, "\n");
2059 }
2060 #endif /* SLJIT_VERBOSE */
2061 CHECK_RETURN_OK;
2062 }
2063
2064 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2065 sljit_s32 src1, sljit_sw src1w,
2066 sljit_s32 src2, sljit_sw src2w)
2067 {
2068 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2069 compiler->last_flags = GET_FLAG_TYPE(op) | (op & SLJIT_32);
2070 #endif /* SLJIT_ARGUMENT_CHECKS */
2071
2072 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2073 compiler->skip_checks = 0;
2074 CHECK_RETURN_OK;
2075 }
2076
2077 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2078 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2079 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) == SLJIT_CMP_F64);
2080 CHECK_ARGUMENT(!(op & SLJIT_SET_Z));
2081 CHECK_ARGUMENT((op & VARIABLE_FLAG_MASK)
2082 || (GET_FLAG_TYPE(op) >= SLJIT_F_EQUAL && GET_FLAG_TYPE(op) <= SLJIT_ORDERED_LESS_EQUAL));
2083 FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2084 FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2085 #endif /* SLJIT_ARGUMENT_CHECKS */
2086 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2087 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2088 fprintf(compiler->verbose, " %s%s", fop1_names[SLJIT_CMP_F64 - SLJIT_FOP1_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2089 if (op & VARIABLE_FLAG_MASK) {
2090 fprintf(compiler->verbose, ".%s", jump_names[GET_FLAG_TYPE(op)]);
2091 }
2092 fprintf(compiler->verbose, " ");
2093 sljit_verbose_fparam(compiler, src1, src1w);
2094 fprintf(compiler->verbose, ", ");
2095 sljit_verbose_fparam(compiler, src2, src2w);
2096 fprintf(compiler->verbose, "\n");
2097 }
2098 #endif /* SLJIT_VERBOSE */
2099 CHECK_RETURN_OK;
2100 }
2101
2102 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2103 sljit_s32 dst, sljit_sw dstw,
2104 sljit_s32 src, sljit_sw srcw)
2105 {
2106 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2107 compiler->skip_checks = 0;
2108 CHECK_RETURN_OK;
2109 }
2110
2111 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2112 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2113 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
2114 FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
2115 FUNCTION_CHECK_DST(dst, dstw);
2116 #endif /* SLJIT_ARGUMENT_CHECKS */
2117 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2118 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2119 fprintf(compiler->verbose, " %s%s.from%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
2120 fop1_conv_types[GET_OPCODE(op) - SLJIT_CONV_SW_FROM_F64],
2121 (op & SLJIT_32) ? ".f32" : ".f64");
2122 sljit_verbose_param(compiler, dst, dstw);
2123 fprintf(compiler->verbose, ", ");
2124 sljit_verbose_fparam(compiler, src, srcw);
2125 fprintf(compiler->verbose, "\n");
2126 }
2127 #endif /* SLJIT_VERBOSE */
2128 CHECK_RETURN_OK;
2129 }
2130
2131 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler *compiler, sljit_s32 op,
2132 sljit_s32 dst, sljit_sw dstw,
2133 sljit_s32 src, sljit_sw srcw)
2134 {
2135 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2136 compiler->skip_checks = 0;
2137 CHECK_RETURN_OK;
2138 }
2139
2140 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2141 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2142 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
2143 FUNCTION_CHECK_SRC(src, srcw);
2144 FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
2145 #endif /* SLJIT_ARGUMENT_CHECKS */
2146 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2147 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2148 fprintf(compiler->verbose, " %s%s.from.%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
2149 (op & SLJIT_32) ? ".f32" : ".f64",
2150 fop1_conv_types[GET_OPCODE(op) - SLJIT_CONV_SW_FROM_F64]);
2151 sljit_verbose_fparam(compiler, dst, dstw);
2152 fprintf(compiler->verbose, ", ");
2153 sljit_verbose_param(compiler, src, srcw);
2154 fprintf(compiler->verbose, "\n");
2155 }
2156 #endif /* SLJIT_VERBOSE */
2157 CHECK_RETURN_OK;
2158 }
2159
2160 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2161 sljit_s32 dst, sljit_sw dstw,
2162 sljit_s32 src1, sljit_sw src1w,
2163 sljit_s32 src2, sljit_sw src2w)
2164 {
2165 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2166 compiler->skip_checks = 0;
2167 CHECK_RETURN_OK;
2168 }
2169
2170 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2171 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2172 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_ADD_F64 && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_DIV_F64);
2173 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
2174 FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2175 FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2176 FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
2177 #endif /* SLJIT_ARGUMENT_CHECKS */
2178 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2179 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2180 fprintf(compiler->verbose, " %s%s ", fop2_names[GET_OPCODE(op) - SLJIT_FOP2_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2181 sljit_verbose_fparam(compiler, dst, dstw);
2182 fprintf(compiler->verbose, ", ");
2183 sljit_verbose_fparam(compiler, src1, src1w);
2184 fprintf(compiler->verbose, ", ");
2185 sljit_verbose_fparam(compiler, src2, src2w);
2186 fprintf(compiler->verbose, "\n");
2187 }
2188 #endif /* SLJIT_VERBOSE */
2189 CHECK_RETURN_OK;
2190 }
2191
2192 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop2r(struct sljit_compiler *compiler, sljit_s32 op,
2193 sljit_s32 dst_freg,
2194 sljit_s32 src1, sljit_sw src1w,
2195 sljit_s32 src2, sljit_sw src2w)
2196 {
2197 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2198 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2199 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) == SLJIT_COPYSIGN_F64);
2200 FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2201 FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2202 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(dst_freg, op & SLJIT_32));
2203 #endif /* SLJIT_ARGUMENT_CHECKS */
2204 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2205 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2206 fprintf(compiler->verbose, " %s%s ", fop2r_names[GET_OPCODE(op) - SLJIT_FOP2R_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2207 sljit_verbose_freg(compiler, dst_freg);
2208 fprintf(compiler->verbose, ", ");
2209 sljit_verbose_fparam(compiler, src1, src1w);
2210 fprintf(compiler->verbose, ", ");
2211 sljit_verbose_fparam(compiler, src2, src2w);
2212 fprintf(compiler->verbose, "\n");
2213 }
2214 #endif /* SLJIT_VERBOSE */
2215 CHECK_RETURN_OK;
2216 }
2217
2218 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fset32(struct sljit_compiler *compiler,
2219 sljit_s32 freg, sljit_f32 value)
2220 {
2221 SLJIT_UNUSED_ARG(value);
2222
2223 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2224 compiler->skip_checks = 0;
2225 CHECK_RETURN_OK;
2226 }
2227
2228 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2229 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2230 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 1));
2231 #endif /* SLJIT_ARGUMENT_CHECKS */
2232 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2233 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2234 fprintf(compiler->verbose, " fset32 ");
2235 sljit_verbose_freg(compiler, freg);
2236 fprintf(compiler->verbose, ", %f\n", value);
2237 }
2238 #endif /* SLJIT_VERBOSE */
2239 CHECK_RETURN_OK;
2240 }
2241
2242 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fset64(struct sljit_compiler *compiler,
2243 sljit_s32 freg, sljit_f64 value)
2244 {
2245 SLJIT_UNUSED_ARG(value);
2246
2247 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2248 compiler->skip_checks = 0;
2249 CHECK_RETURN_OK;
2250 }
2251
2252 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2253 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2254 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 0));
2255 #endif /* SLJIT_ARGUMENT_CHECKS */
2256 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2257 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2258 fprintf(compiler->verbose, " fset64 ");
2259 sljit_verbose_freg(compiler, freg);
2260 fprintf(compiler->verbose, ", %f\n", value);
2261 }
2262 #endif /* SLJIT_VERBOSE */
2263 CHECK_RETURN_OK;
2264 }
2265
2266 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fcopy(struct sljit_compiler *compiler, sljit_s32 op,
2267 sljit_s32 freg, sljit_s32 reg)
2268 {
2269 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2270 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2271 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_COPY_TO_F64 && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_COPY_FROM_F64);
2272 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
2273 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, op & SLJIT_32));
2274
2275 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
2276 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2277 #else /* !SLJIT_64BIT_ARCHITECTURE */
2278 switch (op) {
2279 case SLJIT_COPY32_TO_F32:
2280 case SLJIT_COPY32_FROM_F32:
2281 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2282 break;
2283 case SLJIT_COPY_TO_F64:
2284 case SLJIT_COPY_FROM_F64:
2285 if (reg & REG_PAIR_MASK) {
2286 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_FIRST(reg)));
2287 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_SECOND(reg)));
2288
2289 if (op == SLJIT_COPY_TO_F64)
2290 break;
2291
2292 CHECK_ARGUMENT(REG_PAIR_FIRST(reg) != REG_PAIR_SECOND(reg));
2293 break;
2294 }
2295
2296 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2297 break;
2298 }
2299 #endif /* SLJIT_64BIT_ARCHITECTURE */
2300 #endif /* SLJIT_ARGUMENT_CHECKS */
2301 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2302 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2303 fprintf(compiler->verbose, " copy%s_%s_f%s ", (op & SLJIT_32) ? "32" : "",
2304 GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? "to" : "from", (op & SLJIT_32) ? "32" : "64");
2305
2306 sljit_verbose_freg(compiler, freg);
2307
2308 if (reg & REG_PAIR_MASK) {
2309 fprintf(compiler->verbose, ", {");
2310 sljit_verbose_reg(compiler, REG_PAIR_FIRST(reg));
2311 fprintf(compiler->verbose, ", ");
2312 sljit_verbose_reg(compiler, REG_PAIR_SECOND(reg));
2313 fprintf(compiler->verbose, "}\n");
2314 } else {
2315 fprintf(compiler->verbose, ", ");
2316 sljit_verbose_reg(compiler, reg);
2317 fprintf(compiler->verbose, "\n");
2318 }
2319 }
2320 #endif /* SLJIT_VERBOSE */
2321 CHECK_RETURN_OK;
2322 }
2323
2324 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_label(struct sljit_compiler *compiler)
2325 {
2326 SLJIT_UNUSED_ARG(compiler);
2327
2328 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2329 compiler->skip_checks = 0;
2330 CHECK_RETURN_OK;
2331 }
2332
2333 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2334 compiler->last_flags = 0;
2335 #endif /* SLJIT_ARGUMENT_CHECKS */
2336
2337 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2338 if (SLJIT_UNLIKELY(!!compiler->verbose))
2339 fprintf(compiler->verbose, "label:\n");
2340 #endif /* SLJIT_VERBOSE */
2341 CHECK_RETURN_OK;
2342 }
2343
2344 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2345 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
2346 || (defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM)
2347 #define CHECK_UNORDERED(type, last_flags) \
2348 ((((type) & 0xfe) == SLJIT_ORDERED) && \
2349 ((last_flags) & 0xff) >= SLJIT_UNORDERED && ((last_flags) & 0xff) <= SLJIT_ORDERED_LESS_EQUAL)
2350 #else /* !SLJIT_CONFIG_X86 || SLJIT_CONFIG_ARM */
2351 #define CHECK_UNORDERED(type, last_flags) 0
2352 #endif /* SLJIT_CONFIG_X86 || SLJIT_CONFIG_ARM */
2353 #endif /* SLJIT_ARGUMENT_CHECKS */
2354
2355 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2356 {
2357 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2358 compiler->skip_checks = 0;
2359 CHECK_RETURN_OK;
2360 }
2361
2362 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2363 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP)));
2364 CHECK_ARGUMENT((type & 0xff) >= SLJIT_EQUAL && (type & 0xff) <= SLJIT_FAST_CALL);
2365
2366 if ((type & 0xff) < SLJIT_JUMP) {
2367 if ((type & 0xff) <= SLJIT_NOT_ZERO)
2368 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2369 else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2370 CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2371 compiler->last_flags = 0;
2372 } else
2373 CHECK_ARGUMENT((type & 0xfe) == (compiler->last_flags & 0xff)
2374 || CHECK_UNORDERED(type, compiler->last_flags));
2375 }
2376 #endif /* SLJIT_ARGUMENT_CHECKS */
2377 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2378 if (SLJIT_UNLIKELY(!!compiler->verbose))
2379 fprintf(compiler->verbose, " jump%s %s\n", !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r",
2380 jump_names[type & 0xff]);
2381 #endif /* SLJIT_VERBOSE */
2382 CHECK_RETURN_OK;
2383 }
2384
2385 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2386 sljit_s32 arg_types)
2387 {
2388 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2389 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_CALL_RETURN)));
2390 CHECK_ARGUMENT((type & 0xff) >= SLJIT_CALL && (type & 0xff) <= SLJIT_CALL_REG_ARG);
2391 CHECK_ARGUMENT(function_check_arguments(arg_types, compiler->scratches, -1, compiler->fscratches));
2392
2393 if (type & SLJIT_CALL_RETURN) {
2394 CHECK_ARGUMENT((arg_types & SLJIT_ARG_MASK) == compiler->last_return);
2395
2396 if (compiler->options & SLJIT_ENTER_REG_ARG) {
2397 CHECK_ARGUMENT((type & 0xff) == SLJIT_CALL_REG_ARG);
2398 } else {
2399 CHECK_ARGUMENT((type & 0xff) != SLJIT_CALL_REG_ARG);
2400 }
2401 }
2402 #endif /* SLJIT_ARGUMENT_CHECKS */
2403 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2404 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2405 fprintf(compiler->verbose, " %s%s%s ret[%s", jump_names[type & 0xff],
2406 !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r",
2407 !(type & SLJIT_CALL_RETURN) ? "" : ".ret",
2408 call_arg_names[arg_types & SLJIT_ARG_MASK]);
2409
2410 arg_types >>= SLJIT_ARG_SHIFT;
2411 if (arg_types) {
2412 fprintf(compiler->verbose, "], args[");
2413 do {
2414 fprintf(compiler->verbose, "%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
2415 arg_types >>= SLJIT_ARG_SHIFT;
2416 if (arg_types)
2417 fprintf(compiler->verbose, ",");
2418 } while (arg_types);
2419 }
2420 fprintf(compiler->verbose, "]\n");
2421 }
2422 #endif /* SLJIT_VERBOSE */
2423 CHECK_RETURN_OK;
2424 }
2425
2426 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_cmp(struct sljit_compiler *compiler, sljit_s32 type,
2427 sljit_s32 src1, sljit_sw src1w,
2428 sljit_s32 src2, sljit_sw src2w)
2429 {
2430 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2431 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_32)));
2432 CHECK_ARGUMENT((type & 0xff) >= SLJIT_EQUAL && (type & 0xff) <= SLJIT_SIG_LESS_EQUAL);
2433 FUNCTION_CHECK_SRC(src1, src1w);
2434 FUNCTION_CHECK_SRC(src2, src2w);
2435 compiler->last_flags = 0;
2436 #endif /* SLJIT_ARGUMENT_CHECKS */
2437 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2438 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2439 fprintf(compiler->verbose, " cmp%s%s %s, ", (type & SLJIT_32) ? "32" : "",
2440 !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r", jump_names[type & 0xff]);
2441 sljit_verbose_param(compiler, src1, src1w);
2442 fprintf(compiler->verbose, ", ");
2443 sljit_verbose_param(compiler, src2, src2w);
2444 fprintf(compiler->verbose, "\n");
2445 }
2446 #endif /* SLJIT_VERBOSE */
2447 CHECK_RETURN_OK;
2448 }
2449
2450 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fcmp(struct sljit_compiler *compiler, sljit_s32 type,
2451 sljit_s32 src1, sljit_sw src1w,
2452 sljit_s32 src2, sljit_sw src2w)
2453 {
2454 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2455 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2456 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_32)));
2457 CHECK_ARGUMENT((type & 0xff) >= SLJIT_F_EQUAL && (type & 0xff) <= SLJIT_ORDERED_LESS_EQUAL);
2458 FUNCTION_FCHECK(src1, src1w, type & SLJIT_32);
2459 FUNCTION_FCHECK(src2, src2w, type & SLJIT_32);
2460 compiler->last_flags = 0;
2461 #endif /* SLJIT_ARGUMENT_CHECKS */
2462 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2463 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2464 fprintf(compiler->verbose, " fcmp%s%s %s, ", (type & SLJIT_32) ? ".f32" : ".f64",
2465 !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r", jump_names[type & 0xff]);
2466 sljit_verbose_fparam(compiler, src1, src1w);
2467 fprintf(compiler->verbose, ", ");
2468 sljit_verbose_fparam(compiler, src2, src2w);
2469 fprintf(compiler->verbose, "\n");
2470 }
2471 #endif /* SLJIT_VERBOSE */
2472 CHECK_RETURN_OK;
2473 }
2474
2475 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type,
2476 sljit_s32 src, sljit_sw srcw)
2477 {
2478 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2479 compiler->skip_checks = 0;
2480 CHECK_RETURN_OK;
2481 }
2482
2483 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2484 CHECK_ARGUMENT(type >= SLJIT_JUMP && type <= SLJIT_FAST_CALL);
2485 FUNCTION_CHECK_SRC(src, srcw);
2486 #endif /* SLJIT_ARGUMENT_CHECKS */
2487 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2488 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2489 fprintf(compiler->verbose, " ijump.%s ", jump_names[type]);
2490 sljit_verbose_param(compiler, src, srcw);
2491 fprintf(compiler->verbose, "\n");
2492 }
2493 #endif /* SLJIT_VERBOSE */
2494 CHECK_RETURN_OK;
2495 }
2496
2497 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
2498 sljit_s32 arg_types,
2499 sljit_s32 src, sljit_sw srcw)
2500 {
2501 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2502 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_CALL_RETURN)));
2503 CHECK_ARGUMENT((type & 0xff) >= SLJIT_CALL && (type & 0xff) <= SLJIT_CALL_REG_ARG);
2504 CHECK_ARGUMENT(function_check_arguments(arg_types, compiler->scratches, -1, compiler->fscratches));
2505 FUNCTION_CHECK_SRC(src, srcw);
2506
2507 if (type & SLJIT_CALL_RETURN) {
2508 CHECK_ARGUMENT((arg_types & SLJIT_ARG_MASK) == compiler->last_return);
2509
2510 if (compiler->options & SLJIT_ENTER_REG_ARG) {
2511 CHECK_ARGUMENT((type & 0xff) == SLJIT_CALL_REG_ARG);
2512 } else {
2513 CHECK_ARGUMENT((type & 0xff) != SLJIT_CALL_REG_ARG);
2514 }
2515 }
2516 #endif /* SLJIT_ARGUMENT_CHECKS */
2517 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2518 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2519 fprintf(compiler->verbose, " i%s%s ret[%s", jump_names[type & 0xff],
2520 !(type & SLJIT_CALL_RETURN) ? "" : ".ret",
2521 call_arg_names[arg_types & SLJIT_ARG_MASK]);
2522
2523 arg_types >>= SLJIT_ARG_SHIFT;
2524 if (arg_types) {
2525 fprintf(compiler->verbose, "], args[");
2526 do {
2527 fprintf(compiler->verbose, "%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
2528 arg_types >>= SLJIT_ARG_SHIFT;
2529 if (arg_types)
2530 fprintf(compiler->verbose, ",");
2531 } while (arg_types);
2532 }
2533 fprintf(compiler->verbose, "], ");
2534 sljit_verbose_param(compiler, src, srcw);
2535 fprintf(compiler->verbose, "\n");
2536 }
2537 #endif /* SLJIT_VERBOSE */
2538 CHECK_RETURN_OK;
2539 }
2540
2541 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2542 sljit_s32 dst, sljit_sw dstw,
2543 sljit_s32 type)
2544 {
2545 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2546 CHECK_ARGUMENT(type >= SLJIT_EQUAL && type <= SLJIT_ORDERED_LESS_EQUAL);
2547 CHECK_ARGUMENT(op == SLJIT_MOV || op == SLJIT_MOV32
2548 || (SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_AND && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_XOR));
2549 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK));
2550
2551 if (type <= SLJIT_NOT_ZERO)
2552 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2553 else
2554 CHECK_ARGUMENT((type & 0xfe) == (compiler->last_flags & 0xff)
2555 || CHECK_UNORDERED(type, compiler->last_flags));
2556
2557 FUNCTION_CHECK_DST(dst, dstw);
2558
2559 if (GET_OPCODE(op) >= SLJIT_ADD)
2560 compiler->last_flags = GET_FLAG_TYPE(op) | (op & (SLJIT_32 | SLJIT_SET_Z));
2561 #endif /* SLJIT_ARGUMENT_CHECKS */
2562 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2563 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2564 fprintf(compiler->verbose, " flags.%s%s%s ",
2565 GET_OPCODE(op) < SLJIT_OP2_BASE ? "mov" : op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE],
2566 GET_OPCODE(op) < SLJIT_OP2_BASE ? op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE] : ((op & SLJIT_32) ? "32" : ""),
2567 !(op & SLJIT_SET_Z) ? "" : ".z");
2568 sljit_verbose_param(compiler, dst, dstw);
2569 fprintf(compiler->verbose, ", %s\n", jump_names[type]);
2570 }
2571 #endif /* SLJIT_VERBOSE */
2572 CHECK_RETURN_OK;
2573 }
2574
2575 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
2576 sljit_s32 dst_reg,
2577 sljit_s32 src1, sljit_sw src1w,
2578 sljit_s32 src2_reg)
2579 {
2580 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2581 sljit_s32 cond = type & ~SLJIT_32;
2582
2583 CHECK_ARGUMENT(cond >= SLJIT_EQUAL && cond <= SLJIT_ORDERED_LESS_EQUAL);
2584
2585 CHECK_ARGUMENT(compiler->scratches != -1 && compiler->saveds != -1);
2586 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
2587 FUNCTION_CHECK_SRC(src1, src1w);
2588 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src2_reg));
2589
2590 if (cond <= SLJIT_NOT_ZERO)
2591 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2592 else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2593 CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2594 compiler->last_flags = 0;
2595 } else
2596 CHECK_ARGUMENT((cond & 0xfe) == (compiler->last_flags & 0xff)
2597 || CHECK_UNORDERED(cond, compiler->last_flags));
2598 #endif /* SLJIT_ARGUMENT_CHECKS */
2599 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2600 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2601 fprintf(compiler->verbose, " select%s %s, ",
2602 !(type & SLJIT_32) ? "" : "32",
2603 jump_names[type & ~SLJIT_32]);
2604 sljit_verbose_reg(compiler, dst_reg);
2605 fprintf(compiler->verbose, ", ");
2606 sljit_verbose_param(compiler, src1, src1w);
2607 fprintf(compiler->verbose, ", ");
2608 sljit_verbose_reg(compiler, src2_reg);
2609 fprintf(compiler->verbose, "\n");
2610 }
2611 #endif /* SLJIT_VERBOSE */
2612 CHECK_RETURN_OK;
2613 }
2614
2615 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
2616 sljit_s32 dst_freg,
2617 sljit_s32 src1, sljit_sw src1w,
2618 sljit_s32 src2_freg)
2619 {
2620 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2621 sljit_s32 cond = type & ~SLJIT_32;
2622
2623 CHECK_ARGUMENT(cond >= SLJIT_EQUAL && cond <= SLJIT_ORDERED_LESS_EQUAL);
2624
2625 CHECK_ARGUMENT(compiler->fscratches != -1 && compiler->fsaveds != -1);
2626 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(dst_freg, type & SLJIT_32));
2627 FUNCTION_FCHECK(src1, src1w, type & SLJIT_32);
2628 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(src2_freg, type & SLJIT_32));
2629
2630 if (cond <= SLJIT_NOT_ZERO)
2631 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2632 else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2633 CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2634 compiler->last_flags = 0;
2635 } else
2636 CHECK_ARGUMENT((cond & 0xfe) == (compiler->last_flags & 0xff)
2637 || CHECK_UNORDERED(cond, compiler->last_flags));
2638 #endif /* SLJIT_ARGUMENT_CHECKS */
2639 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2640 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2641 fprintf(compiler->verbose, " fselect%s %s, ",
2642 !(type & SLJIT_32) ? "" : "32",
2643 jump_names[type & ~SLJIT_32]);
2644 sljit_verbose_freg(compiler, dst_freg);
2645 fprintf(compiler->verbose, ", ");
2646 sljit_verbose_fparam(compiler, src1, src1w);
2647 fprintf(compiler->verbose, ", ");
2648 sljit_verbose_freg(compiler, src2_freg);
2649 fprintf(compiler->verbose, "\n");
2650 }
2651 #endif /* SLJIT_VERBOSE */
2652 CHECK_RETURN_OK;
2653 }
2654
2655 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
2656 sljit_s32 reg,
2657 sljit_s32 mem, sljit_sw memw)
2658 {
2659 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2660 sljit_s32 allowed_flags;
2661 #endif /* SLJIT_ARGUMENT_CHECKS */
2662
2663 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2664 compiler->skip_checks = 0;
2665 CHECK_RETURN_OK;
2666 }
2667
2668 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2669 if (type & SLJIT_MEM_UNALIGNED) {
2670 CHECK_ARGUMENT(!(type & (SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2671 } else if (type & SLJIT_MEM_ALIGNED_16) {
2672 CHECK_ARGUMENT(!(type & SLJIT_MEM_ALIGNED_32));
2673 } else {
2674 CHECK_ARGUMENT((reg & REG_PAIR_MASK) || (type & SLJIT_MEM_ALIGNED_32));
2675 }
2676
2677 allowed_flags = SLJIT_MEM_UNALIGNED;
2678
2679 switch (type & 0xff) {
2680 case SLJIT_MOV_P:
2681 case SLJIT_MOV:
2682 allowed_flags |= SLJIT_MEM_ALIGNED_32;
2683 /* fallthrough */
2684 case SLJIT_MOV_U32:
2685 case SLJIT_MOV_S32:
2686 case SLJIT_MOV32:
2687 allowed_flags |= SLJIT_MEM_ALIGNED_16;
2688 break;
2689 }
2690
2691 CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | allowed_flags)) == 0);
2692
2693 if (reg & REG_PAIR_MASK) {
2694 CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV);
2695 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_FIRST(reg)));
2696 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_SECOND(reg)));
2697 CHECK_ARGUMENT(REG_PAIR_FIRST(reg) != REG_PAIR_SECOND(reg));
2698 } else {
2699 CHECK_ARGUMENT((type & 0xff) >= SLJIT_MOV && (type & 0xff) <= SLJIT_MOV_P);
2700 CHECK_ARGUMENT(!(type & SLJIT_32) || ((type & 0xff) >= SLJIT_MOV_U8 && (type & 0xff) <= SLJIT_MOV_S16));
2701 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2702 }
2703
2704 FUNCTION_CHECK_SRC_MEM(mem, memw);
2705 #endif /* SLJIT_ARGUMENT_CHECKS */
2706 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2707 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2708 if ((type & 0xff) == SLJIT_MOV32)
2709 fprintf(compiler->verbose, " %s32",
2710 (type & SLJIT_MEM_STORE) ? "store" : "load");
2711 else
2712 fprintf(compiler->verbose, " %s%s%s",
2713 (type & SLJIT_MEM_STORE) ? "store" : "load",
2714 !(type & SLJIT_32) ? "" : "32", op1_types[(type & 0xff) - SLJIT_OP1_BASE]);
2715
2716 if (type & SLJIT_MEM_UNALIGNED)
2717 printf(".unal");
2718 else if (type & SLJIT_MEM_ALIGNED_16)
2719 printf(".al16");
2720 else if (type & SLJIT_MEM_ALIGNED_32)
2721 printf(".al32");
2722
2723 if (reg & REG_PAIR_MASK) {
2724 fprintf(compiler->verbose, " {");
2725 sljit_verbose_reg(compiler, REG_PAIR_FIRST(reg));
2726 fprintf(compiler->verbose, ", ");
2727 sljit_verbose_reg(compiler, REG_PAIR_SECOND(reg));
2728 fprintf(compiler->verbose, "}, ");
2729 } else {
2730 fprintf(compiler->verbose, " ");
2731 sljit_verbose_reg(compiler, reg);
2732 fprintf(compiler->verbose, ", ");
2733 }
2734 sljit_verbose_param(compiler, mem, memw);
2735 fprintf(compiler->verbose, "\n");
2736 }
2737 #endif /* SLJIT_VERBOSE */
2738 CHECK_RETURN_OK;
2739 }
2740
2741 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
2742 sljit_s32 reg,
2743 sljit_s32 mem, sljit_sw memw)
2744 {
2745 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2746 compiler->skip_checks = 0;
2747 CHECK_RETURN_OK;
2748 }
2749
2750 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2751 CHECK_ARGUMENT((type & 0xff) >= SLJIT_MOV && (type & 0xff) <= SLJIT_MOV_P);
2752 CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_SUPP | SLJIT_MEM_POST)) == 0);
2753 CHECK_ARGUMENT((mem & REG_MASK) != 0 && (mem & REG_MASK) != reg);
2754
2755 FUNCTION_CHECK_SRC_MEM(mem, memw);
2756 #endif /* SLJIT_ARGUMENT_CHECKS */
2757 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2758 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2759 if (type & SLJIT_MEM_SUPP)
2760 CHECK_RETURN_OK;
2761 if (sljit_emit_mem_update(compiler, type | SLJIT_MEM_SUPP, reg, mem, memw) == SLJIT_ERR_UNSUPPORTED) {
2762 fprintf(compiler->verbose, " # mem: unsupported form, no instructions are emitted\n");
2763 CHECK_RETURN_OK;
2764 }
2765
2766 if ((type & 0xff) == SLJIT_MOV32)
2767 fprintf(compiler->verbose, " %s32.%s ",
2768 (type & SLJIT_MEM_STORE) ? "store" : "load",
2769 (type & SLJIT_MEM_POST) ? "post" : "pre");
2770 else
2771 fprintf(compiler->verbose, " %s%s%s.%s ",
2772 (type & SLJIT_MEM_STORE) ? "store" : "load",
2773 !(type & SLJIT_32) ? "" : "32",
2774 op1_types[(type & 0xff) - SLJIT_OP1_BASE],
2775 (type & SLJIT_MEM_POST) ? "post" : "pre");
2776
2777 sljit_verbose_reg(compiler, reg);
2778 fprintf(compiler->verbose, ", ");
2779 sljit_verbose_param(compiler, mem, memw);
2780 fprintf(compiler->verbose, "\n");
2781 }
2782 #endif /* SLJIT_VERBOSE */
2783 CHECK_RETURN_OK;
2784 }
2785
2786 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fmem(struct sljit_compiler *compiler, sljit_s32 type,
2787 sljit_s32 freg,
2788 sljit_s32 mem, sljit_sw memw)
2789 {
2790 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2791 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2792 CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV_F64);
2793
2794 if (type & SLJIT_MEM_UNALIGNED) {
2795 CHECK_ARGUMENT(!(type & (SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2796 } else if (type & SLJIT_MEM_ALIGNED_16) {
2797 CHECK_ARGUMENT(!(type & SLJIT_MEM_ALIGNED_32));
2798 } else {
2799 CHECK_ARGUMENT(type & SLJIT_MEM_ALIGNED_32);
2800 CHECK_ARGUMENT(!(type & SLJIT_32));
2801 }
2802
2803 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_UNALIGNED | SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2804 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, type & SLJIT_32));
2805 FUNCTION_CHECK_SRC_MEM(mem, memw);
2806 #endif /* SLJIT_ARGUMENT_CHECKS */
2807 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2808 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2809 fprintf(compiler->verbose, " %s.%s",
2810 (type & SLJIT_MEM_STORE) ? "store" : "load",
2811 !(type & SLJIT_32) ? "f64" : "f32");
2812
2813 if (type & SLJIT_MEM_UNALIGNED)
2814 printf(".unal");
2815 else if (type & SLJIT_MEM_ALIGNED_16)
2816 printf(".al16");
2817 else if (type & SLJIT_MEM_ALIGNED_32)
2818 printf(".al32");
2819
2820 fprintf(compiler->verbose, " ");
2821 sljit_verbose_freg(compiler, freg);
2822 fprintf(compiler->verbose, ", ");
2823 sljit_verbose_param(compiler, mem, memw);
2824 fprintf(compiler->verbose, "\n");
2825 }
2826 #endif /* SLJIT_VERBOSE */
2827 CHECK_RETURN_OK;
2828 }
2829
2830 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
2831 sljit_s32 freg,
2832 sljit_s32 mem, sljit_sw memw)
2833 {
2834 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2835 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2836 CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV_F64);
2837 CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_SUPP | SLJIT_MEM_POST)) == 0);
2838 FUNCTION_CHECK_SRC_MEM(mem, memw);
2839 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, type & SLJIT_32));
2840 #endif /* SLJIT_ARGUMENT_CHECKS */
2841 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2842 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2843 if (type & SLJIT_MEM_SUPP)
2844 CHECK_RETURN_OK;
2845 if (sljit_emit_fmem_update(compiler, type | SLJIT_MEM_SUPP, freg, mem, memw) == SLJIT_ERR_UNSUPPORTED) {
2846 fprintf(compiler->verbose, " # fmem: unsupported form, no instructions are emitted\n");
2847 CHECK_RETURN_OK;
2848 }
2849
2850 fprintf(compiler->verbose, " %s.%s.%s ",
2851 (type & SLJIT_MEM_STORE) ? "store" : "load",
2852 !(type & SLJIT_32) ? "f64" : "f32",
2853 (type & SLJIT_MEM_POST) ? "post" : "pre");
2854
2855 sljit_verbose_freg(compiler, freg);
2856 fprintf(compiler->verbose, ", ");
2857 sljit_verbose_param(compiler, mem, memw);
2858 fprintf(compiler->verbose, "\n");
2859 }
2860 #endif /* SLJIT_VERBOSE */
2861 CHECK_RETURN_OK;
2862 }
2863
2864 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
2865 sljit_s32 vreg,
2866 sljit_s32 srcdst, sljit_sw srcdstw)
2867 {
2868 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2869 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
2870 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(SLJIT_SIMD_STORE)) == 0);
2871 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
2872 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) <= SLJIT_SIMD_GET_REG_SIZE(type));
2873 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) <= (srcdst & SLJIT_MEM) ? SLJIT_SIMD_GET_REG_SIZE(type) : 0);
2874 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
2875 FUNCTION_VCHECK(srcdst, srcdstw, type);
2876 #endif /* SLJIT_ARGUMENT_CHECKS */
2877 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2878 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2879 if (type & SLJIT_SIMD_TEST)
2880 CHECK_RETURN_OK;
2881 if (sljit_emit_simd_mov(compiler, type | SLJIT_SIMD_TEST, vreg, srcdst, srcdstw) == SLJIT_ERR_UNSUPPORTED) {
2882 fprintf(compiler->verbose, " # simd_mem: unsupported form, no instructions are emitted\n");
2883 CHECK_RETURN_OK;
2884 }
2885
2886 fprintf(compiler->verbose, " simd_%s.%d.%s%d",
2887 (type & SLJIT_SIMD_STORE) ? "store" : "load",
2888 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
2889 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2890 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
2891
2892 if ((type & 0x3f000000) == SLJIT_SIMD_MEM_UNALIGNED)
2893 fprintf(compiler->verbose, ".unal ");
2894 else
2895 fprintf(compiler->verbose, ".al%d ", (8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)));
2896
2897 sljit_verbose_vreg(compiler, vreg);
2898 fprintf(compiler->verbose, ", ");
2899 sljit_verbose_vparam(compiler, srcdst, srcdstw);
2900 fprintf(compiler->verbose, "\n");
2901 }
2902 #endif /* SLJIT_VERBOSE */
2903 CHECK_RETURN_OK;
2904 }
2905
2906 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
2907 sljit_s32 vreg,
2908 sljit_s32 src, sljit_sw srcw)
2909 {
2910 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2911 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
2912 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(0)) == 0);
2913 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
2914 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
2915 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
2916
2917 if (type & SLJIT_SIMD_FLOAT) {
2918 if (src == SLJIT_IMM) {
2919 CHECK_ARGUMENT(srcw == 0);
2920 } else {
2921 FUNCTION_FCHECK(src, srcw, SLJIT_SIMD_GET_ELEM_SIZE(type) == 2);
2922 }
2923 } else if (src != SLJIT_IMM) {
2924 FUNCTION_CHECK_DST(src, srcw);
2925 }
2926 #endif /* SLJIT_ARGUMENT_CHECKS */
2927 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2928 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2929 if (type & SLJIT_SIMD_TEST)
2930 CHECK_RETURN_OK;
2931 if (sljit_emit_simd_replicate(compiler, type | SLJIT_SIMD_TEST, vreg, src, srcw) == SLJIT_ERR_UNSUPPORTED) {
2932 fprintf(compiler->verbose, " # simd_dup: unsupported form, no instructions are emitted\n");
2933 CHECK_RETURN_OK;
2934 }
2935
2936 fprintf(compiler->verbose, " simd_replicate.%d.%s%d ",
2937 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
2938 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2939 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
2940
2941 sljit_verbose_vreg(compiler, vreg);
2942 fprintf(compiler->verbose, ", ");
2943 if (type & SLJIT_SIMD_FLOAT)
2944 sljit_verbose_fparam(compiler, src, srcw);
2945 else
2946 sljit_verbose_param(compiler, src, srcw);
2947 fprintf(compiler->verbose, "\n");
2948 }
2949 #endif /* SLJIT_VERBOSE */
2950 CHECK_RETURN_OK;
2951 }
2952
2953 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
2954 sljit_s32 vreg, sljit_s32 lane_index,
2955 sljit_s32 srcdst, sljit_sw srcdstw)
2956 {
2957 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2958 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
2959 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO | SLJIT_SIMD_LANE_SIGNED | SLJIT_32)) == 0);
2960 CHECK_ARGUMENT((type & (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO)) != (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO));
2961 CHECK_ARGUMENT((type & (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_SIGNED)) != SLJIT_SIMD_LANE_SIGNED);
2962 CHECK_ARGUMENT(!(type & SLJIT_SIMD_FLOAT) || !(type & (SLJIT_SIMD_LANE_SIGNED | SLJIT_32)));
2963 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
2964 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
2965 CHECK_ARGUMENT(!(type & SLJIT_32) || SLJIT_SIMD_GET_ELEM_SIZE(type) <= 2);
2966 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
2967 CHECK_ARGUMENT(lane_index >= 0 && lane_index < (1 << (SLJIT_SIMD_GET_REG_SIZE(type) - SLJIT_SIMD_GET_ELEM_SIZE(type))));
2968
2969 if (type & SLJIT_SIMD_FLOAT) {
2970 FUNCTION_FCHECK(srcdst, srcdstw, SLJIT_SIMD_GET_ELEM_SIZE(type) == 2);
2971 } else if ((type & SLJIT_SIMD_STORE) || srcdst != SLJIT_IMM) {
2972 FUNCTION_CHECK_DST(srcdst, srcdstw);
2973 }
2974 #endif /* SLJIT_ARGUMENT_CHECKS */
2975 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2976 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2977 if (type & SLJIT_SIMD_TEST)
2978 CHECK_RETURN_OK;
2979 if (sljit_emit_simd_lane_mov(compiler, type | SLJIT_SIMD_TEST, vreg, lane_index, srcdst, srcdstw) == SLJIT_ERR_UNSUPPORTED) {
2980 fprintf(compiler->verbose, " # simd_move_lane: unsupported form, no instructions are emitted\n");
2981 CHECK_RETURN_OK;
2982 }
2983
2984 fprintf(compiler->verbose, " simd_%s_lane%s%s%s.%d.%s%d ",
2985 (type & SLJIT_SIMD_STORE) ? "store" : "load",
2986 (type & SLJIT_32) ? "32" : "",
2987 (type & SLJIT_SIMD_LANE_ZERO) ? "_z" : "",
2988 (type & SLJIT_SIMD_LANE_SIGNED) ? "_s" : "",
2989 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
2990 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2991 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
2992
2993 sljit_verbose_vreg(compiler, vreg);
2994 fprintf(compiler->verbose, "[%d], ", lane_index);
2995 if (type & SLJIT_SIMD_FLOAT)
2996 sljit_verbose_fparam(compiler, srcdst, srcdstw);
2997 else
2998 sljit_verbose_param(compiler, srcdst, srcdstw);
2999 fprintf(compiler->verbose, "\n");
3000 }
3001 #endif /* SLJIT_VERBOSE */
3002 CHECK_RETURN_OK;
3003 }
3004
3005 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3006 sljit_s32 vreg,
3007 sljit_s32 src, sljit_s32 src_lane_index)
3008 {
3009 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3010 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3011 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(0)) == 0);
3012 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3013 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3014 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3015 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(src, type));
3016 CHECK_ARGUMENT(src_lane_index >= 0 && src_lane_index < (1 << (SLJIT_SIMD_GET_REG_SIZE(type) - SLJIT_SIMD_GET_ELEM_SIZE(type))));
3017 #endif /* SLJIT_ARGUMENT_CHECKS */
3018 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3019 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3020 if (type & SLJIT_SIMD_TEST)
3021 CHECK_RETURN_OK;
3022 if (sljit_emit_simd_lane_replicate(compiler, type | SLJIT_SIMD_TEST, vreg, src, src_lane_index) == SLJIT_ERR_UNSUPPORTED) {
3023 fprintf(compiler->verbose, " # simd_lane_replicate: unsupported form, no instructions are emitted\n");
3024 CHECK_RETURN_OK;
3025 }
3026
3027 fprintf(compiler->verbose, " simd_lane_replicate.%d.%s%d ",
3028 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3029 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3030 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3031
3032 sljit_verbose_vreg(compiler, vreg);
3033 fprintf(compiler->verbose, ", ");
3034 sljit_verbose_vreg(compiler, src);
3035 fprintf(compiler->verbose, "[%d]\n", src_lane_index);
3036 }
3037 #endif /* SLJIT_VERBOSE */
3038 CHECK_RETURN_OK;
3039 }
3040
3041 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
3042 sljit_s32 vreg,
3043 sljit_s32 src, sljit_sw srcw)
3044 {
3045 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3046 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3047 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(SLJIT_SIMD_EXTEND_SIGNED)) == 0);
3048 CHECK_ARGUMENT((type & (SLJIT_SIMD_EXTEND_SIGNED | SLJIT_SIMD_FLOAT)) != (SLJIT_SIMD_EXTEND_SIGNED | SLJIT_SIMD_FLOAT));
3049 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3050 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3051 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_ELEM2_SIZE(type));
3052 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3053 FUNCTION_VCHECK(src, srcw, type);
3054 #endif /* SLJIT_ARGUMENT_CHECKS */
3055 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3056 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3057 if (type & SLJIT_SIMD_TEST)
3058 CHECK_RETURN_OK;
3059 if (sljit_emit_simd_extend(compiler, type | SLJIT_SIMD_TEST, vreg, src, srcw) == SLJIT_ERR_UNSUPPORTED) {
3060 fprintf(compiler->verbose, " # simd_extend: unsupported form, no instructions are emitted\n");
3061 CHECK_RETURN_OK;
3062 }
3063
3064 fprintf(compiler->verbose, " simd_load_extend%s.%d.%s%d.%s%d ",
3065 (type & SLJIT_SIMD_EXTEND_SIGNED) ? "_s" : "",
3066 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3067 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3068 (8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)),
3069 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3070 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3071
3072 sljit_verbose_vreg(compiler, vreg);
3073 fprintf(compiler->verbose, ", ");
3074 sljit_verbose_vparam(compiler, src, srcw);
3075 fprintf(compiler->verbose, "\n");
3076 }
3077 #endif /* SLJIT_VERBOSE */
3078 CHECK_RETURN_OK;
3079 }
3080
3081 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
3082 sljit_s32 vreg,
3083 sljit_s32 dst, sljit_sw dstw)
3084 {
3085 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3086 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3087 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(SLJIT_32)) == SLJIT_SIMD_STORE);
3088 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3089 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3090 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3091 FUNCTION_CHECK_DST(dst, dstw);
3092 #endif /* SLJIT_ARGUMENT_CHECKS */
3093 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3094 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3095 if (type & SLJIT_SIMD_TEST)
3096 CHECK_RETURN_OK;
3097 if (sljit_emit_simd_sign(compiler, type | SLJIT_SIMD_TEST, vreg, dst, dstw) == SLJIT_ERR_UNSUPPORTED) {
3098 fprintf(compiler->verbose, " # simd_sign: unsupported form, no instructions are emitted\n");
3099 CHECK_RETURN_OK;
3100 }
3101
3102 fprintf(compiler->verbose, " simd_store_sign%s.%d.%s%d ",
3103 (type & SLJIT_32) ? "32" : "",
3104 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3105 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3106 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3107
3108 sljit_verbose_vreg(compiler, vreg);
3109 fprintf(compiler->verbose, ", ");
3110 sljit_verbose_param(compiler, dst, dstw);
3111 fprintf(compiler->verbose, "\n");
3112 }
3113 #endif /* SLJIT_VERBOSE */
3114 CHECK_RETURN_OK;
3115 }
3116
3117 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
3118 sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
3119 {
3120 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3121 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3122 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(0)) >= SLJIT_SIMD_OP2_AND && (type & SLJIT_SIMD_TYPE_MASK2(0)) <= SLJIT_SIMD_OP2_SHUFFLE);
3123 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3124 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) <= SLJIT_SIMD_GET_REG_SIZE(type));
3125 CHECK_ARGUMENT(SLJIT_SIMD_GET_OPCODE(type) != SLJIT_SIMD_OP2_SHUFFLE || (SLJIT_SIMD_GET_ELEM_SIZE(type) == 0 && !(type & SLJIT_SIMD_FLOAT)));
3126 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) <= (src2 & SLJIT_MEM) ? SLJIT_SIMD_GET_REG_SIZE(type) : 0);
3127 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(dst_vreg, type));
3128 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(src1_vreg, type));
3129 FUNCTION_VCHECK(src2, src2w, type);
3130 #endif /* SLJIT_ARGUMENT_CHECKS */
3131 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3132 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3133 if (type & SLJIT_SIMD_TEST)
3134 CHECK_RETURN_OK;
3135 if (sljit_emit_simd_op2(compiler, type | SLJIT_SIMD_TEST, dst_vreg, src1_vreg, src2, src2w) == SLJIT_ERR_UNSUPPORTED) {
3136 fprintf(compiler->verbose, " # simd_op2: unsupported form, no instructions are emitted\n");
3137 CHECK_RETURN_OK;
3138 }
3139
3140 fprintf(compiler->verbose, " simd_%s.%d.%s%d",
3141 simd_op2_names[SLJIT_SIMD_GET_OPCODE(type) - 1],
3142 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3143 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3144 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3145
3146 if ((type & 0x3f000000) != SLJIT_SIMD_MEM_UNALIGNED)
3147 fprintf(compiler->verbose, ".al%d", (8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)));
3148
3149 fprintf(compiler->verbose, " ");
3150 sljit_verbose_vreg(compiler, dst_vreg);
3151 fprintf(compiler->verbose, ", ");
3152 sljit_verbose_vreg(compiler, src1_vreg);
3153 fprintf(compiler->verbose, ", ");
3154 sljit_verbose_vparam(compiler, src2, src2w);
3155 fprintf(compiler->verbose, "\n");
3156 }
3157 #endif /* SLJIT_VERBOSE */
3158 CHECK_RETURN_OK;
3159 }
3160
3161 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
3162 {
3163 /* Any offset is allowed. */
3164 SLJIT_UNUSED_ARG(offset);
3165
3166 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3167 FUNCTION_CHECK_DST(dst, dstw);
3168 #endif /* SLJIT_ARGUMENT_CHECKS */
3169 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3170 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3171 fprintf(compiler->verbose, " local_base ");
3172 sljit_verbose_param(compiler, dst, dstw);
3173 fprintf(compiler->verbose, ", #%" SLJIT_PRINT_D "d\n", offset);
3174 }
3175 #endif /* SLJIT_VERBOSE */
3176 CHECK_RETURN_OK;
3177 }
3178
3179 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
3180 {
3181 SLJIT_UNUSED_ARG(init_value);
3182
3183 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3184 FUNCTION_CHECK_DST(dst, dstw);
3185 #endif /* SLJIT_ARGUMENT_CHECKS */
3186 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3187 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3188 fprintf(compiler->verbose, " const ");
3189 sljit_verbose_param(compiler, dst, dstw);
3190 fprintf(compiler->verbose, ", #%" SLJIT_PRINT_D "d\n", init_value);
3191 }
3192 #endif /* SLJIT_VERBOSE */
3193 CHECK_RETURN_OK;
3194 }
3195
3196 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_mov_addr(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
3197 {
3198 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3199 FUNCTION_CHECK_DST(dst, dstw);
3200 #endif /* SLJIT_ARGUMENT_CHECKS */
3201 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3202 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3203 fprintf(compiler->verbose, " mov_addr ");
3204 sljit_verbose_param(compiler, dst, dstw);
3205 fprintf(compiler->verbose, "\n");
3206 }
3207 #endif /* SLJIT_VERBOSE */
3208 CHECK_RETURN_OK;
3209 }
3210
3211 #else /* !SLJIT_ARGUMENT_CHECKS && !SLJIT_VERBOSE */
3212
3213 #define SLJIT_SKIP_CHECKS(compiler)
3214
3215 #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
3216
3217 #define SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw) \
3218 SLJIT_COMPILE_ASSERT(!(SLJIT_CONV_SW_FROM_F64 & 0x1) && !(SLJIT_CONV_F64_FROM_SW & 0x1) && !(SLJIT_CONV_F64_FROM_UW & 0x1), \
3219 invalid_float_opcodes); \
3220 if (GET_OPCODE(op) >= SLJIT_CONV_SW_FROM_F64 && GET_OPCODE(op) <= SLJIT_CMP_F64) { \
3221 if (GET_OPCODE(op) == SLJIT_CMP_F64) { \
3222 CHECK(check_sljit_emit_fop1_cmp(compiler, op, dst, dstw, src, srcw)); \
3223 ADJUST_LOCAL_OFFSET(dst, dstw); \
3224 ADJUST_LOCAL_OFFSET(src, srcw); \
3225 return sljit_emit_fop1_cmp(compiler, op, dst, dstw, src, srcw); \
3226 } \
3227 if ((GET_OPCODE(op) | 0x1) == SLJIT_CONV_S32_FROM_F64) { \
3228 CHECK(check_sljit_emit_fop1_conv_sw_from_f64(compiler, op, dst, dstw, src, srcw)); \
3229 ADJUST_LOCAL_OFFSET(dst, dstw); \
3230 ADJUST_LOCAL_OFFSET(src, srcw); \
3231 return sljit_emit_fop1_conv_sw_from_f64(compiler, op, dst, dstw, src, srcw); \
3232 } \
3233 if ((GET_OPCODE(op) | 0x1) == SLJIT_CONV_F64_FROM_S32) { \
3234 CHECK(check_sljit_emit_fop1_conv_f64_from_w(compiler, op, dst, dstw, src, srcw)); \
3235 ADJUST_LOCAL_OFFSET(dst, dstw); \
3236 ADJUST_LOCAL_OFFSET(src, srcw); \
3237 return sljit_emit_fop1_conv_f64_from_sw(compiler, op, dst, dstw, src, srcw); \
3238 } \
3239 CHECK(check_sljit_emit_fop1_conv_f64_from_w(compiler, op, dst, dstw, src, srcw)); \
3240 ADJUST_LOCAL_OFFSET(dst, dstw); \
3241 ADJUST_LOCAL_OFFSET(src, srcw); \
3242 return sljit_emit_fop1_conv_f64_from_uw(compiler, op, dst, dstw, src, srcw); \
3243 } \
3244 CHECK(check_sljit_emit_fop1(compiler, op, dst, dstw, src, srcw)); \
3245 ADJUST_LOCAL_OFFSET(dst, dstw); \
3246 ADJUST_LOCAL_OFFSET(src, srcw);
3247
3248 #if (!(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) || (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6))
3249
3250 static sljit_s32 sljit_emit_mem_unaligned(struct sljit_compiler *compiler, sljit_s32 type,
3251 sljit_s32 reg,
3252 sljit_s32 mem, sljit_sw memw)
3253 {
3254 SLJIT_SKIP_CHECKS(compiler);
3255
3256 if (type & SLJIT_MEM_STORE)
3257 return sljit_emit_op1(compiler, type & (0xff | SLJIT_32), mem, memw, reg, 0);
3258 return sljit_emit_op1(compiler, type & (0xff | SLJIT_32), reg, 0, mem, memw);
3259 }
3260
3261 #endif /* (!SLJIT_CONFIG_MIPS || SLJIT_MIPS_REV >= 6) */
3262
3263 #if (!(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) || (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6)) \
3264 && !(defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32)
3265
3266 static sljit_s32 sljit_emit_fmem_unaligned(struct sljit_compiler *compiler, sljit_s32 type,
3267 sljit_s32 freg,
3268 sljit_s32 mem, sljit_sw memw)
3269 {
3270 SLJIT_SKIP_CHECKS(compiler);
3271
3272 if (type & SLJIT_MEM_STORE)
3273 return sljit_emit_fop1(compiler, type & (0xff | SLJIT_32), mem, memw, freg, 0);
3274 return sljit_emit_fop1(compiler, type & (0xff | SLJIT_32), freg, 0, mem, memw);
3275 }
3276
3277 #endif /* (!SLJIT_CONFIG_MIPS || SLJIT_MIPS_REV >= 6) && !SLJIT_CONFIG_ARM */
3278
3279 /* CPU description section */
3280
3281 #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
3282 #define SLJIT_CPUINFO_PART1 " 32bit ("
3283 #elif (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
3284 #define SLJIT_CPUINFO_PART1 " 64bit ("
3285 #else /* !SLJIT_32BIT_ARCHITECTURE && !SLJIT_64BIT_ARCHITECTURE */
3286 #error "Internal error: CPU type info missing"
3287 #endif /* SLJIT_32BIT_ARCHITECTURE */
3288
3289 #if (defined SLJIT_LITTLE_ENDIAN && SLJIT_LITTLE_ENDIAN)
3290 #define SLJIT_CPUINFO_PART2 "little endian + "
3291 #elif (defined SLJIT_BIG_ENDIAN && SLJIT_BIG_ENDIAN)
3292 #define SLJIT_CPUINFO_PART2 "big endian + "
3293 #else /* !SLJIT_LITTLE_ENDIAN && !SLJIT_BIG_ENDIAN */
3294 #error "Internal error: CPU type info missing"
3295 #endif /* SLJIT_LITTLE_ENDIAN */
3296
3297 #if (defined SLJIT_UNALIGNED && SLJIT_UNALIGNED)
3298 #define SLJIT_CPUINFO_PART3 "unaligned)"
3299 #else /* !SLJIT_UNALIGNED */
3300 #define SLJIT_CPUINFO_PART3 "aligned)"
3301 #endif /* SLJIT_UNALIGNED */
3302
3303 #define SLJIT_CPUINFO SLJIT_CPUINFO_PART1 SLJIT_CPUINFO_PART2 SLJIT_CPUINFO_PART3
3304
3305 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
3306 # include "sljitNativeX86_common.c"
3307 #elif (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
3308 # include "sljitNativeARM_32.c"
3309 #elif (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
3310 # include "sljitNativeARM_32.c"
3311 #elif (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
3312 # include "sljitNativeARM_T2_32.c"
3313 #elif (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3314 # include "sljitNativeARM_64.c"
3315 #elif (defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3316 # include "sljitNativePPC_common.c"
3317 #elif (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
3318 # include "sljitNativeMIPS_common.c"
3319 #elif (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
3320 # include "sljitNativeRISCV_common.c"
3321 #elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
3322 # include "sljitNativeS390X.c"
3323 #elif (defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3324 # include "sljitNativeLOONGARCH_64.c"
3325 #endif /* SLJIT_CONFIG_X86 */
3326
3327 #include "sljitSerialize.c"
3328
3329 static SLJIT_INLINE sljit_s32 emit_mov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3330 {
3331 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
3332 /* At the moment the pointer size is always equal to sljit_sw. May be changed in the future. */
3333 if (src == SLJIT_RETURN_REG && (op == SLJIT_MOV || op == SLJIT_MOV_P))
3334 return SLJIT_SUCCESS;
3335 #else /* !SLJIT_64BIT_ARCHITECTURE */
3336 if (src == SLJIT_RETURN_REG && (op == SLJIT_MOV || op == SLJIT_MOV_U32 || op == SLJIT_MOV_S32 || op == SLJIT_MOV_P))
3337 return SLJIT_SUCCESS;
3338 #endif /* SLJIT_64BIT_ARCHITECTURE */
3339
3340 SLJIT_SKIP_CHECKS(compiler);
3341 return sljit_emit_op1(compiler, op, SLJIT_RETURN_REG, 0, src, srcw);
3342 }
3343
3344 #if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) \
3345 && !((defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) && defined __SOFTFP__)
3346
3347 static SLJIT_INLINE sljit_s32 emit_fmov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3348 {
3349 if (src == SLJIT_FR0)
3350 return SLJIT_SUCCESS;
3351
3352 SLJIT_SKIP_CHECKS(compiler);
3353 return sljit_emit_fop1(compiler, op, SLJIT_RETURN_FREG, 0, src, srcw);
3354 }
3355
3356 #endif /* !SLJIT_CONFIG_X86_32 && !(SLJIT_CONFIG_ARM_32 && __SOFTFP__) */
3357
3358 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3359 {
3360 CHECK_ERROR();
3361 CHECK(check_sljit_emit_return(compiler, op, src, srcw));
3362
3363 if (GET_OPCODE(op) < SLJIT_MOV_F64) {
3364 FAIL_IF(emit_mov_before_return(compiler, op, src, srcw));
3365 } else {
3366 FAIL_IF(emit_fmov_before_return(compiler, op, src, srcw));
3367 }
3368
3369 SLJIT_SKIP_CHECKS(compiler);
3370 return sljit_emit_return_void(compiler);
3371 }
3372
3373 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3374 && !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) \
3375 && !(defined(SLJIT_CONFIG_LOONGARCH_64) && SLJIT_CONFIG_LOONGARCH_64)
3376
3377 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2r(struct sljit_compiler *compiler, sljit_s32 op,
3378 sljit_s32 dst_freg,
3379 sljit_s32 src1, sljit_sw src1w,
3380 sljit_s32 src2, sljit_sw src2w)
3381 {
3382 CHECK_ERROR();
3383 CHECK(check_sljit_emit_fop2r(compiler, op, dst_freg, src1, src1w, src2, src2w));
3384 ADJUST_LOCAL_OFFSET(src1, src1w);
3385 ADJUST_LOCAL_OFFSET(src2, src2w);
3386
3387 SLJIT_SKIP_CHECKS(compiler);
3388 return sljit_emit_fop2(compiler, op, dst_freg, 0, src1, src1w, src2, src2w);
3389 }
3390
3391 #endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_S390X && !SLJIT_CONFIG_LOONGARCH_64 */
3392
3393 #if !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) \
3394 && !(defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV) \
3395 && !(defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3396
3397 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_cmp(struct sljit_compiler *compiler, sljit_s32 type,
3398 sljit_s32 src1, sljit_sw src1w,
3399 sljit_s32 src2, sljit_sw src2w)
3400 {
3401 /* Default compare for most architectures. */
3402 sljit_s32 flags, tmp_src, condition;
3403 sljit_sw tmp_srcw;
3404
3405 CHECK_ERROR_PTR();
3406 CHECK_PTR(check_sljit_emit_cmp(compiler, type, src1, src1w, src2, src2w));
3407
3408 condition = type & 0xff;
3409 #if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3410 if ((condition == SLJIT_EQUAL || condition == SLJIT_NOT_EQUAL)) {
3411 if (src1 == SLJIT_IMM && !src1w) {
3412 src1 = src2;
3413 src1w = src2w;
3414 src2 = SLJIT_IMM;
3415 src2w = 0;
3416 }
3417 if (src2 == SLJIT_IMM && !src2w)
3418 return emit_cmp_to0(compiler, type, src1, src1w);
3419 }
3420 #endif /* SLJIT_CONFIG_ARM_64 */
3421
3422 if (SLJIT_UNLIKELY(src1 == SLJIT_IMM && src2 != SLJIT_IMM)) {
3423 /* Immediate is preferred as second argument by most architectures. */
3424 switch (condition) {
3425 case SLJIT_LESS:
3426 condition = SLJIT_GREATER;
3427 break;
3428 case SLJIT_GREATER_EQUAL:
3429 condition = SLJIT_LESS_EQUAL;
3430 break;
3431 case SLJIT_GREATER:
3432 condition = SLJIT_LESS;
3433 break;
3434 case SLJIT_LESS_EQUAL:
3435 condition = SLJIT_GREATER_EQUAL;
3436 break;
3437 case SLJIT_SIG_LESS:
3438 condition = SLJIT_SIG_GREATER;
3439 break;
3440 case SLJIT_SIG_GREATER_EQUAL:
3441 condition = SLJIT_SIG_LESS_EQUAL;
3442 break;
3443 case SLJIT_SIG_GREATER:
3444 condition = SLJIT_SIG_LESS;
3445 break;
3446 case SLJIT_SIG_LESS_EQUAL:
3447 condition = SLJIT_SIG_GREATER_EQUAL;
3448 break;
3449 }
3450
3451 type = condition | (type & (SLJIT_32 | SLJIT_REWRITABLE_JUMP));
3452 tmp_src = src1;
3453 src1 = src2;
3454 src2 = tmp_src;
3455 tmp_srcw = src1w;
3456 src1w = src2w;
3457 src2w = tmp_srcw;
3458 }
3459
3460 if (condition <= SLJIT_NOT_ZERO)
3461 flags = SLJIT_SET_Z;
3462 else
3463 flags = (condition & 0xfe) << VARIABLE_FLAG_SHIFT;
3464
3465 SLJIT_SKIP_CHECKS(compiler);
3466 PTR_FAIL_IF(sljit_emit_op2u(compiler,
3467 SLJIT_SUB | flags | (type & SLJIT_32), src1, src1w, src2, src2w));
3468
3469 SLJIT_SKIP_CHECKS(compiler);
3470 return sljit_emit_jump(compiler, condition | (type & (SLJIT_REWRITABLE_JUMP | SLJIT_32)));
3471 }
3472
3473 #endif /* !SLJIT_CONFIG_MIPS */
3474
3475 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32)
3476
3477 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
3478 {
3479 switch (type) {
3480 case SLJIT_UNORDERED_OR_EQUAL:
3481 case SLJIT_ORDERED_NOT_EQUAL:
3482 return 1;
3483 }
3484
3485 return 0;
3486 }
3487
3488 #endif /* SLJIT_CONFIG_ARM */
3489
3490 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_fcmp(struct sljit_compiler *compiler, sljit_s32 type,
3491 sljit_s32 src1, sljit_sw src1w,
3492 sljit_s32 src2, sljit_sw src2w)
3493 {
3494 CHECK_ERROR_PTR();
3495 CHECK_PTR(check_sljit_emit_fcmp(compiler, type, src1, src1w, src2, src2w));
3496
3497 SLJIT_SKIP_CHECKS(compiler);
3498 sljit_emit_fop1(compiler, SLJIT_CMP_F64 | ((type & 0xfe) << VARIABLE_FLAG_SHIFT) | (type & SLJIT_32), src1, src1w, src2, src2w);
3499
3500 SLJIT_SKIP_CHECKS(compiler);
3501 return sljit_emit_jump(compiler, type);
3502 }
3503
3504 #if !(defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM) \
3505 && !(defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3506
3507 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
3508 sljit_s32 reg,
3509 sljit_s32 mem, sljit_sw memw)
3510 {
3511 CHECK_ERROR();
3512 CHECK(check_sljit_emit_mem_update(compiler, type, reg, mem, memw));
3513 SLJIT_UNUSED_ARG(type);
3514 SLJIT_UNUSED_ARG(reg);
3515 SLJIT_UNUSED_ARG(mem);
3516 SLJIT_UNUSED_ARG(memw);
3517
3518 return SLJIT_ERR_UNSUPPORTED;
3519 }
3520
3521 #endif /* !SLJIT_CONFIG_ARM && !SLJIT_CONFIG_PPC */
3522
3523 #if !(defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
3524 && !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
3525
3526 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem(struct sljit_compiler *compiler, sljit_s32 type,
3527 sljit_s32 freg,
3528 sljit_s32 mem, sljit_sw memw)
3529 {
3530 CHECK_ERROR();
3531 CHECK(check_sljit_emit_fmem(compiler, type, freg, mem, memw));
3532
3533 return sljit_emit_fmem_unaligned(compiler, type, freg, mem, memw);
3534 }
3535
3536 #endif /* !SLJIT_CONFIG_ARM_32 && !SLJIT_CONFIG_MIPS */
3537
3538 #if !(defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64) \
3539 && !(defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3540
3541 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
3542 sljit_s32 freg,
3543 sljit_s32 mem, sljit_sw memw)
3544 {
3545 CHECK_ERROR();
3546 CHECK(check_sljit_emit_fmem_update(compiler, type, freg, mem, memw));
3547 SLJIT_UNUSED_ARG(type);
3548 SLJIT_UNUSED_ARG(freg);
3549 SLJIT_UNUSED_ARG(mem);
3550 SLJIT_UNUSED_ARG(memw);
3551
3552 return SLJIT_ERR_UNSUPPORTED;
3553 }
3554
3555 #endif /* !SLJIT_CONFIG_ARM_64 && !SLJIT_CONFIG_PPC */
3556
3557 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3558 && !(defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM) \
3559 && !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) \
3560 && !(defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV) \
3561 && !(defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3562
3563 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
3564 sljit_s32 vreg,
3565 sljit_s32 srcdst, sljit_sw srcdstw)
3566 {
3567 CHECK_ERROR();
3568 CHECK(check_sljit_emit_simd_mov(compiler, type, vreg, srcdst, srcdstw));
3569 SLJIT_UNUSED_ARG(compiler);
3570 SLJIT_UNUSED_ARG(type);
3571 SLJIT_UNUSED_ARG(vreg);
3572 SLJIT_UNUSED_ARG(srcdst);
3573 SLJIT_UNUSED_ARG(srcdstw);
3574
3575 return SLJIT_ERR_UNSUPPORTED;
3576 }
3577
3578 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3579 sljit_s32 vreg,
3580 sljit_s32 src, sljit_sw srcw)
3581 {
3582 CHECK_ERROR();
3583 CHECK(check_sljit_emit_simd_replicate(compiler, type, vreg, src, srcw));
3584 SLJIT_UNUSED_ARG(compiler);
3585 SLJIT_UNUSED_ARG(type);
3586 SLJIT_UNUSED_ARG(vreg);
3587 SLJIT_UNUSED_ARG(src);
3588 SLJIT_UNUSED_ARG(srcw);
3589
3590 return SLJIT_ERR_UNSUPPORTED;
3591 }
3592
3593 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
3594 sljit_s32 vreg, sljit_s32 lane_index,
3595 sljit_s32 srcdst, sljit_sw srcdstw)
3596 {
3597 CHECK_ERROR();
3598 CHECK(check_sljit_emit_simd_lane_mov(compiler, type, vreg, lane_index, srcdst, srcdstw));
3599 SLJIT_UNUSED_ARG(compiler);
3600 SLJIT_UNUSED_ARG(type);
3601 SLJIT_UNUSED_ARG(vreg);
3602 SLJIT_UNUSED_ARG(lane_index);
3603 SLJIT_UNUSED_ARG(srcdst);
3604 SLJIT_UNUSED_ARG(srcdstw);
3605
3606 return SLJIT_ERR_UNSUPPORTED;
3607 }
3608
3609 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3610 sljit_s32 vreg,
3611 sljit_s32 src, sljit_s32 src_lane_index)
3612 {
3613 CHECK_ERROR();
3614 CHECK(check_sljit_emit_simd_lane_replicate(compiler, type, vreg, src, src_lane_index));
3615 SLJIT_UNUSED_ARG(compiler);
3616 SLJIT_UNUSED_ARG(type);
3617 SLJIT_UNUSED_ARG(vreg);
3618 SLJIT_UNUSED_ARG(src);
3619 SLJIT_UNUSED_ARG(src_lane_index);
3620
3621 return SLJIT_ERR_UNSUPPORTED;
3622 }
3623
3624 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
3625 sljit_s32 vreg,
3626 sljit_s32 src, sljit_sw srcw)
3627 {
3628 CHECK_ERROR();
3629 CHECK(check_sljit_emit_simd_extend(compiler, type, vreg, src, srcw));
3630 SLJIT_UNUSED_ARG(compiler);
3631 SLJIT_UNUSED_ARG(type);
3632 SLJIT_UNUSED_ARG(vreg);
3633 SLJIT_UNUSED_ARG(src);
3634 SLJIT_UNUSED_ARG(srcw);
3635
3636 return SLJIT_ERR_UNSUPPORTED;
3637 }
3638
3639 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
3640 sljit_s32 vreg,
3641 sljit_s32 dst, sljit_sw dstw)
3642 {
3643 CHECK_ERROR();
3644 CHECK(check_sljit_emit_simd_sign(compiler, type, vreg, dst, dstw));
3645 SLJIT_UNUSED_ARG(compiler);
3646 SLJIT_UNUSED_ARG(type);
3647 SLJIT_UNUSED_ARG(vreg);
3648 SLJIT_UNUSED_ARG(dst);
3649 SLJIT_UNUSED_ARG(dstw);
3650
3651 return SLJIT_ERR_UNSUPPORTED;
3652 }
3653
3654 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
3655 sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
3656 {
3657 CHECK_ERROR();
3658 CHECK(check_sljit_emit_simd_op2(compiler, type, dst_vreg, src1_vreg, src2, src2w));
3659 SLJIT_UNUSED_ARG(compiler);
3660 SLJIT_UNUSED_ARG(type);
3661 SLJIT_UNUSED_ARG(dst_vreg);
3662 SLJIT_UNUSED_ARG(src1_vreg);
3663 SLJIT_UNUSED_ARG(src2);
3664 SLJIT_UNUSED_ARG(src2w);
3665
3666 return SLJIT_ERR_UNSUPPORTED;
3667 }
3668
3669 #endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM && !SLJIT_CONFIG_S390X && !SLJIT_CONFIG_LOONGARCH */
3670
3671 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3672 && !(defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3673
3674 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
3675 {
3676 CHECK_ERROR();
3677 CHECK(check_sljit_get_local_base(compiler, dst, dstw, offset));
3678
3679 ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_SP), offset);
3680
3681 SLJIT_SKIP_CHECKS(compiler);
3682
3683 if (offset != 0)
3684 return sljit_emit_op2(compiler, SLJIT_ADD, dst, dstw, SLJIT_SP, 0, SLJIT_IMM, offset);
3685 return sljit_emit_op1(compiler, SLJIT_MOV, dst, dstw, SLJIT_SP, 0);
3686 }
3687
3688 #endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM_64 */
3689
3690 #endif /* !SLJIT_CONFIG_UNSUPPORTED */