Debian Codegraph @

src/3rdparty/pcre2/deps/sljit/sljit_src/sljitLir.c

raw
1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include "sljitLir.h"
28
29 #ifdef _WIN32
30
31 #include <windows.h>
32
33 #endif /* _WIN32 */
34
35 #if !(defined SLJIT_STD_MACROS_DEFINED && SLJIT_STD_MACROS_DEFINED)
36
37 /* These libraries are needed for the macros below. */
38 #include <stdlib.h>
39 #include <string.h>
40
41 #endif /* SLJIT_STD_MACROS_DEFINED */
42
43 #define CHECK_ERROR() \
44 do { \
45 if (SLJIT_UNLIKELY(compiler->error)) \
46 return compiler->error; \
47 } while (0)
48
49 #define CHECK_ERROR_PTR() \
50 do { \
51 if (SLJIT_UNLIKELY(compiler->error)) \
52 return NULL; \
53 } while (0)
54
55 #define FAIL_IF(expr) \
56 do { \
57 if (SLJIT_UNLIKELY(expr)) \
58 return compiler->error; \
59 } while (0)
60
61 #define PTR_FAIL_IF(expr) \
62 do { \
63 if (SLJIT_UNLIKELY(expr)) \
64 return NULL; \
65 } while (0)
66
67 #define FAIL_IF_NULL(ptr) \
68 do { \
69 if (SLJIT_UNLIKELY(!(ptr))) { \
70 compiler->error = SLJIT_ERR_ALLOC_FAILED; \
71 return SLJIT_ERR_ALLOC_FAILED; \
72 } \
73 } while (0)
74
75 #define PTR_FAIL_IF_NULL(ptr) \
76 do { \
77 if (SLJIT_UNLIKELY(!(ptr))) { \
78 compiler->error = SLJIT_ERR_ALLOC_FAILED; \
79 return NULL; \
80 } \
81 } while (0)
82
83 #define PTR_FAIL_WITH_EXEC_IF(ptr) \
84 do { \
85 if (SLJIT_UNLIKELY(!(ptr))) { \
86 compiler->error = SLJIT_ERR_EX_ALLOC_FAILED; \
87 return NULL; \
88 } \
89 } while (0)
90
91 #if !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)
92
93 #define SSIZE_OF(type) ((sljit_s32)sizeof(sljit_ ## type))
94
95 #define VARIABLE_FLAG_SHIFT (10)
96 /* All variable flags are even. */
97 #define VARIABLE_FLAG_MASK (0x3e << VARIABLE_FLAG_SHIFT)
98 #define ALL_STATUS_FLAGS_MASK (VARIABLE_FLAG_MASK | SLJIT_SET_Z)
99 #define GET_FLAG_TYPE(op) ((op) >> VARIABLE_FLAG_SHIFT)
100 #define GET_FLAG_TYPE_MASK(op) (((op) >> VARIABLE_FLAG_SHIFT) & 0x3e)
101 #define GET_OPCODE(op) ((op) & 0xff)
102 #define HAS_FLAGS(op) ((op) & ALL_STATUS_FLAGS_MASK)
103 #define GET_ALL_FLAGS(op) ((op) & (SLJIT_32 | ALL_STATUS_FLAGS_MASK))
104
105 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
106 #define TYPE_CAST_NEEDED(op) \
107 ((op) >= SLJIT_MOV_U8 && (op) <= SLJIT_MOV_S32)
108 #else /* !SLJIT_64BIT_ARCHITECTURE */
109 #define TYPE_CAST_NEEDED(op) \
110 ((op) >= SLJIT_MOV_U8 && (op) <= SLJIT_MOV_S16)
111 #endif /* SLJIT_64BIT_ARCHITECTURE */
112
113 #define BUF_SIZE 4096
114
115 #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
116 #define ABUF_SIZE 2048
117 #else /* !SLJIT_32BIT_ARCHITECTURE */
118 #define ABUF_SIZE 4096
119 #endif /* SLJIT_32BIT_ARCHITECTURE */
120
121 /* Parameter parsing. */
122 #define REG_MASK 0x7f
123 #define OFFS_REG(reg) (((reg) >> 8) & REG_MASK)
124 #define OFFS_REG_MASK (REG_MASK << 8)
125 #define TO_OFFS_REG(reg) ((reg) << 8)
126 #define FAST_IS_REG(reg) ((reg) < REG_MASK)
127
128 /* Mask for argument types. */
129 #define SLJIT_ARG_MASK 0x7
130 #define SLJIT_ARG_FULL_MASK (SLJIT_ARG_MASK | SLJIT_ARG_TYPE_SCRATCH_REG)
131
132 /* Mask for register pairs. */
133 #define REG_PAIR_MASK 0x7f00
134 #define REG_PAIR_FIRST(reg) ((reg) & 0x7f)
135 #define REG_PAIR_SECOND(reg) ((reg) >> 8)
136
137 /* Mask for sljit_emit_enter. */
138 #define ENTER_GET_REGS(regs) ((regs) & 0xff)
139 #define ENTER_GET_FLOAT_REGS(regs) (((regs) >> 8) & 0xff)
140 #define ENTER_GET_VECTOR_REGS(regs) (((regs) >> 16) & 0xff)
141 #define SLJIT_KEPT_SAVEDS_COUNT(options) ((options) & 0x3)
142
143 /* Getters for simd operations, which returns with log2(size). */
144 #define SLJIT_SIMD_GET_OPCODE(type) ((type) & 0xff)
145 #define SLJIT_SIMD_GET_REG_SIZE(type) (((type) >> 12) & 0x3f)
146 #define SLJIT_SIMD_GET_ELEM_SIZE(type) (((type) >> 18) & 0x3f)
147 #define SLJIT_SIMD_GET_ELEM2_SIZE(type) (((type) >> 24) & 0x3f)
148
149 #define SLJIT_SIMD_CHECK_REG(type) (((type) & 0x3f000) >= SLJIT_SIMD_REG_64 && ((type) & 0x3f000) <= SLJIT_SIMD_REG_512)
150 #define SLJIT_SIMD_TYPE_MASK(m) ((sljit_s32)0xff000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))
151 #define SLJIT_SIMD_TYPE_MASK2(m) ((sljit_s32)0xc0000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))
152
153 /* Label definitions. */
154
155 #define SLJIT_LABEL_ALIGNED ((~(sljit_uw)0) - 1)
156
157 struct sljit_extended_label {
158 struct sljit_label label;
159 sljit_uw index;
160 sljit_uw data;
161 };
162
163 /* Jump definitions. */
164
165 /* Jump flag bits. */
166 #define JUMP_ADDR 0x1
167 #define JUMP_MOV_ADDR 0x2
168 /* SLJIT_REWRITABLE_JUMP is 0x10000. */
169
170 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
171 # define PATCH_MB 0x04
172 # define PATCH_MW 0x08
173 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
174 # define PATCH_MD 0x10
175 # define MOV_ADDR_HI 0x20
176 # define JUMP_MAX_SIZE ((sljit_uw)(10 + 3))
177 # define CJUMP_MAX_SIZE ((sljit_uw)(2 + 10 + 3))
178 #else /* !SLJIT_CONFIG_X86_64 */
179 # define JUMP_MAX_SIZE ((sljit_uw)5)
180 # define CJUMP_MAX_SIZE ((sljit_uw)6)
181 #endif /* SLJIT_CONFIG_X86_64 */
182 # define TYPE_SHIFT 17
183 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
184 /* Bits 7..15 is for debug jump size, SLJIT_REWRITABLE_JUMP is 0x10000 */
185 # define JUMP_SIZE_SHIFT 8
186 #endif /* SLJIT_DEBUG */
187 #endif /* SLJIT_CONFIG_X86 */
188
189 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6) || (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
190 /* SLJIT_REWRITABLE_JUMP is 0x10000. */
191 # define IS_BL 0x04
192 # define PATCH_B 0x08
193 #endif /* SLJIT_CONFIG_ARM_V6 || SLJIT_CONFIG_ARM_V7 */
194
195 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
196 # define CPOOL_SIZE 512
197 #endif /* SLJIT_CONFIG_ARM_V6 */
198
199 #if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
200 # define JUMP_SIZE_SHIFT 26
201 # define JUMP_MAX_SIZE ((sljit_uw)3)
202 #endif /* SLJIT_CONFIG_ARM_V7 */
203
204 #if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
205 /* SLJIT_REWRITABLE_JUMP is 0x10000. */
206 # define IS_COND 0x04
207 # define IS_BL 0x08
208 /* mov_addr does not use IS_BL */
209 # define IS_ABS IS_BL
210 /* conditional + imm8 */
211 # define PATCH_TYPE1 0x10
212 /* conditional + imm20 */
213 # define PATCH_TYPE2 0x20
214 /* imm11 */
215 # define PATCH_TYPE3 0x30
216 /* imm24 */
217 # define PATCH_TYPE4 0x40
218 /* BL + imm24 */
219 # define PATCH_TYPE5 0x50
220 /* addwi/subwi */
221 # define PATCH_TYPE6 0x60
222 /* 0xf00 cc code for branches */
223 # define JUMP_SIZE_SHIFT 26
224 # define JUMP_MAX_SIZE ((sljit_uw)5)
225 #endif /* SLJIT_CONFIG_ARM_THUMB2 */
226
227 #if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
228 /* SLJIT_REWRITABLE_JUMP is 0x10000. */
229 # define IS_COND 0x004
230 # define IS_CBZ 0x008
231 # define IS_BL 0x010
232 # define PATCH_COND 0x020
233 # define PATCH_B 0x040
234 # define PATCH_B32 0x080
235 # define PATCH_ABS48 0x100
236 # define PATCH_ABS64 0x200
237 # define JUMP_SIZE_SHIFT 58
238 # define JUMP_MAX_SIZE ((sljit_uw)5)
239 #endif /* SLJIT_CONFIG_ARM_64 */
240
241 #if (defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
242 /* SLJIT_REWRITABLE_JUMP is 0x10000. */
243 # define IS_COND 0x004
244 # define IS_CALL 0x008
245 # define PATCH_B 0x010
246 # define PATCH_ABS_B 0x020
247 #if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
248 # define PATCH_ABS32 0x040
249 # define PATCH_ABS48 0x080
250 # define JUMP_SIZE_SHIFT 58
251 # define JUMP_MAX_SIZE ((sljit_uw)7)
252 #else /* !SLJIT_CONFIG_PPC_64 */
253 # define JUMP_SIZE_SHIFT 26
254 # define JUMP_MAX_SIZE ((sljit_uw)4)
255 #endif /* SLJIT_CONFIG_PPC_64 */
256 #endif /* SLJIT_CONFIG_PPC */
257
258 #if (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
259 /* SLJIT_REWRITABLE_JUMP is 0x10000. */
260 # define IS_MOVABLE 0x004
261 # define IS_JAL 0x008
262 # define IS_CALL 0x010
263 # define IS_BIT26_COND 0x020
264 # define IS_BIT16_COND 0x040
265 # define IS_BIT23_COND 0x080
266
267 # define IS_COND (IS_BIT26_COND | IS_BIT16_COND | IS_BIT23_COND)
268
269 # define PATCH_B 0x100
270 # define PATCH_J 0x200
271
272 #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64)
273 # define PATCH_ABS32 0x400
274 # define PATCH_ABS48 0x800
275 #endif /* SLJIT_CONFIG_MIPS_64 */
276
277 /* instruction types */
278 # define MOVABLE_INS 0
279 /* 1 - 31 last destination register */
280 /* no destination (i.e: store) */
281 # define UNMOVABLE_INS 32
282 /* FPU status register */
283 # define FCSR_FCC 33
284 #endif /* SLJIT_CONFIG_MIPS */
285
286 #if (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
287 /* SLJIT_REWRITABLE_JUMP is 0x10000. */
288 # define IS_COND 0x0004
289 # define IS_CALL 0x0008
290
291 # define IS_COND16 0x0010
292 # define PATCH_B 0x0020
293 # define PATCH_J 0x0040
294 # define PATCH_16 0x0080
295
296 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
297 # define PATCH_REL32 0x0100
298 # define PATCH_ABS32 0x0200
299 # define PATCH_ABS44 0x0400
300 # define PATCH_ABS52 0x0800
301 # define JUMP_SIZE_SHIFT 58
302 # define JUMP_MAX_SIZE ((sljit_uw)12)
303 #else /* !SLJIT_CONFIG_RISCV_64 */
304 # define JUMP_SIZE_SHIFT 26
305 # define JUMP_MAX_SIZE ((sljit_uw)4)
306 #endif /* SLJIT_CONFIG_RISCV_64 */
307 #endif /* SLJIT_CONFIG_RISCV */
308
309 #if (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
310 /* SLJIT_REWRITABLE_JUMP is 0x10000. */
311 # define PATCH_POOL 0x004
312 # define PATCH_IMM32 0x008
313 #endif /* SLJIT_CONFIG_S390X */
314
315 #if (defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
316 /* SLJIT_REWRITABLE_JUMP is 0x10000. */
317 # define IS_COND 0x004
318 # define IS_CALL 0x008
319
320 # define PATCH_B 0x010
321 # define PATCH_J 0x020
322
323 # define PATCH_REL32 0x040
324 # define PATCH_ABS32 0x080
325 # define PATCH_ABS52 0x100
326 # define JUMP_SIZE_SHIFT 58
327 # define JUMP_MAX_SIZE ((sljit_uw)4)
328
329 #endif /* SLJIT_CONFIG_LOONGARCH */
330
331 /* Stack management. */
332
333 #define GET_SAVED_REGISTERS_SIZE(scratches, saveds, extra) \
334 (((scratches < SLJIT_NUMBER_OF_SCRATCH_REGISTERS ? 0 : (scratches - SLJIT_NUMBER_OF_SCRATCH_REGISTERS)) + \
335 (saveds) + (sljit_s32)(extra)) * (sljit_s32)sizeof(sljit_sw))
336
337 #define GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, type) \
338 (((fscratches < SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS ? 0 : (fscratches - SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS)) + \
339 (fsaveds)) * SSIZE_OF(type))
340
341 #define ADJUST_LOCAL_OFFSET(p, i) \
342 if ((p) == (SLJIT_MEM1(SLJIT_SP))) \
343 (i) += SLJIT_LOCALS_OFFSET;
344
345 #endif /* !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED) */
346
347 /* Utils can still be used even if SLJIT_CONFIG_UNSUPPORTED is set. */
348 #include "sljitUtils.c"
349
350 #if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
351 #define SLJIT_CODE_TO_PTR(code) ((void*)((sljit_up)(code) & ~(sljit_up)0x1))
352 #elif (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
353 #define SLJIT_CODE_TO_PTR(code) ((void*)(*(sljit_up*)code))
354 #else /* !SLJIT_CONFIG_ARM_THUMB2 && !SLJIT_INDIRECT_CALL */
355 #define SLJIT_CODE_TO_PTR(code) ((void*)(code))
356 #endif /* SLJIT_CONFIG_ARM_THUMB2 || SLJIT_INDIRECT_CALL */
357
358 #if !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)
359
360 #if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR)
361
362 #if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR)
363
364 #if defined(__NetBSD__)
365 #include "allocator_src/sljitProtExecAllocatorNetBSD.c"
366 #else /* !__NetBSD__ */
367 #include "allocator_src/sljitProtExecAllocatorPosix.c"
368 #endif /* __NetBSD__ */
369
370 #elif (defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR)
371
372 #if defined(_WIN32)
373 #include "allocator_src/sljitWXExecAllocatorWindows.c"
374 #else /* !_WIN32 */
375 #include "allocator_src/sljitWXExecAllocatorPosix.c"
376 #endif /* _WIN32 */
377
378 #else /* !SLJIT_PROT_EXECUTABLAE_ALLOCATOR && !SLJIT_WX_EXECUTABLE_ALLOCATOR */
379
380 #if defined(_WIN32)
381 #include "allocator_src/sljitExecAllocatorWindows.c"
382 #elif defined(__APPLE__)
383 #include "allocator_src/sljitExecAllocatorApple.c"
384 #elif defined(__FreeBSD__)
385 #include "allocator_src/sljitExecAllocatorFreeBSD.c"
386 #else /* !_WIN32 && !__APPLE__ && !__FreeBSD__ */
387 #include "allocator_src/sljitExecAllocatorPosix.c"
388 #endif /* _WIN32 */
389
390 #endif /* SLJIT_PROT_EXECUTABLE_ALLOCATOR */
391
392 #else /* !SLJIT_EXECUTABLE_ALLOCATOR */
393
394 #ifndef SLJIT_UPDATE_WX_FLAGS
395 #define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec)
396 #endif /* SLJIT_UPDATE_WX_FLAGS */
397
398 #endif /* SLJIT_EXECUTABLE_ALLOCATOR */
399
400 #if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR)
401 #define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr) + (exec_offset))
402 #else /* !SLJIT_PROT_EXECUTABLE_ALLOCATOR */
403 #define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr))
404 #endif /* SLJIT_PROT_EXECUTABLE_ALLOCATOR */
405
406 /* Argument checking features. */
407
408 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
409
410 /* Returns with error when an invalid argument is passed. */
411
412 #define CHECK_ARGUMENT(x) \
413 do { \
414 if (SLJIT_UNLIKELY(!(x))) \
415 return 1; \
416 } while (0)
417
418 #define CHECK_RETURN_TYPE sljit_s32
419 #define CHECK_RETURN_OK return 0
420
421 #define CHECK(x) \
422 do { \
423 if (SLJIT_UNLIKELY(x)) { \
424 compiler->error = SLJIT_ERR_BAD_ARGUMENT; \
425 return SLJIT_ERR_BAD_ARGUMENT; \
426 } \
427 } while (0)
428
429 #define CHECK_PTR(x) \
430 do { \
431 if (SLJIT_UNLIKELY(x)) { \
432 compiler->error = SLJIT_ERR_BAD_ARGUMENT; \
433 return NULL; \
434 } \
435 } while (0)
436
437 #define CHECK_REG_INDEX(x) \
438 do { \
439 if (SLJIT_UNLIKELY(x)) { \
440 return -2; \
441 } \
442 } while (0)
443
444 #elif (defined SLJIT_DEBUG && SLJIT_DEBUG)
445
446 /* Assertion failure occures if an invalid argument is passed. */
447 #undef SLJIT_ARGUMENT_CHECKS
448 #define SLJIT_ARGUMENT_CHECKS 1
449
450 #define CHECK_ARGUMENT(x) SLJIT_ASSERT(x)
451 #define CHECK_RETURN_TYPE void
452 #define CHECK_RETURN_OK return
453 #define CHECK(x) x
454 #define CHECK_PTR(x) x
455 #define CHECK_REG_INDEX(x) x
456
457 #elif (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
458
459 /* Arguments are not checked. */
460 #define CHECK_RETURN_TYPE void
461 #define CHECK_RETURN_OK return
462 #define CHECK(x) x
463 #define CHECK_PTR(x) x
464 #define CHECK_REG_INDEX(x) x
465
466 #else /* !SLJIT_ARGUMENT_CHECKS && !SLJIT_DEBUG && !SLJIT_VERBOSE */
467
468 /* Arguments are not checked. */
469 #define CHECK(x)
470 #define CHECK_PTR(x)
471 #define CHECK_REG_INDEX(x)
472
473 #endif /* SLJIT_ARGUMENT_CHECKS */
474
475 /* --------------------------------------------------------------------- */
476 /* Public functions */
477 /* --------------------------------------------------------------------- */
478
479 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) || (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
480 #define SLJIT_NEEDS_COMPILER_INIT 1
481 static sljit_s32 compiler_initialized = 0;
482 /* A thread safe initialization. */
483 static void init_compiler(void);
484 #endif /* SLJIT_CONFIG_X86 || SLJIT_CONFIG_RISCV */
485
486 SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data)
487 {
488 struct sljit_compiler *compiler = (struct sljit_compiler*)SLJIT_MALLOC(sizeof(struct sljit_compiler), allocator_data);
489 if (!compiler)
490 return NULL;
491 SLJIT_ZEROMEM(compiler, sizeof(struct sljit_compiler));
492
493 SLJIT_COMPILE_ASSERT(
494 sizeof(sljit_s8) == 1 && sizeof(sljit_u8) == 1
495 && sizeof(sljit_s16) == 2 && sizeof(sljit_u16) == 2
496 && sizeof(sljit_s32) == 4 && sizeof(sljit_u32) == 4
497 && (sizeof(sljit_up) == 4 || sizeof(sljit_up) == 8)
498 && sizeof(sljit_up) <= sizeof(sljit_sw)
499 && sizeof(sljit_up) == sizeof(sljit_sp)
500 && (sizeof(sljit_sw) == 4 || sizeof(sljit_sw) == 8)
501 && (sizeof(sljit_uw) == sizeof(sljit_sw)),
502 invalid_integer_types);
503 SLJIT_COMPILE_ASSERT((SLJIT_REWRITABLE_JUMP & SLJIT_32) == 0
504 && (ALL_STATUS_FLAGS_MASK & SLJIT_32) == 0
505 && (SLJIT_REWRITABLE_JUMP & ALL_STATUS_FLAGS_MASK) == 0,
506 rewritable_jump_and_status_flag_bits_and_sljit_32_must_not_have_common_bits);
507 SLJIT_COMPILE_ASSERT((VARIABLE_FLAG_MASK & SLJIT_SET_Z) == 0,
508 rewritable_jump_and_status_flag_bits_must_not_have_common_bits);
509 SLJIT_COMPILE_ASSERT(!(SLJIT_EQUAL & 0x1) && !(SLJIT_LESS & 0x1) && !(SLJIT_F_EQUAL & 0x1) && !(SLJIT_JUMP & 0x1),
510 conditional_flags_must_be_even_numbers);
511
512 /* Only the non-zero members must be set. */
513 compiler->error = SLJIT_SUCCESS;
514
515 compiler->allocator_data = allocator_data;
516 compiler->buf = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, allocator_data);
517 compiler->abuf = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, allocator_data);
518
519 if (!compiler->buf || !compiler->abuf) {
520 if (compiler->buf)
521 SLJIT_FREE(compiler->buf, allocator_data);
522 if (compiler->abuf)
523 SLJIT_FREE(compiler->abuf, allocator_data);
524 SLJIT_FREE(compiler, allocator_data);
525 return NULL;
526 }
527
528 compiler->buf->next = NULL;
529 compiler->buf->used_size = 0;
530 compiler->abuf->next = NULL;
531 compiler->abuf->used_size = 0;
532
533 compiler->scratches = -1;
534 compiler->saveds = -1;
535 compiler->fscratches = -1;
536 compiler->fsaveds = -1;
537 #if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS) \
538 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
539 || (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
540 compiler->vscratches = -1;
541 compiler->vsaveds = -1;
542 #endif /* SLJIT_SEPARATE_VECTOR_REGISTERS || SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
543 compiler->local_size = -1;
544
545 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
546 compiler->args_size = -1;
547 #endif /* SLJIT_CONFIG_X86_32 */
548
549 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
550 compiler->cpool = (sljit_uw*)SLJIT_MALLOC(CPOOL_SIZE * sizeof(sljit_uw)
551 + CPOOL_SIZE * sizeof(sljit_u8), allocator_data);
552 if (!compiler->cpool) {
553 SLJIT_FREE(compiler->buf, allocator_data);
554 SLJIT_FREE(compiler->abuf, allocator_data);
555 SLJIT_FREE(compiler, allocator_data);
556 return NULL;
557 }
558 compiler->cpool_unique = (sljit_u8*)(compiler->cpool + CPOOL_SIZE);
559 compiler->cpool_diff = 0xffffffff;
560 #endif /* SLJIT_CONFIG_ARM_V6 */
561
562 #if (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
563 compiler->delay_slot = UNMOVABLE_INS;
564 #endif /* SLJIT_CONFIG_MIPS */
565
566 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
567 || (defined SLJIT_DEBUG && SLJIT_DEBUG)
568 SLJIT_ASSERT(compiler->last_flags == 0 && compiler->logical_local_size == 0);
569 compiler->last_return = -1;
570 #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_DEBUG */
571
572 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
573 || (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
574 #if !(defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
575 compiler->real_fscratches = -1;
576 compiler->real_fsaveds = -1;
577 #endif /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
578 SLJIT_ASSERT(compiler->skip_checks == 0);
579 #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
580
581 #if (defined SLJIT_NEEDS_COMPILER_INIT && SLJIT_NEEDS_COMPILER_INIT)
582 if (!compiler_initialized) {
583 init_compiler();
584 compiler_initialized = 1;
585 }
586 #endif /* SLJIT_NEEDS_COMPILER_INIT */
587
588 return compiler;
589 }
590
591 SLJIT_API_FUNC_ATTRIBUTE void sljit_free_compiler(struct sljit_compiler *compiler)
592 {
593 struct sljit_memory_fragment *buf;
594 struct sljit_memory_fragment *curr;
595 void *allocator_data = compiler->allocator_data;
596 SLJIT_UNUSED_ARG(allocator_data);
597
598 buf = compiler->buf;
599 while (buf) {
600 curr = buf;
601 buf = buf->next;
602 SLJIT_FREE(curr, allocator_data);
603 }
604
605 buf = compiler->abuf;
606 while (buf) {
607 curr = buf;
608 buf = buf->next;
609 SLJIT_FREE(curr, allocator_data);
610 }
611
612 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
613 SLJIT_FREE(compiler->cpool, allocator_data);
614 #endif /* SLJIT_CONFIG_ARM_V6 */
615 SLJIT_FREE(compiler, allocator_data);
616 }
617
618 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_compiler_memory_error(struct sljit_compiler *compiler)
619 {
620 if (compiler->error == SLJIT_SUCCESS)
621 compiler->error = SLJIT_ERR_ALLOC_FAILED;
622 }
623
624 SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data)
625 {
626 SLJIT_UNUSED_ARG(exec_allocator_data);
627
628 SLJIT_FREE_EXEC(SLJIT_CODE_TO_PTR(code), exec_allocator_data);
629 }
630
631 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_label(struct sljit_jump *jump, struct sljit_label* label)
632 {
633 if (SLJIT_LIKELY(!!jump) && SLJIT_LIKELY(!!label)) {
634 jump->flags &= (sljit_uw)~JUMP_ADDR;
635 jump->u.label = label;
636 }
637 }
638
639 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_target(struct sljit_jump *jump, sljit_uw target)
640 {
641 if (SLJIT_LIKELY(!!jump)) {
642 jump->flags |= JUMP_ADDR;
643 jump->u.target = target;
644 }
645 }
646
647 #define SLJIT_CURRENT_FLAGS_ALL (SLJIT_CURRENT_FLAGS_32 | SLJIT_CURRENT_FLAGS_ADD \
648 | SLJIT_CURRENT_FLAGS_SUB | SLJIT_CURRENT_FLAGS_COMPARE | SLJIT_CURRENT_FLAGS_OP2CMPZ)
649
650 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_current_flags(struct sljit_compiler *compiler, sljit_s32 current_flags)
651 {
652 SLJIT_UNUSED_ARG(compiler);
653 SLJIT_UNUSED_ARG(current_flags);
654
655 #if (defined SLJIT_HAS_STATUS_FLAGS_STATE && SLJIT_HAS_STATUS_FLAGS_STATE)
656 #if (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
657 if (current_flags & SLJIT_CURRENT_FLAGS_OP2CMPZ) {
658 current_flags |= SLJIT_SET_Z;
659 }
660 #endif /* SLJIT_CONFIG_S390X */
661
662 compiler->status_flags_state = current_flags;
663 #endif /* SLJIT_HAS_STATUS_FLAGS_STATE */
664
665 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
666 compiler->last_flags = 0;
667 if ((current_flags & ~(ALL_STATUS_FLAGS_MASK | SLJIT_CURRENT_FLAGS_ALL)) == 0) {
668 compiler->last_flags = GET_FLAG_TYPE(current_flags) | (current_flags & (SLJIT_32 | SLJIT_SET_Z));
669 }
670 #endif /* SLJIT_ARGUMENT_CHECKS */
671 }
672
673 /* --------------------------------------------------------------------- */
674 /* Private functions */
675 /* --------------------------------------------------------------------- */
676
677 static void* ensure_buf(struct sljit_compiler *compiler, sljit_uw size)
678 {
679 sljit_u8 *ret;
680 struct sljit_memory_fragment *new_frag;
681
682 SLJIT_ASSERT(size <= 256);
683 if (compiler->buf->used_size + size <= (BUF_SIZE - (sljit_uw)SLJIT_OFFSETOF(struct sljit_memory_fragment, memory))) {
684 ret = compiler->buf->memory + compiler->buf->used_size;
685 compiler->buf->used_size += size;
686 return ret;
687 }
688 new_frag = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, compiler->allocator_data);
689 PTR_FAIL_IF_NULL(new_frag);
690 new_frag->next = compiler->buf;
691 compiler->buf = new_frag;
692 new_frag->used_size = size;
693 return new_frag->memory;
694 }
695
696 static void* ensure_abuf(struct sljit_compiler *compiler, sljit_uw size)
697 {
698 sljit_u8 *ret;
699 struct sljit_memory_fragment *new_frag;
700
701 SLJIT_ASSERT(size <= 256);
702 if (compiler->abuf->used_size + size <= (ABUF_SIZE - (sljit_uw)SLJIT_OFFSETOF(struct sljit_memory_fragment, memory))) {
703 ret = compiler->abuf->memory + compiler->abuf->used_size;
704 compiler->abuf->used_size += size;
705 return ret;
706 }
707 new_frag = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, compiler->allocator_data);
708 PTR_FAIL_IF_NULL(new_frag);
709 new_frag->next = compiler->abuf;
710 compiler->abuf = new_frag;
711 new_frag->used_size = size;
712 return new_frag->memory;
713 }
714
715 SLJIT_API_FUNC_ATTRIBUTE void* sljit_alloc_memory(struct sljit_compiler *compiler, sljit_s32 size)
716 {
717 CHECK_ERROR_PTR();
718
719 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
720 if (size <= 0 || size > 128)
721 return NULL;
722 size = (size + 7) & ~7;
723 #else /* !SLJIT_64BIT_ARCHITECTURE */
724 if (size <= 0 || size > 64)
725 return NULL;
726 size = (size + 3) & ~3;
727 #endif /* SLJIT_64BIT_ARCHITECTURE */
728 return ensure_abuf(compiler, (sljit_uw)size);
729 }
730
731 static SLJIT_INLINE void reverse_buf(struct sljit_compiler *compiler)
732 {
733 struct sljit_memory_fragment *buf = compiler->buf;
734 struct sljit_memory_fragment *prev = NULL;
735 struct sljit_memory_fragment *tmp;
736
737 do {
738 tmp = buf->next;
739 buf->next = prev;
740 prev = buf;
741 buf = tmp;
742 } while (buf != NULL);
743
744 compiler->buf = prev;
745 }
746
747 static SLJIT_INLINE void* allocate_executable_memory(sljit_uw size, sljit_s32 options,
748 void *exec_allocator_data, sljit_sw *executable_offset)
749 {
750 void *code;
751 struct sljit_generate_code_buffer *buffer;
752
753 if (SLJIT_LIKELY(!(options & SLJIT_GENERATE_CODE_BUFFER))) {
754 code = SLJIT_MALLOC_EXEC(size, exec_allocator_data);
755 *executable_offset = SLJIT_EXEC_OFFSET(code);
756 return code;
757 }
758
759 buffer = (struct sljit_generate_code_buffer*)exec_allocator_data;
760
761 if (size <= buffer->size) {
762 *executable_offset = buffer->executable_offset;
763 return buffer->buffer;
764 }
765
766 return NULL;
767 }
768
769 #define SLJIT_MAX_ADDRESS ~(sljit_uw)0
770
771 #define SLJIT_GET_NEXT_SIZE(ptr) (ptr != NULL) ? ((ptr)->size) : SLJIT_MAX_ADDRESS
772 #define SLJIT_GET_NEXT_ADDRESS(ptr) (ptr != NULL) ? ((ptr)->addr) : SLJIT_MAX_ADDRESS
773
774 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
775
776 #define SLJIT_NEXT_DEFINE_TYPES \
777 sljit_uw next_label_size; \
778 sljit_uw next_jump_addr; \
779 sljit_uw next_const_addr; \
780 sljit_uw next_min_addr
781
782 #define SLJIT_NEXT_INIT_TYPES() \
783 next_label_size = SLJIT_GET_NEXT_SIZE(label); \
784 next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump); \
785 next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
786
787 #define SLJIT_GET_NEXT_MIN() \
788 next_min_addr = sljit_get_next_min(next_label_size, next_jump_addr, next_const_addr);
789
790 static SLJIT_INLINE sljit_uw sljit_get_next_min(sljit_uw next_label_size,
791 sljit_uw next_jump_addr, sljit_uw next_const_addr)
792 {
793 sljit_uw result = next_jump_addr;
794
795 SLJIT_ASSERT(result == SLJIT_MAX_ADDRESS || result != next_const_addr);
796
797 if (next_const_addr < result)
798 result = next_const_addr;
799
800 if (next_label_size < result)
801 result = next_label_size;
802
803 return result;
804 }
805
806 #endif /* !SLJIT_CONFIG_X86 */
807
808 #if !(defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
809
810 static void update_float_register_count(struct sljit_compiler *compiler, sljit_s32 scratches, sljit_s32 saveds)
811 {
812 sljit_s32 vscratches = ENTER_GET_VECTOR_REGS(scratches);
813 sljit_s32 vsaveds = ENTER_GET_VECTOR_REGS(saveds);
814
815 if (compiler->fscratches < vscratches)
816 compiler->fscratches = vscratches;
817
818 if (compiler->fsaveds < vsaveds)
819 compiler->fsaveds = vsaveds;
820
821 if (compiler->fsaveds + compiler->fscratches > SLJIT_NUMBER_OF_FLOAT_REGISTERS)
822 compiler->fscratches = SLJIT_NUMBER_OF_FLOAT_REGISTERS - compiler->fsaveds;
823 }
824
825 #endif /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
826
827 static SLJIT_INLINE void set_emit_enter(struct sljit_compiler *compiler,
828 sljit_s32 options, sljit_s32 args,
829 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
830 {
831 SLJIT_UNUSED_ARG(args);
832 SLJIT_UNUSED_ARG(local_size);
833
834 compiler->options = options;
835 compiler->scratches = ENTER_GET_REGS(scratches);
836 compiler->saveds = ENTER_GET_REGS(saveds);
837 /* These members may be copied to real_* members below. */
838 compiler->fscratches = ENTER_GET_FLOAT_REGS(scratches);
839 compiler->fsaveds = ENTER_GET_FLOAT_REGS(saveds);
840 #if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
841 compiler->vscratches = ENTER_GET_VECTOR_REGS(scratches);
842 compiler->vsaveds = ENTER_GET_VECTOR_REGS(saveds);
843 #else /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
844 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
845 || (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
846 compiler->real_fscratches = compiler->fscratches;
847 compiler->real_fsaveds = compiler->fsaveds;
848 compiler->vscratches = ENTER_GET_VECTOR_REGS(scratches);
849 compiler->vsaveds = ENTER_GET_VECTOR_REGS(saveds);
850 #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
851 update_float_register_count(compiler, scratches, saveds);
852 #endif /* SLJIT_SEPARATE_VECTOR_REGISTERS */
853 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
854 compiler->last_return = args & SLJIT_ARG_MASK;
855 compiler->logical_local_size = local_size;
856 #endif /* SLJIT_ARGUMENT_CHECKS */
857 }
858
859 static SLJIT_INLINE void set_label(struct sljit_label *label, struct sljit_compiler *compiler)
860 {
861 label->next = NULL;
862 label->u.index = compiler->label_count++;
863 label->size = compiler->size;
864
865 if (compiler->last_label != NULL)
866 compiler->last_label->next = label;
867 else
868 compiler->labels = label;
869
870 compiler->last_label = label;
871 }
872
873 static SLJIT_INLINE void set_extended_label(struct sljit_extended_label *ext_label, struct sljit_compiler *compiler, sljit_uw type, sljit_uw data)
874 {
875 set_label(&ext_label->label, compiler);
876 ext_label->index = ext_label->label.u.index;
877 ext_label->label.u.index = type;
878 ext_label->data = data;
879 }
880
881 static SLJIT_INLINE void set_jump(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_u32 flags)
882 {
883 jump->next = NULL;
884 jump->flags = flags;
885 jump->u.label = NULL;
886
887 if (compiler->last_jump != NULL)
888 compiler->last_jump->next = jump;
889 else
890 compiler->jumps = jump;
891
892 compiler->last_jump = jump;
893 }
894
895 static SLJIT_INLINE void set_mov_addr(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_uw offset)
896 {
897 jump->next = NULL;
898 jump->addr = compiler->size - offset;
899 jump->flags = JUMP_MOV_ADDR;
900 jump->u.label = NULL;
901 if (compiler->last_jump != NULL)
902 compiler->last_jump->next = jump;
903 else
904 compiler->jumps = jump;
905 compiler->last_jump = jump;
906 }
907
908 static SLJIT_INLINE void set_const(struct sljit_const *const_, struct sljit_compiler *compiler)
909 {
910 const_->next = NULL;
911 const_->addr = compiler->size;
912 if (compiler->last_const != NULL)
913 compiler->last_const->next = const_;
914 else
915 compiler->consts = const_;
916 compiler->last_const = const_;
917 }
918
919 #define ADDRESSING_DEPENDS_ON(exp, reg) \
920 (((exp) & SLJIT_MEM) && (((exp) & REG_MASK) == reg || OFFS_REG(exp) == reg))
921
922 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
923
924 static sljit_s32 function_check_arguments(sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds, sljit_s32 fscratches)
925 {
926 sljit_s32 word_arg_count, scratch_arg_end, saved_arg_count, float_arg_count, curr_type;
927
928 curr_type = (arg_types & SLJIT_ARG_FULL_MASK);
929
930 if (curr_type >= SLJIT_ARG_TYPE_F64) {
931 if (curr_type > SLJIT_ARG_TYPE_F32 || fscratches == 0)
932 return 0;
933 } else if (curr_type >= SLJIT_ARG_TYPE_W) {
934 if (scratches == 0)
935 return 0;
936 }
937
938 arg_types >>= SLJIT_ARG_SHIFT;
939
940 word_arg_count = 0;
941 scratch_arg_end = 0;
942 saved_arg_count = 0;
943 float_arg_count = 0;
944 while (arg_types != 0) {
945 if (word_arg_count + float_arg_count >= 4)
946 return 0;
947
948 curr_type = (arg_types & SLJIT_ARG_MASK);
949
950 if (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) {
951 if (saveds == -1 || curr_type < SLJIT_ARG_TYPE_W || curr_type > SLJIT_ARG_TYPE_P)
952 return 0;
953
954 word_arg_count++;
955 scratch_arg_end = word_arg_count;
956 } else {
957 if (curr_type < SLJIT_ARG_TYPE_W || curr_type > SLJIT_ARG_TYPE_F32)
958 return 0;
959
960 if (curr_type < SLJIT_ARG_TYPE_F64) {
961 word_arg_count++;
962 saved_arg_count++;
963 } else
964 float_arg_count++;
965 }
966
967 arg_types >>= SLJIT_ARG_SHIFT;
968 }
969
970 if (saveds == -1)
971 return (word_arg_count <= scratches && float_arg_count <= fscratches);
972
973 return (saved_arg_count <= saveds && scratch_arg_end <= scratches && float_arg_count <= fscratches);
974 }
975
976 #define FUNCTION_CHECK_IS_REG(r) \
977 (((r) >= SLJIT_R0 && (r) < (SLJIT_R0 + compiler->scratches)) \
978 || ((r) > (SLJIT_S0 - compiler->saveds) && (r) <= SLJIT_S0) \
979 || ((r) >= SLJIT_TMP_REGISTER_BASE && (r) < (SLJIT_TMP_REGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_REGISTERS)))
980
981 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
982 #define CHECK_IF_VIRTUAL_REGISTER(p) ((p) <= SLJIT_S3 && (p) >= SLJIT_S8)
983 #else /* !SLJIT_CONFIG_X86_32 */
984 #define CHECK_IF_VIRTUAL_REGISTER(p) 0
985 #endif /* SLJIT_CONFIG_X86_32 */
986
987 static sljit_s32 function_check_src_mem(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
988 {
989 if (compiler->scratches == -1)
990 return 0;
991
992 if (!(p & SLJIT_MEM))
993 return 0;
994
995 if (p == SLJIT_MEM1(SLJIT_SP))
996 return (i >= 0 && i < compiler->logical_local_size);
997
998 if (!(!(p & REG_MASK) || FUNCTION_CHECK_IS_REG(p & REG_MASK)))
999 return 0;
1000
1001 if (CHECK_IF_VIRTUAL_REGISTER(p & REG_MASK))
1002 return 0;
1003
1004 if (p & OFFS_REG_MASK) {
1005 if (!(p & REG_MASK))
1006 return 0;
1007
1008 if (!(FUNCTION_CHECK_IS_REG(OFFS_REG(p))))
1009 return 0;
1010
1011 if (CHECK_IF_VIRTUAL_REGISTER(OFFS_REG(p)))
1012 return 0;
1013
1014 if ((i & ~0x3) != 0)
1015 return 0;
1016 }
1017
1018 return (p & ~(SLJIT_MEM | REG_MASK | OFFS_REG_MASK)) == 0;
1019 }
1020
1021 #define FUNCTION_CHECK_SRC_MEM(p, i) \
1022 CHECK_ARGUMENT(function_check_src_mem(compiler, p, i));
1023
1024 static sljit_s32 function_check_src(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1025 {
1026 if (compiler->scratches == -1)
1027 return 0;
1028
1029 if (FUNCTION_CHECK_IS_REG(p))
1030 return (i == 0);
1031
1032 if (p == SLJIT_IMM)
1033 return 1;
1034
1035 return function_check_src_mem(compiler, p, i);
1036 }
1037
1038 #define FUNCTION_CHECK_SRC(p, i) \
1039 CHECK_ARGUMENT(function_check_src(compiler, p, i));
1040
1041 static sljit_s32 function_check_dst(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1042 {
1043 if (compiler->scratches == -1)
1044 return 0;
1045
1046 if (FUNCTION_CHECK_IS_REG(p))
1047 return (i == 0);
1048
1049 return function_check_src_mem(compiler, p, i);
1050 }
1051
1052 #define FUNCTION_CHECK_DST(p, i) \
1053 CHECK_ARGUMENT(function_check_dst(compiler, p, i));
1054
1055 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
1056 || (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
1057
1058 #define FUNCTION_CHECK_IS_FREG(fr, is_32) \
1059 function_check_is_freg(compiler, (fr), (is_32))
1060
1061 static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr, sljit_s32 is_32);
1062
1063 #define FUNCTION_FCHECK(p, i, is_32) \
1064 CHECK_ARGUMENT(function_fcheck(compiler, (p), (i), (is_32)));
1065
1066 static sljit_s32 function_fcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i, sljit_s32 is_32)
1067 {
1068 if (compiler->scratches == -1)
1069 return 0;
1070
1071 if (FUNCTION_CHECK_IS_FREG(p, is_32))
1072 return (i == 0);
1073
1074 return function_check_src_mem(compiler, p, i);
1075 }
1076
1077 static sljit_s32 function_check_is_vreg(struct sljit_compiler *compiler, sljit_s32 vr, sljit_s32 type);
1078
1079 #define FUNCTION_CHECK_IS_VREG(vr, type) \
1080 function_check_is_vreg(compiler, (vr), (type))
1081
1082 #define FUNCTION_VCHECK(p, i, type) \
1083 CHECK_ARGUMENT(function_vcheck(compiler, (p), (i), (type)))
1084
1085 static sljit_s32 function_vcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i, sljit_s32 type)
1086 {
1087 if (compiler->scratches == -1)
1088 return 0;
1089
1090 if (FUNCTION_CHECK_IS_VREG(p, type))
1091 return (i == 0);
1092
1093 return function_check_src_mem(compiler, p, i);
1094 }
1095
1096 #else /* !SLJIT_CONFIG_ARM_32 && !SLJIT_CONFIG_MIPS_32 */
1097
1098 #define FUNCTION_CHECK_IS_FREG(fr, is_32) \
1099 function_check_is_freg(compiler, (fr))
1100
1101 static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr)
1102 {
1103 sljit_s32 fscratches, fsaveds;
1104
1105 if (compiler->scratches == -1)
1106 return 0;
1107
1108 #if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
1109 fscratches = compiler->fscratches;
1110 fsaveds = compiler->fsaveds;
1111 #else /* SLJIT_SEPARATE_VECTOR_REGISTERS */
1112 fscratches = compiler->real_fscratches;
1113 fsaveds = compiler->real_fsaveds;
1114 #endif /* !SLJIT_SEPARATE_VECTOR_REGISTERS */
1115
1116 return (fr >= SLJIT_FR0 && fr < (SLJIT_FR0 + fscratches))
1117 || (fr > (SLJIT_FS0 - fsaveds) && fr <= SLJIT_FS0)
1118 || (fr >= SLJIT_TMP_FREGISTER_BASE && fr < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS));
1119 }
1120
1121 #define FUNCTION_FCHECK(p, i, is_32) \
1122 CHECK_ARGUMENT(function_fcheck(compiler, (p), (i)));
1123
1124 static sljit_s32 function_fcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1125 {
1126 if (compiler->scratches == -1)
1127 return 0;
1128
1129 if (function_check_is_freg(compiler, p))
1130 return (i == 0);
1131
1132 return function_check_src_mem(compiler, p, i);
1133 }
1134
1135 #define FUNCTION_CHECK_IS_VREG(vr, type) \
1136 function_check_is_vreg(compiler, (vr))
1137
1138 static sljit_s32 function_check_is_vreg(struct sljit_compiler *compiler, sljit_s32 vr)
1139 {
1140 if (compiler->scratches == -1)
1141 return 0;
1142
1143 return (vr >= SLJIT_VR0 && vr < (SLJIT_VR0 + compiler->vscratches))
1144 || (vr > (SLJIT_VS0 - compiler->vsaveds) && vr <= SLJIT_VS0)
1145 || (vr >= SLJIT_TMP_VREGISTER_BASE && vr < (SLJIT_TMP_VREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_VECTOR_REGISTERS));
1146 }
1147
1148 #define FUNCTION_VCHECK(p, i, type) \
1149 CHECK_ARGUMENT(function_vcheck(compiler, (p), (i)))
1150
1151 static sljit_s32 function_vcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1152 {
1153 if (compiler->scratches == -1)
1154 return 0;
1155
1156 if (function_check_is_vreg(compiler, p))
1157 return (i == 0);
1158
1159 return function_check_src_mem(compiler, p, i);
1160 }
1161
1162 #endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1163
1164 #endif /* SLJIT_ARGUMENT_CHECKS */
1165
1166 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1167
1168 SLJIT_API_FUNC_ATTRIBUTE void sljit_compiler_verbose(struct sljit_compiler *compiler, FILE* verbose)
1169 {
1170 compiler->verbose = verbose;
1171 }
1172
1173 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
1174 #ifdef _WIN64
1175 #ifdef __GNUC__
1176 # define SLJIT_PRINT_D "ll"
1177 #else /* !__GNUC__ */
1178 # define SLJIT_PRINT_D "I64"
1179 #endif /* __GNUC__ */
1180 #else /* !_WIN64 */
1181 # define SLJIT_PRINT_D "l"
1182 #endif /* _WIN64 */
1183 #else /* !SLJIT_64BIT_ARCHITECTURE */
1184 # define SLJIT_PRINT_D ""
1185 #endif /* SLJIT_64BIT_ARCHITECTURE */
1186
1187 static void sljit_verbose_reg(struct sljit_compiler *compiler, sljit_s32 r)
1188 {
1189 if (r < (SLJIT_R0 + compiler->scratches))
1190 fprintf(compiler->verbose, "r%d", r - SLJIT_R0);
1191 else if (r < SLJIT_SP)
1192 fprintf(compiler->verbose, "s%d", SLJIT_NUMBER_OF_REGISTERS - r);
1193 else if (r == SLJIT_SP)
1194 fprintf(compiler->verbose, "sp");
1195 else
1196 fprintf(compiler->verbose, "t%d", r - SLJIT_TMP_REGISTER_BASE);
1197 }
1198
1199 static void sljit_verbose_freg(struct sljit_compiler *compiler, sljit_s32 r)
1200 {
1201 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
1202 || (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
1203 if (r >= SLJIT_F64_SECOND(SLJIT_FR0)) {
1204 fprintf(compiler->verbose, "^");
1205 r -= SLJIT_F64_SECOND(0);
1206 }
1207 #endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1208
1209 if (r < (SLJIT_FR0 + compiler->fscratches))
1210 fprintf(compiler->verbose, "fr%d", r - SLJIT_FR0);
1211 else if (r < SLJIT_TMP_FREGISTER_BASE)
1212 fprintf(compiler->verbose, "fs%d", SLJIT_NUMBER_OF_FLOAT_REGISTERS - r);
1213 else
1214 fprintf(compiler->verbose, "ft%d", r - SLJIT_TMP_FREGISTER_BASE);
1215 }
1216
1217 static void sljit_verbose_vreg(struct sljit_compiler *compiler, sljit_s32 r)
1218 {
1219 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
1220 || (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
1221 if (r >= SLJIT_F64_SECOND(SLJIT_VR0)) {
1222 fprintf(compiler->verbose, "^");
1223 r -= SLJIT_F64_SECOND(0);
1224 }
1225 #endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1226
1227 if (r < (SLJIT_VR0 + compiler->vscratches))
1228 fprintf(compiler->verbose, "vr%d", r - SLJIT_VR0);
1229 else if (r < SLJIT_TMP_VREGISTER_BASE)
1230 fprintf(compiler->verbose, "vs%d", SLJIT_NUMBER_OF_VECTOR_REGISTERS - r);
1231 else
1232 fprintf(compiler->verbose, "vt%d", r - SLJIT_TMP_VREGISTER_BASE);
1233 }
1234
1235 static void sljit_verbose_mem(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1236 {
1237 if (!(p & REG_MASK)) {
1238 fprintf(compiler->verbose, "[%" SLJIT_PRINT_D "d]", i);
1239 return;
1240 }
1241
1242 fputc('[', compiler->verbose);
1243 sljit_verbose_reg(compiler, (p) & REG_MASK);
1244 if (p & OFFS_REG_MASK) {
1245 fprintf(compiler->verbose, " + ");
1246 sljit_verbose_reg(compiler, OFFS_REG(p));
1247 if (i)
1248 fprintf(compiler->verbose, " * %d", 1 << (i));
1249 } else if (i)
1250 fprintf(compiler->verbose, " + %" SLJIT_PRINT_D "d", (i));
1251 fputc(']', compiler->verbose);
1252 }
1253
1254 static void sljit_verbose_param(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1255 {
1256 if (p == SLJIT_IMM)
1257 fprintf(compiler->verbose, "#%" SLJIT_PRINT_D "d", i);
1258 else if (p & SLJIT_MEM)
1259 sljit_verbose_mem(compiler, p, i);
1260 else
1261 sljit_verbose_reg(compiler, p);
1262 }
1263
1264 static void sljit_verbose_fparam(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1265 {
1266 if (p & SLJIT_MEM)
1267 sljit_verbose_mem(compiler, p, i);
1268 else
1269 sljit_verbose_freg(compiler, p);
1270 }
1271
1272 static void sljit_verbose_vparam(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1273 {
1274 if (p & SLJIT_MEM)
1275 sljit_verbose_mem(compiler, p, i);
1276 else
1277 sljit_verbose_vreg(compiler, p);
1278 }
1279
1280 static const char* op0_names[] = {
1281 "breakpoint", "nop", "lmul.uw", "lmul.sw",
1282 "divmod.u", "divmod.s", "div.u", "div.s",
1283 "memory_barrier", "endbr", "skip_frames_before_return"
1284 };
1285
1286 static const char* op1_names[] = {
1287 "mov", "mov", "mov", "mov",
1288 "mov", "mov", "mov", "mov",
1289 "mov", "clz", "ctz", "rev",
1290 "rev", "rev", "rev", "rev"
1291 };
1292
1293 static const char* op1_types[] = {
1294 "", ".u8", ".s8", ".u16",
1295 ".s16", ".u32", ".s32", "32",
1296 ".p", "", "", "",
1297 ".u16", ".s16", ".u32", ".s32"
1298 };
1299
1300 static const char* op2_names[] = {
1301 "add", "addc", "sub", "subc",
1302 "mul", "and", "or", "xor",
1303 "shl", "mshl", "lshr", "mlshr",
1304 "ashr", "mashr", "rotl", "rotr"
1305 };
1306
1307 static const char* op2r_names[] = {
1308 "muladd"
1309 };
1310
1311 static const char* op_src_dst_names[] = {
1312 "fast_return", "skip_frames_before_fast_return",
1313 "prefetch_l1", "prefetch_l2",
1314 "prefetch_l3", "prefetch_once",
1315 "fast_enter", "get_return_address"
1316 };
1317
1318 static const char* fop1_names[] = {
1319 "mov", "conv", "conv", "conv",
1320 "conv", "conv", "conv", "conv",
1321 "cmp", "neg", "abs",
1322 };
1323
1324 static const char* fop1_conv_types[] = {
1325 "sw", "s32", "sw", "s32",
1326 "uw", "u32"
1327 };
1328
1329 static const char* fop2_names[] = {
1330 "add", "sub", "mul", "div"
1331 };
1332
1333 static const char* fop2r_names[] = {
1334 "copysign"
1335 };
1336
1337 static const char* simd_op2_names[] = {
1338 "and", "or", "xor", "shuffle"
1339 };
1340
1341 static const char* jump_names[] = {
1342 "equal", "not_equal",
1343 "less", "greater_equal",
1344 "greater", "less_equal",
1345 "sig_less", "sig_greater_equal",
1346 "sig_greater", "sig_less_equal",
1347 "overflow", "not_overflow",
1348 "carry", "not_carry",
1349 "atomic_stored", "atomic_not_stored",
1350 "f_equal", "f_not_equal",
1351 "f_less", "f_greater_equal",
1352 "f_greater", "f_less_equal",
1353 "unordered", "ordered",
1354 "ordered_equal", "unordered_or_not_equal",
1355 "ordered_less", "unordered_or_greater_equal",
1356 "ordered_greater", "unordered_or_less_equal",
1357 "unordered_or_equal", "ordered_not_equal",
1358 "unordered_or_less", "ordered_greater_equal",
1359 "unordered_or_greater", "ordered_less_equal",
1360 "jump", "fast_call",
1361 "call", "call_reg_arg"
1362 };
1363
1364 static const char* call_arg_names[] = {
1365 "void", "w", "32", "p", "f64", "f32"
1366 };
1367
1368 static const char* op_addr_types[] = {
1369 "mov_addr", "mov_abs_addr", "add_abs_addr"
1370 };
1371
1372 #endif /* SLJIT_VERBOSE */
1373
1374 /* --------------------------------------------------------------------- */
1375 /* Arch dependent */
1376 /* --------------------------------------------------------------------- */
1377
1378 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
1379 || (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1380
1381 #define SLJIT_SKIP_CHECKS(compiler) (compiler)->skip_checks = 1
1382 #define SLJIT_CHECK_OPCODE(op, flags) ((op) & ~(SLJIT_32 | ALL_STATUS_FLAGS_MASK | (flags)))
1383
1384 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_generate_code(struct sljit_compiler *compiler, sljit_s32 options)
1385 {
1386 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1387 struct sljit_jump *jump;
1388 #endif /* SLJIT_ARGUMENT_CHECKS */
1389
1390 SLJIT_UNUSED_ARG(compiler);
1391 SLJIT_UNUSED_ARG(options);
1392
1393 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1394 CHECK_ARGUMENT(compiler->size > 0);
1395 CHECK_ARGUMENT((options & ~(SLJIT_GENERATE_CODE_BUFFER | SLJIT_GENERATE_CODE_NO_CONTEXT)) == 0);
1396
1397 jump = compiler->jumps;
1398 while (jump) {
1399 /* All jumps have target. */
1400 CHECK_ARGUMENT((jump->flags & JUMP_ADDR) || jump->u.label != NULL);
1401 jump = jump->next;
1402 }
1403 #endif /* SLJIT_ARGUMENT_CHECKS */
1404 CHECK_RETURN_OK;
1405 }
1406
1407 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1408 #define SLJIT_ENTER_CPU_SPECIFIC_OPTIONS (SLJIT_ENTER_USE_VEX)
1409 #else /* !SLJIT_CONFIG_X86 */
1410 #define SLJIT_ENTER_CPU_SPECIFIC_OPTIONS (0)
1411 #endif /* !SLJIT_CONFIG_X86 */
1412
1413 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_enter(struct sljit_compiler *compiler,
1414 sljit_s32 options, sljit_s32 arg_types,
1415 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1416 {
1417 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1418 sljit_s32 real_scratches = ENTER_GET_REGS(scratches);
1419 sljit_s32 real_saveds = ENTER_GET_REGS(saveds);
1420 sljit_s32 real_fscratches = ENTER_GET_FLOAT_REGS(scratches);
1421 sljit_s32 real_fsaveds = ENTER_GET_FLOAT_REGS(saveds);
1422 sljit_s32 real_vscratches = ENTER_GET_VECTOR_REGS(scratches);
1423 sljit_s32 real_vsaveds = ENTER_GET_VECTOR_REGS(saveds);
1424 #endif /* SLJIT_ARGUMENT_CHECKS */
1425 SLJIT_UNUSED_ARG(compiler);
1426
1427 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1428 if (options & SLJIT_ENTER_REG_ARG) {
1429 CHECK_ARGUMENT(!(options & ~(0x3 | SLJIT_ENTER_REG_ARG | SLJIT_ENTER_CPU_SPECIFIC_OPTIONS)));
1430 } else {
1431 CHECK_ARGUMENT((options & ~SLJIT_ENTER_CPU_SPECIFIC_OPTIONS) == 0);
1432 }
1433 CHECK_ARGUMENT(SLJIT_KEPT_SAVEDS_COUNT(options) <= 3 && SLJIT_KEPT_SAVEDS_COUNT(options) <= saveds);
1434 CHECK_ARGUMENT((scratches & ~0xffffff) == 0 && (saveds & ~0xffffff) == 0);
1435 CHECK_ARGUMENT(real_scratches >= 0 && real_scratches <= SLJIT_NUMBER_OF_REGISTERS);
1436 CHECK_ARGUMENT(real_saveds >= 0 && real_saveds <= SLJIT_NUMBER_OF_SAVED_REGISTERS);
1437 CHECK_ARGUMENT(real_scratches + real_saveds <= SLJIT_NUMBER_OF_REGISTERS);
1438 CHECK_ARGUMENT(real_fscratches >= 0 && real_fscratches <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1439 CHECK_ARGUMENT(real_fsaveds >= 0 && real_fsaveds <= SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS);
1440 CHECK_ARGUMENT(real_fscratches + real_fsaveds <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1441 CHECK_ARGUMENT(real_vscratches >= 0 && real_vscratches <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1442 CHECK_ARGUMENT(real_vsaveds >= 0 && real_vsaveds <= SLJIT_NUMBER_OF_SAVED_VECTOR_REGISTERS);
1443 CHECK_ARGUMENT(real_vscratches + real_vsaveds <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1444 CHECK_ARGUMENT(local_size >= 0 && local_size <= SLJIT_MAX_LOCAL_SIZE);
1445 CHECK_ARGUMENT((arg_types & SLJIT_ARG_FULL_MASK) <= SLJIT_ARG_TYPE_F32);
1446 CHECK_ARGUMENT(function_check_arguments(arg_types, real_scratches,
1447 (options & SLJIT_ENTER_REG_ARG) ? 0 : real_saveds, real_fscratches));
1448
1449 compiler->last_flags = 0;
1450 #endif /* SLJIT_ARGUMENT_CHECKS */
1451 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1452 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1453 fprintf(compiler->verbose, " enter ret[%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
1454
1455 arg_types >>= SLJIT_ARG_SHIFT;
1456 if (arg_types) {
1457 fprintf(compiler->verbose, "], args[");
1458 do {
1459 fprintf(compiler->verbose, "%s%s", call_arg_names[arg_types & SLJIT_ARG_MASK],
1460 (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) ? "_r" : "");
1461 arg_types >>= SLJIT_ARG_SHIFT;
1462 if (arg_types)
1463 fprintf(compiler->verbose, ",");
1464 } while (arg_types);
1465 }
1466
1467 fprintf(compiler->verbose, "],");
1468
1469 if (options & SLJIT_ENTER_REG_ARG) {
1470 if (SLJIT_KEPT_SAVEDS_COUNT(options) > 0)
1471 fprintf(compiler->verbose, " opt:reg_arg(%d),", SLJIT_KEPT_SAVEDS_COUNT(options));
1472 else
1473 fprintf(compiler->verbose, " opt:reg_arg,");
1474 }
1475
1476 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1477 if (options & SLJIT_ENTER_USE_VEX) {
1478 fprintf(compiler->verbose, " opt:use_vex,");
1479 }
1480 #endif /* !SLJIT_CONFIG_X86 */
1481
1482 fprintf(compiler->verbose, " scratches:%d, saveds:%d, fscratches:%d, fsaveds:%d, vscratches:%d, vsaveds:%d, local_size:%d\n",
1483 ENTER_GET_REGS(scratches), ENTER_GET_REGS(saveds), ENTER_GET_FLOAT_REGS(scratches), ENTER_GET_FLOAT_REGS(saveds),
1484 ENTER_GET_VECTOR_REGS(scratches), ENTER_GET_VECTOR_REGS(saveds), local_size);
1485 }
1486 #endif /* SLJIT_VERBOSE */
1487 CHECK_RETURN_OK;
1488 }
1489
1490 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_set_context(struct sljit_compiler *compiler,
1491 sljit_s32 options, sljit_s32 arg_types,
1492 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
1493 {
1494 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1495 sljit_s32 real_scratches = ENTER_GET_REGS(scratches);
1496 sljit_s32 real_saveds = ENTER_GET_REGS(saveds);
1497 sljit_s32 real_fscratches = ENTER_GET_FLOAT_REGS(scratches);
1498 sljit_s32 real_fsaveds = ENTER_GET_FLOAT_REGS(saveds);
1499 sljit_s32 real_vscratches = ENTER_GET_VECTOR_REGS(scratches);
1500 sljit_s32 real_vsaveds = ENTER_GET_VECTOR_REGS(saveds);
1501 #endif /* SLJIT_ARGUMENT_CHECKS */
1502 SLJIT_UNUSED_ARG(compiler);
1503
1504 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1505 if (options & SLJIT_ENTER_REG_ARG) {
1506 CHECK_ARGUMENT(!(options & ~(0x3 | SLJIT_ENTER_REG_ARG | SLJIT_ENTER_CPU_SPECIFIC_OPTIONS)));
1507 } else {
1508 CHECK_ARGUMENT((options & ~SLJIT_ENTER_CPU_SPECIFIC_OPTIONS) == 0);
1509 }
1510 CHECK_ARGUMENT(SLJIT_KEPT_SAVEDS_COUNT(options) <= 3 && SLJIT_KEPT_SAVEDS_COUNT(options) <= saveds);
1511 CHECK_ARGUMENT((scratches & ~0xffffff) == 0 && (saveds & ~0xffffff) == 0);
1512 CHECK_ARGUMENT(real_scratches >= 0 && real_scratches <= SLJIT_NUMBER_OF_REGISTERS);
1513 CHECK_ARGUMENT(real_saveds >= 0 && real_saveds <= SLJIT_NUMBER_OF_SAVED_REGISTERS);
1514 CHECK_ARGUMENT(real_scratches + real_saveds <= SLJIT_NUMBER_OF_REGISTERS);
1515 CHECK_ARGUMENT(real_fscratches >= 0 && real_fscratches <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1516 CHECK_ARGUMENT(real_fsaveds >= 0 && real_fsaveds <= SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS);
1517 CHECK_ARGUMENT(real_fscratches + real_fsaveds <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1518 CHECK_ARGUMENT(real_vscratches >= 0 && real_vscratches <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1519 CHECK_ARGUMENT(real_vsaveds >= 0 && real_vsaveds <= SLJIT_NUMBER_OF_SAVED_VECTOR_REGISTERS);
1520 CHECK_ARGUMENT(real_vscratches + real_vsaveds <= SLJIT_NUMBER_OF_VECTOR_REGISTERS);
1521 CHECK_ARGUMENT(local_size >= 0 && local_size <= SLJIT_MAX_LOCAL_SIZE);
1522 CHECK_ARGUMENT((arg_types & SLJIT_ARG_FULL_MASK) < SLJIT_ARG_TYPE_F64);
1523 CHECK_ARGUMENT(function_check_arguments(arg_types, real_scratches,
1524 (options & SLJIT_ENTER_REG_ARG) ? 0 : real_saveds, real_fscratches));
1525
1526 compiler->last_flags = 0;
1527 #endif /* SLJIT_ARGUMENT_CHECKS */
1528 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1529 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1530 fprintf(compiler->verbose, " set_context ret[%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
1531
1532 arg_types >>= SLJIT_ARG_SHIFT;
1533 if (arg_types) {
1534 fprintf(compiler->verbose, "], args[");
1535 do {
1536 fprintf(compiler->verbose, "%s%s", call_arg_names[arg_types & SLJIT_ARG_MASK],
1537 (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) ? "_r" : "");
1538 arg_types >>= SLJIT_ARG_SHIFT;
1539 if (arg_types)
1540 fprintf(compiler->verbose, ",");
1541 } while (arg_types);
1542 }
1543
1544 fprintf(compiler->verbose, "],");
1545
1546 if (options & SLJIT_ENTER_REG_ARG) {
1547 if (SLJIT_KEPT_SAVEDS_COUNT(options) > 0)
1548 fprintf(compiler->verbose, " opt:reg_arg(%d),", SLJIT_KEPT_SAVEDS_COUNT(options));
1549 else
1550 fprintf(compiler->verbose, " opt:reg_arg,");
1551 }
1552
1553 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1554 if (options & SLJIT_ENTER_USE_VEX) {
1555 fprintf(compiler->verbose, " opt:use_vex,");
1556 }
1557 #endif /* !SLJIT_CONFIG_X86 */
1558
1559 fprintf(compiler->verbose, " scratches:%d, saveds:%d, fscratches:%d, fsaveds:%d, vscratches:%d, vsaveds:%d, local_size:%d\n",
1560 ENTER_GET_REGS(scratches), ENTER_GET_REGS(saveds), ENTER_GET_FLOAT_REGS(scratches), ENTER_GET_FLOAT_REGS(saveds),
1561 ENTER_GET_VECTOR_REGS(scratches), ENTER_GET_VECTOR_REGS(saveds), local_size);
1562 }
1563 #endif /* SLJIT_VERBOSE */
1564 CHECK_RETURN_OK;
1565 }
1566
1567 #undef SLJIT_ENTER_CPU_SPECIFIC_OPTIONS
1568
1569 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return_void(struct sljit_compiler *compiler)
1570 {
1571 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1572 compiler->skip_checks = 0;
1573 CHECK_RETURN_OK;
1574 }
1575
1576 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1577 CHECK_ARGUMENT(compiler->last_return == SLJIT_ARG_TYPE_RET_VOID);
1578 #endif /* SLJIT_ARGUMENT_CHECKS */
1579
1580 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1581 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1582 fprintf(compiler->verbose, " return_void\n");
1583 }
1584 #endif /* SLJIT_VERBOSE */
1585 CHECK_RETURN_OK;
1586 }
1587
1588 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
1589 {
1590 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1591 CHECK_ARGUMENT(compiler->scratches >= 0);
1592
1593 switch (compiler->last_return) {
1594 case SLJIT_ARG_TYPE_W:
1595 CHECK_ARGUMENT(op >= SLJIT_MOV && op <= SLJIT_MOV_S32);
1596 break;
1597 case SLJIT_ARG_TYPE_32:
1598 CHECK_ARGUMENT(op == SLJIT_MOV32 || (op >= SLJIT_MOV32_U8 && op <= SLJIT_MOV32_S16));
1599 break;
1600 case SLJIT_ARG_TYPE_P:
1601 CHECK_ARGUMENT(op == SLJIT_MOV_P);
1602 break;
1603 case SLJIT_ARG_TYPE_F64:
1604 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1605 CHECK_ARGUMENT(op == SLJIT_MOV_F64);
1606 break;
1607 case SLJIT_ARG_TYPE_F32:
1608 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1609 CHECK_ARGUMENT(op == SLJIT_MOV_F32);
1610 break;
1611 default:
1612 /* Context not initialized, void, etc. */
1613 CHECK_ARGUMENT(0);
1614 break;
1615 }
1616
1617 if (SLJIT_CHECK_OPCODE(op, 0) < SLJIT_MOV_F64) {
1618 FUNCTION_CHECK_SRC(src, srcw);
1619 } else {
1620 FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
1621 }
1622 compiler->last_flags = 0;
1623 #endif /* SLJIT_ARGUMENT_CHECKS */
1624 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1625 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1626 if (GET_OPCODE(op) < SLJIT_MOV_F64) {
1627 fprintf(compiler->verbose, " return%s%s ", !(op & SLJIT_32) ? "" : "32",
1628 op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1629 sljit_verbose_param(compiler, src, srcw);
1630 } else {
1631 fprintf(compiler->verbose, " return%s ", !(op & SLJIT_32) ? ".f64" : ".f32");
1632 sljit_verbose_fparam(compiler, src, srcw);
1633 }
1634 fprintf(compiler->verbose, "\n");
1635 }
1636 #endif /* SLJIT_VERBOSE */
1637 CHECK_RETURN_OK;
1638 }
1639
1640 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return_to(struct sljit_compiler *compiler,
1641 sljit_s32 src, sljit_sw srcw)
1642 {
1643 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1644 FUNCTION_CHECK_SRC(src, srcw);
1645 #endif /* SLJIT_ARGUMENT_CHECKS */
1646 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1647 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1648 fprintf(compiler->verbose, " return_to ");
1649 sljit_verbose_param(compiler, src, srcw);
1650 fprintf(compiler->verbose, "\n");
1651 }
1652 #endif /* SLJIT_VERBOSE */
1653 CHECK_RETURN_OK;
1654 }
1655
1656 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1657 {
1658 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1659 CHECK_ARGUMENT((op >= SLJIT_BREAKPOINT && op <= SLJIT_LMUL_SW)
1660 || ((op & ~SLJIT_32) >= SLJIT_DIVMOD_UW && (op & ~SLJIT_32) <= SLJIT_DIV_SW)
1661 || (op >= SLJIT_MEMORY_BARRIER && op <= SLJIT_SKIP_FRAMES_BEFORE_RETURN));
1662 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) < SLJIT_LMUL_UW || SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_MEMORY_BARRIER || compiler->scratches >= 2);
1663 if ((SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_LMUL_UW && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_DIV_SW) || op == SLJIT_SKIP_FRAMES_BEFORE_RETURN)
1664 compiler->last_flags = 0;
1665 #endif /* SLJIT_ARGUMENT_CHECKS */
1666 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1667 if (SLJIT_UNLIKELY(!!compiler->verbose))
1668 {
1669 fprintf(compiler->verbose, " %s", op0_names[GET_OPCODE(op) - SLJIT_OP0_BASE]);
1670 if (GET_OPCODE(op) >= SLJIT_DIVMOD_UW && GET_OPCODE(op) <= SLJIT_DIV_SW) {
1671 fprintf(compiler->verbose, (op & SLJIT_32) ? "32" : "w");
1672 }
1673 fprintf(compiler->verbose, "\n");
1674 }
1675 #endif /* SLJIT_VERBOSE */
1676 CHECK_RETURN_OK;
1677 }
1678
1679 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1680 sljit_s32 dst, sljit_sw dstw,
1681 sljit_s32 src, sljit_sw srcw)
1682 {
1683 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1684 compiler->skip_checks = 0;
1685 CHECK_RETURN_OK;
1686 }
1687
1688 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1689 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_MOV && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_REV_S32);
1690
1691 switch (GET_OPCODE(op)) {
1692 case SLJIT_MOV:
1693 case SLJIT_MOV_U32:
1694 case SLJIT_MOV_S32:
1695 case SLJIT_MOV32:
1696 case SLJIT_MOV_P:
1697 case SLJIT_REV_U32:
1698 case SLJIT_REV_S32:
1699 /* Nothing allowed */
1700 CHECK_ARGUMENT(!(op & (SLJIT_32 | ALL_STATUS_FLAGS_MASK)));
1701 break;
1702 default:
1703 /* Only SLJIT_32 is allowed. */
1704 CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
1705 break;
1706 }
1707
1708 FUNCTION_CHECK_DST(dst, dstw);
1709 FUNCTION_CHECK_SRC(src, srcw);
1710 #endif /* SLJIT_ARGUMENT_CHECKS */
1711 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1712 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1713 fprintf(compiler->verbose, " %s%s%s ", op1_names[GET_OPCODE(op) - SLJIT_OP1_BASE],
1714 !(op & SLJIT_32) ? "" : "32", op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1715
1716 sljit_verbose_param(compiler, dst, dstw);
1717 fprintf(compiler->verbose, ", ");
1718 sljit_verbose_param(compiler, src, srcw);
1719 fprintf(compiler->verbose, "\n");
1720 }
1721 #endif /* SLJIT_VERBOSE */
1722 CHECK_RETURN_OK;
1723 }
1724
1725 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
1726 sljit_s32 dst_reg,
1727 sljit_s32 mem_reg)
1728 {
1729 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1730 compiler->skip_checks = 0;
1731 CHECK_RETURN_OK;
1732 }
1733
1734 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1735 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_ATOMIC));
1736 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | ALL_STATUS_FLAGS_MASK) >= SLJIT_MOV
1737 && SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | ALL_STATUS_FLAGS_MASK) <= SLJIT_MOV_P);
1738 CHECK_ARGUMENT((op & (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS)) != (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS));
1739
1740 /* All arguments must be valid registers. */
1741 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1742 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(mem_reg) && !CHECK_IF_VIRTUAL_REGISTER(mem_reg));
1743
1744 if (GET_OPCODE(op) < SLJIT_MOV_U8 || GET_OPCODE(op) > SLJIT_MOV_S16) {
1745 /* Nothing allowed. */
1746 CHECK_ARGUMENT(!(op & SLJIT_32));
1747 }
1748
1749 compiler->last_flags = 0;
1750 #endif /* SLJIT_ARGUMENT_CHECKS */
1751 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1752 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1753 if (op & SLJIT_ATOMIC_TEST)
1754 CHECK_RETURN_OK;
1755 if (sljit_emit_atomic_load(compiler, op | SLJIT_ATOMIC_TEST, dst_reg, mem_reg)) {
1756 fprintf(compiler->verbose, " # atomic_load: unsupported form, no instructions are emitted\n");
1757 CHECK_RETURN_OK;
1758 }
1759
1760 fprintf(compiler->verbose, " atomic_load");
1761 if (op & SLJIT_ATOMIC_USE_CAS)
1762 fprintf(compiler->verbose, "_cas");
1763 if (op & SLJIT_ATOMIC_USE_LS)
1764 fprintf(compiler->verbose, "_ls");
1765
1766 fprintf(compiler->verbose, "%s%s ", !(op & SLJIT_32) ? "" : "32",
1767 op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1768 sljit_verbose_reg(compiler, dst_reg);
1769 fprintf(compiler->verbose, ", [");
1770 sljit_verbose_reg(compiler, mem_reg);
1771 fprintf(compiler->verbose, "]\n");
1772 }
1773 #endif /* SLJIT_VERBOSE */
1774 CHECK_RETURN_OK;
1775 }
1776
1777 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
1778 sljit_s32 src_reg,
1779 sljit_s32 mem_reg,
1780 sljit_s32 temp_reg)
1781 {
1782 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1783 compiler->skip_checks = 0;
1784 CHECK_RETURN_OK;
1785 }
1786
1787 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1788 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_ATOMIC));
1789 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | SLJIT_SET_Z) >= SLJIT_MOV
1790 && SLJIT_CHECK_OPCODE(op, SLJIT_ATOMIC_TEST | SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS | SLJIT_SET_Z) <= SLJIT_MOV_P);
1791 CHECK_ARGUMENT((op & (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS)) != (SLJIT_ATOMIC_USE_CAS | SLJIT_ATOMIC_USE_LS));
1792
1793 /* All arguments must be valid registers. */
1794 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src_reg));
1795 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(mem_reg) && !CHECK_IF_VIRTUAL_REGISTER(mem_reg));
1796 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(temp_reg) && (src_reg != temp_reg || (op & SLJIT_ATOMIC_USE_LS)));
1797
1798 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK) || GET_FLAG_TYPE_MASK(op) == SLJIT_ATOMIC_STORED);
1799
1800 if (GET_OPCODE(op) < SLJIT_MOV_U8 || GET_OPCODE(op) > SLJIT_MOV_S16) {
1801 /* Nothing allowed. */
1802 CHECK_ARGUMENT(!(op & SLJIT_32));
1803 }
1804
1805 compiler->last_flags = GET_FLAG_TYPE_MASK(op) | (op & SLJIT_32);
1806 #endif /* SLJIT_ARGUMENT_CHECKS */
1807 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1808 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1809 if (op & SLJIT_ATOMIC_TEST)
1810 CHECK_RETURN_OK;
1811 if (sljit_emit_atomic_store(compiler, op | SLJIT_ATOMIC_TEST, src_reg, mem_reg, temp_reg)) {
1812 fprintf(compiler->verbose, " # atomic_store: unsupported form, no instructions are emitted\n");
1813 CHECK_RETURN_OK;
1814 }
1815
1816 fprintf(compiler->verbose, " atomic_store");
1817 if (op & SLJIT_ATOMIC_USE_CAS)
1818 fprintf(compiler->verbose, "_cas");
1819 if (op & SLJIT_ATOMIC_USE_LS)
1820 fprintf(compiler->verbose, "_ls");
1821
1822 fprintf(compiler->verbose, "%s%s%s ", !(op & SLJIT_32) ? "" : "32",
1823 op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE], !(op & VARIABLE_FLAG_MASK) ? "" : ".stored");
1824 sljit_verbose_reg(compiler, src_reg);
1825 fprintf(compiler->verbose, ", [");
1826 sljit_verbose_reg(compiler, mem_reg);
1827 fprintf(compiler->verbose, "], ");
1828 sljit_verbose_reg(compiler, temp_reg);
1829 fprintf(compiler->verbose, "\n");
1830 }
1831 #endif /* SLJIT_VERBOSE */
1832 CHECK_RETURN_OK;
1833 }
1834
1835 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1836
1837 static sljit_s32 check_sljit_emit_op2_operation(struct sljit_compiler *compiler, sljit_s32 op)
1838 {
1839 switch (GET_OPCODE(op)) {
1840 case SLJIT_AND:
1841 case SLJIT_OR:
1842 case SLJIT_XOR:
1843 case SLJIT_SHL:
1844 case SLJIT_MSHL:
1845 case SLJIT_LSHR:
1846 case SLJIT_MLSHR:
1847 case SLJIT_ASHR:
1848 case SLJIT_MASHR:
1849 return !(op & VARIABLE_FLAG_MASK);
1850 case SLJIT_MUL:
1851 return !(op & SLJIT_SET_Z) && (!(op & VARIABLE_FLAG_MASK) || GET_FLAG_TYPE(op) == SLJIT_OVERFLOW);
1852 case SLJIT_ADD:
1853 return !(op & VARIABLE_FLAG_MASK)
1854 || GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY)
1855 || GET_FLAG_TYPE(op) == SLJIT_OVERFLOW;
1856 case SLJIT_SUB:
1857 return !(op & VARIABLE_FLAG_MASK)
1858 || (GET_FLAG_TYPE(op) >= SLJIT_LESS && GET_FLAG_TYPE(op) <= SLJIT_OVERFLOW)
1859 || GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY);
1860 case SLJIT_ADDC:
1861 case SLJIT_SUBC:
1862 return (!(op & VARIABLE_FLAG_MASK) || GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY))
1863 && (compiler->last_flags & 0xff) == GET_FLAG_TYPE(SLJIT_SET_CARRY)
1864 && (op & SLJIT_32) == (compiler->last_flags & SLJIT_32);
1865 break;
1866 case SLJIT_ROTL:
1867 case SLJIT_ROTR:
1868 return !(op & ALL_STATUS_FLAGS_MASK);
1869 }
1870
1871 /* Operation type should be checked earlier. */
1872 SLJIT_UNREACHABLE();
1873 return 1;
1874 }
1875
1876 #endif /* SLJIT_ARGUMENT_CHECKS */
1877
1878 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 unset,
1879 sljit_s32 dst, sljit_sw dstw,
1880 sljit_s32 src1, sljit_sw src1w,
1881 sljit_s32 src2, sljit_sw src2w)
1882 {
1883 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1884 compiler->skip_checks = 0;
1885 CHECK_RETURN_OK;
1886 }
1887
1888 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1889 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_ADD && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_ROTR);
1890 CHECK_ARGUMENT(check_sljit_emit_op2_operation(compiler, op));
1891
1892 if (unset) {
1893 CHECK_ARGUMENT(HAS_FLAGS(op));
1894 } else {
1895 FUNCTION_CHECK_DST(dst, dstw);
1896 }
1897 FUNCTION_CHECK_SRC(src1, src1w);
1898 FUNCTION_CHECK_SRC(src2, src2w);
1899 compiler->last_flags = GET_FLAG_TYPE(op) | (op & (SLJIT_32 | SLJIT_SET_Z));
1900 #endif /* SLJIT_ARGUMENT_CHECKS */
1901 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1902 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1903 fprintf(compiler->verbose, " %s%s%s%s%s ", op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE], !(op & SLJIT_32) ? "" : "32",
1904 !(op & SLJIT_SET_Z) ? "" : ".z", !(op & VARIABLE_FLAG_MASK) ? "" : ".",
1905 !(op & VARIABLE_FLAG_MASK) ? "" : jump_names[GET_FLAG_TYPE(op)]);
1906 if (unset)
1907 fprintf(compiler->verbose, "unset");
1908 else
1909 sljit_verbose_param(compiler, dst, dstw);
1910 fprintf(compiler->verbose, ", ");
1911 sljit_verbose_param(compiler, src1, src1w);
1912 fprintf(compiler->verbose, ", ");
1913 sljit_verbose_param(compiler, src2, src2w);
1914 fprintf(compiler->verbose, "\n");
1915 }
1916 #endif /* SLJIT_VERBOSE */
1917 CHECK_RETURN_OK;
1918 }
1919
1920 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
1921 sljit_s32 dst_reg,
1922 sljit_s32 src1, sljit_sw src1w,
1923 sljit_s32 src2, sljit_sw src2w)
1924 {
1925 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1926 CHECK_ARGUMENT((op | SLJIT_32) == SLJIT_MULADD32);
1927 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1928 FUNCTION_CHECK_SRC(src1, src1w);
1929 FUNCTION_CHECK_SRC(src2, src2w);
1930 compiler->last_flags = 0;
1931 #endif /* SLJIT_ARGUMENT_CHECKS */
1932 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1933 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1934 fprintf(compiler->verbose, " %s%s ", op2r_names[GET_OPCODE(op) - SLJIT_OP2R_BASE], !(op & SLJIT_32) ? "" : "32");
1935
1936 sljit_verbose_reg(compiler, dst_reg);
1937 fprintf(compiler->verbose, ", ");
1938 sljit_verbose_param(compiler, src1, src1w);
1939 fprintf(compiler->verbose, ", ");
1940 sljit_verbose_param(compiler, src2, src2w);
1941 fprintf(compiler->verbose, "\n");
1942 }
1943 #endif /* SLJIT_VERBOSE */
1944 CHECK_RETURN_OK;
1945 }
1946
1947 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
1948 sljit_s32 dst_reg,
1949 sljit_s32 src1_reg,
1950 sljit_s32 src2_reg,
1951 sljit_s32 src3, sljit_sw src3w)
1952 {
1953 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1954 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) == SLJIT_SHL || SLJIT_CHECK_OPCODE(op, 0) == SLJIT_LSHR
1955 || SLJIT_CHECK_OPCODE(op, 0) == SLJIT_MSHL || SLJIT_CHECK_OPCODE(op, 0) == SLJIT_MLSHR);
1956 CHECK_ARGUMENT((op & ~(0xff | SLJIT_32 | SLJIT_SHIFT_INTO_NON_ZERO)) == 0);
1957 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1958 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src1_reg));
1959 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src2_reg));
1960 FUNCTION_CHECK_SRC(src3, src3w);
1961 CHECK_ARGUMENT(dst_reg != src2_reg);
1962 #endif /* SLJIT_ARGUMENT_CHECKS */
1963 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1964 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1965 fprintf(compiler->verbose, " %s%s.into%s ", op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE], !(op & SLJIT_32) ? "" : "32",
1966 (op & SLJIT_SHIFT_INTO_NON_ZERO) ? ".nz" : "");
1967
1968 sljit_verbose_reg(compiler, dst_reg);
1969 fprintf(compiler->verbose, ", ");
1970 sljit_verbose_reg(compiler, src1_reg);
1971 fprintf(compiler->verbose, ", ");
1972 sljit_verbose_reg(compiler, src2_reg);
1973 fprintf(compiler->verbose, ", ");
1974 sljit_verbose_param(compiler, src3, src3w);
1975 fprintf(compiler->verbose, "\n");
1976 }
1977 #endif /* SLJIT_VERBOSE */
1978 CHECK_RETURN_OK;
1979 }
1980
1981 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2_shift(struct sljit_compiler *compiler, sljit_s32 op,
1982 sljit_s32 dst, sljit_sw dstw,
1983 sljit_s32 src1, sljit_sw src1w,
1984 sljit_s32 src2, sljit_sw src2w,
1985 sljit_sw shift_arg)
1986 {
1987 SLJIT_UNUSED_ARG(shift_arg);
1988 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1989 CHECK_ARGUMENT((op & ~SLJIT_SRC2_UNDEFINED) == (SLJIT_ADD | SLJIT_SHL_IMM));
1990 FUNCTION_CHECK_DST(dst, dstw);
1991 FUNCTION_CHECK_SRC(src1, src1w);
1992 FUNCTION_CHECK_SRC(src2, src2w);
1993 compiler->last_flags = 0;
1994 #endif /* SLJIT_ARGUMENT_CHECKS */
1995 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1996 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1997 fprintf(compiler->verbose, " %s.shl_imm%s ", op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE],
1998 (op & SLJIT_SRC2_UNDEFINED) ? ".src2und" : "");
1999
2000 sljit_verbose_param(compiler, dst, dstw);
2001 fprintf(compiler->verbose, ", ");
2002 sljit_verbose_param(compiler, src1, src1w);
2003 fprintf(compiler->verbose, ", ");
2004 sljit_verbose_param(compiler, src2, src2w);
2005 fprintf(compiler->verbose, ", #%" SLJIT_PRINT_D "d\n", shift_arg);
2006 }
2007 #endif /* SLJIT_VERBOSE */
2008 CHECK_RETURN_OK;
2009 }
2010
2011 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
2012 sljit_s32 src, sljit_sw srcw)
2013 {
2014 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2015 CHECK_ARGUMENT(op >= SLJIT_FAST_RETURN && op <= SLJIT_PREFETCH_ONCE);
2016 FUNCTION_CHECK_SRC(src, srcw);
2017
2018 if (op == SLJIT_FAST_RETURN || op == SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN) {
2019 CHECK_ARGUMENT(src != SLJIT_IMM);
2020 compiler->last_flags = 0;
2021 } else if (op >= SLJIT_PREFETCH_L1 && op <= SLJIT_PREFETCH_ONCE) {
2022 CHECK_ARGUMENT(src & SLJIT_MEM);
2023 }
2024 #endif /* SLJIT_ARGUMENT_CHECKS */
2025 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2026 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2027 fprintf(compiler->verbose, " %s ", op_src_dst_names[op - SLJIT_OP_SRC_DST_BASE]);
2028 sljit_verbose_param(compiler, src, srcw);
2029 fprintf(compiler->verbose, "\n");
2030 }
2031 #endif /* SLJIT_VERBOSE */
2032 CHECK_RETURN_OK;
2033 }
2034
2035 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
2036 sljit_s32 dst, sljit_sw dstw)
2037 {
2038 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2039 CHECK_ARGUMENT(op >= SLJIT_FAST_ENTER && op <= SLJIT_GET_RETURN_ADDRESS);
2040 FUNCTION_CHECK_DST(dst, dstw);
2041
2042 if (op == SLJIT_FAST_ENTER)
2043 compiler->last_flags = 0;
2044 #endif /* SLJIT_ARGUMENT_CHECKS */
2045 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2046 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2047 fprintf(compiler->verbose, " %s ", op_src_dst_names[op - SLJIT_OP_SRC_DST_BASE]);
2048 sljit_verbose_param(compiler, dst, dstw);
2049 fprintf(compiler->verbose, "\n");
2050 }
2051 #endif /* SLJIT_VERBOSE */
2052 CHECK_RETURN_OK;
2053 }
2054
2055 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
2056 {
2057 SLJIT_UNUSED_ARG(type);
2058 SLJIT_UNUSED_ARG(reg);
2059 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2060 if (type == SLJIT_GP_REGISTER) {
2061 CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_REGISTERS)
2062 || (reg >= SLJIT_TMP_REGISTER_BASE && reg < (SLJIT_TMP_REGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_REGISTERS)));
2063 }
2064 #if (defined SLJIT_SEPARATE_VECTOR_REGISTERS && SLJIT_SEPARATE_VECTOR_REGISTERS)
2065 else if (((type >> 12) == 0 || ((type >> 12) >= 3 && (type >> 12) <= 6))) {
2066 CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_VECTOR_REGISTERS)
2067 || (reg >= SLJIT_TMP_VREGISTER_BASE && reg < (SLJIT_TMP_VREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_VECTOR_REGISTERS)));
2068 }
2069 #endif /* SLJIT_SEPARATE_VECTOR_REGISTERS */
2070 else {
2071 CHECK_ARGUMENT(type == SLJIT_FLOAT_REGISTER || ((type >> 12) == 0 || ((type >> 12) >= 3 && (type >> 12) <= 6) || (type & (3 << 12)) || (type & (4 << 12)) || (type & (5 << 12)) || (type & (6 << 12))));
2072 CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_FLOAT_REGISTERS)
2073 || (reg >= SLJIT_TMP_FREGISTER_BASE && reg < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS)));
2074 }
2075 #endif /* SLJIT_ARGUMENT_CHECKS */
2076 CHECK_RETURN_OK;
2077 }
2078
2079 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_custom(struct sljit_compiler *compiler,
2080 void *instruction, sljit_u32 size)
2081 {
2082 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2083 sljit_u32 i;
2084 #endif /* SLJIT_VERBOSE */
2085
2086 SLJIT_UNUSED_ARG(compiler);
2087
2088 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2089 CHECK_ARGUMENT(instruction);
2090
2091 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
2092 CHECK_ARGUMENT(size > 0 && size < 16);
2093 #elif (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2) || (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
2094 CHECK_ARGUMENT((size == 2 && (((sljit_sw)instruction) & 0x1) == 0)
2095 || (size == 4 && (((sljit_sw)instruction) & 0x3) == 0));
2096 #elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
2097 CHECK_ARGUMENT(size == 2 || size == 4 || size == 6);
2098 #else /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM_THUMB2 && !SLJIT_CONFIG_RISCV && !SLJIT_CONFIG_S390X */
2099 CHECK_ARGUMENT(size == 4 && (((sljit_sw)instruction) & 0x3) == 0);
2100 #endif /* SLJIT_CONFIG_X86 */
2101
2102 compiler->last_flags = 0;
2103 #endif /* SLJIT_ARGUMENT_CHECKS */
2104 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2105 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2106 fprintf(compiler->verbose, " op_custom");
2107 for (i = 0; i < size; i++)
2108 fprintf(compiler->verbose, " 0x%x", ((sljit_u8*)instruction)[i]);
2109 fprintf(compiler->verbose, "\n");
2110 }
2111 #endif /* SLJIT_VERBOSE */
2112 CHECK_RETURN_OK;
2113 }
2114
2115 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2116 sljit_s32 dst, sljit_sw dstw,
2117 sljit_s32 src, sljit_sw srcw)
2118 {
2119 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2120 compiler->skip_checks = 0;
2121 CHECK_RETURN_OK;
2122 }
2123
2124 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2125 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2126 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_MOV_F64 && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_ABS_F64);
2127 CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
2128 FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
2129 FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
2130 #endif /* SLJIT_ARGUMENT_CHECKS */
2131 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2132 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2133 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32)
2134 fprintf(compiler->verbose, " %s%s ", fop1_names[SLJIT_CONV_F64_FROM_F32 - SLJIT_FOP1_BASE],
2135 (op & SLJIT_32) ? ".f32.from.f64" : ".f64.from.f32");
2136 else
2137 fprintf(compiler->verbose, " %s%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
2138 (op & SLJIT_32) ? ".f32" : ".f64");
2139
2140 sljit_verbose_fparam(compiler, dst, dstw);
2141 fprintf(compiler->verbose, ", ");
2142 sljit_verbose_fparam(compiler, src, srcw);
2143 fprintf(compiler->verbose, "\n");
2144 }
2145 #endif /* SLJIT_VERBOSE */
2146 CHECK_RETURN_OK;
2147 }
2148
2149 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2150 sljit_s32 src1, sljit_sw src1w,
2151 sljit_s32 src2, sljit_sw src2w)
2152 {
2153 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2154 compiler->last_flags = GET_FLAG_TYPE(op) | (op & SLJIT_32);
2155 #endif /* SLJIT_ARGUMENT_CHECKS */
2156
2157 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2158 compiler->skip_checks = 0;
2159 CHECK_RETURN_OK;
2160 }
2161
2162 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2163 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2164 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) == SLJIT_CMP_F64);
2165 CHECK_ARGUMENT(!(op & SLJIT_SET_Z));
2166 CHECK_ARGUMENT((op & VARIABLE_FLAG_MASK)
2167 || (GET_FLAG_TYPE(op) >= SLJIT_F_EQUAL && GET_FLAG_TYPE(op) <= SLJIT_ORDERED_LESS_EQUAL));
2168 FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2169 FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2170 #endif /* SLJIT_ARGUMENT_CHECKS */
2171 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2172 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2173 fprintf(compiler->verbose, " %s%s", fop1_names[SLJIT_CMP_F64 - SLJIT_FOP1_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2174 if (op & VARIABLE_FLAG_MASK) {
2175 fprintf(compiler->verbose, ".%s", jump_names[GET_FLAG_TYPE(op)]);
2176 }
2177 fprintf(compiler->verbose, " ");
2178 sljit_verbose_fparam(compiler, src1, src1w);
2179 fprintf(compiler->verbose, ", ");
2180 sljit_verbose_fparam(compiler, src2, src2w);
2181 fprintf(compiler->verbose, "\n");
2182 }
2183 #endif /* SLJIT_VERBOSE */
2184 CHECK_RETURN_OK;
2185 }
2186
2187 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2188 sljit_s32 dst, sljit_sw dstw,
2189 sljit_s32 src, sljit_sw srcw)
2190 {
2191 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2192 compiler->skip_checks = 0;
2193 CHECK_RETURN_OK;
2194 }
2195
2196 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2197 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2198 CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
2199 FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
2200 FUNCTION_CHECK_DST(dst, dstw);
2201 #endif /* SLJIT_ARGUMENT_CHECKS */
2202 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2203 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2204 fprintf(compiler->verbose, " %s%s.from%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
2205 fop1_conv_types[GET_OPCODE(op) - SLJIT_CONV_SW_FROM_F64],
2206 (op & SLJIT_32) ? ".f32" : ".f64");
2207 sljit_verbose_param(compiler, dst, dstw);
2208 fprintf(compiler->verbose, ", ");
2209 sljit_verbose_fparam(compiler, src, srcw);
2210 fprintf(compiler->verbose, "\n");
2211 }
2212 #endif /* SLJIT_VERBOSE */
2213 CHECK_RETURN_OK;
2214 }
2215
2216 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler *compiler, sljit_s32 op,
2217 sljit_s32 dst, sljit_sw dstw,
2218 sljit_s32 src, sljit_sw srcw)
2219 {
2220 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2221 compiler->skip_checks = 0;
2222 CHECK_RETURN_OK;
2223 }
2224
2225 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2226 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2227 CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
2228 FUNCTION_CHECK_SRC(src, srcw);
2229 FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
2230 #endif /* SLJIT_ARGUMENT_CHECKS */
2231 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2232 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2233 fprintf(compiler->verbose, " %s%s.from.%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
2234 (op & SLJIT_32) ? ".f32" : ".f64",
2235 fop1_conv_types[GET_OPCODE(op) - SLJIT_CONV_SW_FROM_F64]);
2236 sljit_verbose_fparam(compiler, dst, dstw);
2237 fprintf(compiler->verbose, ", ");
2238 sljit_verbose_param(compiler, src, srcw);
2239 fprintf(compiler->verbose, "\n");
2240 }
2241 #endif /* SLJIT_VERBOSE */
2242 CHECK_RETURN_OK;
2243 }
2244
2245 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2246 sljit_s32 dst, sljit_sw dstw,
2247 sljit_s32 src1, sljit_sw src1w,
2248 sljit_s32 src2, sljit_sw src2w)
2249 {
2250 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2251 compiler->skip_checks = 0;
2252 CHECK_RETURN_OK;
2253 }
2254
2255 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2256 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2257 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_ADD_F64 && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_DIV_F64);
2258 CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
2259 FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2260 FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2261 FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
2262 #endif /* SLJIT_ARGUMENT_CHECKS */
2263 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2264 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2265 fprintf(compiler->verbose, " %s%s ", fop2_names[GET_OPCODE(op) - SLJIT_FOP2_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2266 sljit_verbose_fparam(compiler, dst, dstw);
2267 fprintf(compiler->verbose, ", ");
2268 sljit_verbose_fparam(compiler, src1, src1w);
2269 fprintf(compiler->verbose, ", ");
2270 sljit_verbose_fparam(compiler, src2, src2w);
2271 fprintf(compiler->verbose, "\n");
2272 }
2273 #endif /* SLJIT_VERBOSE */
2274 CHECK_RETURN_OK;
2275 }
2276
2277 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop2r(struct sljit_compiler *compiler, sljit_s32 op,
2278 sljit_s32 dst_freg,
2279 sljit_s32 src1, sljit_sw src1w,
2280 sljit_s32 src2, sljit_sw src2w)
2281 {
2282 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2283 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2284 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) == SLJIT_COPYSIGN_F64);
2285 FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2286 FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2287 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(dst_freg, op & SLJIT_32));
2288 #endif /* SLJIT_ARGUMENT_CHECKS */
2289 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2290 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2291 fprintf(compiler->verbose, " %s%s ", fop2r_names[GET_OPCODE(op) - SLJIT_FOP2R_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2292 sljit_verbose_freg(compiler, dst_freg);
2293 fprintf(compiler->verbose, ", ");
2294 sljit_verbose_fparam(compiler, src1, src1w);
2295 fprintf(compiler->verbose, ", ");
2296 sljit_verbose_fparam(compiler, src2, src2w);
2297 fprintf(compiler->verbose, "\n");
2298 }
2299 #endif /* SLJIT_VERBOSE */
2300 CHECK_RETURN_OK;
2301 }
2302
2303 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fset32(struct sljit_compiler *compiler,
2304 sljit_s32 freg, sljit_f32 value)
2305 {
2306 SLJIT_UNUSED_ARG(value);
2307
2308 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2309 compiler->skip_checks = 0;
2310 CHECK_RETURN_OK;
2311 }
2312
2313 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2314 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2315 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 1));
2316 #endif /* SLJIT_ARGUMENT_CHECKS */
2317 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2318 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2319 fprintf(compiler->verbose, " fset32 ");
2320 sljit_verbose_freg(compiler, freg);
2321 fprintf(compiler->verbose, ", %f\n", value);
2322 }
2323 #endif /* SLJIT_VERBOSE */
2324 CHECK_RETURN_OK;
2325 }
2326
2327 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fset64(struct sljit_compiler *compiler,
2328 sljit_s32 freg, sljit_f64 value)
2329 {
2330 SLJIT_UNUSED_ARG(value);
2331
2332 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2333 compiler->skip_checks = 0;
2334 CHECK_RETURN_OK;
2335 }
2336
2337 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2338 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2339 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 0));
2340 #endif /* SLJIT_ARGUMENT_CHECKS */
2341 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2342 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2343 fprintf(compiler->verbose, " fset64 ");
2344 sljit_verbose_freg(compiler, freg);
2345 fprintf(compiler->verbose, ", %f\n", value);
2346 }
2347 #endif /* SLJIT_VERBOSE */
2348 CHECK_RETURN_OK;
2349 }
2350
2351 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fcopy(struct sljit_compiler *compiler, sljit_s32 op,
2352 sljit_s32 freg, sljit_s32 reg)
2353 {
2354 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2355 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2356 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_COPY_TO_F64 && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_COPY_FROM_F64);
2357 CHECK_ARGUMENT(!(op & ALL_STATUS_FLAGS_MASK));
2358 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, op & SLJIT_32));
2359
2360 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
2361 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2362 #else /* !SLJIT_64BIT_ARCHITECTURE */
2363 switch (op) {
2364 case SLJIT_COPY32_TO_F32:
2365 case SLJIT_COPY32_FROM_F32:
2366 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2367 break;
2368 case SLJIT_COPY_TO_F64:
2369 case SLJIT_COPY_FROM_F64:
2370 if (reg & REG_PAIR_MASK) {
2371 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_FIRST(reg)));
2372 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_SECOND(reg)));
2373
2374 if (op == SLJIT_COPY_TO_F64)
2375 break;
2376
2377 CHECK_ARGUMENT(REG_PAIR_FIRST(reg) != REG_PAIR_SECOND(reg));
2378 break;
2379 }
2380
2381 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2382 break;
2383 }
2384 #endif /* SLJIT_64BIT_ARCHITECTURE */
2385 #endif /* SLJIT_ARGUMENT_CHECKS */
2386 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2387 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2388 fprintf(compiler->verbose, " copy%s_%s_f%s ", (op & SLJIT_32) ? "32" : "",
2389 GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? "to" : "from", (op & SLJIT_32) ? "32" : "64");
2390
2391 sljit_verbose_freg(compiler, freg);
2392
2393 if (reg & REG_PAIR_MASK) {
2394 fprintf(compiler->verbose, ", {");
2395 sljit_verbose_reg(compiler, REG_PAIR_FIRST(reg));
2396 fprintf(compiler->verbose, ", ");
2397 sljit_verbose_reg(compiler, REG_PAIR_SECOND(reg));
2398 fprintf(compiler->verbose, "}\n");
2399 } else {
2400 fprintf(compiler->verbose, ", ");
2401 sljit_verbose_reg(compiler, reg);
2402 fprintf(compiler->verbose, "\n");
2403 }
2404 }
2405 #endif /* SLJIT_VERBOSE */
2406 CHECK_RETURN_OK;
2407 }
2408
2409 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_label(struct sljit_compiler *compiler)
2410 {
2411 SLJIT_UNUSED_ARG(compiler);
2412
2413 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2414 compiler->skip_checks = 0;
2415 CHECK_RETURN_OK;
2416 }
2417
2418 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2419 compiler->last_flags = 0;
2420 #endif /* SLJIT_ARGUMENT_CHECKS */
2421
2422 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2423 if (SLJIT_UNLIKELY(!!compiler->verbose))
2424 fprintf(compiler->verbose, "label:\n");
2425 #endif /* SLJIT_VERBOSE */
2426 CHECK_RETURN_OK;
2427 }
2428
2429 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_aligned_label(struct sljit_compiler *compiler,
2430 sljit_s32 alignment, struct sljit_read_only_buffer *buffers)
2431 {
2432 SLJIT_UNUSED_ARG(compiler);
2433 SLJIT_UNUSED_ARG(buffers);
2434
2435 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2436 CHECK_ARGUMENT(alignment >= SLJIT_LABEL_ALIGN_1 && alignment <= SLJIT_LABEL_ALIGN_16);
2437 compiler->last_flags = 0;
2438 #endif /* SLJIT_ARGUMENT_CHECKS */
2439
2440 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2441 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2442 fprintf(compiler->verbose, "label.al%d:%s", 1 << alignment, buffers == NULL ? "\n" : " [");
2443
2444 if (buffers != NULL) {
2445 fprintf(compiler->verbose, "%ld", (long int)buffers->size);
2446 buffers = buffers->next;
2447
2448 while (buffers != NULL) {
2449 fprintf(compiler->verbose, ", %ld", (long int)buffers->size);
2450 buffers = buffers->next;
2451 }
2452
2453 fprintf(compiler->verbose, "]\n");
2454 }
2455 }
2456 #endif /* SLJIT_VERBOSE */
2457 CHECK_RETURN_OK;
2458 }
2459
2460 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2461 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
2462 || (defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM)
2463 #define CHECK_UNORDERED(type, last_flags) \
2464 ((((type) & 0xfe) == SLJIT_ORDERED) && \
2465 ((last_flags) & 0xff) >= SLJIT_UNORDERED && ((last_flags) & 0xff) <= SLJIT_ORDERED_LESS_EQUAL)
2466 #else /* !SLJIT_CONFIG_X86 || SLJIT_CONFIG_ARM */
2467 #define CHECK_UNORDERED(type, last_flags) 0
2468 #endif /* SLJIT_CONFIG_X86 || SLJIT_CONFIG_ARM */
2469 #endif /* SLJIT_ARGUMENT_CHECKS */
2470
2471 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2472 {
2473 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2474 compiler->skip_checks = 0;
2475 CHECK_RETURN_OK;
2476 }
2477
2478 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2479 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP)));
2480 CHECK_ARGUMENT((type & 0xff) >= SLJIT_EQUAL && (type & 0xff) <= SLJIT_FAST_CALL);
2481
2482 if ((type & 0xff) < SLJIT_JUMP) {
2483 if ((type & 0xff) <= SLJIT_NOT_ZERO)
2484 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2485 else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2486 CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2487 compiler->last_flags = 0;
2488 } else
2489 CHECK_ARGUMENT((type & 0xfe) == (compiler->last_flags & 0xff)
2490 || CHECK_UNORDERED(type, compiler->last_flags));
2491 }
2492 #endif /* SLJIT_ARGUMENT_CHECKS */
2493 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2494 if (SLJIT_UNLIKELY(!!compiler->verbose))
2495 fprintf(compiler->verbose, " jump%s %s\n", !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r",
2496 jump_names[type & 0xff]);
2497 #endif /* SLJIT_VERBOSE */
2498 CHECK_RETURN_OK;
2499 }
2500
2501 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2502 sljit_s32 arg_types)
2503 {
2504 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2505 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_CALL_RETURN)));
2506 CHECK_ARGUMENT((type & 0xff) >= SLJIT_CALL && (type & 0xff) <= SLJIT_CALL_REG_ARG);
2507 CHECK_ARGUMENT(function_check_arguments(arg_types, compiler->scratches, -1, compiler->fscratches));
2508
2509 if (type & SLJIT_CALL_RETURN) {
2510 CHECK_ARGUMENT((arg_types & SLJIT_ARG_MASK) == compiler->last_return);
2511
2512 if (compiler->options & SLJIT_ENTER_REG_ARG) {
2513 CHECK_ARGUMENT((type & 0xff) == SLJIT_CALL_REG_ARG);
2514 } else {
2515 CHECK_ARGUMENT((type & 0xff) != SLJIT_CALL_REG_ARG);
2516 }
2517 }
2518 #endif /* SLJIT_ARGUMENT_CHECKS */
2519 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2520 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2521 fprintf(compiler->verbose, " %s%s%s ret[%s", jump_names[type & 0xff],
2522 !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r",
2523 !(type & SLJIT_CALL_RETURN) ? "" : ".ret",
2524 call_arg_names[arg_types & SLJIT_ARG_MASK]);
2525
2526 arg_types >>= SLJIT_ARG_SHIFT;
2527 if (arg_types) {
2528 fprintf(compiler->verbose, "], args[");
2529 do {
2530 fprintf(compiler->verbose, "%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
2531 arg_types >>= SLJIT_ARG_SHIFT;
2532 if (arg_types)
2533 fprintf(compiler->verbose, ",");
2534 } while (arg_types);
2535 }
2536 fprintf(compiler->verbose, "]\n");
2537 }
2538 #endif /* SLJIT_VERBOSE */
2539 CHECK_RETURN_OK;
2540 }
2541
2542 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_cmp(struct sljit_compiler *compiler, sljit_s32 type,
2543 sljit_s32 src1, sljit_sw src1w,
2544 sljit_s32 src2, sljit_sw src2w)
2545 {
2546 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2547 compiler->skip_checks = 0;
2548 CHECK_RETURN_OK;
2549 }
2550
2551 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2552 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_32)));
2553 CHECK_ARGUMENT((type & 0xff) >= SLJIT_EQUAL && (type & 0xff) <= SLJIT_SIG_LESS_EQUAL);
2554 FUNCTION_CHECK_SRC(src1, src1w);
2555 FUNCTION_CHECK_SRC(src2, src2w);
2556 compiler->last_flags = 0;
2557 #endif /* SLJIT_ARGUMENT_CHECKS */
2558 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2559 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2560 fprintf(compiler->verbose, " cmp%s%s %s, ", (type & SLJIT_32) ? "32" : "",
2561 !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r", jump_names[type & 0xff]);
2562 sljit_verbose_param(compiler, src1, src1w);
2563 fprintf(compiler->verbose, ", ");
2564 sljit_verbose_param(compiler, src2, src2w);
2565 fprintf(compiler->verbose, "\n");
2566 }
2567 #endif /* SLJIT_VERBOSE */
2568 CHECK_RETURN_OK;
2569 }
2570
2571 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fcmp(struct sljit_compiler *compiler, sljit_s32 type,
2572 sljit_s32 src1, sljit_sw src1w,
2573 sljit_s32 src2, sljit_sw src2w)
2574 {
2575 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2576 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2577 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_32)));
2578 CHECK_ARGUMENT((type & 0xff) >= SLJIT_F_EQUAL && (type & 0xff) <= SLJIT_ORDERED_LESS_EQUAL);
2579 FUNCTION_FCHECK(src1, src1w, type & SLJIT_32);
2580 FUNCTION_FCHECK(src2, src2w, type & SLJIT_32);
2581 compiler->last_flags = 0;
2582 #endif /* SLJIT_ARGUMENT_CHECKS */
2583 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2584 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2585 fprintf(compiler->verbose, " fcmp%s%s %s, ", (type & SLJIT_32) ? ".f32" : ".f64",
2586 !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r", jump_names[type & 0xff]);
2587 sljit_verbose_fparam(compiler, src1, src1w);
2588 fprintf(compiler->verbose, ", ");
2589 sljit_verbose_fparam(compiler, src2, src2w);
2590 fprintf(compiler->verbose, "\n");
2591 }
2592 #endif /* SLJIT_VERBOSE */
2593 CHECK_RETURN_OK;
2594 }
2595
2596 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2cmpz(struct sljit_compiler *compiler, sljit_s32 op,
2597 sljit_s32 dst, sljit_sw dstw,
2598 sljit_s32 src1, sljit_sw src1w,
2599 sljit_s32 src2, sljit_sw src2w)
2600 {
2601 SLJIT_COMPILE_ASSERT((SLJIT_JUMP_IF_ZERO == SLJIT_SET_Z) && (SLJIT_JUMP_IF_NON_ZERO == 0),
2602 incorrect_values_for_sljit_jump_zero_or_non_zero);
2603
2604 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2605 CHECK_ARGUMENT(SLJIT_CHECK_OPCODE(op, SLJIT_REWRITABLE_JUMP) >= SLJIT_ADD && SLJIT_CHECK_OPCODE(op, SLJIT_REWRITABLE_JUMP) <= SLJIT_ROTR);
2606 /* Not all opcodes allow setting zero flags. */
2607 CHECK_ARGUMENT(check_sljit_emit_op2_operation(compiler, ((op | SLJIT_SET_Z) & ~SLJIT_REWRITABLE_JUMP)));
2608 FUNCTION_CHECK_DST(dst, dstw);
2609 FUNCTION_CHECK_SRC(src1, src1w);
2610 FUNCTION_CHECK_SRC(src2, src2w);
2611 compiler->last_flags = GET_FLAG_TYPE(op) | (op & SLJIT_32);
2612 #endif /* SLJIT_ARGUMENT_CHECKS */
2613 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2614 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2615 fprintf(compiler->verbose, " cmp%sz.%s%s%s%s%s ", (op & SLJIT_JUMP_IF_ZERO) ? "" : "n",
2616 op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE], !(op & SLJIT_32) ? "" : "32",
2617 !(op & VARIABLE_FLAG_MASK) ? "" : ".", !(op & VARIABLE_FLAG_MASK) ? "" : jump_names[GET_FLAG_TYPE(op & ~SLJIT_REWRITABLE_JUMP)],
2618 !(op & SLJIT_REWRITABLE_JUMP) ? "" : ".r");
2619 sljit_verbose_param(compiler, dst, dstw);
2620 fprintf(compiler->verbose, ", ");
2621 sljit_verbose_param(compiler, src1, src1w);
2622 fprintf(compiler->verbose, ", ");
2623 sljit_verbose_param(compiler, src2, src2w);
2624 fprintf(compiler->verbose, "\n");
2625 }
2626 #endif /* SLJIT_VERBOSE */
2627 CHECK_RETURN_OK;
2628 }
2629
2630 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type,
2631 sljit_s32 src, sljit_sw srcw)
2632 {
2633 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2634 compiler->skip_checks = 0;
2635 CHECK_RETURN_OK;
2636 }
2637
2638 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2639 CHECK_ARGUMENT(type >= SLJIT_JUMP && type <= SLJIT_FAST_CALL);
2640 FUNCTION_CHECK_SRC(src, srcw);
2641 #endif /* SLJIT_ARGUMENT_CHECKS */
2642 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2643 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2644 fprintf(compiler->verbose, " ijump.%s ", jump_names[type]);
2645 sljit_verbose_param(compiler, src, srcw);
2646 fprintf(compiler->verbose, "\n");
2647 }
2648 #endif /* SLJIT_VERBOSE */
2649 CHECK_RETURN_OK;
2650 }
2651
2652 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
2653 sljit_s32 arg_types,
2654 sljit_s32 src, sljit_sw srcw)
2655 {
2656 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2657 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_CALL_RETURN)));
2658 CHECK_ARGUMENT((type & 0xff) >= SLJIT_CALL && (type & 0xff) <= SLJIT_CALL_REG_ARG);
2659 CHECK_ARGUMENT(function_check_arguments(arg_types, compiler->scratches, -1, compiler->fscratches));
2660 FUNCTION_CHECK_SRC(src, srcw);
2661
2662 if (type & SLJIT_CALL_RETURN) {
2663 CHECK_ARGUMENT((arg_types & SLJIT_ARG_MASK) == compiler->last_return);
2664
2665 if (compiler->options & SLJIT_ENTER_REG_ARG) {
2666 CHECK_ARGUMENT((type & 0xff) == SLJIT_CALL_REG_ARG);
2667 } else {
2668 CHECK_ARGUMENT((type & 0xff) != SLJIT_CALL_REG_ARG);
2669 }
2670 }
2671 #endif /* SLJIT_ARGUMENT_CHECKS */
2672 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2673 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2674 fprintf(compiler->verbose, " i%s%s ret[%s", jump_names[type & 0xff],
2675 !(type & SLJIT_CALL_RETURN) ? "" : ".ret",
2676 call_arg_names[arg_types & SLJIT_ARG_MASK]);
2677
2678 arg_types >>= SLJIT_ARG_SHIFT;
2679 if (arg_types) {
2680 fprintf(compiler->verbose, "], args[");
2681 do {
2682 fprintf(compiler->verbose, "%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
2683 arg_types >>= SLJIT_ARG_SHIFT;
2684 if (arg_types)
2685 fprintf(compiler->verbose, ",");
2686 } while (arg_types);
2687 }
2688 fprintf(compiler->verbose, "], ");
2689 sljit_verbose_param(compiler, src, srcw);
2690 fprintf(compiler->verbose, "\n");
2691 }
2692 #endif /* SLJIT_VERBOSE */
2693 CHECK_RETURN_OK;
2694 }
2695
2696 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2697 sljit_s32 dst, sljit_sw dstw,
2698 sljit_s32 type)
2699 {
2700 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2701 CHECK_ARGUMENT(type >= SLJIT_EQUAL && type <= SLJIT_ORDERED_LESS_EQUAL);
2702 CHECK_ARGUMENT(op == SLJIT_MOV || op == SLJIT_MOV32
2703 || (SLJIT_CHECK_OPCODE(op, 0) >= SLJIT_AND && SLJIT_CHECK_OPCODE(op, 0) <= SLJIT_XOR));
2704 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK));
2705
2706 if (type <= SLJIT_NOT_ZERO)
2707 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2708 else
2709 CHECK_ARGUMENT((type & 0xfe) == (compiler->last_flags & 0xff)
2710 || CHECK_UNORDERED(type, compiler->last_flags));
2711
2712 FUNCTION_CHECK_DST(dst, dstw);
2713
2714 if (GET_OPCODE(op) >= SLJIT_ADD)
2715 compiler->last_flags = GET_FLAG_TYPE(op) | (op & (SLJIT_32 | SLJIT_SET_Z));
2716 #endif /* SLJIT_ARGUMENT_CHECKS */
2717 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2718 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2719 fprintf(compiler->verbose, " flags.%s%s%s ",
2720 GET_OPCODE(op) < SLJIT_OP2_BASE ? "mov" : op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE],
2721 GET_OPCODE(op) < SLJIT_OP2_BASE ? op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE] : ((op & SLJIT_32) ? "32" : ""),
2722 !(op & SLJIT_SET_Z) ? "" : ".z");
2723 sljit_verbose_param(compiler, dst, dstw);
2724 fprintf(compiler->verbose, ", %s\n", jump_names[type]);
2725 }
2726 #endif /* SLJIT_VERBOSE */
2727 CHECK_RETURN_OK;
2728 }
2729
2730 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
2731 sljit_s32 dst_reg,
2732 sljit_s32 src1, sljit_sw src1w,
2733 sljit_s32 src2_reg)
2734 {
2735 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2736 sljit_s32 cond = type & ~(SLJIT_32 | SLJIT_COMPARE_SELECT);
2737 CHECK_ARGUMENT((type & SLJIT_COMPARE_SELECT) ? (cond >= SLJIT_LESS && cond <= SLJIT_SET_SIG_LESS_EQUAL)
2738 : (cond >= SLJIT_EQUAL && cond <= SLJIT_ORDERED_LESS_EQUAL));
2739 CHECK_ARGUMENT(compiler->scratches != -1 && compiler->saveds != -1);
2740 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
2741 FUNCTION_CHECK_SRC(src1, src1w);
2742 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src2_reg));
2743
2744 if (!(type & SLJIT_COMPARE_SELECT)) {
2745 if (cond <= SLJIT_NOT_ZERO)
2746 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2747 else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2748 CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2749 compiler->last_flags = 0;
2750 } else
2751 CHECK_ARGUMENT((cond & 0xfe) == (compiler->last_flags & 0xff)
2752 || CHECK_UNORDERED(cond, compiler->last_flags));
2753 } else
2754 compiler->last_flags = 0;
2755 #endif /* SLJIT_ARGUMENT_CHECKS */
2756 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2757 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2758 fprintf(compiler->verbose, " %sselect%s %s, ",
2759 !(type & SLJIT_COMPARE_SELECT) ? "" : "cmp_",
2760 !(type & SLJIT_32) ? "" : "32",
2761 jump_names[type & ~SLJIT_32]);
2762 sljit_verbose_reg(compiler, dst_reg);
2763 fprintf(compiler->verbose, ", ");
2764 sljit_verbose_param(compiler, src1, src1w);
2765 fprintf(compiler->verbose, ", ");
2766 sljit_verbose_reg(compiler, src2_reg);
2767 fprintf(compiler->verbose, "\n");
2768 }
2769 #endif /* SLJIT_VERBOSE */
2770 CHECK_RETURN_OK;
2771 }
2772
2773 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
2774 sljit_s32 dst_freg,
2775 sljit_s32 src1, sljit_sw src1w,
2776 sljit_s32 src2_freg)
2777 {
2778 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2779 sljit_s32 cond = type & ~SLJIT_32;
2780
2781 CHECK_ARGUMENT(cond >= SLJIT_EQUAL && cond <= SLJIT_ORDERED_LESS_EQUAL);
2782
2783 CHECK_ARGUMENT(compiler->fscratches != -1 && compiler->fsaveds != -1);
2784 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(dst_freg, type & SLJIT_32));
2785 FUNCTION_FCHECK(src1, src1w, type & SLJIT_32);
2786 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(src2_freg, type & SLJIT_32));
2787
2788 if (cond <= SLJIT_NOT_ZERO)
2789 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2790 else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2791 CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2792 compiler->last_flags = 0;
2793 } else
2794 CHECK_ARGUMENT((cond & 0xfe) == (compiler->last_flags & 0xff)
2795 || CHECK_UNORDERED(cond, compiler->last_flags));
2796 #endif /* SLJIT_ARGUMENT_CHECKS */
2797 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2798 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2799 fprintf(compiler->verbose, " fselect%s %s, ",
2800 !(type & SLJIT_32) ? "" : "32",
2801 jump_names[type & ~SLJIT_32]);
2802 sljit_verbose_freg(compiler, dst_freg);
2803 fprintf(compiler->verbose, ", ");
2804 sljit_verbose_fparam(compiler, src1, src1w);
2805 fprintf(compiler->verbose, ", ");
2806 sljit_verbose_freg(compiler, src2_freg);
2807 fprintf(compiler->verbose, "\n");
2808 }
2809 #endif /* SLJIT_VERBOSE */
2810 CHECK_RETURN_OK;
2811 }
2812
2813 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
2814 sljit_s32 reg,
2815 sljit_s32 mem, sljit_sw memw)
2816 {
2817 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2818 sljit_s32 allowed_flags;
2819 #endif /* SLJIT_ARGUMENT_CHECKS */
2820
2821 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2822 compiler->skip_checks = 0;
2823 CHECK_RETURN_OK;
2824 }
2825
2826 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2827 if (type & SLJIT_MEM_UNALIGNED) {
2828 CHECK_ARGUMENT(!(type & (SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2829 } else if (type & SLJIT_MEM_ALIGNED_16) {
2830 CHECK_ARGUMENT(!(type & SLJIT_MEM_ALIGNED_32));
2831 } else {
2832 CHECK_ARGUMENT((reg & REG_PAIR_MASK) || (type & SLJIT_MEM_ALIGNED_32));
2833 }
2834
2835 allowed_flags = SLJIT_MEM_UNALIGNED;
2836
2837 switch (type & 0xff) {
2838 case SLJIT_MOV_P:
2839 case SLJIT_MOV:
2840 allowed_flags |= SLJIT_MEM_ALIGNED_32;
2841 SLJIT_FALLTHROUGH
2842 case SLJIT_MOV_U32:
2843 case SLJIT_MOV_S32:
2844 case SLJIT_MOV32:
2845 allowed_flags |= SLJIT_MEM_ALIGNED_16;
2846 break;
2847 }
2848
2849 CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | allowed_flags)) == 0);
2850
2851 if (reg & REG_PAIR_MASK) {
2852 CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV);
2853 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_FIRST(reg)));
2854 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_SECOND(reg)));
2855 CHECK_ARGUMENT(REG_PAIR_FIRST(reg) != REG_PAIR_SECOND(reg));
2856 } else {
2857 CHECK_ARGUMENT((type & 0xff) >= SLJIT_MOV && (type & 0xff) <= SLJIT_MOV_P);
2858 CHECK_ARGUMENT(!(type & SLJIT_32) || ((type & 0xff) >= SLJIT_MOV_U8 && (type & 0xff) <= SLJIT_MOV_S16));
2859 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2860 }
2861
2862 FUNCTION_CHECK_SRC_MEM(mem, memw);
2863 #endif /* SLJIT_ARGUMENT_CHECKS */
2864 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2865 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2866 if ((type & 0xff) == SLJIT_MOV32)
2867 fprintf(compiler->verbose, " %s32",
2868 (type & SLJIT_MEM_STORE) ? "store" : "load");
2869 else
2870 fprintf(compiler->verbose, " %s%s%s",
2871 (type & SLJIT_MEM_STORE) ? "store" : "load",
2872 !(type & SLJIT_32) ? "" : "32", op1_types[(type & 0xff) - SLJIT_OP1_BASE]);
2873
2874 if (type & SLJIT_MEM_UNALIGNED)
2875 fprintf(compiler->verbose, ".unal");
2876 else if (type & SLJIT_MEM_ALIGNED_16)
2877 fprintf(compiler->verbose, ".al16");
2878 else if (type & SLJIT_MEM_ALIGNED_32)
2879 fprintf(compiler->verbose, ".al32");
2880
2881 if (reg & REG_PAIR_MASK) {
2882 fprintf(compiler->verbose, " {");
2883 sljit_verbose_reg(compiler, REG_PAIR_FIRST(reg));
2884 fprintf(compiler->verbose, ", ");
2885 sljit_verbose_reg(compiler, REG_PAIR_SECOND(reg));
2886 fprintf(compiler->verbose, "}, ");
2887 } else {
2888 fprintf(compiler->verbose, " ");
2889 sljit_verbose_reg(compiler, reg);
2890 fprintf(compiler->verbose, ", ");
2891 }
2892 sljit_verbose_param(compiler, mem, memw);
2893 fprintf(compiler->verbose, "\n");
2894 }
2895 #endif /* SLJIT_VERBOSE */
2896 CHECK_RETURN_OK;
2897 }
2898
2899 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
2900 sljit_s32 reg,
2901 sljit_s32 mem, sljit_sw memw)
2902 {
2903 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2904 compiler->skip_checks = 0;
2905 CHECK_RETURN_OK;
2906 }
2907
2908 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2909 CHECK_ARGUMENT((type & 0xff) >= SLJIT_MOV && (type & 0xff) <= SLJIT_MOV_P);
2910 CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_SUPP | SLJIT_MEM_POST)) == 0);
2911 CHECK_ARGUMENT((mem & REG_MASK) != 0 && (mem & REG_MASK) != reg);
2912
2913 FUNCTION_CHECK_SRC_MEM(mem, memw);
2914 #endif /* SLJIT_ARGUMENT_CHECKS */
2915 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2916 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2917 if (type & SLJIT_MEM_SUPP)
2918 CHECK_RETURN_OK;
2919 if (sljit_emit_mem_update(compiler, type | SLJIT_MEM_SUPP, reg, mem, memw) == SLJIT_ERR_UNSUPPORTED) {
2920 fprintf(compiler->verbose, " # mem: unsupported form, no instructions are emitted\n");
2921 CHECK_RETURN_OK;
2922 }
2923
2924 if ((type & 0xff) == SLJIT_MOV32)
2925 fprintf(compiler->verbose, " %s32.%s ",
2926 (type & SLJIT_MEM_STORE) ? "store" : "load",
2927 (type & SLJIT_MEM_POST) ? "post" : "pre");
2928 else
2929 fprintf(compiler->verbose, " %s%s%s.%s ",
2930 (type & SLJIT_MEM_STORE) ? "store" : "load",
2931 !(type & SLJIT_32) ? "" : "32",
2932 op1_types[(type & 0xff) - SLJIT_OP1_BASE],
2933 (type & SLJIT_MEM_POST) ? "post" : "pre");
2934
2935 sljit_verbose_reg(compiler, reg);
2936 fprintf(compiler->verbose, ", ");
2937 sljit_verbose_param(compiler, mem, memw);
2938 fprintf(compiler->verbose, "\n");
2939 }
2940 #endif /* SLJIT_VERBOSE */
2941 CHECK_RETURN_OK;
2942 }
2943
2944 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fmem(struct sljit_compiler *compiler, sljit_s32 type,
2945 sljit_s32 freg,
2946 sljit_s32 mem, sljit_sw memw)
2947 {
2948 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2949 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2950 CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV_F64);
2951
2952 if (type & SLJIT_MEM_UNALIGNED) {
2953 CHECK_ARGUMENT(!(type & (SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2954 } else if (type & SLJIT_MEM_ALIGNED_16) {
2955 CHECK_ARGUMENT(!(type & SLJIT_MEM_ALIGNED_32));
2956 } else {
2957 CHECK_ARGUMENT(type & SLJIT_MEM_ALIGNED_32);
2958 CHECK_ARGUMENT(!(type & SLJIT_32));
2959 }
2960
2961 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_UNALIGNED | SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2962 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, type & SLJIT_32));
2963 FUNCTION_CHECK_SRC_MEM(mem, memw);
2964 #endif /* SLJIT_ARGUMENT_CHECKS */
2965 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2966 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2967 fprintf(compiler->verbose, " %s.%s",
2968 (type & SLJIT_MEM_STORE) ? "store" : "load",
2969 !(type & SLJIT_32) ? "f64" : "f32");
2970
2971 if (type & SLJIT_MEM_UNALIGNED)
2972 printf(".unal");
2973 else if (type & SLJIT_MEM_ALIGNED_16)
2974 printf(".al16");
2975 else if (type & SLJIT_MEM_ALIGNED_32)
2976 printf(".al32");
2977
2978 fprintf(compiler->verbose, " ");
2979 sljit_verbose_freg(compiler, freg);
2980 fprintf(compiler->verbose, ", ");
2981 sljit_verbose_param(compiler, mem, memw);
2982 fprintf(compiler->verbose, "\n");
2983 }
2984 #endif /* SLJIT_VERBOSE */
2985 CHECK_RETURN_OK;
2986 }
2987
2988 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
2989 sljit_s32 freg,
2990 sljit_s32 mem, sljit_sw memw)
2991 {
2992 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2993 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2994 CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV_F64);
2995 CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_SUPP | SLJIT_MEM_POST)) == 0);
2996 FUNCTION_CHECK_SRC_MEM(mem, memw);
2997 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, type & SLJIT_32));
2998 #endif /* SLJIT_ARGUMENT_CHECKS */
2999 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3000 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3001 if (type & SLJIT_MEM_SUPP)
3002 CHECK_RETURN_OK;
3003 if (sljit_emit_fmem_update(compiler, type | SLJIT_MEM_SUPP, freg, mem, memw) == SLJIT_ERR_UNSUPPORTED) {
3004 fprintf(compiler->verbose, " # fmem: unsupported form, no instructions are emitted\n");
3005 CHECK_RETURN_OK;
3006 }
3007
3008 fprintf(compiler->verbose, " %s.%s.%s ",
3009 (type & SLJIT_MEM_STORE) ? "store" : "load",
3010 !(type & SLJIT_32) ? "f64" : "f32",
3011 (type & SLJIT_MEM_POST) ? "post" : "pre");
3012
3013 sljit_verbose_freg(compiler, freg);
3014 fprintf(compiler->verbose, ", ");
3015 sljit_verbose_param(compiler, mem, memw);
3016 fprintf(compiler->verbose, "\n");
3017 }
3018 #endif /* SLJIT_VERBOSE */
3019 CHECK_RETURN_OK;
3020 }
3021
3022 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
3023 sljit_s32 vreg,
3024 sljit_s32 srcdst, sljit_sw srcdstw)
3025 {
3026 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3027 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3028 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(SLJIT_SIMD_STORE)) == 0);
3029 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3030 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) <= SLJIT_SIMD_GET_REG_SIZE(type));
3031 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) <= (srcdst & SLJIT_MEM) ? SLJIT_SIMD_GET_REG_SIZE(type) : 0);
3032 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3033 FUNCTION_VCHECK(srcdst, srcdstw, type);
3034 #endif /* SLJIT_ARGUMENT_CHECKS */
3035 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3036 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3037 if (type & SLJIT_SIMD_TEST)
3038 CHECK_RETURN_OK;
3039 if (sljit_emit_simd_mov(compiler, type | SLJIT_SIMD_TEST, vreg, srcdst, srcdstw) == SLJIT_ERR_UNSUPPORTED) {
3040 fprintf(compiler->verbose, " # simd_mem: unsupported form, no instructions are emitted\n");
3041 CHECK_RETURN_OK;
3042 }
3043
3044 fprintf(compiler->verbose, " simd_%s.%d.%s%d",
3045 (type & SLJIT_SIMD_STORE) ? "store" : "load",
3046 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3047 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3048 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3049
3050 if ((type & 0x3f000000) == SLJIT_SIMD_MEM_UNALIGNED)
3051 fprintf(compiler->verbose, ".unal ");
3052 else
3053 fprintf(compiler->verbose, ".al%d ", (8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)));
3054
3055 sljit_verbose_vreg(compiler, vreg);
3056 fprintf(compiler->verbose, ", ");
3057 sljit_verbose_vparam(compiler, srcdst, srcdstw);
3058 fprintf(compiler->verbose, "\n");
3059 }
3060 #endif /* SLJIT_VERBOSE */
3061 CHECK_RETURN_OK;
3062 }
3063
3064 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3065 sljit_s32 vreg,
3066 sljit_s32 src, sljit_sw srcw)
3067 {
3068 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3069 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3070 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(0)) == 0);
3071 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3072 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3073 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3074
3075 if (type & SLJIT_SIMD_FLOAT) {
3076 if (src == SLJIT_IMM) {
3077 CHECK_ARGUMENT(srcw == 0);
3078 } else {
3079 FUNCTION_FCHECK(src, srcw, SLJIT_SIMD_GET_ELEM_SIZE(type) == 2);
3080 }
3081 } else if (src != SLJIT_IMM) {
3082 FUNCTION_CHECK_DST(src, srcw);
3083 }
3084 #endif /* SLJIT_ARGUMENT_CHECKS */
3085 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3086 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3087 if (type & SLJIT_SIMD_TEST)
3088 CHECK_RETURN_OK;
3089 if (sljit_emit_simd_replicate(compiler, type | SLJIT_SIMD_TEST, vreg, src, srcw) == SLJIT_ERR_UNSUPPORTED) {
3090 fprintf(compiler->verbose, " # simd_dup: unsupported form, no instructions are emitted\n");
3091 CHECK_RETURN_OK;
3092 }
3093
3094 fprintf(compiler->verbose, " simd_replicate.%d.%s%d ",
3095 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3096 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3097 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3098
3099 sljit_verbose_vreg(compiler, vreg);
3100 fprintf(compiler->verbose, ", ");
3101 if (type & SLJIT_SIMD_FLOAT)
3102 sljit_verbose_fparam(compiler, src, srcw);
3103 else
3104 sljit_verbose_param(compiler, src, srcw);
3105 fprintf(compiler->verbose, "\n");
3106 }
3107 #endif /* SLJIT_VERBOSE */
3108 CHECK_RETURN_OK;
3109 }
3110
3111 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
3112 sljit_s32 vreg, sljit_s32 lane_index,
3113 sljit_s32 srcdst, sljit_sw srcdstw)
3114 {
3115 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3116 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3117 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO | SLJIT_SIMD_LANE_SIGNED | SLJIT_32)) == 0);
3118 CHECK_ARGUMENT((type & (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO)) != (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO));
3119 CHECK_ARGUMENT((type & (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_SIGNED)) != SLJIT_SIMD_LANE_SIGNED);
3120 CHECK_ARGUMENT(!(type & SLJIT_SIMD_FLOAT) || !(type & (SLJIT_SIMD_LANE_SIGNED | SLJIT_32)));
3121 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3122 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3123 CHECK_ARGUMENT(!(type & SLJIT_32) || SLJIT_SIMD_GET_ELEM_SIZE(type) <= 2);
3124 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3125 CHECK_ARGUMENT(lane_index >= 0 && lane_index < (1 << (SLJIT_SIMD_GET_REG_SIZE(type) - SLJIT_SIMD_GET_ELEM_SIZE(type))));
3126
3127 if (type & SLJIT_SIMD_FLOAT) {
3128 FUNCTION_FCHECK(srcdst, srcdstw, SLJIT_SIMD_GET_ELEM_SIZE(type) == 2);
3129 } else if ((type & SLJIT_SIMD_STORE) || srcdst != SLJIT_IMM) {
3130 FUNCTION_CHECK_DST(srcdst, srcdstw);
3131 }
3132 #endif /* SLJIT_ARGUMENT_CHECKS */
3133 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3134 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3135 if (type & SLJIT_SIMD_TEST)
3136 CHECK_RETURN_OK;
3137 if (sljit_emit_simd_lane_mov(compiler, type | SLJIT_SIMD_TEST, vreg, lane_index, srcdst, srcdstw) == SLJIT_ERR_UNSUPPORTED) {
3138 fprintf(compiler->verbose, " # simd_move_lane: unsupported form, no instructions are emitted\n");
3139 CHECK_RETURN_OK;
3140 }
3141
3142 fprintf(compiler->verbose, " simd_%s_lane%s%s%s.%d.%s%d ",
3143 (type & SLJIT_SIMD_STORE) ? "store" : "load",
3144 (type & SLJIT_32) ? "32" : "",
3145 (type & SLJIT_SIMD_LANE_ZERO) ? "_z" : "",
3146 (type & SLJIT_SIMD_LANE_SIGNED) ? "_s" : "",
3147 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3148 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3149 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3150
3151 sljit_verbose_vreg(compiler, vreg);
3152 fprintf(compiler->verbose, "[%d], ", lane_index);
3153 if (type & SLJIT_SIMD_FLOAT)
3154 sljit_verbose_fparam(compiler, srcdst, srcdstw);
3155 else
3156 sljit_verbose_param(compiler, srcdst, srcdstw);
3157 fprintf(compiler->verbose, "\n");
3158 }
3159 #endif /* SLJIT_VERBOSE */
3160 CHECK_RETURN_OK;
3161 }
3162
3163 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3164 sljit_s32 vreg,
3165 sljit_s32 src, sljit_s32 src_lane_index)
3166 {
3167 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3168 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3169 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(0)) == 0);
3170 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3171 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3172 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3173 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(src, type));
3174 CHECK_ARGUMENT(src_lane_index >= 0 && src_lane_index < (1 << (SLJIT_SIMD_GET_REG_SIZE(type) - SLJIT_SIMD_GET_ELEM_SIZE(type))));
3175 #endif /* SLJIT_ARGUMENT_CHECKS */
3176 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3177 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3178 if (type & SLJIT_SIMD_TEST)
3179 CHECK_RETURN_OK;
3180 if (sljit_emit_simd_lane_replicate(compiler, type | SLJIT_SIMD_TEST, vreg, src, src_lane_index) == SLJIT_ERR_UNSUPPORTED) {
3181 fprintf(compiler->verbose, " # simd_lane_replicate: unsupported form, no instructions are emitted\n");
3182 CHECK_RETURN_OK;
3183 }
3184
3185 fprintf(compiler->verbose, " simd_lane_replicate.%d.%s%d ",
3186 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3187 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3188 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3189
3190 sljit_verbose_vreg(compiler, vreg);
3191 fprintf(compiler->verbose, ", ");
3192 sljit_verbose_vreg(compiler, src);
3193 fprintf(compiler->verbose, "[%d]\n", src_lane_index);
3194 }
3195 #endif /* SLJIT_VERBOSE */
3196 CHECK_RETURN_OK;
3197 }
3198
3199 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
3200 sljit_s32 vreg,
3201 sljit_s32 src, sljit_sw srcw)
3202 {
3203 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3204 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3205 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(SLJIT_SIMD_EXTEND_SIGNED)) == 0);
3206 CHECK_ARGUMENT((type & (SLJIT_SIMD_EXTEND_SIGNED | SLJIT_SIMD_FLOAT)) != (SLJIT_SIMD_EXTEND_SIGNED | SLJIT_SIMD_FLOAT));
3207 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3208 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3209 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_ELEM2_SIZE(type));
3210 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3211 FUNCTION_VCHECK(src, srcw, type);
3212 #endif /* SLJIT_ARGUMENT_CHECKS */
3213 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3214 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3215 if (type & SLJIT_SIMD_TEST)
3216 CHECK_RETURN_OK;
3217 if (sljit_emit_simd_extend(compiler, type | SLJIT_SIMD_TEST, vreg, src, srcw) == SLJIT_ERR_UNSUPPORTED) {
3218 fprintf(compiler->verbose, " # simd_extend: unsupported form, no instructions are emitted\n");
3219 CHECK_RETURN_OK;
3220 }
3221
3222 fprintf(compiler->verbose, " simd_load_extend%s.%d.%s%d.%s%d ",
3223 (type & SLJIT_SIMD_EXTEND_SIGNED) ? "_s" : "",
3224 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3225 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3226 (8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)),
3227 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3228 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3229
3230 sljit_verbose_vreg(compiler, vreg);
3231 fprintf(compiler->verbose, ", ");
3232 sljit_verbose_vparam(compiler, src, srcw);
3233 fprintf(compiler->verbose, "\n");
3234 }
3235 #endif /* SLJIT_VERBOSE */
3236 CHECK_RETURN_OK;
3237 }
3238
3239 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
3240 sljit_s32 vreg,
3241 sljit_s32 dst, sljit_sw dstw)
3242 {
3243 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3244 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3245 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(SLJIT_32)) == SLJIT_SIMD_STORE);
3246 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3247 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
3248 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(vreg, type));
3249 FUNCTION_CHECK_DST(dst, dstw);
3250 #endif /* SLJIT_ARGUMENT_CHECKS */
3251 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3252 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3253 if (type & SLJIT_SIMD_TEST)
3254 CHECK_RETURN_OK;
3255 if (sljit_emit_simd_sign(compiler, type | SLJIT_SIMD_TEST, vreg, dst, dstw) == SLJIT_ERR_UNSUPPORTED) {
3256 fprintf(compiler->verbose, " # simd_sign: unsupported form, no instructions are emitted\n");
3257 CHECK_RETURN_OK;
3258 }
3259
3260 fprintf(compiler->verbose, " simd_store_sign%s.%d.%s%d ",
3261 (type & SLJIT_32) ? "32" : "",
3262 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3263 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3264 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3265
3266 sljit_verbose_vreg(compiler, vreg);
3267 fprintf(compiler->verbose, ", ");
3268 sljit_verbose_param(compiler, dst, dstw);
3269 fprintf(compiler->verbose, "\n");
3270 }
3271 #endif /* SLJIT_VERBOSE */
3272 CHECK_RETURN_OK;
3273 }
3274
3275 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
3276 sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
3277 {
3278 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3279 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
3280 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(0)) >= SLJIT_SIMD_OP2_AND && (type & SLJIT_SIMD_TYPE_MASK2(0)) <= SLJIT_SIMD_OP2_SHUFFLE);
3281 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
3282 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) <= SLJIT_SIMD_GET_REG_SIZE(type));
3283 CHECK_ARGUMENT(SLJIT_SIMD_GET_OPCODE(type) != SLJIT_SIMD_OP2_SHUFFLE || (SLJIT_SIMD_GET_ELEM_SIZE(type) == 0 && !(type & SLJIT_SIMD_FLOAT)));
3284 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) <= (src2 & SLJIT_MEM) ? SLJIT_SIMD_GET_REG_SIZE(type) : 0);
3285 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(dst_vreg, type));
3286 CHECK_ARGUMENT(FUNCTION_CHECK_IS_VREG(src1_vreg, type));
3287 FUNCTION_VCHECK(src2, src2w, type);
3288 #endif /* SLJIT_ARGUMENT_CHECKS */
3289 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3290 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3291 if (type & SLJIT_SIMD_TEST)
3292 CHECK_RETURN_OK;
3293 if (sljit_emit_simd_op2(compiler, type | SLJIT_SIMD_TEST, dst_vreg, src1_vreg, src2, src2w) == SLJIT_ERR_UNSUPPORTED) {
3294 fprintf(compiler->verbose, " # simd_op2: unsupported form, no instructions are emitted\n");
3295 CHECK_RETURN_OK;
3296 }
3297
3298 fprintf(compiler->verbose, " simd_%s.%d.%s%d",
3299 simd_op2_names[SLJIT_SIMD_GET_OPCODE(type) - 1],
3300 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
3301 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
3302 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
3303
3304 if ((type & 0x3f000000) != SLJIT_SIMD_MEM_UNALIGNED)
3305 fprintf(compiler->verbose, ".al%d", (8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)));
3306
3307 fprintf(compiler->verbose, " ");
3308 sljit_verbose_vreg(compiler, dst_vreg);
3309 fprintf(compiler->verbose, ", ");
3310 sljit_verbose_vreg(compiler, src1_vreg);
3311 fprintf(compiler->verbose, ", ");
3312 sljit_verbose_vparam(compiler, src2, src2w);
3313 fprintf(compiler->verbose, "\n");
3314 }
3315 #endif /* SLJIT_VERBOSE */
3316 CHECK_RETURN_OK;
3317 }
3318
3319 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
3320 {
3321 /* Any offset is allowed. */
3322 SLJIT_UNUSED_ARG(offset);
3323
3324 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3325 FUNCTION_CHECK_DST(dst, dstw);
3326 #endif /* SLJIT_ARGUMENT_CHECKS */
3327 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3328 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3329 fprintf(compiler->verbose, " local_base ");
3330 sljit_verbose_param(compiler, dst, dstw);
3331 fprintf(compiler->verbose, ", #%" SLJIT_PRINT_D "d\n", offset);
3332 }
3333 #endif /* SLJIT_VERBOSE */
3334 CHECK_RETURN_OK;
3335 }
3336
3337 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 op,
3338 sljit_s32 dst, sljit_sw dstw,
3339 sljit_sw init_value)
3340 {
3341 SLJIT_UNUSED_ARG(init_value);
3342
3343 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3344 CHECK_ARGUMENT(op == SLJIT_MOV || op == SLJIT_MOV_S32
3345 || op == SLJIT_MOV32 || (op | SLJIT_32) == SLJIT_MOV32_U8);
3346 FUNCTION_CHECK_DST(dst, dstw);
3347 #endif /* SLJIT_ARGUMENT_CHECKS */
3348 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3349 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3350 fprintf(compiler->verbose, " const%s%s ",
3351 !(op & SLJIT_32) ? "" : "32", op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
3352 sljit_verbose_param(compiler, dst, dstw);
3353 fprintf(compiler->verbose, ", #%" SLJIT_PRINT_D "d\n", init_value);
3354 }
3355 #endif /* SLJIT_VERBOSE */
3356 CHECK_RETURN_OK;
3357 }
3358
3359 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_addr(struct sljit_compiler *compiler, sljit_s32 op,
3360 sljit_s32 dst, sljit_sw dstw)
3361 {
3362 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3363 CHECK_ARGUMENT(op == SLJIT_MOV_ADDR || op == SLJIT_MOV_ABS_ADDR || op == SLJIT_ADD_ABS_ADDR);
3364 FUNCTION_CHECK_DST(dst, dstw);
3365 #endif /* SLJIT_ARGUMENT_CHECKS */
3366 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3367 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3368 fprintf(compiler->verbose, " %s ", op_addr_types[op]);
3369 sljit_verbose_param(compiler, dst, dstw);
3370 fprintf(compiler->verbose, "\n");
3371 }
3372 #endif /* SLJIT_VERBOSE */
3373 CHECK_RETURN_OK;
3374 }
3375
3376 #else /* !SLJIT_ARGUMENT_CHECKS && !SLJIT_VERBOSE */
3377
3378 #define SLJIT_SKIP_CHECKS(compiler)
3379
3380 #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
3381
3382 #define SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw) \
3383 SLJIT_COMPILE_ASSERT(!(SLJIT_CONV_SW_FROM_F64 & 0x1) && !(SLJIT_CONV_F64_FROM_SW & 0x1) && !(SLJIT_CONV_F64_FROM_UW & 0x1), \
3384 invalid_float_opcodes); \
3385 if (GET_OPCODE(op) >= SLJIT_CONV_SW_FROM_F64 && GET_OPCODE(op) <= SLJIT_CMP_F64) { \
3386 if (GET_OPCODE(op) == SLJIT_CMP_F64) { \
3387 CHECK(check_sljit_emit_fop1_cmp(compiler, op, dst, dstw, src, srcw)); \
3388 ADJUST_LOCAL_OFFSET(dst, dstw); \
3389 ADJUST_LOCAL_OFFSET(src, srcw); \
3390 return sljit_emit_fop1_cmp(compiler, op, dst, dstw, src, srcw); \
3391 } \
3392 if ((GET_OPCODE(op) | 0x1) == SLJIT_CONV_S32_FROM_F64) { \
3393 CHECK(check_sljit_emit_fop1_conv_sw_from_f64(compiler, op, dst, dstw, src, srcw)); \
3394 ADJUST_LOCAL_OFFSET(dst, dstw); \
3395 ADJUST_LOCAL_OFFSET(src, srcw); \
3396 return sljit_emit_fop1_conv_sw_from_f64(compiler, op, dst, dstw, src, srcw); \
3397 } \
3398 if ((GET_OPCODE(op) | 0x1) == SLJIT_CONV_F64_FROM_S32) { \
3399 CHECK(check_sljit_emit_fop1_conv_f64_from_w(compiler, op, dst, dstw, src, srcw)); \
3400 ADJUST_LOCAL_OFFSET(dst, dstw); \
3401 ADJUST_LOCAL_OFFSET(src, srcw); \
3402 return sljit_emit_fop1_conv_f64_from_sw(compiler, op, dst, dstw, src, srcw); \
3403 } \
3404 CHECK(check_sljit_emit_fop1_conv_f64_from_w(compiler, op, dst, dstw, src, srcw)); \
3405 ADJUST_LOCAL_OFFSET(dst, dstw); \
3406 ADJUST_LOCAL_OFFSET(src, srcw); \
3407 return sljit_emit_fop1_conv_f64_from_uw(compiler, op, dst, dstw, src, srcw); \
3408 } \
3409 CHECK(check_sljit_emit_fop1(compiler, op, dst, dstw, src, srcw)); \
3410 ADJUST_LOCAL_OFFSET(dst, dstw); \
3411 ADJUST_LOCAL_OFFSET(src, srcw);
3412
3413 #if (!(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) || (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6))
3414
3415 static sljit_s32 sljit_emit_mem_unaligned(struct sljit_compiler *compiler, sljit_s32 type,
3416 sljit_s32 reg,
3417 sljit_s32 mem, sljit_sw memw)
3418 {
3419 SLJIT_SKIP_CHECKS(compiler);
3420
3421 if (type & SLJIT_MEM_STORE)
3422 return sljit_emit_op1(compiler, type & (0xff | SLJIT_32), mem, memw, reg, 0);
3423 return sljit_emit_op1(compiler, type & (0xff | SLJIT_32), reg, 0, mem, memw);
3424 }
3425
3426 #endif /* (!SLJIT_CONFIG_MIPS || SLJIT_MIPS_REV >= 6) */
3427
3428 #if (!(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) || (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6)) \
3429 && !(defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32)
3430
3431 static sljit_s32 sljit_emit_fmem_unaligned(struct sljit_compiler *compiler, sljit_s32 type,
3432 sljit_s32 freg,
3433 sljit_s32 mem, sljit_sw memw)
3434 {
3435 SLJIT_SKIP_CHECKS(compiler);
3436
3437 if (type & SLJIT_MEM_STORE)
3438 return sljit_emit_fop1(compiler, type & (0xff | SLJIT_32), mem, memw, freg, 0);
3439 return sljit_emit_fop1(compiler, type & (0xff | SLJIT_32), freg, 0, mem, memw);
3440 }
3441
3442 #endif /* (!SLJIT_CONFIG_MIPS || SLJIT_MIPS_REV >= 6) && !SLJIT_CONFIG_ARM */
3443
3444 static void sljit_reset_read_only_buffers(struct sljit_read_only_buffer *buffers)
3445 {
3446 while (buffers != NULL) {
3447 buffers->u.label = NULL;
3448 buffers = buffers->next;
3449 }
3450 }
3451
3452 /* CPU description section */
3453
3454 #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
3455 #define SLJIT_CPUINFO_PART1 " 32bit ("
3456 #elif (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
3457 #define SLJIT_CPUINFO_PART1 " 64bit ("
3458 #else /* !SLJIT_32BIT_ARCHITECTURE && !SLJIT_64BIT_ARCHITECTURE */
3459 #error "Internal error: CPU type info missing"
3460 #endif /* SLJIT_32BIT_ARCHITECTURE */
3461
3462 #if (defined SLJIT_LITTLE_ENDIAN && SLJIT_LITTLE_ENDIAN)
3463 #define SLJIT_CPUINFO_PART2 "little endian + "
3464 #elif (defined SLJIT_BIG_ENDIAN && SLJIT_BIG_ENDIAN)
3465 #define SLJIT_CPUINFO_PART2 "big endian + "
3466 #else /* !SLJIT_LITTLE_ENDIAN && !SLJIT_BIG_ENDIAN */
3467 #error "Internal error: CPU type info missing"
3468 #endif /* SLJIT_LITTLE_ENDIAN */
3469
3470 #if (defined SLJIT_UNALIGNED && SLJIT_UNALIGNED)
3471 #define SLJIT_CPUINFO_PART3 "unaligned)"
3472 #else /* !SLJIT_UNALIGNED */
3473 #define SLJIT_CPUINFO_PART3 "aligned)"
3474 #endif /* SLJIT_UNALIGNED */
3475
3476 #define SLJIT_CPUINFO SLJIT_CPUINFO_PART1 SLJIT_CPUINFO_PART2 SLJIT_CPUINFO_PART3
3477
3478 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
3479 # include "sljitNativeX86_common.c"
3480 #elif (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
3481 # include "sljitNativeARM_32.c"
3482 #elif (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
3483 # include "sljitNativeARM_32.c"
3484 #elif (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
3485 # include "sljitNativeARM_T2_32.c"
3486 #elif (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3487 # include "sljitNativeARM_64.c"
3488 #elif (defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3489 # include "sljitNativePPC_common.c"
3490 #elif (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
3491 # include "sljitNativeMIPS_common.c"
3492 #elif (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
3493 # include "sljitNativeRISCV_common.c"
3494 #elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
3495 # include "sljitNativeS390X.c"
3496 #elif (defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3497 # include "sljitNativeLOONGARCH_64.c"
3498 #endif /* SLJIT_CONFIG_X86 */
3499
3500 #include "sljitSerialize.c"
3501
3502 static SLJIT_INLINE sljit_s32 emit_mov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3503 {
3504 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
3505 /* At the moment the pointer size is always equal to sljit_sw. May be changed in the future. */
3506 if (src == SLJIT_RETURN_REG && (op == SLJIT_MOV || op == SLJIT_MOV_P))
3507 return SLJIT_SUCCESS;
3508 #else /* !SLJIT_64BIT_ARCHITECTURE */
3509 if (src == SLJIT_RETURN_REG && (op == SLJIT_MOV || op == SLJIT_MOV_U32 || op == SLJIT_MOV_S32 || op == SLJIT_MOV_P))
3510 return SLJIT_SUCCESS;
3511 #endif /* SLJIT_64BIT_ARCHITECTURE */
3512
3513 SLJIT_SKIP_CHECKS(compiler);
3514 return sljit_emit_op1(compiler, op, SLJIT_RETURN_REG, 0, src, srcw);
3515 }
3516
3517 #if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) \
3518 && !((defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) && defined __SOFTFP__)
3519
3520 static SLJIT_INLINE sljit_s32 emit_fmov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3521 {
3522 if (src == SLJIT_FR0)
3523 return SLJIT_SUCCESS;
3524
3525 SLJIT_SKIP_CHECKS(compiler);
3526 return sljit_emit_fop1(compiler, op, SLJIT_RETURN_FREG, 0, src, srcw);
3527 }
3528
3529 #endif /* !SLJIT_CONFIG_X86_32 && !(SLJIT_CONFIG_ARM_32 && __SOFTFP__) */
3530
3531 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3532 {
3533 CHECK_ERROR();
3534 CHECK(check_sljit_emit_return(compiler, op, src, srcw));
3535
3536 if (GET_OPCODE(op) < SLJIT_MOV_F64) {
3537 FAIL_IF(emit_mov_before_return(compiler, op, src, srcw));
3538 } else {
3539 FAIL_IF(emit_fmov_before_return(compiler, op, src, srcw));
3540 }
3541
3542 SLJIT_SKIP_CHECKS(compiler);
3543 return sljit_emit_return_void(compiler);
3544 }
3545
3546 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3547 && !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) \
3548 && !(defined(SLJIT_CONFIG_LOONGARCH_64) && SLJIT_CONFIG_LOONGARCH_64)
3549
3550 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2r(struct sljit_compiler *compiler, sljit_s32 op,
3551 sljit_s32 dst_freg,
3552 sljit_s32 src1, sljit_sw src1w,
3553 sljit_s32 src2, sljit_sw src2w)
3554 {
3555 CHECK_ERROR();
3556 CHECK(check_sljit_emit_fop2r(compiler, op, dst_freg, src1, src1w, src2, src2w));
3557 ADJUST_LOCAL_OFFSET(src1, src1w);
3558 ADJUST_LOCAL_OFFSET(src2, src2w);
3559
3560 SLJIT_SKIP_CHECKS(compiler);
3561 return sljit_emit_fop2(compiler, op, dst_freg, 0, src1, src1w, src2, src2w);
3562 }
3563
3564 #endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_S390X && !SLJIT_CONFIG_LOONGARCH_64 */
3565
3566 #if !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) \
3567 && !(defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV) \
3568 && !(defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3569
3570 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_cmp(struct sljit_compiler *compiler, sljit_s32 type,
3571 sljit_s32 src1, sljit_sw src1w,
3572 sljit_s32 src2, sljit_sw src2w)
3573 {
3574 /* Default compare for most architectures. */
3575 sljit_s32 flags, tmp_src, condition;
3576 sljit_sw tmp_srcw;
3577
3578 CHECK_ERROR_PTR();
3579 CHECK_PTR(check_sljit_emit_cmp(compiler, type, src1, src1w, src2, src2w));
3580
3581 condition = type & 0xff;
3582 #if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3583 if ((condition == SLJIT_EQUAL || condition == SLJIT_NOT_EQUAL)) {
3584 if (src1 == SLJIT_IMM && !src1w) {
3585 src1 = src2;
3586 src1w = src2w;
3587 src2 = SLJIT_IMM;
3588 src2w = 0;
3589 }
3590 if (src2 == SLJIT_IMM && !src2w)
3591 return emit_cmp_to0(compiler, type, src1, src1w);
3592 }
3593 #endif /* SLJIT_CONFIG_ARM_64 */
3594
3595 if (SLJIT_UNLIKELY(src1 == SLJIT_IMM && src2 != SLJIT_IMM)) {
3596 /* Immediate is preferred as second argument by most architectures. */
3597 switch (condition) {
3598 case SLJIT_LESS:
3599 condition = SLJIT_GREATER;
3600 break;
3601 case SLJIT_GREATER_EQUAL:
3602 condition = SLJIT_LESS_EQUAL;
3603 break;
3604 case SLJIT_GREATER:
3605 condition = SLJIT_LESS;
3606 break;
3607 case SLJIT_LESS_EQUAL:
3608 condition = SLJIT_GREATER_EQUAL;
3609 break;
3610 case SLJIT_SIG_LESS:
3611 condition = SLJIT_SIG_GREATER;
3612 break;
3613 case SLJIT_SIG_GREATER_EQUAL:
3614 condition = SLJIT_SIG_LESS_EQUAL;
3615 break;
3616 case SLJIT_SIG_GREATER:
3617 condition = SLJIT_SIG_LESS;
3618 break;
3619 case SLJIT_SIG_LESS_EQUAL:
3620 condition = SLJIT_SIG_GREATER_EQUAL;
3621 break;
3622 }
3623
3624 type = condition | (type & (SLJIT_32 | SLJIT_REWRITABLE_JUMP));
3625 tmp_src = src1;
3626 src1 = src2;
3627 src2 = tmp_src;
3628 tmp_srcw = src1w;
3629 src1w = src2w;
3630 src2w = tmp_srcw;
3631 }
3632
3633 if (condition <= SLJIT_NOT_ZERO)
3634 flags = SLJIT_SET_Z;
3635 else
3636 flags = (condition & 0xfe) << VARIABLE_FLAG_SHIFT;
3637
3638 SLJIT_SKIP_CHECKS(compiler);
3639 PTR_FAIL_IF(sljit_emit_op2u(compiler,
3640 SLJIT_SUB | flags | (type & SLJIT_32), src1, src1w, src2, src2w));
3641
3642 SLJIT_SKIP_CHECKS(compiler);
3643 return sljit_emit_jump(compiler, condition | (type & (SLJIT_REWRITABLE_JUMP | SLJIT_32)));
3644 }
3645
3646 #endif /* !SLJIT_CONFIG_MIPS */
3647
3648 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32)
3649
3650 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
3651 {
3652 switch (type) {
3653 case SLJIT_UNORDERED_OR_EQUAL:
3654 case SLJIT_ORDERED_NOT_EQUAL:
3655 return 1;
3656 }
3657
3658 return 0;
3659 }
3660
3661 #endif /* SLJIT_CONFIG_ARM */
3662
3663 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_fcmp(struct sljit_compiler *compiler, sljit_s32 type,
3664 sljit_s32 src1, sljit_sw src1w,
3665 sljit_s32 src2, sljit_sw src2w)
3666 {
3667 CHECK_ERROR_PTR();
3668 CHECK_PTR(check_sljit_emit_fcmp(compiler, type, src1, src1w, src2, src2w));
3669
3670 SLJIT_SKIP_CHECKS(compiler);
3671 sljit_emit_fop1(compiler, SLJIT_CMP_F64 | ((type & 0xfe) << VARIABLE_FLAG_SHIFT) | (type & SLJIT_32), src1, src1w, src2, src2w);
3672
3673 SLJIT_SKIP_CHECKS(compiler);
3674 return sljit_emit_jump(compiler, type);
3675 }
3676
3677 #if !(defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV) \
3678 && !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) \
3679 && !(defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3680
3681 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_op2cmpz(struct sljit_compiler *compiler, sljit_s32 op,
3682 sljit_s32 dst, sljit_sw dstw,
3683 sljit_s32 src1, sljit_sw src1w,
3684 sljit_s32 src2, sljit_sw src2w)
3685 {
3686 CHECK_ERROR_PTR();
3687 CHECK_PTR(check_sljit_emit_op2cmpz(compiler, op, dst, dstw, src1, src1w, src2, src2w));
3688
3689 SLJIT_SKIP_CHECKS(compiler);
3690 PTR_FAIL_IF(sljit_emit_op2(compiler, ((op | SLJIT_SET_Z) & ~SLJIT_REWRITABLE_JUMP), dst, dstw, src1, src1w, src2, src2w));
3691
3692 SLJIT_SKIP_CHECKS(compiler);
3693 return sljit_emit_jump(compiler, ((op & SLJIT_JUMP_IF_ZERO) ? SLJIT_ZERO : SLJIT_NOT_ZERO) | (op & SLJIT_REWRITABLE_JUMP));
3694 }
3695
3696 #else /* SLJIT_CONFIG_RISCV || SLJIT_CONFIG_MIPS || SLJIT_CONFIG_LOONGARCH */
3697
3698 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_op2cmpz(struct sljit_compiler *compiler, sljit_s32 op,
3699 sljit_s32 dst, sljit_sw dstw,
3700 sljit_s32 src1, sljit_sw src1w,
3701 sljit_s32 src2, sljit_sw src2w)
3702 {
3703 sljit_s32 reg;
3704
3705 CHECK_ERROR_PTR();
3706 CHECK_PTR(check_sljit_emit_op2cmpz(compiler, op, dst, dstw, src1, src1w, src2, src2w));
3707
3708 SLJIT_SKIP_CHECKS(compiler);
3709 PTR_FAIL_IF(sljit_emit_op2(compiler, (op & ~(SLJIT_SET_Z | SLJIT_REWRITABLE_JUMP)), dst, dstw, src1, src1w, src2, src2w));
3710
3711 /* When dst is a memory operand, its value is stored in SLJIT_TMP_DEST_FREG. */
3712 reg = FAST_IS_REG(dst) ? dst : SLJIT_TMP_DEST_REG;
3713
3714 SLJIT_SKIP_CHECKS(compiler);
3715 return sljit_emit_cmp(compiler, ((op & SLJIT_JUMP_IF_ZERO) ? SLJIT_ZERO : SLJIT_NOT_ZERO) | (op & (SLJIT_32 | SLJIT_REWRITABLE_JUMP)), reg, 0, SLJIT_IMM, 0);
3716 }
3717
3718 #endif /* !SLJIT_CONFIG_RISCV && !SLJIT_CONFIG_MIPS && !SLJIT_CONFIG_LOONGARCH */
3719
3720 #if !(defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM) \
3721 && !(defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3722
3723 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
3724 sljit_s32 reg,
3725 sljit_s32 mem, sljit_sw memw)
3726 {
3727 CHECK_ERROR();
3728 CHECK(check_sljit_emit_mem_update(compiler, type, reg, mem, memw));
3729 SLJIT_UNUSED_ARG(type);
3730 SLJIT_UNUSED_ARG(reg);
3731 SLJIT_UNUSED_ARG(mem);
3732 SLJIT_UNUSED_ARG(memw);
3733
3734 return SLJIT_ERR_UNSUPPORTED;
3735 }
3736
3737 #endif /* !SLJIT_CONFIG_ARM && !SLJIT_CONFIG_PPC */
3738
3739 #if !(defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
3740 && !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
3741
3742 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem(struct sljit_compiler *compiler, sljit_s32 type,
3743 sljit_s32 freg,
3744 sljit_s32 mem, sljit_sw memw)
3745 {
3746 CHECK_ERROR();
3747 CHECK(check_sljit_emit_fmem(compiler, type, freg, mem, memw));
3748
3749 return sljit_emit_fmem_unaligned(compiler, type, freg, mem, memw);
3750 }
3751
3752 #endif /* !SLJIT_CONFIG_ARM_32 && !SLJIT_CONFIG_MIPS */
3753
3754 #if !(defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64) \
3755 && !(defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3756
3757 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
3758 sljit_s32 freg,
3759 sljit_s32 mem, sljit_sw memw)
3760 {
3761 CHECK_ERROR();
3762 CHECK(check_sljit_emit_fmem_update(compiler, type, freg, mem, memw));
3763 SLJIT_UNUSED_ARG(type);
3764 SLJIT_UNUSED_ARG(freg);
3765 SLJIT_UNUSED_ARG(mem);
3766 SLJIT_UNUSED_ARG(memw);
3767
3768 return SLJIT_ERR_UNSUPPORTED;
3769 }
3770
3771 #endif /* !SLJIT_CONFIG_ARM_64 && !SLJIT_CONFIG_PPC */
3772
3773 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3774 && !(defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM) \
3775 && !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) \
3776 && !(defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV) \
3777 && !(defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3778
3779 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
3780 sljit_s32 vreg,
3781 sljit_s32 srcdst, sljit_sw srcdstw)
3782 {
3783 CHECK_ERROR();
3784 CHECK(check_sljit_emit_simd_mov(compiler, type, vreg, srcdst, srcdstw));
3785 SLJIT_UNUSED_ARG(compiler);
3786 SLJIT_UNUSED_ARG(type);
3787 SLJIT_UNUSED_ARG(vreg);
3788 SLJIT_UNUSED_ARG(srcdst);
3789 SLJIT_UNUSED_ARG(srcdstw);
3790
3791 return SLJIT_ERR_UNSUPPORTED;
3792 }
3793
3794 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3795 sljit_s32 vreg,
3796 sljit_s32 src, sljit_sw srcw)
3797 {
3798 CHECK_ERROR();
3799 CHECK(check_sljit_emit_simd_replicate(compiler, type, vreg, src, srcw));
3800 SLJIT_UNUSED_ARG(compiler);
3801 SLJIT_UNUSED_ARG(type);
3802 SLJIT_UNUSED_ARG(vreg);
3803 SLJIT_UNUSED_ARG(src);
3804 SLJIT_UNUSED_ARG(srcw);
3805
3806 return SLJIT_ERR_UNSUPPORTED;
3807 }
3808
3809 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
3810 sljit_s32 vreg, sljit_s32 lane_index,
3811 sljit_s32 srcdst, sljit_sw srcdstw)
3812 {
3813 CHECK_ERROR();
3814 CHECK(check_sljit_emit_simd_lane_mov(compiler, type, vreg, lane_index, srcdst, srcdstw));
3815 SLJIT_UNUSED_ARG(compiler);
3816 SLJIT_UNUSED_ARG(type);
3817 SLJIT_UNUSED_ARG(vreg);
3818 SLJIT_UNUSED_ARG(lane_index);
3819 SLJIT_UNUSED_ARG(srcdst);
3820 SLJIT_UNUSED_ARG(srcdstw);
3821
3822 return SLJIT_ERR_UNSUPPORTED;
3823 }
3824
3825 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3826 sljit_s32 vreg,
3827 sljit_s32 src, sljit_s32 src_lane_index)
3828 {
3829 CHECK_ERROR();
3830 CHECK(check_sljit_emit_simd_lane_replicate(compiler, type, vreg, src, src_lane_index));
3831 SLJIT_UNUSED_ARG(compiler);
3832 SLJIT_UNUSED_ARG(type);
3833 SLJIT_UNUSED_ARG(vreg);
3834 SLJIT_UNUSED_ARG(src);
3835 SLJIT_UNUSED_ARG(src_lane_index);
3836
3837 return SLJIT_ERR_UNSUPPORTED;
3838 }
3839
3840 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
3841 sljit_s32 vreg,
3842 sljit_s32 src, sljit_sw srcw)
3843 {
3844 CHECK_ERROR();
3845 CHECK(check_sljit_emit_simd_extend(compiler, type, vreg, src, srcw));
3846 SLJIT_UNUSED_ARG(compiler);
3847 SLJIT_UNUSED_ARG(type);
3848 SLJIT_UNUSED_ARG(vreg);
3849 SLJIT_UNUSED_ARG(src);
3850 SLJIT_UNUSED_ARG(srcw);
3851
3852 return SLJIT_ERR_UNSUPPORTED;
3853 }
3854
3855 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
3856 sljit_s32 vreg,
3857 sljit_s32 dst, sljit_sw dstw)
3858 {
3859 CHECK_ERROR();
3860 CHECK(check_sljit_emit_simd_sign(compiler, type, vreg, dst, dstw));
3861 SLJIT_UNUSED_ARG(compiler);
3862 SLJIT_UNUSED_ARG(type);
3863 SLJIT_UNUSED_ARG(vreg);
3864 SLJIT_UNUSED_ARG(dst);
3865 SLJIT_UNUSED_ARG(dstw);
3866
3867 return SLJIT_ERR_UNSUPPORTED;
3868 }
3869
3870 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
3871 sljit_s32 dst_vreg, sljit_s32 src1_vreg, sljit_s32 src2, sljit_sw src2w)
3872 {
3873 CHECK_ERROR();
3874 CHECK(check_sljit_emit_simd_op2(compiler, type, dst_vreg, src1_vreg, src2, src2w));
3875 SLJIT_UNUSED_ARG(compiler);
3876 SLJIT_UNUSED_ARG(type);
3877 SLJIT_UNUSED_ARG(dst_vreg);
3878 SLJIT_UNUSED_ARG(src1_vreg);
3879 SLJIT_UNUSED_ARG(src2);
3880 SLJIT_UNUSED_ARG(src2w);
3881
3882 return SLJIT_ERR_UNSUPPORTED;
3883 }
3884
3885 #endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM && !SLJIT_CONFIG_S390X && !SLJIT_CONFIG_LOONGARCH */
3886
3887 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3888 && !(defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3889
3890 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
3891 {
3892 CHECK_ERROR();
3893 CHECK(check_sljit_get_local_base(compiler, dst, dstw, offset));
3894
3895 ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_SP), offset);
3896
3897 SLJIT_SKIP_CHECKS(compiler);
3898
3899 if (offset != 0)
3900 return sljit_emit_op2(compiler, SLJIT_ADD, dst, dstw, SLJIT_SP, 0, SLJIT_IMM, offset);
3901 return sljit_emit_op1(compiler, SLJIT_MOV, dst, dstw, SLJIT_SP, 0);
3902 }
3903
3904 #endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM_64 */
3905
3906 SLJIT_API_FUNC_ATTRIBUTE void* sljit_read_only_buffer_start_writing(sljit_uw addr, sljit_uw size, sljit_sw executable_offset)
3907 {
3908 SLJIT_UNUSED_ARG(size);
3909 SLJIT_UNUSED_ARG(executable_offset);
3910
3911 SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + size), 0);
3912 return SLJIT_ADD_EXEC_OFFSET(addr, -executable_offset);
3913 }
3914
3915 SLJIT_API_FUNC_ATTRIBUTE void sljit_read_only_buffer_end_writing(sljit_uw addr, sljit_uw size, sljit_sw executable_offset)
3916 {
3917 SLJIT_UNUSED_ARG(addr);
3918 SLJIT_UNUSED_ARG(size);
3919 SLJIT_UNUSED_ARG(executable_offset);
3920
3921 SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + size), 1);
3922 SLJIT_CACHE_FLUSH((void*)addr, (void*)(addr + size));
3923 }
3924
3925 #endif /* !SLJIT_CONFIG_UNSUPPORTED */