Debian Codegraph @

src/3rdparty/pcre2/deps/sljit/sljit_src/sljitNativeX86_32.c

raw
1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg ([email protected]). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 /* x86 32-bit arch dependent functions. */
28
29 /* --------------------------------------------------------------------- */
30 /* Operators */
31 /* --------------------------------------------------------------------- */
32
33 static sljit_s32 emit_do_imm(struct sljit_compiler *compiler, sljit_u8 opcode, sljit_sw imm)
34 {
35 sljit_u8 *inst;
36
37 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + sizeof(sljit_sw));
38 FAIL_IF(!inst);
39 INC_SIZE(1 + sizeof(sljit_sw));
40 *inst++ = opcode;
41 sljit_unaligned_store_sw(inst, imm);
42 return SLJIT_SUCCESS;
43 }
44
45 /* Size contains the flags as well. */
46 static sljit_u8* emit_x86_instruction(struct sljit_compiler *compiler, sljit_uw size,
47 /* The register or immediate operand. */
48 sljit_s32 a, sljit_sw imma,
49 /* The general operand (not immediate). */
50 sljit_s32 b, sljit_sw immb)
51 {
52 sljit_u8 *inst;
53 sljit_u8 *buf_ptr;
54 sljit_u8 reg_map_b;
55 sljit_uw flags = size;
56 sljit_uw inst_size;
57
58 /* Both cannot be switched on. */
59 SLJIT_ASSERT((flags & (EX86_BIN_INS | EX86_SHIFT_INS)) != (EX86_BIN_INS | EX86_SHIFT_INS));
60 /* Size flags not allowed for typed instructions. */
61 SLJIT_ASSERT(!(flags & (EX86_BIN_INS | EX86_SHIFT_INS)) || (flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) == 0);
62 /* Both size flags cannot be switched on. */
63 SLJIT_ASSERT((flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) != (EX86_BYTE_ARG | EX86_HALF_ARG));
64 /* SSE2 and immediate is not possible. */
65 SLJIT_ASSERT(a != SLJIT_IMM || !(flags & EX86_SSE2));
66 SLJIT_ASSERT(((flags & (EX86_PREF_F2 | EX86_PREF_F3 | EX86_PREF_66))
67 & ((flags & (EX86_PREF_F2 | EX86_PREF_F3 | EX86_PREF_66)) - 1)) == 0);
68 SLJIT_ASSERT((flags & (EX86_VEX_EXT | EX86_REX)) != EX86_VEX_EXT);
69
70 size &= 0xf;
71 /* The mod r/m byte is always present. */
72 inst_size = size + 1;
73
74 if (flags & (EX86_PREF_F2 | EX86_PREF_F3 | EX86_PREF_66))
75 inst_size++;
76
77 /* Calculate size of b. */
78 if (b & SLJIT_MEM) {
79 if (!(b & REG_MASK))
80 inst_size += sizeof(sljit_sw);
81 else {
82 if (immb != 0 && !(b & OFFS_REG_MASK)) {
83 /* Immediate operand. */
84 if (immb <= 127 && immb >= -128)
85 inst_size += sizeof(sljit_s8);
86 else
87 inst_size += sizeof(sljit_sw);
88 } else if (reg_map[b & REG_MASK] == 5) {
89 /* Swap registers if possible. */
90 if ((b & OFFS_REG_MASK) && (immb & 0x3) == 0 && reg_map[OFFS_REG(b)] != 5)
91 b = SLJIT_MEM | OFFS_REG(b) | TO_OFFS_REG(b & REG_MASK);
92 else
93 inst_size += sizeof(sljit_s8);
94 }
95
96 if (reg_map[b & REG_MASK] == 4 && !(b & OFFS_REG_MASK))
97 b |= TO_OFFS_REG(SLJIT_SP);
98
99 if (b & OFFS_REG_MASK)
100 inst_size += 1; /* SIB byte. */
101 }
102 }
103
104 /* Calculate size of a. */
105 if (a == SLJIT_IMM) {
106 if (flags & EX86_BIN_INS) {
107 if (imma <= 127 && imma >= -128) {
108 inst_size += 1;
109 flags |= EX86_BYTE_ARG;
110 } else
111 inst_size += 4;
112 } else if (flags & EX86_SHIFT_INS) {
113 SLJIT_ASSERT(imma <= 0x1f);
114 if (imma != 1) {
115 inst_size++;
116 flags |= EX86_BYTE_ARG;
117 }
118 } else if (flags & EX86_BYTE_ARG)
119 inst_size++;
120 else if (flags & EX86_HALF_ARG)
121 inst_size += sizeof(short);
122 else
123 inst_size += sizeof(sljit_sw);
124 } else
125 SLJIT_ASSERT(!(flags & EX86_SHIFT_INS) || a == SLJIT_PREF_SHIFT_REG);
126
127 inst = (sljit_u8*)ensure_buf(compiler, 1 + inst_size);
128 PTR_FAIL_IF(!inst);
129
130 /* Encoding the byte. */
131 INC_SIZE(inst_size);
132 if (flags & EX86_PREF_F2)
133 *inst++ = 0xf2;
134 else if (flags & EX86_PREF_F3)
135 *inst++ = 0xf3;
136 else if (flags & EX86_PREF_66)
137 *inst++ = 0x66;
138
139 buf_ptr = inst + size;
140
141 /* Encode mod/rm byte. */
142 if (!(flags & EX86_SHIFT_INS)) {
143 if ((flags & EX86_BIN_INS) && a == SLJIT_IMM)
144 *inst = (flags & EX86_BYTE_ARG) ? GROUP_BINARY_83 : GROUP_BINARY_81;
145
146 if (a == SLJIT_IMM)
147 *buf_ptr = 0;
148 else if (!(flags & EX86_SSE2_OP1))
149 *buf_ptr = U8(reg_map[a] << 3);
150 else
151 *buf_ptr = U8(freg_map[a] << 3);
152 } else {
153 if (a == SLJIT_IMM) {
154 if (imma == 1)
155 *inst = GROUP_SHIFT_1;
156 else
157 *inst = GROUP_SHIFT_N;
158 } else
159 *inst = GROUP_SHIFT_CL;
160 *buf_ptr = 0;
161 }
162
163 if (!(b & SLJIT_MEM)) {
164 *buf_ptr = U8(*buf_ptr | MOD_REG | (!(flags & EX86_SSE2_OP2) ? reg_map[b] : freg_map[b]));
165 buf_ptr++;
166 } else if (b & REG_MASK) {
167 reg_map_b = reg_map[b & REG_MASK];
168
169 if (!(b & OFFS_REG_MASK) || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_SP)) {
170 if (immb != 0 || reg_map_b == 5) {
171 if (immb <= 127 && immb >= -128)
172 *buf_ptr |= 0x40;
173 else
174 *buf_ptr |= 0x80;
175 }
176
177 if (!(b & OFFS_REG_MASK))
178 *buf_ptr++ |= reg_map_b;
179 else {
180 buf_ptr[0] |= 0x04;
181 buf_ptr[1] = U8(reg_map_b | (reg_map[OFFS_REG(b)] << 3));
182 buf_ptr += 2;
183 }
184
185 if (immb != 0 || reg_map_b == 5) {
186 if (immb <= 127 && immb >= -128)
187 *buf_ptr++ = U8(immb); /* 8 bit displacement. */
188 else {
189 sljit_unaligned_store_sw(buf_ptr, immb); /* 32 bit displacement. */
190 buf_ptr += sizeof(sljit_sw);
191 }
192 }
193 } else {
194 if (reg_map_b == 5)
195 *buf_ptr |= 0x40;
196
197 buf_ptr[0] |= 0x04;
198 buf_ptr[1] = U8(reg_map_b | (reg_map[OFFS_REG(b)] << 3) | (immb << 6));
199 buf_ptr += 2;
200
201 if (reg_map_b == 5)
202 *buf_ptr++ = 0;
203 }
204 } else {
205 *buf_ptr++ |= 0x05;
206 sljit_unaligned_store_sw(buf_ptr, immb); /* 32 bit displacement. */
207 buf_ptr += sizeof(sljit_sw);
208 }
209
210 if (a == SLJIT_IMM) {
211 if (flags & EX86_BYTE_ARG)
212 *buf_ptr = U8(imma);
213 else if (flags & EX86_HALF_ARG)
214 sljit_unaligned_store_s16(buf_ptr, (sljit_s16)imma);
215 else if (!(flags & EX86_SHIFT_INS))
216 sljit_unaligned_store_sw(buf_ptr, imma);
217 }
218
219 return inst;
220 }
221
222 static sljit_s32 emit_vex_instruction(struct sljit_compiler *compiler, sljit_uw op,
223 /* The first and second register operand. */
224 sljit_s32 a, sljit_s32 v,
225 /* The general operand (not immediate). */
226 sljit_s32 b, sljit_sw immb)
227 {
228 sljit_u8 *inst;
229 sljit_u8 vex = 0;
230 sljit_u8 vex_m = 0;
231 sljit_uw size;
232
233 SLJIT_ASSERT(((op & (EX86_PREF_F2 | EX86_PREF_F3 | EX86_PREF_66))
234 & ((op & (EX86_PREF_F2 | EX86_PREF_F3 | EX86_PREF_66)) - 1)) == 0);
235
236 if (op & VEX_OP_0F38)
237 vex_m = 0x2;
238 else if (op & VEX_OP_0F3A)
239 vex_m = 0x3;
240
241 if (op & VEX_W) {
242 if (vex_m == 0)
243 vex_m = 0x1;
244
245 vex |= 0x80;
246 }
247
248 if (op & EX86_PREF_66)
249 vex |= 0x1;
250 else if (op & EX86_PREF_F2)
251 vex |= 0x3;
252 else if (op & EX86_PREF_F3)
253 vex |= 0x2;
254
255 op &= ~(EX86_PREF_66 | EX86_PREF_F2 | EX86_PREF_F3);
256
257 if (op & VEX_256)
258 vex |= 0x4;
259
260 vex = U8(vex | ((((op & VEX_SSE2_OPV) ? freg_map[v] : reg_map[v]) ^ 0xf) << 3));
261
262 size = op & ~(sljit_uw)0xff;
263 size |= (vex_m == 0) ? 3 : 4;
264
265 inst = emit_x86_instruction(compiler, size, a, 0, b, immb);
266 FAIL_IF(!inst);
267
268 if (vex_m == 0) {
269 inst[0] = 0xc5;
270 inst[1] = U8(vex | 0x80);
271 inst[2] = U8(op);
272 return SLJIT_SUCCESS;
273 }
274
275 inst[0] = 0xc4;
276 inst[1] = U8(vex_m | 0xe0);
277 inst[2] = vex;
278 inst[3] = U8(op);
279 return SLJIT_SUCCESS;
280 }
281
282 /* --------------------------------------------------------------------- */
283 /* Enter / return */
284 /* --------------------------------------------------------------------- */
285
286 static sljit_u8* detect_far_jump_type(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_sw executable_offset)
287 {
288 sljit_uw type = jump->flags >> TYPE_SHIFT;
289
290 if (type == SLJIT_JUMP) {
291 *code_ptr++ = JMP_i32;
292 } else if (type >= SLJIT_FAST_CALL) {
293 *code_ptr++ = CALL_i32;
294 } else {
295 *code_ptr++ = GROUP_0F;
296 *code_ptr++ = get_jump_code(type);
297 }
298
299 jump->addr = (sljit_uw)code_ptr;
300
301 if (jump->flags & JUMP_ADDR)
302 sljit_unaligned_store_sw(code_ptr, (sljit_sw)(jump->u.target - (jump->addr + 4) - (sljit_uw)executable_offset));
303 else
304 jump->flags |= PATCH_MW;
305 code_ptr += 4;
306
307 return code_ptr;
308 }
309
310 #define ENTER_TMP_TO_R4 0x00001
311 #define ENTER_TMP_TO_S 0x00002
312
313 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
314 sljit_s32 options, sljit_s32 arg_types,
315 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
316 {
317 sljit_s32 word_arg_count, saved_arg_count, float_arg_count;
318 sljit_s32 size, args_size, types, status;
319 sljit_s32 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(options);
320 sljit_u8 *inst;
321 #ifdef _WIN32
322 sljit_s32 r2_offset = -1;
323 #endif
324
325 CHECK_ERROR();
326 CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, local_size));
327 set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
328
329 scratches = ENTER_GET_REGS(scratches);
330
331 /* Emit ENDBR32 at function entry if needed. */
332 FAIL_IF(emit_endbranch(compiler));
333
334 SLJIT_COMPILE_ASSERT(SLJIT_FR0 == 1, float_register_index_start);
335
336 arg_types >>= SLJIT_ARG_SHIFT;
337 word_arg_count = 0;
338 status = 0;
339
340 if (options & SLJIT_ENTER_REG_ARG) {
341 args_size = 3 * SSIZE_OF(sw);
342
343 while (arg_types) {
344 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64) {
345 word_arg_count++;
346 if (word_arg_count >= 4)
347 status |= ENTER_TMP_TO_R4;
348 }
349
350 arg_types >>= SLJIT_ARG_SHIFT;
351 }
352
353 compiler->args_size = 0;
354 } else {
355 types = arg_types;
356 saved_arg_count = 0;
357 float_arg_count = 0;
358 args_size = SSIZE_OF(sw);
359 while (types) {
360 switch (types & SLJIT_ARG_MASK) {
361 case SLJIT_ARG_TYPE_F64:
362 float_arg_count++;
363 FAIL_IF(emit_sse2_load(compiler, 0, float_arg_count, SLJIT_MEM1(SLJIT_SP), args_size));
364 args_size += SSIZE_OF(f64);
365 break;
366 case SLJIT_ARG_TYPE_F32:
367 float_arg_count++;
368 FAIL_IF(emit_sse2_load(compiler, 1, float_arg_count, SLJIT_MEM1(SLJIT_SP), args_size));
369 args_size += SSIZE_OF(f32);
370 break;
371 default:
372 word_arg_count++;
373
374 if (!(types & SLJIT_ARG_TYPE_SCRATCH_REG))
375 saved_arg_count++;
376
377 if (word_arg_count == 4) {
378 if (types & SLJIT_ARG_TYPE_SCRATCH_REG) {
379 status |= ENTER_TMP_TO_R4;
380 arg_types &= ~(SLJIT_ARG_FULL_MASK << 3 * SLJIT_ARG_SHIFT);
381 } else if (saved_arg_count == 4) {
382 status |= ENTER_TMP_TO_S;
383 arg_types &= ~(SLJIT_ARG_FULL_MASK << 3 * SLJIT_ARG_SHIFT);
384 }
385 }
386
387 args_size += SSIZE_OF(sw);
388 break;
389 }
390 types >>= SLJIT_ARG_SHIFT;
391 }
392
393 args_size -= SSIZE_OF(sw);
394 compiler->args_size = args_size;
395 }
396
397 size = (scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3) - kept_saveds_count;
398 if (!(options & SLJIT_ENTER_REG_ARG))
399 size++;
400
401 if (size != 0) {
402 inst = (sljit_u8*)ensure_buf(compiler, (sljit_uw)(size + 1));
403 FAIL_IF(!inst);
404
405 INC_SIZE((sljit_uw)size);
406
407 if (!(options & SLJIT_ENTER_REG_ARG))
408 PUSH_REG(reg_map[TMP_REG1]);
409
410 if ((saveds > 2 && kept_saveds_count <= 2) || scratches > 9)
411 PUSH_REG(reg_map[SLJIT_S2]);
412 if ((saveds > 1 && kept_saveds_count <= 1) || scratches > 10)
413 PUSH_REG(reg_map[SLJIT_S1]);
414 if ((saveds > 0 && kept_saveds_count == 0) || scratches > 11)
415 PUSH_REG(reg_map[SLJIT_S0]);
416
417 size *= SSIZE_OF(sw);
418 }
419
420 if (status & (ENTER_TMP_TO_R4 | ENTER_TMP_TO_S))
421 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), args_size + size);
422
423 size += SSIZE_OF(sw);
424
425 local_size = ((SLJIT_LOCALS_OFFSET_BASE + local_size + size + 0xf) & ~0xf) - size;
426 compiler->local_size = local_size;
427
428 word_arg_count = 0;
429 saved_arg_count = 0;
430 args_size = size;
431 while (arg_types) {
432 switch (arg_types & SLJIT_ARG_MASK) {
433 case SLJIT_ARG_TYPE_F64:
434 args_size += SSIZE_OF(f64);
435 break;
436 case SLJIT_ARG_TYPE_F32:
437 args_size += SSIZE_OF(f32);
438 break;
439 default:
440 word_arg_count++;
441 SLJIT_ASSERT(word_arg_count <= 3 || (word_arg_count == 4 && !(status & (ENTER_TMP_TO_R4 | ENTER_TMP_TO_S))));
442
443 if (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) {
444 #ifdef _WIN32
445 if (word_arg_count == 3 && local_size > 4 * 4096)
446 r2_offset = local_size + args_size;
447 else
448 #endif
449 EMIT_MOV(compiler, word_arg_count, 0, SLJIT_MEM1(SLJIT_SP), args_size);
450
451 } else {
452 EMIT_MOV(compiler, SLJIT_S0 - saved_arg_count, 0, SLJIT_MEM1(SLJIT_SP), args_size);
453 saved_arg_count++;
454 }
455
456 args_size += SSIZE_OF(sw);
457 break;
458 }
459 arg_types >>= SLJIT_ARG_SHIFT;
460 }
461
462 SLJIT_ASSERT(SLJIT_LOCALS_OFFSET > 0);
463
464 #ifdef _WIN32
465 SLJIT_ASSERT(r2_offset == -1 || local_size > 4 * 4096);
466
467 if (local_size > 4096) {
468 if (local_size <= 4 * 4096) {
469 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096);
470
471 if (local_size > 2 * 4096)
472 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096 * 2);
473 if (local_size > 3 * 4096)
474 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096 * 3);
475 }
476 else {
477 if (options & SLJIT_ENTER_REG_ARG) {
478 SLJIT_ASSERT(r2_offset == -1);
479
480 inst = (sljit_u8*)ensure_buf(compiler, (sljit_uw)(1 + 1));
481 FAIL_IF(!inst);
482 INC_SIZE(1);
483 PUSH_REG(reg_map[SLJIT_R2]);
484
485 local_size -= SSIZE_OF(sw);
486 r2_offset = local_size;
487 }
488
489 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_IMM, local_size >> 12);
490
491 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096);
492 BINARY_IMM32(SUB, 4096, SLJIT_SP, 0);
493
494 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
495 FAIL_IF(!inst);
496
497 INC_SIZE(2);
498 inst[0] = LOOP_i8;
499 inst[1] = (sljit_u8)-16;
500 local_size &= 0xfff;
501 }
502 }
503
504 if (local_size > 0) {
505 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -local_size);
506 BINARY_IMM32(SUB, local_size, SLJIT_SP, 0);
507 }
508
509 if (r2_offset != -1)
510 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), r2_offset);
511
512 #else /* !_WIN32 */
513
514 SLJIT_ASSERT(local_size > 0);
515
516 BINARY_IMM32(SUB, local_size, SLJIT_SP, 0);
517
518 #endif /* _WIN32 */
519
520 size = SLJIT_LOCALS_OFFSET_BASE - SSIZE_OF(sw);
521 kept_saveds_count = SLJIT_R3 - kept_saveds_count;
522
523 while (saved_arg_count > 3) {
524 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), size, kept_saveds_count, 0);
525 kept_saveds_count++;
526 size -= SSIZE_OF(sw);
527 saved_arg_count--;
528 }
529
530 if (status & (ENTER_TMP_TO_R4 | ENTER_TMP_TO_S)) {
531 if (status & ENTER_TMP_TO_R4)
532 size = 2 * SSIZE_OF(sw);
533
534 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), size, TMP_REG1, 0);
535 }
536
537 return SLJIT_SUCCESS;
538 }
539
540 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
541 sljit_s32 options, sljit_s32 arg_types,
542 sljit_s32 scratches, sljit_s32 saveds, sljit_s32 local_size)
543 {
544 sljit_s32 args_size;
545
546 CHECK_ERROR();
547 CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, local_size));
548 set_emit_enter(compiler, options, arg_types, scratches, saveds, local_size);
549
550 scratches = ENTER_GET_REGS(scratches);
551
552 arg_types >>= SLJIT_ARG_SHIFT;
553 args_size = 0;
554
555 if (!(options & SLJIT_ENTER_REG_ARG)) {
556 while (arg_types) {
557 switch (arg_types & SLJIT_ARG_MASK) {
558 case SLJIT_ARG_TYPE_F64:
559 args_size += SSIZE_OF(f64);
560 break;
561 case SLJIT_ARG_TYPE_F32:
562 args_size += SSIZE_OF(f32);
563 break;
564 default:
565 args_size += SSIZE_OF(sw);
566 break;
567 }
568 arg_types >>= SLJIT_ARG_SHIFT;
569 }
570 }
571
572 compiler->args_size = args_size;
573
574 /* [esp+0] for saving temporaries and for function calls. */
575
576 saveds = (1 + (scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3) - SLJIT_KEPT_SAVEDS_COUNT(options)) * SSIZE_OF(sw);
577
578 /* Saving ebp. */
579 if (!(options & SLJIT_ENTER_REG_ARG))
580 saveds += SSIZE_OF(sw);
581
582 compiler->local_size = ((SLJIT_LOCALS_OFFSET_BASE + local_size + saveds + 0xf) & ~0xf) - saveds;
583 return SLJIT_SUCCESS;
584 }
585
586 static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler, sljit_s32 is_return_to)
587 {
588 sljit_s32 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
589 sljit_s32 local_size, saveds;
590 sljit_uw size;
591 sljit_u8 *inst;
592
593 size = (sljit_uw)((compiler->scratches > 9 ? (compiler->scratches - 9) : 0) +
594 (compiler->saveds <= 3 ? compiler->saveds : 3) - kept_saveds_count);
595
596 local_size = compiler->local_size;
597
598 if (!(compiler->options & SLJIT_ENTER_REG_ARG))
599 size++;
600 else if (is_return_to && size == 0) {
601 local_size += SSIZE_OF(sw);
602 is_return_to = 0;
603 }
604
605 if (local_size > 0)
606 BINARY_IMM32(ADD, local_size, SLJIT_SP, 0);
607
608 if (size == 0)
609 return SLJIT_SUCCESS;
610
611 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
612 FAIL_IF(!inst);
613
614 INC_SIZE(size);
615
616 saveds = compiler->saveds;
617
618 if ((saveds > 0 && kept_saveds_count == 0) || compiler->scratches > 11)
619 POP_REG(reg_map[SLJIT_S0]);
620 if ((saveds > 1 && kept_saveds_count <= 1) || compiler->scratches > 10)
621 POP_REG(reg_map[SLJIT_S1]);
622 if ((saveds > 2 && kept_saveds_count <= 2) || compiler->scratches > 9)
623 POP_REG(reg_map[SLJIT_S2]);
624
625 if (!(compiler->options & SLJIT_ENTER_REG_ARG))
626 POP_REG(reg_map[TMP_REG1]);
627
628 if (is_return_to)
629 BINARY_IMM32(ADD, sizeof(sljit_sw), SLJIT_SP, 0);
630
631 return SLJIT_SUCCESS;
632 }
633
634 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
635 {
636 CHECK_ERROR();
637 CHECK(check_sljit_emit_return_void(compiler));
638
639 SLJIT_ASSERT(compiler->args_size >= 0);
640 SLJIT_ASSERT(compiler->local_size > 0);
641
642 FAIL_IF(emit_stack_frame_release(compiler, 0));
643
644 return emit_byte(compiler, RET_near);
645 }
646
647 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_to(struct sljit_compiler *compiler,
648 sljit_s32 src, sljit_sw srcw)
649 {
650 sljit_s32 src_r;
651
652 CHECK_ERROR();
653 CHECK(check_sljit_emit_return_to(compiler, src, srcw));
654
655 if ((src & SLJIT_MEM) || (src > SLJIT_R2 && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options)))) {
656 ADJUST_LOCAL_OFFSET(src, srcw);
657 CHECK_EXTRA_REGS(src, srcw, (void)0);
658
659 src_r = (compiler->options & SLJIT_ENTER_REG_ARG) ? TMP_REG1 : SLJIT_R1;
660
661 EMIT_MOV(compiler, src_r, 0, src, srcw);
662 src = src_r;
663 srcw = 0;
664 }
665
666 FAIL_IF(emit_stack_frame_release(compiler, 1));
667
668 SLJIT_SKIP_CHECKS(compiler);
669 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
670 }
671
672 /* --------------------------------------------------------------------- */
673 /* Call / return instructions */
674 /* --------------------------------------------------------------------- */
675
676 static sljit_s32 call_get_stack_size(sljit_s32 arg_types, sljit_s32 *word_arg_count_ptr)
677 {
678 sljit_sw stack_size = 0;
679 sljit_s32 word_arg_count = 0;
680
681 arg_types >>= SLJIT_ARG_SHIFT;
682
683 while (arg_types) {
684 switch (arg_types & SLJIT_ARG_MASK) {
685 case SLJIT_ARG_TYPE_F64:
686 stack_size += SSIZE_OF(f64);
687 break;
688 case SLJIT_ARG_TYPE_F32:
689 stack_size += SSIZE_OF(f32);
690 break;
691 default:
692 word_arg_count++;
693 stack_size += SSIZE_OF(sw);
694 break;
695 }
696
697 arg_types >>= SLJIT_ARG_SHIFT;
698 }
699
700 if (word_arg_count_ptr)
701 *word_arg_count_ptr = word_arg_count;
702
703 if (stack_size <= 4 * SSIZE_OF(sw))
704 return 0;
705
706 return ((stack_size - (4 * SSIZE_OF(sw)) + 0xf) & ~0xf);
707 }
708
709 static sljit_s32 call_with_args(struct sljit_compiler *compiler,
710 sljit_s32 arg_types, sljit_sw stack_size, sljit_s32 word_arg_count, sljit_s32 keep_tmp1)
711 {
712 sljit_s32 float_arg_count = 0, arg4_reg = 0, arg_offset;
713 sljit_u8 *inst;
714
715 if (word_arg_count >= 4) {
716 arg4_reg = SLJIT_R0;
717
718 if (!keep_tmp1) {
719 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw));
720 arg4_reg = TMP_REG1;
721 }
722 }
723
724 if (stack_size > 0)
725 BINARY_IMM32(SUB, stack_size, SLJIT_SP, 0);
726
727 arg_offset = 0;
728 word_arg_count = 0;
729 arg_types >>= SLJIT_ARG_SHIFT;
730
731 while (arg_types) {
732 switch (arg_types & SLJIT_ARG_MASK) {
733 case SLJIT_ARG_TYPE_F64:
734 float_arg_count++;
735 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), arg_offset, float_arg_count));
736 arg_offset += SSIZE_OF(f64);
737 break;
738 case SLJIT_ARG_TYPE_F32:
739 float_arg_count++;
740 FAIL_IF(emit_sse2_store(compiler, 1, SLJIT_MEM1(SLJIT_SP), arg_offset, float_arg_count));
741 arg_offset += SSIZE_OF(f32);
742 break;
743 default:
744 word_arg_count++;
745 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), arg_offset, (word_arg_count >= 4) ? arg4_reg : word_arg_count, 0);
746
747 if (word_arg_count == 1 && arg4_reg == SLJIT_R0)
748 EMIT_MOV(compiler, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw) + stack_size);
749
750 arg_offset += SSIZE_OF(sw);
751 break;
752 }
753
754 arg_types >>= SLJIT_ARG_SHIFT;
755 }
756
757 return SLJIT_SUCCESS;
758 }
759
760 static sljit_s32 post_call_with_args(struct sljit_compiler *compiler,
761 sljit_s32 arg_types, sljit_s32 stack_size)
762 {
763 sljit_u8 *inst;
764 sljit_s32 single;
765
766 if (stack_size > 0)
767 BINARY_IMM32(ADD, stack_size, SLJIT_SP, 0);
768
769 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64)
770 return SLJIT_SUCCESS;
771
772 single = ((arg_types & SLJIT_ARG_MASK) == SLJIT_ARG_TYPE_F32);
773
774 inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
775 FAIL_IF(!inst);
776 INC_SIZE(3);
777 inst[0] = single ? FSTPS : FSTPD;
778 inst[1] = (0x03 << 3) | 0x04;
779 inst[2] = (0x04 << 3) | reg_map[SLJIT_SP];
780
781 return emit_sse2_load(compiler, single, SLJIT_FR0, SLJIT_MEM1(SLJIT_SP), 0);
782 }
783
784 static sljit_s32 tail_call_with_args(struct sljit_compiler *compiler,
785 sljit_s32 *extra_space, sljit_s32 arg_types,
786 sljit_s32 src, sljit_sw srcw)
787 {
788 sljit_sw args_size, saved_regs_size;
789 sljit_sw types, word_arg_count, float_arg_count;
790 sljit_sw stack_size, prev_stack_size, min_size, offset;
791 sljit_sw word_arg4_offset;
792 sljit_u8 r2_offset = 0;
793 sljit_s32 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
794 sljit_u8* inst;
795
796 ADJUST_LOCAL_OFFSET(src, srcw);
797 CHECK_EXTRA_REGS(src, srcw, (void)0);
798
799 saved_regs_size = (1 + (compiler->scratches > 9 ? (compiler->scratches - 9) : 0)
800 + (compiler->saveds <= 3 ? compiler->saveds : 3) - kept_saveds_count) * SSIZE_OF(sw);
801
802 word_arg_count = 0;
803 float_arg_count = 0;
804 arg_types >>= SLJIT_ARG_SHIFT;
805 types = 0;
806 args_size = 0;
807
808 while (arg_types != 0) {
809 types = (types << SLJIT_ARG_SHIFT) | (arg_types & SLJIT_ARG_MASK);
810
811 switch (arg_types & SLJIT_ARG_MASK) {
812 case SLJIT_ARG_TYPE_F64:
813 args_size += SSIZE_OF(f64);
814 float_arg_count++;
815 break;
816 case SLJIT_ARG_TYPE_F32:
817 args_size += SSIZE_OF(f32);
818 float_arg_count++;
819 break;
820 default:
821 word_arg_count++;
822 args_size += SSIZE_OF(sw);
823 break;
824 }
825 arg_types >>= SLJIT_ARG_SHIFT;
826 }
827
828 if (args_size <= compiler->args_size) {
829 *extra_space = 0;
830 stack_size = args_size + SSIZE_OF(sw) + saved_regs_size;
831
832 offset = stack_size + compiler->local_size;
833
834 if (src != SLJIT_IMM && src != SLJIT_R0) {
835 if (word_arg_count >= 1) {
836 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_R0, 0);
837 r2_offset = sizeof(sljit_sw);
838 }
839 EMIT_MOV(compiler, SLJIT_R0, 0, src, srcw);
840 }
841
842 while (types != 0) {
843 switch (types & SLJIT_ARG_MASK) {
844 case SLJIT_ARG_TYPE_F64:
845 offset -= SSIZE_OF(f64);
846 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
847 float_arg_count--;
848 break;
849 case SLJIT_ARG_TYPE_F32:
850 offset -= SSIZE_OF(f32);
851 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
852 float_arg_count--;
853 break;
854 default:
855 switch (word_arg_count) {
856 case 1:
857 offset -= SSIZE_OF(sw);
858 if (r2_offset != 0) {
859 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), 0);
860 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
861 } else
862 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R0, 0);
863 break;
864 case 2:
865 offset -= SSIZE_OF(sw);
866 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R1, 0);
867 break;
868 case 3:
869 offset -= SSIZE_OF(sw);
870 break;
871 case 4:
872 offset -= SSIZE_OF(sw);
873 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw));
874 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
875 break;
876 }
877 word_arg_count--;
878 break;
879 }
880 types >>= SLJIT_ARG_SHIFT;
881 }
882
883 return emit_stack_frame_release(compiler, 0);
884 }
885
886 stack_size = args_size + SSIZE_OF(sw);
887
888 if (word_arg_count >= 1 && src != SLJIT_IMM && src != SLJIT_R0) {
889 r2_offset = SSIZE_OF(sw);
890 stack_size += SSIZE_OF(sw);
891 }
892
893 if (word_arg_count >= 3)
894 stack_size += SSIZE_OF(sw);
895
896 prev_stack_size = SSIZE_OF(sw) + saved_regs_size;
897 min_size = prev_stack_size + compiler->local_size;
898
899 word_arg4_offset = 2 * SSIZE_OF(sw);
900
901 if (stack_size > min_size) {
902 BINARY_IMM32(SUB, stack_size - min_size, SLJIT_SP, 0);
903 if (src == SLJIT_MEM1(SLJIT_SP))
904 srcw += stack_size - min_size;
905 word_arg4_offset += stack_size - min_size;
906 }
907 else
908 stack_size = min_size;
909
910 if (word_arg_count >= 3) {
911 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), r2_offset, SLJIT_R2, 0);
912
913 if (word_arg_count >= 4)
914 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), word_arg4_offset);
915 }
916
917 if (src != SLJIT_IMM && src != SLJIT_R0) {
918 if (word_arg_count >= 1) {
919 SLJIT_ASSERT(r2_offset == sizeof(sljit_sw));
920 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_R0, 0);
921 }
922 EMIT_MOV(compiler, SLJIT_R0, 0, src, srcw);
923 }
924
925 /* Restore saved registers. */
926 offset = stack_size - 2 * SSIZE_OF(sw);
927 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), offset);
928
929 if (compiler->saveds > 2 || compiler->scratches > 9) {
930 offset -= SSIZE_OF(sw);
931 EMIT_MOV(compiler, SLJIT_S2, 0, SLJIT_MEM1(SLJIT_SP), offset);
932 }
933 if ((compiler->saveds > 1 && kept_saveds_count <= 1) || compiler->scratches > 10) {
934 offset -= SSIZE_OF(sw);
935 EMIT_MOV(compiler, SLJIT_S1, 0, SLJIT_MEM1(SLJIT_SP), offset);
936 }
937 if ((compiler->saveds > 0 && kept_saveds_count == 0) || compiler->scratches > 11) {
938 offset -= SSIZE_OF(sw);
939 EMIT_MOV(compiler, SLJIT_S0, 0, SLJIT_MEM1(SLJIT_SP), offset);
940 }
941
942 /* Copy fourth argument and return address. */
943 offset = stack_size - SSIZE_OF(sw);
944 *extra_space = args_size;
945
946 if (word_arg_count >= 4) {
947 offset -= SSIZE_OF(sw);
948 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
949 }
950
951 while (types != 0) {
952 switch (types & SLJIT_ARG_MASK) {
953 case SLJIT_ARG_TYPE_F64:
954 offset -= SSIZE_OF(f64);
955 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
956 float_arg_count--;
957 break;
958 case SLJIT_ARG_TYPE_F32:
959 offset -= SSIZE_OF(f32);
960 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
961 float_arg_count--;
962 break;
963 default:
964 switch (word_arg_count) {
965 case 1:
966 offset -= SSIZE_OF(sw);
967 if (r2_offset != 0) {
968 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), 0);
969 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
970 } else
971 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R0, 0);
972 break;
973 case 2:
974 offset -= SSIZE_OF(sw);
975 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R1, 0);
976 break;
977 case 3:
978 offset -= SSIZE_OF(sw);
979 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), r2_offset);
980 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
981 break;
982 }
983 word_arg_count--;
984 break;
985 }
986 types >>= SLJIT_ARG_SHIFT;
987 }
988
989 SLJIT_ASSERT(offset >= 0);
990
991 if (offset == 0)
992 return SLJIT_SUCCESS;
993
994 BINARY_IMM32(ADD, offset, SLJIT_SP, 0);
995 return SLJIT_SUCCESS;
996 }
997
998 static sljit_s32 emit_tail_call_end(struct sljit_compiler *compiler, sljit_s32 extra_space)
999 {
1000 /* Called when stack consumption cannot be reduced to 0. */
1001 sljit_u8 *inst;
1002
1003 BINARY_IMM32(ADD, extra_space, SLJIT_SP, 0);
1004 return emit_byte(compiler, RET_near);
1005 }
1006
1007 static sljit_s32 tail_call_reg_arg_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types)
1008 {
1009 sljit_s32 word_arg_count = 0;
1010 sljit_s32 kept_saveds_count, offset;
1011
1012 arg_types >>= SLJIT_ARG_SHIFT;
1013
1014 while (arg_types) {
1015 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64)
1016 word_arg_count++;
1017
1018 arg_types >>= SLJIT_ARG_SHIFT;
1019 }
1020
1021 if (word_arg_count < 4)
1022 return SLJIT_SUCCESS;
1023
1024 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw));
1025
1026 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
1027 offset = compiler->local_size + 3 * SSIZE_OF(sw);
1028
1029 if ((compiler->saveds > 0 && kept_saveds_count == 0) || compiler->scratches > 11)
1030 offset += SSIZE_OF(sw);
1031 if ((compiler->saveds > 1 && kept_saveds_count <= 1) || compiler->scratches > 10)
1032 offset += SSIZE_OF(sw);
1033 if ((compiler->saveds > 2 && kept_saveds_count <= 2) || compiler->scratches > 9)
1034 offset += SSIZE_OF(sw);
1035
1036 return emit_mov(compiler, SLJIT_MEM1(SLJIT_SP), offset, TMP_REG1, 0);
1037 }
1038
1039 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
1040 sljit_s32 arg_types)
1041 {
1042 struct sljit_jump *jump;
1043 sljit_sw stack_size = 0;
1044 sljit_s32 word_arg_count;
1045
1046 CHECK_ERROR_PTR();
1047 CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
1048
1049 if (type & SLJIT_CALL_RETURN) {
1050 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
1051 PTR_FAIL_IF(tail_call_reg_arg_with_args(compiler, arg_types));
1052 PTR_FAIL_IF(emit_stack_frame_release(compiler, 0));
1053
1054 SLJIT_SKIP_CHECKS(compiler);
1055 return sljit_emit_jump(compiler, SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP));
1056 }
1057
1058 stack_size = type;
1059 PTR_FAIL_IF(tail_call_with_args(compiler, &stack_size, arg_types, SLJIT_IMM, 0));
1060
1061 SLJIT_SKIP_CHECKS(compiler);
1062
1063 if (stack_size == 0)
1064 return sljit_emit_jump(compiler, SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP));
1065
1066 jump = sljit_emit_jump(compiler, type);
1067 PTR_FAIL_IF(jump == NULL);
1068
1069 PTR_FAIL_IF(emit_tail_call_end(compiler, stack_size));
1070 return jump;
1071 }
1072
1073 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
1074 SLJIT_SKIP_CHECKS(compiler);
1075 return sljit_emit_jump(compiler, type);
1076 }
1077
1078 stack_size = call_get_stack_size(arg_types, &word_arg_count);
1079 PTR_FAIL_IF(call_with_args(compiler, arg_types, stack_size, word_arg_count, 0));
1080
1081 SLJIT_SKIP_CHECKS(compiler);
1082 jump = sljit_emit_jump(compiler, type);
1083 PTR_FAIL_IF(jump == NULL);
1084
1085 PTR_FAIL_IF(post_call_with_args(compiler, arg_types, stack_size));
1086 return jump;
1087 }
1088
1089 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
1090 sljit_s32 arg_types,
1091 sljit_s32 src, sljit_sw srcw)
1092 {
1093 sljit_sw stack_size = 0;
1094 sljit_s32 word_arg_count;
1095
1096 CHECK_ERROR();
1097 CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
1098
1099 if (type & SLJIT_CALL_RETURN) {
1100 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
1101 FAIL_IF(tail_call_reg_arg_with_args(compiler, arg_types));
1102
1103 if ((src & SLJIT_MEM) || (src > SLJIT_R2 && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options)))) {
1104 ADJUST_LOCAL_OFFSET(src, srcw);
1105 CHECK_EXTRA_REGS(src, srcw, (void)0);
1106
1107 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1108 src = TMP_REG1;
1109 srcw = 0;
1110 }
1111
1112 FAIL_IF(emit_stack_frame_release(compiler, 0));
1113
1114 SLJIT_SKIP_CHECKS(compiler);
1115 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1116 }
1117
1118 stack_size = type;
1119 FAIL_IF(tail_call_with_args(compiler, &stack_size, arg_types, src, srcw));
1120
1121 if (src != SLJIT_IMM) {
1122 src = SLJIT_R0;
1123 srcw = 0;
1124 }
1125
1126 SLJIT_SKIP_CHECKS(compiler);
1127
1128 if (stack_size == 0)
1129 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1130
1131 FAIL_IF(sljit_emit_ijump(compiler, type, src, srcw));
1132 return emit_tail_call_end(compiler, stack_size);
1133 }
1134
1135 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
1136 SLJIT_SKIP_CHECKS(compiler);
1137 return sljit_emit_ijump(compiler, type, src, srcw);
1138 }
1139
1140 ADJUST_LOCAL_OFFSET(src, srcw);
1141 CHECK_EXTRA_REGS(src, srcw, (void)0);
1142
1143 if (src & SLJIT_MEM) {
1144 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1145 src = TMP_REG1;
1146 srcw = 0;
1147 }
1148
1149 stack_size = call_get_stack_size(arg_types, &word_arg_count);
1150 FAIL_IF(call_with_args(compiler, arg_types, stack_size, word_arg_count, src == TMP_REG1));
1151
1152 if (stack_size > 0 && src == SLJIT_MEM1(SLJIT_SP))
1153 srcw += stack_size;
1154
1155 SLJIT_SKIP_CHECKS(compiler);
1156 FAIL_IF(sljit_emit_ijump(compiler, type, src, srcw));
1157
1158 return post_call_with_args(compiler, arg_types, stack_size);
1159 }
1160
1161 static SLJIT_INLINE sljit_s32 emit_fmov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
1162 {
1163 sljit_u8* inst;
1164
1165 if (compiler->options & SLJIT_ENTER_REG_ARG) {
1166 if (src == SLJIT_FR0)
1167 return SLJIT_SUCCESS;
1168
1169 SLJIT_SKIP_CHECKS(compiler);
1170 return sljit_emit_fop1(compiler, op, SLJIT_RETURN_FREG, 0, src, srcw);
1171 }
1172
1173 if (FAST_IS_REG(src)) {
1174 FAIL_IF(emit_sse2_store(compiler, op & SLJIT_32, SLJIT_MEM1(SLJIT_SP), 0, src));
1175
1176 src = SLJIT_MEM1(SLJIT_SP);
1177 srcw = 0;
1178 } else {
1179 ADJUST_LOCAL_OFFSET(src, srcw);
1180 }
1181
1182 inst = emit_x86_instruction(compiler, 1 | EX86_SSE2_OP1, 0, 0, src, srcw);
1183 *inst = (op & SLJIT_32) ? FLDS : FLDL;
1184
1185 return SLJIT_SUCCESS;
1186 }
1187
1188 static sljit_s32 emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
1189 {
1190 sljit_u8 *inst;
1191
1192 CHECK_EXTRA_REGS(dst, dstw, (void)0);
1193
1194 /* Unused dest is possible here. */
1195 if (FAST_IS_REG(dst))
1196 return emit_byte(compiler, U8(POP_r + reg_map[dst]));
1197
1198 /* Memory. */
1199 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
1200 FAIL_IF(!inst);
1201 *inst = POP_rm;
1202 return SLJIT_SUCCESS;
1203 }
1204
1205 static sljit_s32 emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw)
1206 {
1207 sljit_u8 *inst;
1208
1209 CHECK_EXTRA_REGS(src, srcw, (void)0);
1210
1211 if (FAST_IS_REG(src)) {
1212 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 1);
1213 FAIL_IF(!inst);
1214
1215 INC_SIZE(1 + 1);
1216 PUSH_REG(reg_map[src]);
1217 }
1218 else {
1219 inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
1220 FAIL_IF(!inst);
1221 inst[0] = GROUP_FF;
1222 inst[1] |= PUSH_rm;
1223
1224 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1225 FAIL_IF(!inst);
1226 INC_SIZE(1);
1227 }
1228
1229 RET();
1230 return SLJIT_SUCCESS;
1231 }
1232
1233 static sljit_s32 sljit_emit_get_return_address(struct sljit_compiler *compiler,
1234 sljit_s32 dst, sljit_sw dstw)
1235 {
1236 sljit_s32 options = compiler->options;
1237 sljit_s32 saveds = compiler->saveds;
1238 sljit_s32 scratches = compiler->scratches;
1239
1240 saveds = ((scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3) - SLJIT_KEPT_SAVEDS_COUNT(options)) * SSIZE_OF(sw);
1241
1242 /* Saving ebp. */
1243 if (!(options & SLJIT_ENTER_REG_ARG))
1244 saveds += SSIZE_OF(sw);
1245
1246 return emit_mov(compiler, dst, dstw, SLJIT_MEM1(SLJIT_SP), compiler->local_size + saveds);
1247 }
1248
1249 /* --------------------------------------------------------------------- */
1250 /* Other operations */
1251 /* --------------------------------------------------------------------- */
1252
1253 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
1254 sljit_s32 dst_reg,
1255 sljit_s32 src1, sljit_sw src1w,
1256 sljit_s32 src2_reg)
1257 {
1258 sljit_s32 dst = dst_reg;
1259 sljit_sw dstw = 0;
1260 sljit_sw src2w = 0;
1261
1262 CHECK_ERROR();
1263 CHECK(check_sljit_emit_select(compiler, type, dst_reg, src1, src1w, src2_reg));
1264
1265 ADJUST_LOCAL_OFFSET(src1, src1w);
1266
1267 CHECK_EXTRA_REGS(dst, dstw, (void)0);
1268 CHECK_EXTRA_REGS(src1, src1w, (void)0);
1269 CHECK_EXTRA_REGS(src2_reg, src2w, (void)0);
1270
1271 type &= ~SLJIT_32;
1272
1273 if (dst & SLJIT_MEM) {
1274 if (src1 == SLJIT_IMM || (!(src1 & SLJIT_MEM) && (src2_reg & SLJIT_MEM))) {
1275 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1276 src1 = src2_reg;
1277 src1w = src2w;
1278 type ^= 0x1;
1279 } else
1280 EMIT_MOV(compiler, TMP_REG1, 0, src2_reg, src2w);
1281
1282 dst_reg = TMP_REG1;
1283 } else {
1284 if (dst_reg != src2_reg) {
1285 if (dst_reg == src1) {
1286 src1 = src2_reg;
1287 src1w = src2w;
1288 type ^= 0x1;
1289 } else if (ADDRESSING_DEPENDS_ON(src1, dst_reg)) {
1290 EMIT_MOV(compiler, dst_reg, 0, src1, src1w);
1291 src1 = src2_reg;
1292 src1w = src2w;
1293 type ^= 0x1;
1294 } else
1295 EMIT_MOV(compiler, dst_reg, 0, src2_reg, src2w);
1296 }
1297 }
1298
1299 if (sljit_has_cpu_feature(SLJIT_HAS_CMOV) && (src1 != SLJIT_IMM || dst_reg != TMP_REG1)) {
1300 if (SLJIT_UNLIKELY(src1 == SLJIT_IMM)) {
1301 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1302 src1 = TMP_REG1;
1303 src1w = 0;
1304 }
1305
1306 FAIL_IF(emit_groupf(compiler, U8(get_jump_code((sljit_uw)type) - 0x40), dst_reg, src1, src1w));
1307 } else
1308 FAIL_IF(emit_cmov_generic(compiler, type, dst_reg, src1, src1w));
1309
1310 if (dst & SLJIT_MEM)
1311 return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
1312 return SLJIT_SUCCESS;
1313 }
1314
1315 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
1316 sljit_s32 reg,
1317 sljit_s32 mem, sljit_sw memw)
1318 {
1319 sljit_u8* inst;
1320 sljit_s32 i, next, reg_idx, offset;
1321 sljit_u8 regs[2];
1322
1323 CHECK_ERROR();
1324 CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw));
1325
1326 if (!(reg & REG_PAIR_MASK))
1327 return sljit_emit_mem_unaligned(compiler, type, reg, mem, memw);
1328
1329 ADJUST_LOCAL_OFFSET(mem, memw);
1330
1331 regs[0] = U8(REG_PAIR_FIRST(reg));
1332 regs[1] = U8(REG_PAIR_SECOND(reg));
1333
1334 next = SSIZE_OF(sw);
1335
1336 if (!(type & SLJIT_MEM_STORE) && (regs[0] == (mem & REG_MASK) || regs[0] == OFFS_REG(mem))) {
1337 if (regs[1] == (mem & REG_MASK) || regs[1] == OFFS_REG(mem)) {
1338 /* None of them are virtual register so TMP_REG1 will not be used. */
1339 EMIT_MOV(compiler, TMP_REG1, 0, OFFS_REG(mem), 0);
1340
1341 if (regs[1] == OFFS_REG(mem))
1342 next = -SSIZE_OF(sw);
1343
1344 mem = (mem & ~OFFS_REG_MASK) | TO_OFFS_REG(TMP_REG1);
1345 } else {
1346 next = -SSIZE_OF(sw);
1347
1348 if (!(mem & OFFS_REG_MASK))
1349 memw += SSIZE_OF(sw);
1350 }
1351 }
1352
1353 for (i = 0; i < 2; i++) {
1354 reg_idx = next > 0 ? i : (i ^ 0x1);
1355 reg = regs[reg_idx];
1356
1357 offset = -1;
1358
1359 if (reg >= SLJIT_R3 && reg <= SLJIT_S3) {
1360 offset = (2 * SSIZE_OF(sw)) + ((reg) - SLJIT_R3) * SSIZE_OF(sw);
1361 reg = TMP_REG1;
1362
1363 if (type & SLJIT_MEM_STORE)
1364 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), offset);
1365 }
1366
1367 if ((mem & OFFS_REG_MASK) && (reg_idx == 1)) {
1368 inst = (sljit_u8*)ensure_buf(compiler, (sljit_uw)(1 + 4));
1369 FAIL_IF(!inst);
1370
1371 INC_SIZE(4);
1372
1373 inst[0] = (type & SLJIT_MEM_STORE) ? MOV_rm_r : MOV_r_rm;
1374 inst[1] = 0x44 | U8(reg_map[reg] << 3);
1375 inst[2] = U8(memw << 6) | U8(reg_map[OFFS_REG(mem)] << 3) | reg_map[mem & REG_MASK];
1376 inst[3] = sizeof(sljit_sw);
1377 } else if (type & SLJIT_MEM_STORE) {
1378 EMIT_MOV(compiler, mem, memw, reg, 0);
1379 } else {
1380 EMIT_MOV(compiler, reg, 0, mem, memw);
1381 }
1382
1383 if (!(mem & OFFS_REG_MASK))
1384 memw += next;
1385
1386 if (!(type & SLJIT_MEM_STORE) && offset != -1)
1387 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, TMP_REG1, 0);
1388 }
1389
1390 return SLJIT_SUCCESS;
1391 }
1392
1393 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_uw(struct sljit_compiler *compiler, sljit_s32 op,
1394 sljit_s32 dst, sljit_sw dstw,
1395 sljit_s32 src, sljit_sw srcw)
1396 {
1397 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG;
1398 sljit_u8 *inst, *jump_inst1, *jump_inst2;
1399 sljit_uw size1, size2;
1400
1401 /* Binary representation of 0x80000000. */
1402 static const sljit_f64 f64_high_bit = (sljit_f64)0x80000000ul;
1403
1404 CHECK_EXTRA_REGS(src, srcw, (void)0);
1405
1406 if (!(op & SLJIT_32)) {
1407 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1408
1409 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 1, TMP_REG1, 0);
1410 FAIL_IF(!inst);
1411 inst[1] |= ROL;
1412
1413 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 1, TMP_REG1, 0);
1414 FAIL_IF(!inst);
1415 inst[1] |= SHR;
1416
1417 FAIL_IF(emit_groupf(compiler, CVTSI2SD_x_rm | EX86_PREF_F2 | EX86_SSE2_OP1, dst_r, TMP_REG1, 0));
1418
1419 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1420 FAIL_IF(!inst);
1421 INC_SIZE(2);
1422 inst[0] = U8(get_jump_code(SLJIT_NOT_CARRY) - 0x10);
1423
1424 size1 = compiler->size;
1425 FAIL_IF(emit_groupf(compiler, ADDSD_x_xm | EX86_PREF_F2 | EX86_SSE2, dst_r, SLJIT_MEM0(), (sljit_sw)&f64_high_bit));
1426
1427 inst[1] = U8(compiler->size - size1);
1428
1429 if (dst_r == TMP_FREG)
1430 return emit_sse2_store(compiler, 0, dst, dstw, TMP_FREG);
1431 return SLJIT_SUCCESS;
1432 }
1433
1434 if (!FAST_IS_REG(src)) {
1435 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1436 src = TMP_REG1;
1437 }
1438
1439 BINARY_IMM32(CMP, 0, src, 0);
1440
1441 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1442 FAIL_IF(!inst);
1443 INC_SIZE(2);
1444 inst[0] = JL_i8;
1445 jump_inst1 = inst;
1446
1447 size1 = compiler->size;
1448
1449 FAIL_IF(emit_groupf(compiler, CVTSI2SD_x_rm | EX86_SELECT_F2_F3(op) | EX86_SSE2_OP1, dst_r, src, 0));
1450
1451 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1452 FAIL_IF(!inst);
1453 INC_SIZE(2);
1454 inst[0] = JMP_i8;
1455 jump_inst2 = inst;
1456
1457 size2 = compiler->size;
1458
1459 jump_inst1[1] = U8(size2 - size1);
1460
1461 if (src != TMP_REG1)
1462 EMIT_MOV(compiler, TMP_REG1, 0, src, 0);
1463
1464 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 1, TMP_REG1, 0);
1465 FAIL_IF(!inst);
1466 inst[1] |= SHR;
1467
1468 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1469 FAIL_IF(!inst);
1470 INC_SIZE(2);
1471 inst[0] = JNC_i8;
1472 jump_inst1 = inst;
1473
1474 size1 = compiler->size;
1475
1476 BINARY_IMM32(OR, 1, TMP_REG1, 0);
1477 jump_inst1[1] = U8(compiler->size - size1);
1478
1479 FAIL_IF(emit_groupf(compiler, CVTSI2SD_x_rm | EX86_SELECT_F2_F3(op) | EX86_SSE2_OP1, dst_r, TMP_REG1, 0));
1480 FAIL_IF(emit_groupf(compiler, ADDSD_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, dst_r, dst_r, 0));
1481
1482 jump_inst2[1] = U8(compiler->size - size2);
1483
1484 if (dst_r == TMP_FREG)
1485 return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
1486 return SLJIT_SUCCESS;
1487 }
1488
1489 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset32(struct sljit_compiler *compiler,
1490 sljit_s32 freg, sljit_f32 value)
1491 {
1492 sljit_u8 *inst;
1493 union {
1494 sljit_s32 imm;
1495 sljit_f32 value;
1496 } u;
1497
1498 CHECK_ERROR();
1499 CHECK(check_sljit_emit_fset32(compiler, freg, value));
1500
1501 u.value = value;
1502
1503 if (u.imm != 0)
1504 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, u.imm);
1505
1506 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1507 FAIL_IF(!inst);
1508 INC_SIZE(4);
1509
1510 inst[0] = GROUP_66;
1511 inst[1] = GROUP_0F;
1512
1513 if (u.imm == 0) {
1514 inst[2] = PXOR_x_xm;
1515 inst[3] = U8(freg_map[freg] | (freg_map[freg] << 3) | MOD_REG);
1516 } else {
1517 inst[2] = MOVD_x_rm;
1518 inst[3] = U8(reg_map[TMP_REG1] | (freg_map[freg] << 3) | MOD_REG);
1519 }
1520
1521 return SLJIT_SUCCESS;
1522 }
1523
1524 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset64(struct sljit_compiler *compiler,
1525 sljit_s32 freg, sljit_f64 value)
1526 {
1527 sljit_u8 *inst;
1528 union {
1529 sljit_s32 imm[2];
1530 sljit_f64 value;
1531 } u;
1532
1533 CHECK_ERROR();
1534 CHECK(check_sljit_emit_fset64(compiler, freg, value));
1535
1536 u.value = value;
1537
1538 if (u.imm[0] == 0) {
1539 if (u.imm[1] == 0)
1540 return emit_groupf(compiler, PXOR_x_xm | EX86_PREF_66 | EX86_SSE2, freg, freg, 0);
1541
1542 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, u.imm[1]);
1543 } else {
1544 SLJIT_ASSERT(cpu_feature_list != 0);
1545
1546 if (!(cpu_feature_list & CPU_FEATURE_SSE41) && u.imm[1] != 0 && u.imm[0] != u.imm[1]) {
1547 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_IMM, u.imm[0]);
1548 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_sw), SLJIT_IMM, u.imm[1]);
1549
1550 return emit_groupf(compiler, MOVLPD_x_m | EX86_SSE2, freg, SLJIT_MEM1(SLJIT_SP), 0);
1551 }
1552
1553 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, u.imm[0]);
1554 }
1555
1556 FAIL_IF(emit_groupf(compiler, MOVD_x_rm | EX86_PREF_66 | EX86_SSE2_OP1, freg, TMP_REG1, 0));
1557
1558 if (u.imm[1] == 0)
1559 return SLJIT_SUCCESS;
1560
1561 if (u.imm[0] == 0) {
1562 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1563 FAIL_IF(!inst);
1564 INC_SIZE(4);
1565
1566 inst[0] = GROUP_0F;
1567 inst[1] = SHUFPS_x_xm;
1568 inst[2] = U8(MOD_REG | (freg_map[freg] << 3) | freg_map[freg]);
1569 inst[3] = 0x51;
1570 return SLJIT_SUCCESS;
1571 }
1572
1573 if (u.imm[0] != u.imm[1]) {
1574 SLJIT_ASSERT(cpu_feature_list & CPU_FEATURE_SSE41);
1575 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, u.imm[1]);
1576
1577 FAIL_IF(emit_groupf_ext(compiler, PINSRD_x_rm_i8 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2_OP1, freg, TMP_REG1, 0));
1578 return emit_byte(compiler, 1);
1579 }
1580
1581 inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
1582 FAIL_IF(!inst);
1583 INC_SIZE(3);
1584
1585 inst[0] = GROUP_0F;
1586 inst[1] = UNPCKLPS_x_xm;
1587 inst[2] = U8(MOD_REG | (freg_map[freg] << 3) | freg_map[freg]);
1588 return SLJIT_SUCCESS;
1589 }
1590
1591 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fcopy(struct sljit_compiler *compiler, sljit_s32 op,
1592 sljit_s32 freg, sljit_s32 reg)
1593 {
1594 sljit_u8 *inst;
1595 sljit_s32 reg2;
1596 sljit_sw regw, reg2w;
1597
1598 CHECK_ERROR();
1599 CHECK(check_sljit_emit_fcopy(compiler, op, freg, reg));
1600
1601 regw = 0;
1602 reg2 = 0;
1603 reg2w = 0;
1604
1605 SLJIT_ASSERT(cpu_feature_list != 0);
1606
1607 if (!(op & SLJIT_32) && (cpu_feature_list & CPU_FEATURE_SSE41)) {
1608 if (reg & REG_PAIR_MASK) {
1609 reg2 = REG_PAIR_FIRST(reg);
1610 reg = REG_PAIR_SECOND(reg);
1611
1612 CHECK_EXTRA_REGS(reg, regw, (void)0);
1613
1614 FAIL_IF(emit_groupf(compiler, (GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? MOVD_x_rm : MOVD_rm_x)
1615 | EX86_PREF_66 | EX86_SSE2_OP1, freg, reg, regw));
1616 } else
1617 reg2 = reg;
1618
1619 CHECK_EXTRA_REGS(reg2, reg2w, (void)0);
1620
1621 FAIL_IF(emit_groupf_ext(compiler, (GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? PINSRD_x_rm_i8 : PEXTRD_rm_x_i8)
1622 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2_OP1, freg, reg2, reg2w));
1623 return emit_byte(compiler, 1);
1624 }
1625
1626 if (reg & REG_PAIR_MASK) {
1627 reg2 = REG_PAIR_SECOND(reg);
1628 reg = REG_PAIR_FIRST(reg);
1629
1630 if (reg == reg2)
1631 reg = 0;
1632
1633 CHECK_EXTRA_REGS(reg2, reg2w, (void)0);
1634 }
1635
1636 CHECK_EXTRA_REGS(reg, regw, (void)0);
1637
1638 if (op & SLJIT_32)
1639 return emit_groupf(compiler, (GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? MOVD_x_rm : MOVD_rm_x)
1640 | EX86_PREF_66 | EX86_SSE2_OP1, freg, reg, regw);
1641
1642 if (op == SLJIT_COPY_FROM_F64) {
1643 inst = (sljit_u8*)ensure_buf(compiler, 1 + 5);
1644 FAIL_IF(!inst);
1645 INC_SIZE(5);
1646
1647 inst[0] = GROUP_66;
1648 inst[1] = GROUP_0F;
1649 inst[2] = PSHUFD_x_xm;
1650 inst[3] = U8(MOD_REG | (TMP_FREG << 3) | freg_map[freg]);
1651 inst[4] = 1;
1652 } else if (reg != 0)
1653 FAIL_IF(emit_groupf(compiler, MOVD_x_rm | EX86_PREF_66 | EX86_SSE2_OP1, TMP_FREG, reg, regw));
1654
1655 if (reg2 != 0)
1656 FAIL_IF(emit_groupf(compiler, (GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? MOVD_x_rm : MOVD_rm_x)
1657 | EX86_PREF_66 | EX86_SSE2_OP1, freg, reg2, reg2w));
1658
1659 if (GET_OPCODE(op) == SLJIT_COPY_TO_F64) {
1660 inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
1661 FAIL_IF(!inst);
1662 INC_SIZE(3);
1663
1664 inst[0] = GROUP_0F;
1665 inst[1] = UNPCKLPS_x_xm;
1666 inst[2] = U8(MOD_REG | (freg_map[freg] << 3) | freg_map[reg == 0 ? freg : TMP_FREG]);
1667 } else
1668 FAIL_IF(emit_groupf(compiler, MOVD_rm_x | EX86_PREF_66 | EX86_SSE2_OP1, TMP_FREG, reg, regw));
1669
1670 return SLJIT_SUCCESS;
1671 }
1672
1673 static sljit_s32 skip_frames_before_return(struct sljit_compiler *compiler)
1674 {
1675 sljit_sw size;
1676
1677 /* Don't adjust shadow stack if it isn't enabled. */
1678 if (!cpu_has_shadow_stack())
1679 return SLJIT_SUCCESS;
1680
1681 SLJIT_ASSERT(compiler->args_size >= 0);
1682 SLJIT_ASSERT(compiler->local_size > 0);
1683
1684 size = compiler->local_size;
1685 size += (1 + (compiler->scratches > 9 ? (compiler->scratches - 9) : 0)
1686 + (compiler->saveds <= 3 ? compiler->saveds : 3)) * SSIZE_OF(sw);
1687
1688 return adjust_shadow_stack(compiler, SLJIT_MEM1(SLJIT_SP), size);
1689 }