| 1 |
#include "Python.h" |
| 2 |
|
| 3 |
#include "pycore_backoff.h" |
| 4 |
#include "pycore_call.h" |
| 5 |
#include "pycore_ceval.h" |
| 6 |
#include "pycore_cell.h" |
| 7 |
#include "pycore_dict.h" |
| 8 |
#include "pycore_emscripten_signal.h" |
| 9 |
#include "pycore_intrinsics.h" |
| 10 |
#include "pycore_jit.h" |
| 11 |
#include "pycore_long.h" |
| 12 |
#include "pycore_opcode_metadata.h" |
| 13 |
#include "pycore_opcode_utils.h" |
| 14 |
#include "pycore_optimizer.h" |
| 15 |
#include "pycore_pyatomic_ft_wrappers.h" |
| 16 |
#include "pycore_range.h" |
| 17 |
#include "pycore_setobject.h" |
| 18 |
#include "pycore_sliceobject.h" |
| 19 |
#include "pycore_descrobject.h" |
| 20 |
|
| 21 |
#include "ceval_macros.h" |
| 22 |
|
| 23 |
#undef CURRENT_OPARG |
| 24 |
#define CURRENT_OPARG() (_oparg) |
| 25 |
|
| 26 |
#undef CURRENT_OPERAND |
| 27 |
#define CURRENT_OPERAND() (_operand) |
| 28 |
|
| 29 |
#undef DEOPT_IF |
| 30 |
#define DEOPT_IF(COND, INSTNAME) \ |
| 31 |
do { \ |
| 32 |
if ((COND)) { \ |
| 33 |
goto deoptimize; \ |
| 34 |
} \ |
| 35 |
} while (0) |
| 36 |
|
| 37 |
#undef ENABLE_SPECIALIZATION |
| 38 |
#define ENABLE_SPECIALIZATION (0) |
| 39 |
|
| 40 |
#undef GOTO_ERROR |
| 41 |
#define GOTO_ERROR(LABEL) \ |
| 42 |
do { \ |
| 43 |
goto LABEL ## _tier_two; \ |
| 44 |
} while (0) |
| 45 |
|
| 46 |
#undef GOTO_TIER_TWO |
| 47 |
#define GOTO_TIER_TWO(EXECUTOR) \ |
| 48 |
do { \ |
| 49 |
OPT_STAT_INC(traces_executed); \ |
| 50 |
__attribute__((musttail)) \ |
| 51 |
return ((jit_func)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \ |
| 52 |
} while (0) |
| 53 |
|
| 54 |
#undef GOTO_TIER_ONE |
| 55 |
#define GOTO_TIER_ONE(TARGET) \ |
| 56 |
do { \ |
| 57 |
_PyFrame_SetStackPointer(frame, stack_pointer); \ |
| 58 |
return TARGET; \ |
| 59 |
} while (0) |
| 60 |
|
| 61 |
#undef LOAD_IP |
| 62 |
#define LOAD_IP(UNUSED) \ |
| 63 |
do { \ |
| 64 |
} while (0) |
| 65 |
|
| 66 |
#define PATCH_VALUE(TYPE, NAME, ALIAS) \ |
| 67 |
PyAPI_DATA(void) ALIAS; \ |
| 68 |
TYPE NAME = (TYPE)(uintptr_t)&ALIAS; |
| 69 |
|
| 70 |
#define PATCH_JUMP(ALIAS) \ |
| 71 |
do { \ |
| 72 |
PyAPI_DATA(void) ALIAS; \ |
| 73 |
__attribute__((musttail)) \ |
| 74 |
return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \ |
| 75 |
} while (0) |
| 76 |
|
| 77 |
#undef JUMP_TO_JUMP_TARGET |
| 78 |
#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET) |
| 79 |
|
| 80 |
#undef JUMP_TO_ERROR |
| 81 |
#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET) |
| 82 |
|
| 83 |
_Py_CODEUNIT * |
| 84 |
_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) |
| 85 |
{ |
| 86 |
// Locals that the instruction implementations expect to exist: |
| 87 |
PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) |
| 88 |
int oparg; |
| 89 |
int uopcode = _JIT_OPCODE; |
| 90 |
_Py_CODEUNIT *next_instr; |
| 91 |
// Other stuff we need handy: |
| 92 |
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) |
| 93 |
#if SIZEOF_VOID_P == 8 |
| 94 |
PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) |
| 95 |
#else |
| 96 |
assert(SIZEOF_VOID_P == 4); |
| 97 |
PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI) |
| 98 |
PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO) |
| 99 |
uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo; |
| 100 |
#endif |
| 101 |
PATCH_VALUE(uint32_t, _target, _JIT_TARGET) |
| 102 |
PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX) |
| 103 |
|
| 104 |
OPT_STAT_INC(uops_executed); |
| 105 |
UOP_STAT_INC(uopcode, execution_count); |
| 106 |
|
| 107 |
// The actual instruction definitions (only one will be used): |
| 108 |
if (uopcode == _JUMP_TO_TOP) { |
| 109 |
PATCH_JUMP(_JIT_TOP); |
| 110 |
} |
| 111 |
switch (uopcode) { |
| 112 |
#include "executor_cases.c.h" |
| 113 |
default: |
| 114 |
Py_UNREACHABLE(); |
| 115 |
} |
| 116 |
PATCH_JUMP(_JIT_CONTINUE); |
| 117 |
// Labels that the instruction implementations expect to exist: |
| 118 |
|
| 119 |
error_tier_two: |
| 120 |
tstate->previous_executor = (PyObject *)current_executor; |
| 121 |
GOTO_TIER_ONE(NULL); |
| 122 |
exit_to_tier1: |
| 123 |
tstate->previous_executor = (PyObject *)current_executor; |
| 124 |
GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); |
| 125 |
exit_to_tier1_dynamic: |
| 126 |
tstate->previous_executor = (PyObject *)current_executor; |
| 127 |
GOTO_TIER_ONE(frame->instr_ptr); |
| 128 |
exit_to_trace: |
| 129 |
{ |
| 130 |
_PyExitData *exit = ¤t_executor->exits[_exit_index]; |
| 131 |
Py_INCREF(exit->executor); |
| 132 |
tstate->previous_executor = (PyObject *)current_executor; |
| 133 |
GOTO_TIER_TWO(exit->executor); |
| 134 |
} |
| 135 |
} |