| 1 |
#include "Python.h" |
| 2 |
|
| 3 |
#include "pycore_backoff.h" |
| 4 |
#include "pycore_call.h" |
| 5 |
#include "pycore_cell.h" |
| 6 |
#include "pycore_ceval.h" |
| 7 |
#include "pycore_code.h" |
| 8 |
#include "pycore_descrobject.h" |
| 9 |
#include "pycore_dict.h" |
| 10 |
#include "pycore_emscripten_signal.h" |
| 11 |
#include "pycore_floatobject.h" |
| 12 |
#include "pycore_frame.h" |
| 13 |
#include "pycore_function.h" |
| 14 |
#include "pycore_genobject.h" |
| 15 |
#include "pycore_interpframe.h" |
| 16 |
#include "pycore_interpolation.h" |
| 17 |
#include "pycore_intrinsics.h" |
| 18 |
#include "pycore_jit.h" |
| 19 |
#include "pycore_list.h" |
| 20 |
#include "pycore_long.h" |
| 21 |
#include "pycore_opcode_metadata.h" |
| 22 |
#include "pycore_opcode_utils.h" |
| 23 |
#include "pycore_optimizer.h" |
| 24 |
#include "pycore_pyatomic_ft_wrappers.h" |
| 25 |
#include "pycore_range.h" |
| 26 |
#include "pycore_setobject.h" |
| 27 |
#include "pycore_sliceobject.h" |
| 28 |
#include "pycore_stackref.h" |
| 29 |
#include "pycore_template.h" |
| 30 |
#include "pycore_tuple.h" |
| 31 |
#include "pycore_unicodeobject.h" |
| 32 |
|
| 33 |
#include "ceval_macros.h" |
| 34 |
|
| 35 |
#include "jit.h" |
| 36 |
|
| 37 |
#undef CURRENT_OPARG |
| 38 |
#define CURRENT_OPARG() (_oparg) |
| 39 |
|
| 40 |
#undef CURRENT_OPERAND0 |
| 41 |
#define CURRENT_OPERAND0() (_operand0) |
| 42 |
|
| 43 |
#undef CURRENT_OPERAND1 |
| 44 |
#define CURRENT_OPERAND1() (_operand1) |
| 45 |
|
| 46 |
#undef CURRENT_TARGET |
| 47 |
#define CURRENT_TARGET() (_target) |
| 48 |
|
| 49 |
#undef GOTO_TIER_TWO |
| 50 |
#define GOTO_TIER_TWO(EXECUTOR) \ |
| 51 |
do { \ |
| 52 |
OPT_STAT_INC(traces_executed); \ |
| 53 |
_PyExecutorObject *_executor = (EXECUTOR); \ |
| 54 |
tstate->current_executor = (PyObject *)_executor; \ |
| 55 |
jit_func_preserve_none jitted = _executor->jit_side_entry; \ |
| 56 |
__attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ |
| 57 |
} while (0) |
| 58 |
|
| 59 |
#undef GOTO_TIER_ONE |
| 60 |
#define GOTO_TIER_ONE(TARGET) \ |
| 61 |
do { \ |
| 62 |
tstate->current_executor = NULL; \ |
| 63 |
_PyFrame_SetStackPointer(frame, stack_pointer); \ |
| 64 |
return TARGET; \ |
| 65 |
} while (0) |
| 66 |
|
| 67 |
#undef LOAD_IP |
| 68 |
#define LOAD_IP(UNUSED) \ |
| 69 |
do { \ |
| 70 |
} while (0) |
| 71 |
|
| 72 |
#undef LLTRACE_RESUME_FRAME |
| 73 |
#ifdef Py_DEBUG |
| 74 |
#define LLTRACE_RESUME_FRAME() (frame->lltrace = 0) |
| 75 |
#else |
| 76 |
#define LLTRACE_RESUME_FRAME() do {} while (0) |
| 77 |
#endif |
| 78 |
|
| 79 |
#define PATCH_JUMP(ALIAS) \ |
| 80 |
do { \ |
| 81 |
PATCH_VALUE(jit_func_preserve_none, jump, ALIAS); \ |
| 82 |
__attribute__((musttail)) return jump(frame, stack_pointer, tstate); \ |
| 83 |
} while (0) |
| 84 |
|
| 85 |
#undef JUMP_TO_JUMP_TARGET |
| 86 |
#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET) |
| 87 |
|
| 88 |
#undef JUMP_TO_ERROR |
| 89 |
#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET) |
| 90 |
|
| 91 |
#define TIER_TWO 2 |
| 92 |
|
| 93 |
__attribute__((preserve_none)) _Py_CODEUNIT * |
| 94 |
_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) |
| 95 |
{ |
| 96 |
// Locals that the instruction implementations expect to exist: |
| 97 |
PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) |
| 98 |
int oparg; |
| 99 |
int uopcode = _JIT_OPCODE; |
| 100 |
_Py_CODEUNIT *next_instr; |
| 101 |
// Other stuff we need handy: |
| 102 |
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) |
| 103 |
#if SIZEOF_VOID_P == 8 |
| 104 |
PATCH_VALUE(uint64_t, _operand0, _JIT_OPERAND0) |
| 105 |
PATCH_VALUE(uint64_t, _operand1, _JIT_OPERAND1) |
| 106 |
#else |
| 107 |
assert(SIZEOF_VOID_P == 4); |
| 108 |
PATCH_VALUE(uint32_t, _operand0_hi, _JIT_OPERAND0_HI) |
| 109 |
PATCH_VALUE(uint32_t, _operand0_lo, _JIT_OPERAND0_LO) |
| 110 |
uint64_t _operand0 = ((uint64_t)_operand0_hi << 32) | _operand0_lo; |
| 111 |
PATCH_VALUE(uint32_t, _operand1_hi, _JIT_OPERAND1_HI) |
| 112 |
PATCH_VALUE(uint32_t, _operand1_lo, _JIT_OPERAND1_LO) |
| 113 |
uint64_t _operand1 = ((uint64_t)_operand1_hi << 32) | _operand1_lo; |
| 114 |
#endif |
| 115 |
PATCH_VALUE(uint32_t, _target, _JIT_TARGET) |
| 116 |
OPT_STAT_INC(uops_executed); |
| 117 |
UOP_STAT_INC(uopcode, execution_count); |
| 118 |
switch (uopcode) { |
| 119 |
// The actual instruction definition gets inserted here: |
| 120 |
CASE |
| 121 |
default: |
| 122 |
Py_UNREACHABLE(); |
| 123 |
} |
| 124 |
PATCH_JUMP(_JIT_CONTINUE); |
| 125 |
} |