| 1 |
#ifndef Py_INTERNAL_CODE_H |
| 2 |
#define Py_INTERNAL_CODE_H |
| 3 |
#ifdef __cplusplus |
| 4 |
extern "C" { |
| 5 |
#endif |
| 6 |
|
| 7 |
#ifndef Py_BUILD_CORE |
| 8 |
# error "this header requires Py_BUILD_CORE define" |
| 9 |
#endif |
| 10 |
|
| 11 |
#include "pycore_backoff.h" // _Py_BackoffCounter |
| 12 |
#include "pycore_structs.h" // _Py_CODEUNIT |
| 13 |
#include "pycore_tstate.h" // _PyThreadStateImpl |
| 14 |
|
| 15 |
|
| 16 |
#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) |
| 17 |
#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) |
| 18 |
|
| 19 |
|
| 20 |
/* These macros only remain defined for compatibility. */ |
| 21 |
#define _Py_OPCODE(word) ((word).op.code) |
| 22 |
#define _Py_OPARG(word) ((word).op.arg) |
| 23 |
|
| 24 |
static inline _Py_CODEUNIT |
| 25 |
_py_make_codeunit(uint8_t opcode, uint8_t oparg) |
| 26 |
{ |
| 27 |
// No designated initialisers because of C++ compat |
| 28 |
_Py_CODEUNIT word; |
| 29 |
word.op.code = opcode; |
| 30 |
word.op.arg = oparg; |
| 31 |
return word; |
| 32 |
} |
| 33 |
|
| 34 |
static inline void |
| 35 |
_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) |
| 36 |
{ |
| 37 |
word->op.code = opcode; |
| 38 |
} |
| 39 |
|
| 40 |
#define _Py_MAKE_CODEUNIT(opcode, oparg) _py_make_codeunit((opcode), (oparg)) |
| 41 |
#define _Py_SET_OPCODE(word, opcode) _py_set_opcode(&(word), (opcode)) |
| 42 |
|
| 43 |
|
| 44 |
// We hide some of the newer PyCodeObject fields behind macros. |
| 45 |
// This helps with backporting certain changes to 3.12. |
| 46 |
#define _PyCode_HAS_EXECUTORS(CODE) \ |
| 47 |
(CODE->co_executors != NULL) |
| 48 |
#define _PyCode_HAS_INSTRUMENTATION(CODE) \ |
| 49 |
(CODE->_co_instrumentation_version > 0) |
| 50 |
|
| 51 |
|
| 52 |
extern PyStatus _PyCode_Init(PyInterpreterState *interp); |
| 53 |
extern void _PyCode_Fini(PyInterpreterState *interp); |
| 54 |
|
| 55 |
|
| 56 |
/* PEP 659 |
| 57 |
* Specialization and quickening structs and helper functions |
| 58 |
*/ |
| 59 |
|
| 60 |
|
| 61 |
// Inline caches. If you change the number of cache entries for an instruction, |
| 62 |
// you must *also* update the number of cache entries in Lib/opcode.py and bump |
| 63 |
// the magic number in Lib/importlib/_bootstrap_external.py! |
| 64 |
|
| 65 |
#define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT)) |
| 66 |
|
| 67 |
typedef struct { |
| 68 |
_Py_BackoffCounter counter; |
| 69 |
uint16_t module_keys_version; |
| 70 |
uint16_t builtin_keys_version; |
| 71 |
uint16_t index; |
| 72 |
} _PyLoadGlobalCache; |
| 73 |
|
| 74 |
#define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache) |
| 75 |
|
| 76 |
typedef struct { |
| 77 |
_Py_BackoffCounter counter; |
| 78 |
uint16_t external_cache[4]; |
| 79 |
} _PyBinaryOpCache; |
| 80 |
|
| 81 |
#define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache) |
| 82 |
|
| 83 |
typedef struct { |
| 84 |
_Py_BackoffCounter counter; |
| 85 |
} _PyUnpackSequenceCache; |
| 86 |
|
| 87 |
#define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \ |
| 88 |
CACHE_ENTRIES(_PyUnpackSequenceCache) |
| 89 |
|
| 90 |
typedef struct { |
| 91 |
_Py_BackoffCounter counter; |
| 92 |
} _PyCompareOpCache; |
| 93 |
|
| 94 |
#define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache) |
| 95 |
|
| 96 |
typedef struct { |
| 97 |
_Py_BackoffCounter counter; |
| 98 |
} _PySuperAttrCache; |
| 99 |
|
| 100 |
#define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache) |
| 101 |
|
| 102 |
typedef struct { |
| 103 |
_Py_BackoffCounter counter; |
| 104 |
uint16_t version[2]; |
| 105 |
uint16_t index; |
| 106 |
} _PyAttrCache; |
| 107 |
|
| 108 |
typedef struct { |
| 109 |
_Py_BackoffCounter counter; |
| 110 |
uint16_t type_version[2]; |
| 111 |
union { |
| 112 |
uint16_t keys_version[2]; |
| 113 |
uint16_t dict_offset; |
| 114 |
}; |
| 115 |
uint16_t descr[4]; |
| 116 |
} _PyLoadMethodCache; |
| 117 |
|
| 118 |
|
| 119 |
// MUST be the max(_PyAttrCache, _PyLoadMethodCache) |
| 120 |
#define INLINE_CACHE_ENTRIES_LOAD_ATTR CACHE_ENTRIES(_PyLoadMethodCache) |
| 121 |
|
| 122 |
#define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache) |
| 123 |
|
| 124 |
typedef struct { |
| 125 |
_Py_BackoffCounter counter; |
| 126 |
uint16_t func_version[2]; |
| 127 |
} _PyCallCache; |
| 128 |
|
| 129 |
#define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) |
| 130 |
#define INLINE_CACHE_ENTRIES_CALL_KW CACHE_ENTRIES(_PyCallCache) |
| 131 |
|
| 132 |
typedef struct { |
| 133 |
_Py_BackoffCounter counter; |
| 134 |
} _PyStoreSubscrCache; |
| 135 |
|
| 136 |
#define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) |
| 137 |
|
| 138 |
typedef struct { |
| 139 |
_Py_BackoffCounter counter; |
| 140 |
} _PyForIterCache; |
| 141 |
|
| 142 |
#define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) |
| 143 |
|
| 144 |
typedef struct { |
| 145 |
_Py_BackoffCounter counter; |
| 146 |
} _PySendCache; |
| 147 |
|
| 148 |
#define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache) |
| 149 |
|
| 150 |
typedef struct { |
| 151 |
_Py_BackoffCounter counter; |
| 152 |
uint16_t version[2]; |
| 153 |
} _PyToBoolCache; |
| 154 |
|
| 155 |
#define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache) |
| 156 |
|
| 157 |
typedef struct { |
| 158 |
_Py_BackoffCounter counter; |
| 159 |
} _PyContainsOpCache; |
| 160 |
|
| 161 |
#define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache) |
| 162 |
|
| 163 |
/* "Locals plus" for a code object is the set of locals + cell vars + |
| 164 |
* free vars. This relates to variable names as well as offsets into |
| 165 |
* the "fast locals" storage array of execution frames. The compiler |
| 166 |
* builds the list of names, their offsets, and the corresponding |
| 167 |
* kind of local. |
| 168 |
* |
| 169 |
* Those kinds represent the source of the initial value and the |
| 170 |
* variable's scope (as related to closures). A "local" is an |
| 171 |
* argument or other variable defined in the current scope. A "free" |
| 172 |
* variable is one that is defined in an outer scope and comes from |
| 173 |
* the function's closure. A "cell" variable is a local that escapes |
| 174 |
* into an inner function as part of a closure, and thus must be |
| 175 |
* wrapped in a cell. Any "local" can also be a "cell", but the |
| 176 |
* "free" kind is mutually exclusive with both. |
| 177 |
*/ |
| 178 |
|
| 179 |
// Note that these all fit within a byte, as do combinations. |
| 180 |
#define CO_FAST_ARG_POS (0x02) // pos-only, pos-or-kw, varargs |
| 181 |
#define CO_FAST_ARG_KW (0x04) // kw-only, pos-or-kw, varkwargs |
| 182 |
#define CO_FAST_ARG_VAR (0x08) // varargs, varkwargs |
| 183 |
#define CO_FAST_ARG (CO_FAST_ARG_POS | CO_FAST_ARG_KW | CO_FAST_ARG_VAR) |
| 184 |
#define CO_FAST_HIDDEN (0x10) |
| 185 |
#define CO_FAST_LOCAL (0x20) |
| 186 |
#define CO_FAST_CELL (0x40) |
| 187 |
#define CO_FAST_FREE (0x80) |
| 188 |
|
| 189 |
typedef unsigned char _PyLocals_Kind; |
| 190 |
|
| 191 |
static inline _PyLocals_Kind |
| 192 |
_PyLocals_GetKind(PyObject *kinds, int i) |
| 193 |
{ |
| 194 |
assert(PyBytes_Check(kinds)); |
| 195 |
assert(0 <= i && i < PyBytes_GET_SIZE(kinds)); |
| 196 |
char *ptr = PyBytes_AS_STRING(kinds); |
| 197 |
return (_PyLocals_Kind)(ptr[i]); |
| 198 |
} |
| 199 |
|
| 200 |
static inline void |
| 201 |
_PyLocals_SetKind(PyObject *kinds, int i, _PyLocals_Kind kind) |
| 202 |
{ |
| 203 |
assert(PyBytes_Check(kinds)); |
| 204 |
assert(0 <= i && i < PyBytes_GET_SIZE(kinds)); |
| 205 |
char *ptr = PyBytes_AS_STRING(kinds); |
| 206 |
ptr[i] = (char) kind; |
| 207 |
} |
| 208 |
|
| 209 |
|
| 210 |
struct _PyCodeConstructor { |
| 211 |
/* metadata */ |
| 212 |
PyObject *filename; |
| 213 |
PyObject *name; |
| 214 |
PyObject *qualname; |
| 215 |
int flags; |
| 216 |
|
| 217 |
/* the code */ |
| 218 |
PyObject *code; |
| 219 |
int firstlineno; |
| 220 |
PyObject *linetable; |
| 221 |
|
| 222 |
/* used by the code */ |
| 223 |
PyObject *consts; |
| 224 |
PyObject *names; |
| 225 |
|
| 226 |
/* mapping frame offsets to information */ |
| 227 |
PyObject *localsplusnames; // Tuple of strings |
| 228 |
PyObject *localspluskinds; // Bytes object, one byte per variable |
| 229 |
|
| 230 |
/* args (within varnames) */ |
| 231 |
int argcount; |
| 232 |
int posonlyargcount; |
| 233 |
// XXX Replace argcount with posorkwargcount (argcount - posonlyargcount). |
| 234 |
int kwonlyargcount; |
| 235 |
|
| 236 |
/* needed to create the frame */ |
| 237 |
int stacksize; |
| 238 |
|
| 239 |
/* used by the eval loop */ |
| 240 |
PyObject *exceptiontable; |
| 241 |
}; |
| 242 |
|
| 243 |
// Using an "arguments struct" like this is helpful for maintainability |
| 244 |
// in a case such as this with many parameters. It does bear a risk: |
| 245 |
// if the struct changes and callers are not updated properly then the |
| 246 |
// compiler will not catch problems (like a missing argument). This can |
| 247 |
// cause hard-to-debug problems. The risk is mitigated by the use of |
| 248 |
// check_code() in codeobject.c. However, we may decide to switch |
| 249 |
// back to a regular function signature. Regardless, this approach |
| 250 |
// wouldn't be appropriate if this weren't a strictly internal API. |
| 251 |
// (See the comments in https://github.com/python/cpython/pull/26258.) |
| 252 |
extern int _PyCode_Validate(struct _PyCodeConstructor *); |
| 253 |
extern PyCodeObject* _PyCode_New(struct _PyCodeConstructor *); |
| 254 |
|
| 255 |
|
| 256 |
/* Private API */ |
| 257 |
|
| 258 |
/* Getters for internal PyCodeObject data. */ |
| 259 |
extern PyObject* _PyCode_GetVarnames(PyCodeObject *); |
| 260 |
extern PyObject* _PyCode_GetCellvars(PyCodeObject *); |
| 261 |
extern PyObject* _PyCode_GetFreevars(PyCodeObject *); |
| 262 |
extern PyObject* _PyCode_GetCode(PyCodeObject *); |
| 263 |
|
| 264 |
/** API for initializing the line number tables. */ |
| 265 |
extern int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds); |
| 266 |
|
| 267 |
/** Out of process API for initializing the location table. */ |
| 268 |
extern void _PyLineTable_InitAddressRange( |
| 269 |
const char *linetable, |
| 270 |
Py_ssize_t length, |
| 271 |
int firstlineno, |
| 272 |
PyCodeAddressRange *range); |
| 273 |
|
| 274 |
/** API for traversing the line number table. */ |
| 275 |
extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); |
| 276 |
extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); |
| 277 |
|
| 278 |
// Similar to PyCode_Addr2Line(), but return -1 if the code object is invalid |
| 279 |
// and can be called without an attached tstate. Used by dump_frame() in |
| 280 |
// Python/traceback.c. The function uses heuristics to detect freed memory, |
| 281 |
// it's not 100% reliable. |
| 282 |
extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr); |
| 283 |
|
| 284 |
|
| 285 |
/** API for executors */ |
| 286 |
extern void _PyCode_Clear_Executors(PyCodeObject *code); |
| 287 |
|
| 288 |
|
| 289 |
#ifdef Py_GIL_DISABLED |
| 290 |
// gh-115999 tracks progress on addressing this. |
| 291 |
#define ENABLE_SPECIALIZATION 0 |
| 292 |
// Use this to enable specialization families once they are thread-safe. All |
| 293 |
// uses will be replaced with ENABLE_SPECIALIZATION once all families are |
| 294 |
// thread-safe. |
| 295 |
#define ENABLE_SPECIALIZATION_FT 1 |
| 296 |
#else |
| 297 |
#define ENABLE_SPECIALIZATION 1 |
| 298 |
#define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION |
| 299 |
#endif |
| 300 |
|
| 301 |
/* Specialization functions */ |
| 302 |
|
| 303 |
extern void _Py_Specialize_LoadSuperAttr(_PyStackRef global_super, _PyStackRef cls, |
| 304 |
_Py_CODEUNIT *instr, int load_method); |
| 305 |
extern void _Py_Specialize_LoadAttr(_PyStackRef owner, _Py_CODEUNIT *instr, |
| 306 |
PyObject *name); |
| 307 |
extern void _Py_Specialize_StoreAttr(_PyStackRef owner, _Py_CODEUNIT *instr, |
| 308 |
PyObject *name); |
| 309 |
extern void _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, |
| 310 |
_Py_CODEUNIT *instr, PyObject *name); |
| 311 |
extern void _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, |
| 312 |
_Py_CODEUNIT *instr); |
| 313 |
extern void _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, |
| 314 |
int nargs); |
| 315 |
extern void _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, |
| 316 |
int nargs); |
| 317 |
extern void _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, |
| 318 |
int oparg, _PyStackRef *locals); |
| 319 |
extern void _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, |
| 320 |
_Py_CODEUNIT *instr, int oparg); |
| 321 |
extern void _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *instr, |
| 322 |
int oparg); |
| 323 |
extern void _Py_Specialize_ForIter(_PyStackRef iter, _Py_CODEUNIT *instr, int oparg); |
| 324 |
extern void _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); |
| 325 |
extern void _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); |
| 326 |
extern void _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); |
| 327 |
extern void _Py_GatherStats_GetIter(_PyStackRef iterable); |
| 328 |
|
| 329 |
// Utility functions for reading/writing 32/64-bit values in the inline caches. |
| 330 |
// Great care should be taken to ensure that these functions remain correct and |
| 331 |
// performant! They should compile to just "move" instructions on all supported |
| 332 |
// compilers and platforms. |
| 333 |
|
| 334 |
// We use memcpy to let the C compiler handle unaligned accesses and endianness |
| 335 |
// issues for us. It also seems to produce better code than manual copying for |
| 336 |
// most compilers (see https://blog.regehr.org/archives/959 for more info). |
| 337 |
|
| 338 |
static inline void |
| 339 |
write_u32(uint16_t *p, uint32_t val) |
| 340 |
{ |
| 341 |
memcpy(p, &val, sizeof(val)); |
| 342 |
} |
| 343 |
|
| 344 |
static inline void |
| 345 |
write_u64(uint16_t *p, uint64_t val) |
| 346 |
{ |
| 347 |
memcpy(p, &val, sizeof(val)); |
| 348 |
} |
| 349 |
|
| 350 |
static inline void |
| 351 |
write_ptr(uint16_t *p, void *val) |
| 352 |
{ |
| 353 |
memcpy(p, &val, sizeof(val)); |
| 354 |
} |
| 355 |
|
| 356 |
static inline uint16_t |
| 357 |
read_u16(uint16_t *p) |
| 358 |
{ |
| 359 |
return *p; |
| 360 |
} |
| 361 |
|
| 362 |
static inline uint32_t |
| 363 |
read_u32(uint16_t *p) |
| 364 |
{ |
| 365 |
uint32_t val; |
| 366 |
memcpy(&val, p, sizeof(val)); |
| 367 |
return val; |
| 368 |
} |
| 369 |
|
| 370 |
static inline uint64_t |
| 371 |
read_u64(uint16_t *p) |
| 372 |
{ |
| 373 |
uint64_t val; |
| 374 |
memcpy(&val, p, sizeof(val)); |
| 375 |
return val; |
| 376 |
} |
| 377 |
|
| 378 |
static inline PyObject * |
| 379 |
read_obj(uint16_t *p) |
| 380 |
{ |
| 381 |
PyObject *val; |
| 382 |
memcpy(&val, p, sizeof(val)); |
| 383 |
return val; |
| 384 |
} |
| 385 |
|
| 386 |
/* See InternalDocs/exception_handling.md for details. |
| 387 |
*/ |
| 388 |
static inline unsigned char * |
| 389 |
parse_varint(unsigned char *p, int *result) { |
| 390 |
int val = p[0] & 63; |
| 391 |
while (p[0] & 64) { |
| 392 |
p++; |
| 393 |
val = (val << 6) | (p[0] & 63); |
| 394 |
} |
| 395 |
*result = val; |
| 396 |
return p+1; |
| 397 |
} |
| 398 |
|
| 399 |
static inline int |
| 400 |
write_varint(uint8_t *ptr, unsigned int val) |
| 401 |
{ |
| 402 |
int written = 1; |
| 403 |
while (val >= 64) { |
| 404 |
*ptr++ = 64 | (val & 63); |
| 405 |
val >>= 6; |
| 406 |
written++; |
| 407 |
} |
| 408 |
*ptr = (uint8_t)val; |
| 409 |
return written; |
| 410 |
} |
| 411 |
|
| 412 |
static inline int |
| 413 |
write_signed_varint(uint8_t *ptr, int val) |
| 414 |
{ |
| 415 |
unsigned int uval; |
| 416 |
if (val < 0) { |
| 417 |
// (unsigned int)(-val) has an undefined behavior for INT_MIN |
| 418 |
uval = ((0 - (unsigned int)val) << 1) | 1; |
| 419 |
} |
| 420 |
else { |
| 421 |
uval = (unsigned int)val << 1; |
| 422 |
} |
| 423 |
return write_varint(ptr, uval); |
| 424 |
} |
| 425 |
|
| 426 |
static inline int |
| 427 |
write_location_entry_start(uint8_t *ptr, int code, int length) |
| 428 |
{ |
| 429 |
assert((code & 15) == code); |
| 430 |
*ptr = 128 | (uint8_t)(code << 3) | (uint8_t)(length - 1); |
| 431 |
return 1; |
| 432 |
} |
| 433 |
|
| 434 |
|
| 435 |
/** Counters |
| 436 |
* The first 16-bit value in each inline cache is a counter. |
| 437 |
* |
| 438 |
* When counting executions until the next specialization attempt, |
| 439 |
* exponential backoff is used to reduce the number of specialization failures. |
| 440 |
* See pycore_backoff.h for more details. |
| 441 |
* On a specialization failure, the backoff counter is restarted. |
| 442 |
*/ |
| 443 |
|
| 444 |
// A value of 1 means that we attempt to specialize the *second* time each |
| 445 |
// instruction is executed. Executing twice is a much better indicator of |
| 446 |
// "hotness" than executing once, but additional warmup delays only prevent |
| 447 |
// specialization. Most types stabilize by the second execution, too: |
| 448 |
#define ADAPTIVE_WARMUP_VALUE 1 |
| 449 |
#define ADAPTIVE_WARMUP_BACKOFF 1 |
| 450 |
|
| 451 |
// A value of 52 means that we attempt to re-specialize after 53 misses (a prime |
| 452 |
// number, useful for avoiding artifacts if every nth value is a different type |
| 453 |
// or something). Setting the backoff to 0 means that the counter is reset to |
| 454 |
// the same state as a warming-up instruction (value == 1, backoff == 1) after |
| 455 |
// deoptimization. This isn't strictly necessary, but it is bit easier to reason |
| 456 |
// about when thinking about the opcode transitions as a state machine: |
| 457 |
#define ADAPTIVE_COOLDOWN_VALUE 52 |
| 458 |
#define ADAPTIVE_COOLDOWN_BACKOFF 0 |
| 459 |
|
| 460 |
// Can't assert this in pycore_backoff.h because of header order dependencies |
| 461 |
#if SIDE_EXIT_INITIAL_VALUE <= ADAPTIVE_COOLDOWN_VALUE |
| 462 |
# error "Cold exit value should be larger than adaptive cooldown value" |
| 463 |
#endif |
| 464 |
|
| 465 |
static inline _Py_BackoffCounter |
| 466 |
adaptive_counter_bits(uint16_t value, uint16_t backoff) { |
| 467 |
return make_backoff_counter(value, backoff); |
| 468 |
} |
| 469 |
|
| 470 |
static inline _Py_BackoffCounter |
| 471 |
adaptive_counter_warmup(void) { |
| 472 |
return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE, |
| 473 |
ADAPTIVE_WARMUP_BACKOFF); |
| 474 |
} |
| 475 |
|
| 476 |
static inline _Py_BackoffCounter |
| 477 |
adaptive_counter_cooldown(void) { |
| 478 |
return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE, |
| 479 |
ADAPTIVE_COOLDOWN_BACKOFF); |
| 480 |
} |
| 481 |
|
| 482 |
static inline _Py_BackoffCounter |
| 483 |
adaptive_counter_backoff(_Py_BackoffCounter counter) { |
| 484 |
return restart_backoff_counter(counter); |
| 485 |
} |
| 486 |
|
| 487 |
/* Specialization Extensions */ |
| 488 |
|
| 489 |
/* callbacks for an external specialization */ |
| 490 |
typedef int (*binaryopguardfunc)(PyObject *lhs, PyObject *rhs); |
| 491 |
typedef PyObject *(*binaryopactionfunc)(PyObject *lhs, PyObject *rhs); |
| 492 |
|
| 493 |
typedef struct { |
| 494 |
int oparg; |
| 495 |
binaryopguardfunc guard; |
| 496 |
binaryopactionfunc action; |
| 497 |
} _PyBinaryOpSpecializationDescr; |
| 498 |
|
| 499 |
/* Comparison bit masks. */ |
| 500 |
|
| 501 |
/* Note this evaluates its arguments twice each */ |
| 502 |
#define COMPARISON_BIT(x, y) (1 << (2 * ((x) >= (y)) + ((x) <= (y)))) |
| 503 |
|
| 504 |
/* |
| 505 |
* The following bits are chosen so that the value of |
| 506 |
* COMPARSION_BIT(left, right) |
| 507 |
* masked by the values below will be non-zero if the |
| 508 |
* comparison is true, and zero if it is false */ |
| 509 |
|
| 510 |
/* This is for values that are unordered, ie. NaN, not types that are unordered, e.g. sets */ |
| 511 |
#define COMPARISON_UNORDERED 1 |
| 512 |
|
| 513 |
#define COMPARISON_LESS_THAN 2 |
| 514 |
#define COMPARISON_GREATER_THAN 4 |
| 515 |
#define COMPARISON_EQUALS 8 |
| 516 |
|
| 517 |
#define COMPARISON_NOT_EQUALS (COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN) |
| 518 |
|
| 519 |
extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); |
| 520 |
|
| 521 |
extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); |
| 522 |
|
| 523 |
extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); |
| 524 |
|
| 525 |
extern PyObject *_PyInstrumentation_BranchesIterator(PyCodeObject *code); |
| 526 |
|
| 527 |
struct _PyCode8 _PyCode_DEF(8); |
| 528 |
|
| 529 |
PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; |
| 530 |
|
| 531 |
#ifdef Py_GIL_DISABLED |
| 532 |
|
| 533 |
static inline _PyCodeArray * |
| 534 |
_PyCode_GetTLBCArray(PyCodeObject *co) |
| 535 |
{ |
| 536 |
return _Py_STATIC_CAST(_PyCodeArray *, |
| 537 |
_Py_atomic_load_ptr_acquire(&co->co_tlbc)); |
| 538 |
} |
| 539 |
|
| 540 |
// Return a pointer to the thread-local bytecode for the current thread, if it |
| 541 |
// exists. |
| 542 |
static inline _Py_CODEUNIT * |
| 543 |
_PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co) |
| 544 |
{ |
| 545 |
_PyCodeArray *code = _PyCode_GetTLBCArray(co); |
| 546 |
int32_t idx = ((_PyThreadStateImpl*) tstate)->tlbc_index; |
| 547 |
if (idx < code->size && code->entries[idx] != NULL) { |
| 548 |
return (_Py_CODEUNIT *) code->entries[idx]; |
| 549 |
} |
| 550 |
return NULL; |
| 551 |
} |
| 552 |
|
| 553 |
// Return a pointer to the thread-local bytecode for the current thread, |
| 554 |
// creating it if necessary. |
| 555 |
extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co); |
| 556 |
|
| 557 |
// Reserve an index for the current thread into thread-local bytecode |
| 558 |
// arrays |
| 559 |
// |
| 560 |
// Returns the reserved index or -1 on error. |
| 561 |
extern int32_t _Py_ReserveTLBCIndex(PyInterpreterState *interp); |
| 562 |
|
| 563 |
// Release the current thread's index into thread-local bytecode arrays |
| 564 |
extern void _Py_ClearTLBCIndex(_PyThreadStateImpl *tstate); |
| 565 |
|
| 566 |
// Free all TLBC copies not associated with live threads. |
| 567 |
// |
| 568 |
// Returns 0 on success or -1 on error. |
| 569 |
extern int _Py_ClearUnusedTLBC(PyInterpreterState *interp); |
| 570 |
#endif |
| 571 |
|
| 572 |
|
| 573 |
typedef struct { |
| 574 |
int total; |
| 575 |
struct co_locals_counts { |
| 576 |
int total; |
| 577 |
struct { |
| 578 |
int total; |
| 579 |
int numposonly; |
| 580 |
int numposorkw; |
| 581 |
int numkwonly; |
| 582 |
int varargs; |
| 583 |
int varkwargs; |
| 584 |
} args; |
| 585 |
int numpure; |
| 586 |
struct { |
| 587 |
int total; |
| 588 |
// numargs does not contribute to locals.total. |
| 589 |
int numargs; |
| 590 |
int numothers; |
| 591 |
} cells; |
| 592 |
struct { |
| 593 |
int total; |
| 594 |
int numpure; |
| 595 |
int numcells; |
| 596 |
} hidden; |
| 597 |
} locals; |
| 598 |
int numfree; // nonlocal |
| 599 |
struct co_unbound_counts { |
| 600 |
int total; |
| 601 |
struct { |
| 602 |
int total; |
| 603 |
int numglobal; |
| 604 |
int numbuiltin; |
| 605 |
int numunknown; |
| 606 |
} globals; |
| 607 |
int numattrs; |
| 608 |
int numunknown; |
| 609 |
} unbound; |
| 610 |
} _PyCode_var_counts_t; |
| 611 |
|
| 612 |
PyAPI_FUNC(void) _PyCode_GetVarCounts( |
| 613 |
PyCodeObject *, |
| 614 |
_PyCode_var_counts_t *); |
| 615 |
PyAPI_FUNC(int) _PyCode_SetUnboundVarCounts( |
| 616 |
PyThreadState *, |
| 617 |
PyCodeObject *, |
| 618 |
_PyCode_var_counts_t *, |
| 619 |
PyObject *globalnames, |
| 620 |
PyObject *attrnames, |
| 621 |
PyObject *globalsns, |
| 622 |
PyObject *builtinsns); |
| 623 |
|
| 624 |
|
| 625 |
/* "Stateless" code is a function or code object which does not rely on |
| 626 |
* external state or internal state. It may rely on arguments and |
| 627 |
* builtins, but not globals or a closure. Thus it does not rely |
| 628 |
* on __globals__ or __closure__, and a stateless function |
| 629 |
* is equivalent to its code object. |
| 630 |
* |
| 631 |
* Stateless code also does not keep any persistent state |
| 632 |
* of its own, so it can't have any executors, monitoring, |
| 633 |
* instrumentation, or "extras" (i.e. co_extra). |
| 634 |
* |
| 635 |
* Stateless code may create nested functions, including closures. |
| 636 |
* However, nested functions must themselves be stateless, except they |
| 637 |
* *can* close on the enclosing locals. |
| 638 |
* |
| 639 |
* Stateless code may return any value, including nested functions and closures. |
| 640 |
* |
| 641 |
* Stateless code that takes no arguments and doesn't return anything |
| 642 |
* may be treated like a script. |
| 643 |
* |
| 644 |
* We consider stateless code to be "portable" if it does not return |
| 645 |
* any object that holds a reference to any of the code's locals. Thus |
| 646 |
* generators and coroutines are not portable. Likewise a function |
| 647 |
* that returns a closure is not portable. The concept of |
| 648 |
* portability is useful in cases where the code is run |
| 649 |
* in a different execution context than where |
| 650 |
* the return value will be used. */ |
| 651 |
|
| 652 |
PyAPI_FUNC(int) _PyCode_CheckNoInternalState(PyCodeObject *, const char **); |
| 653 |
PyAPI_FUNC(int) _PyCode_CheckNoExternalState( |
| 654 |
PyCodeObject *, |
| 655 |
_PyCode_var_counts_t *, |
| 656 |
const char **); |
| 657 |
PyAPI_FUNC(int) _PyCode_VerifyStateless( |
| 658 |
PyThreadState *, |
| 659 |
PyCodeObject *, |
| 660 |
PyObject *globalnames, |
| 661 |
PyObject *globalsns, |
| 662 |
PyObject *builtinsns); |
| 663 |
|
| 664 |
PyAPI_FUNC(int) _PyCode_CheckPureFunction(PyCodeObject *, const char **); |
| 665 |
PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *); |
| 666 |
|
| 667 |
|
| 668 |
#ifdef __cplusplus |
| 669 |
} |
| 670 |
#endif |
| 671 |
#endif /* !Py_INTERNAL_CODE_H */ |