mirror/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-01-26 12:55:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

gh-142982: Specialize CALL_FUNCTION_EX (GH-143391)

Browse Source
This commit is contained in:
Ken Jin 2026-01-07 04:34:08 +08:00 committed by GitHub
parent ff7d1cec41
commit df355348f0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 2074 additions and 1188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Include/internal/pycore_code.h
View File

@ -160,6 +160,12 @@ typedef struct {
#define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache)
typedef struct {
_Py_BackoffCounter counter;
} _PyCallFunctionExCache;
#define INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX CACHE_ENTRIES(_PyCallFunctionExCache)
/* "Locals plus" for a code object is the set of locals + cell vars +
* free vars. This relates to variable names as well as offsets into
* the "fast locals" storage array of execution frames. The compiler
@ -326,6 +332,7 @@ PyAPI_FUNC(void) _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr);
PyAPI_FUNC(void) _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr);
PyAPI_FUNC(void) _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr);
PyAPI_FUNC(void) _Py_GatherStats_GetIter(_PyStackRef iterable);
PyAPI_FUNC(void) _Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr);
// Utility functions for reading/writing 32/64-bit values in the inline caches.
// Great care should be taken to ensure that these functions remain correct and

4
Include/internal/pycore_interpframe.h
View File

@ -395,6 +395,10 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func,
size_t argcount, PyObject *kwnames,
_PyInterpreterFrame *previous);
PyAPI_FUNC(_PyInterpreterFrame *)
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func,
PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous);
#ifdef __cplusplus
}
#endif

3
Include/internal/pycore_magic_number.h
View File

@ -289,6 +289,7 @@ Known values:
Python 3.15a2 3656 (Add TRACE_RECORD instruction, for platforms with switch based interpreter)
Python 3.15a4 3657 (Add BINARY_OP_SUBSCR_USTR_INT)
Python 3.15a4 3658 (Optimize bytecode for list/set called on genexp)
Python 3.15a4 3659 (Add CALL_FUNCTION_EX specialization)
Python 3.16 will start with 3700
@ -302,7 +303,7 @@ PC/launcher.c must also be updated.
*/
#define PYC_MAGIC_NUMBER 3658
#define PYC_MAGIC_NUMBER 3659
/* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
(little-endian) and then appending b'\r\n'. */
#define PYC_MAGIC_NUMBER_TOKEN \

25
Include/internal/pycore_opcode_metadata.h generated
View File

@ -108,6 +108,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) {
return 2 + oparg;
case CALL_BUILTIN_O:
return 2 + oparg;
case CALL_EX_NON_PY_GENERAL:
return 4;
case CALL_EX_PY:
return 4;
case CALL_FUNCTION_EX:
return 4;
case CALL_INTRINSIC_1:
@ -595,6 +599,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) {
return 1;
case CALL_BUILTIN_O:
return 1;
case CALL_EX_NON_PY_GENERAL:
return 1;
case CALL_EX_PY:
return 0;
case CALL_FUNCTION_EX:
return 1;
case CALL_INTRINSIC_1:
@ -1117,7 +1125,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = {
[CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
[CALL_EX_NON_PY_GENERAL] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
[CALL_EX_PY] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
[CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
[CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG },
@ -1180,7 +1190,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = {
[IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
[INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
[INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
[INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
[INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG },
[INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG },
@ -1366,6 +1376,8 @@ _PyOpcode_macro_expansion[256] = {
[CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } },
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } },
[CALL_BUILTIN_O] = { .nuops = 4, .uops = { { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } },
[CALL_EX_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CALL_FUNCTION_EX_NON_PY_GENERAL, OPARG_SIMPLE, 1 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 1 } } },
[CALL_EX_PY] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } },
[CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 } } },
[CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 } } },
[CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } },
@ -1561,6 +1573,8 @@ const char *_PyOpcode_OpName[267] = {
[CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST",
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS",
[CALL_BUILTIN_O] = "CALL_BUILTIN_O",
[CALL_EX_NON_PY_GENERAL] = "CALL_EX_NON_PY_GENERAL",
[CALL_EX_PY] = "CALL_EX_PY",
[CALL_FUNCTION_EX] = "CALL_FUNCTION_EX",
[CALL_INTRINSIC_1] = "CALL_INTRINSIC_1",
[CALL_INTRINSIC_2] = "CALL_INTRINSIC_2",
@ -1787,6 +1801,7 @@ const uint8_t _PyOpcode_Caches[256] = {
[FOR_ITER] = 1,
[CALL] = 3,
[CALL_KW] = 3,
[CALL_FUNCTION_EX] = 1,
[BINARY_OP] = 5,
};
#endif
@ -1801,8 +1816,6 @@ const uint8_t _PyOpcode_Deopt[256] = {
[125] = 125,
[126] = 126,
[127] = 127,
[211] = 211,
[212] = 212,
[213] = 213,
[214] = 214,
[215] = 215,
@ -1858,6 +1871,8 @@ const uint8_t _PyOpcode_Deopt[256] = {
[CALL_BUILTIN_FAST] = CALL,
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL,
[CALL_BUILTIN_O] = CALL,
[CALL_EX_NON_PY_GENERAL] = CALL_FUNCTION_EX,
[CALL_EX_PY] = CALL_FUNCTION_EX,
[CALL_FUNCTION_EX] = CALL_FUNCTION_EX,
[CALL_INTRINSIC_1] = CALL_INTRINSIC_1,
[CALL_INTRINSIC_2] = CALL_INTRINSIC_2,
@ -2062,8 +2077,6 @@ const uint8_t _PyOpcode_Deopt[256] = {
case 125: \
case 126: \
case 127: \
case 211: \
case 212: \
case 213: \
case 214: \
case 215: \

2082
Include/internal/pycore_uop_ids.h generated
View File

File diff suppressed because it is too large Load Diff

72
Include/internal/pycore_uop_metadata.h generated
View File

@ -302,6 +302,10 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG,
[_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG,
[_CHECK_IS_PY_CALLABLE_EX] = HAS_EXIT_FLAG,
[_PY_FRAME_EX] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG,
[_CHECK_IS_NOT_PY_CALLABLE_EX] = HAS_EXIT_FLAG,
[_CALL_FUNCTION_EX_NON_PY_GENERAL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
[_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG,
[_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG,
@ -2774,6 +2778,42 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = {
{ 3, 3, _MAKE_CALLARGS_A_TUPLE_r33 },
},
},
[_CHECK_IS_PY_CALLABLE_EX] = {
.best = { 0, 1, 2, 3 },
.entries = {
{ 3, 0, _CHECK_IS_PY_CALLABLE_EX_r03 },
{ 3, 1, _CHECK_IS_PY_CALLABLE_EX_r13 },
{ 3, 2, _CHECK_IS_PY_CALLABLE_EX_r23 },
{ 3, 3, _CHECK_IS_PY_CALLABLE_EX_r33 },
},
},
[_PY_FRAME_EX] = {
.best = { 3, 3, 3, 3 },
.entries = {
{ -1, -1, -1 },
{ -1, -1, -1 },
{ -1, -1, -1 },
{ 1, 1, _PY_FRAME_EX_r31 },
},
},
[_CHECK_IS_NOT_PY_CALLABLE_EX] = {
.best = { 0, 1, 2, 3 },
.entries = {
{ 3, 0, _CHECK_IS_NOT_PY_CALLABLE_EX_r03 },
{ 3, 1, _CHECK_IS_NOT_PY_CALLABLE_EX_r13 },
{ 3, 2, _CHECK_IS_NOT_PY_CALLABLE_EX_r23 },
{ 3, 3, _CHECK_IS_NOT_PY_CALLABLE_EX_r33 },
},
},
[_CALL_FUNCTION_EX_NON_PY_GENERAL] = {
.best = { 3, 3, 3, 3 },
.entries = {
{ -1, -1, -1 },
{ -1, -1, -1 },
{ -1, -1, -1 },
{ 1, 3, _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 },
},
},
[_MAKE_FUNCTION] = {
.best = { 1, 1, 1, 1 },
.entries = {
@ -3842,6 +3882,16 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = {
[_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = _CHECK_IS_NOT_PY_CALLABLE_KW,
[_CALL_KW_NON_PY_r11] = _CALL_KW_NON_PY,
[_MAKE_CALLARGS_A_TUPLE_r33] = _MAKE_CALLARGS_A_TUPLE,
[_CHECK_IS_PY_CALLABLE_EX_r03] = _CHECK_IS_PY_CALLABLE_EX,
[_CHECK_IS_PY_CALLABLE_EX_r13] = _CHECK_IS_PY_CALLABLE_EX,
[_CHECK_IS_PY_CALLABLE_EX_r23] = _CHECK_IS_PY_CALLABLE_EX,
[_CHECK_IS_PY_CALLABLE_EX_r33] = _CHECK_IS_PY_CALLABLE_EX,
[_PY_FRAME_EX_r31] = _PY_FRAME_EX,
[_CHECK_IS_NOT_PY_CALLABLE_EX_r03] = _CHECK_IS_NOT_PY_CALLABLE_EX,
[_CHECK_IS_NOT_PY_CALLABLE_EX_r13] = _CHECK_IS_NOT_PY_CALLABLE_EX,
[_CHECK_IS_NOT_PY_CALLABLE_EX_r23] = _CHECK_IS_NOT_PY_CALLABLE_EX,
[_CHECK_IS_NOT_PY_CALLABLE_EX_r33] = _CHECK_IS_NOT_PY_CALLABLE_EX,
[_CALL_FUNCTION_EX_NON_PY_GENERAL_r31] = _CALL_FUNCTION_EX_NON_PY_GENERAL,
[_MAKE_FUNCTION_r11] = _MAKE_FUNCTION,
[_SET_FUNCTION_ATTRIBUTE_r01] = _SET_FUNCTION_ATTRIBUTE,
[_SET_FUNCTION_ATTRIBUTE_r11] = _SET_FUNCTION_ATTRIBUTE,
@ -4094,6 +4144,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = {
[_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01",
[_CALL_BUILTIN_O] = "_CALL_BUILTIN_O",
[_CALL_BUILTIN_O_r03] = "_CALL_BUILTIN_O_r03",
[_CALL_FUNCTION_EX_NON_PY_GENERAL] = "_CALL_FUNCTION_EX_NON_PY_GENERAL",
[_CALL_FUNCTION_EX_NON_PY_GENERAL_r31] = "_CALL_FUNCTION_EX_NON_PY_GENERAL_r31",
[_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1",
[_CALL_INTRINSIC_1_r11] = "_CALL_INTRINSIC_1_r11",
[_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2",
@ -4159,8 +4211,18 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = {
[_CHECK_FUNCTION_VERSION_KW_r11] = "_CHECK_FUNCTION_VERSION_KW_r11",
[_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE",
[_CHECK_IS_NOT_PY_CALLABLE_r00] = "_CHECK_IS_NOT_PY_CALLABLE_r00",
[_CHECK_IS_NOT_PY_CALLABLE_EX] = "_CHECK_IS_NOT_PY_CALLABLE_EX",
[_CHECK_IS_NOT_PY_CALLABLE_EX_r03] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r03",
[_CHECK_IS_NOT_PY_CALLABLE_EX_r13] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r13",
[_CHECK_IS_NOT_PY_CALLABLE_EX_r23] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r23",
[_CHECK_IS_NOT_PY_CALLABLE_EX_r33] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r33",
[_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW",
[_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = "_CHECK_IS_NOT_PY_CALLABLE_KW_r11",
[_CHECK_IS_PY_CALLABLE_EX] = "_CHECK_IS_PY_CALLABLE_EX",
[_CHECK_IS_PY_CALLABLE_EX_r03] = "_CHECK_IS_PY_CALLABLE_EX_r03",
[_CHECK_IS_PY_CALLABLE_EX_r13] = "_CHECK_IS_PY_CALLABLE_EX_r13",
[_CHECK_IS_PY_CALLABLE_EX_r23] = "_CHECK_IS_PY_CALLABLE_EX_r23",
[_CHECK_IS_PY_CALLABLE_EX_r33] = "_CHECK_IS_PY_CALLABLE_EX_r33",
[_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES",
[_CHECK_MANAGED_OBJECT_HAS_VALUES_r01] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r01",
[_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r11",
@ -4869,6 +4931,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = {
[_PUSH_NULL_r23] = "_PUSH_NULL_r23",
[_PUSH_NULL_CONDITIONAL] = "_PUSH_NULL_CONDITIONAL",
[_PUSH_NULL_CONDITIONAL_r00] = "_PUSH_NULL_CONDITIONAL_r00",
[_PY_FRAME_EX] = "_PY_FRAME_EX",
[_PY_FRAME_EX_r31] = "_PY_FRAME_EX_r31",
[_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL",
[_PY_FRAME_GENERAL_r01] = "_PY_FRAME_GENERAL_r01",
[_PY_FRAME_KW] = "_PY_FRAME_KW",
@ -5597,6 +5661,14 @@ int _PyUop_num_popped(int opcode, int oparg)
return 3 + oparg;
case _MAKE_CALLARGS_A_TUPLE:
return 0;
case _CHECK_IS_PY_CALLABLE_EX:
return 0;
case _PY_FRAME_EX:
return 4;
case _CHECK_IS_NOT_PY_CALLABLE_EX:
return 0;
case _CALL_FUNCTION_EX_NON_PY_GENERAL:
return 4;
case _MAKE_FUNCTION:
return 1;
case _SET_FUNCTION_ATTRIBUTE:

122
Include/opcode_ids.h generated
View File

@ -154,66 +154,68 @@ extern "C" {
#define CALL_BUILTIN_FAST 148
#define CALL_BUILTIN_FAST_WITH_KEYWORDS 149
#define CALL_BUILTIN_O 150
#define CALL_ISINSTANCE 151
#define CALL_KW_BOUND_METHOD 152
#define CALL_KW_NON_PY 153
#define CALL_KW_PY 154
#define CALL_LEN 155
#define CALL_LIST_APPEND 156
#define CALL_METHOD_DESCRIPTOR_FAST 157
#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 158
#define CALL_METHOD_DESCRIPTOR_NOARGS 159
#define CALL_METHOD_DESCRIPTOR_O 160
#define CALL_NON_PY_GENERAL 161
#define CALL_PY_EXACT_ARGS 162
#define CALL_PY_GENERAL 163
#define CALL_STR_1 164
#define CALL_TUPLE_1 165
#define CALL_TYPE_1 166
#define COMPARE_OP_FLOAT 167
#define COMPARE_OP_INT 168
#define COMPARE_OP_STR 169
#define CONTAINS_OP_DICT 170
#define CONTAINS_OP_SET 171
#define FOR_ITER_GEN 172
#define FOR_ITER_LIST 173
#define FOR_ITER_RANGE 174
#define FOR_ITER_TUPLE 175
#define JUMP_BACKWARD_JIT 176
#define JUMP_BACKWARD_NO_JIT 177
#define LOAD_ATTR_CLASS 178
#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 179
#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 180
#define LOAD_ATTR_INSTANCE_VALUE 181
#define LOAD_ATTR_METHOD_LAZY_DICT 182
#define LOAD_ATTR_METHOD_NO_DICT 183
#define LOAD_ATTR_METHOD_WITH_VALUES 184
#define LOAD_ATTR_MODULE 185
#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 186
#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 187
#define LOAD_ATTR_PROPERTY 188
#define LOAD_ATTR_SLOT 189
#define LOAD_ATTR_WITH_HINT 190
#define LOAD_GLOBAL_BUILTIN 191
#define LOAD_GLOBAL_MODULE 192
#define LOAD_SUPER_ATTR_ATTR 193
#define LOAD_SUPER_ATTR_METHOD 194
#define RESUME_CHECK 195
#define SEND_GEN 196
#define STORE_ATTR_INSTANCE_VALUE 197
#define STORE_ATTR_SLOT 198
#define STORE_ATTR_WITH_HINT 199
#define STORE_SUBSCR_DICT 200
#define STORE_SUBSCR_LIST_INT 201
#define TO_BOOL_ALWAYS_TRUE 202
#define TO_BOOL_BOOL 203
#define TO_BOOL_INT 204
#define TO_BOOL_LIST 205
#define TO_BOOL_NONE 206
#define TO_BOOL_STR 207
#define UNPACK_SEQUENCE_LIST 208
#define UNPACK_SEQUENCE_TUPLE 209
#define UNPACK_SEQUENCE_TWO_TUPLE 210
#define CALL_EX_NON_PY_GENERAL 151
#define CALL_EX_PY 152
#define CALL_ISINSTANCE 153
#define CALL_KW_BOUND_METHOD 154
#define CALL_KW_NON_PY 155
#define CALL_KW_PY 156
#define CALL_LEN 157
#define CALL_LIST_APPEND 158
#define CALL_METHOD_DESCRIPTOR_FAST 159
#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 160
#define CALL_METHOD_DESCRIPTOR_NOARGS 161
#define CALL_METHOD_DESCRIPTOR_O 162
#define CALL_NON_PY_GENERAL 163
#define CALL_PY_EXACT_ARGS 164
#define CALL_PY_GENERAL 165
#define CALL_STR_1 166
#define CALL_TUPLE_1 167
#define CALL_TYPE_1 168
#define COMPARE_OP_FLOAT 169
#define COMPARE_OP_INT 170
#define COMPARE_OP_STR 171
#define CONTAINS_OP_DICT 172
#define CONTAINS_OP_SET 173
#define FOR_ITER_GEN 174
#define FOR_ITER_LIST 175
#define FOR_ITER_RANGE 176
#define FOR_ITER_TUPLE 177
#define JUMP_BACKWARD_JIT 178
#define JUMP_BACKWARD_NO_JIT 179
#define LOAD_ATTR_CLASS 180
#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 181
#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 182
#define LOAD_ATTR_INSTANCE_VALUE 183
#define LOAD_ATTR_METHOD_LAZY_DICT 184
#define LOAD_ATTR_METHOD_NO_DICT 185
#define LOAD_ATTR_METHOD_WITH_VALUES 186
#define LOAD_ATTR_MODULE 187
#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 188
#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 189
#define LOAD_ATTR_PROPERTY 190
#define LOAD_ATTR_SLOT 191
#define LOAD_ATTR_WITH_HINT 192
#define LOAD_GLOBAL_BUILTIN 193
#define LOAD_GLOBAL_MODULE 194
#define LOAD_SUPER_ATTR_ATTR 195
#define LOAD_SUPER_ATTR_METHOD 196
#define RESUME_CHECK 197
#define SEND_GEN 198
#define STORE_ATTR_INSTANCE_VALUE 199
#define STORE_ATTR_SLOT 200
#define STORE_ATTR_WITH_HINT 201
#define STORE_SUBSCR_DICT 202
#define STORE_SUBSCR_LIST_INT 203
#define TO_BOOL_ALWAYS_TRUE 204
#define TO_BOOL_BOOL 205
#define TO_BOOL_INT 206
#define TO_BOOL_LIST 207
#define TO_BOOL_NONE 208
#define TO_BOOL_STR 209
#define UNPACK_SEQUENCE_LIST 210
#define UNPACK_SEQUENCE_TUPLE 211
#define UNPACK_SEQUENCE_TWO_TUPLE 212
#define INSTRUMENTED_END_FOR 233
#define INSTRUMENTED_POP_ITER 234
#define INSTRUMENTED_END_SEND 235

126
Lib/_opcode_metadata.py generated
View File

@ -119,6 +119,10 @@ _specializations = {
"CALL_KW_PY",
"CALL_KW_NON_PY",
],
"CALL_FUNCTION_EX": [
"CALL_EX_PY",
"CALL_EX_NON_PY_GENERAL",
],
}
_specialized_opmap = {
@ -146,66 +150,68 @@ _specialized_opmap = {
'CALL_BUILTIN_FAST': 148,
'CALL_BUILTIN_FAST_WITH_KEYWORDS': 149,
'CALL_BUILTIN_O': 150,
'CALL_ISINSTANCE': 151,
'CALL_KW_BOUND_METHOD': 152,
'CALL_KW_NON_PY': 153,
'CALL_KW_PY': 154,
'CALL_LEN': 155,
'CALL_LIST_APPEND': 156,
'CALL_METHOD_DESCRIPTOR_FAST': 157,
'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 158,
'CALL_METHOD_DESCRIPTOR_NOARGS': 159,
'CALL_METHOD_DESCRIPTOR_O': 160,
'CALL_NON_PY_GENERAL': 161,
'CALL_PY_EXACT_ARGS': 162,
'CALL_PY_GENERAL': 163,
'CALL_STR_1': 164,
'CALL_TUPLE_1': 165,
'CALL_TYPE_1': 166,
'COMPARE_OP_FLOAT': 167,
'COMPARE_OP_INT': 168,
'COMPARE_OP_STR': 169,
'CONTAINS_OP_DICT': 170,
'CONTAINS_OP_SET': 171,
'FOR_ITER_GEN': 172,
'FOR_ITER_LIST': 173,
'FOR_ITER_RANGE': 174,
'FOR_ITER_TUPLE': 175,
'JUMP_BACKWARD_JIT': 176,
'JUMP_BACKWARD_NO_JIT': 177,
'LOAD_ATTR_CLASS': 178,
'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 179,
'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 180,
'LOAD_ATTR_INSTANCE_VALUE': 181,
'LOAD_ATTR_METHOD_LAZY_DICT': 182,
'LOAD_ATTR_METHOD_NO_DICT': 183,
'LOAD_ATTR_METHOD_WITH_VALUES': 184,
'LOAD_ATTR_MODULE': 185,
'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 186,
'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 187,
'LOAD_ATTR_PROPERTY': 188,
'LOAD_ATTR_SLOT': 189,
'LOAD_ATTR_WITH_HINT': 190,
'LOAD_GLOBAL_BUILTIN': 191,
'LOAD_GLOBAL_MODULE': 192,
'LOAD_SUPER_ATTR_ATTR': 193,
'LOAD_SUPER_ATTR_METHOD': 194,
'RESUME_CHECK': 195,
'SEND_GEN': 196,
'STORE_ATTR_INSTANCE_VALUE': 197,
'STORE_ATTR_SLOT': 198,
'STORE_ATTR_WITH_HINT': 199,
'STORE_SUBSCR_DICT': 200,
'STORE_SUBSCR_LIST_INT': 201,
'TO_BOOL_ALWAYS_TRUE': 202,
'TO_BOOL_BOOL': 203,
'TO_BOOL_INT': 204,
'TO_BOOL_LIST': 205,
'TO_BOOL_NONE': 206,
'TO_BOOL_STR': 207,
'UNPACK_SEQUENCE_LIST': 208,
'UNPACK_SEQUENCE_TUPLE': 209,
'UNPACK_SEQUENCE_TWO_TUPLE': 210,
'CALL_EX_NON_PY_GENERAL': 151,
'CALL_EX_PY': 152,
'CALL_ISINSTANCE': 153,
'CALL_KW_BOUND_METHOD': 154,
'CALL_KW_NON_PY': 155,
'CALL_KW_PY': 156,
'CALL_LEN': 157,
'CALL_LIST_APPEND': 158,
'CALL_METHOD_DESCRIPTOR_FAST': 159,
'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 160,
'CALL_METHOD_DESCRIPTOR_NOARGS': 161,
'CALL_METHOD_DESCRIPTOR_O': 162,
'CALL_NON_PY_GENERAL': 163,
'CALL_PY_EXACT_ARGS': 164,
'CALL_PY_GENERAL': 165,
'CALL_STR_1': 166,
'CALL_TUPLE_1': 167,
'CALL_TYPE_1': 168,
'COMPARE_OP_FLOAT': 169,
'COMPARE_OP_INT': 170,
'COMPARE_OP_STR': 171,
'CONTAINS_OP_DICT': 172,
'CONTAINS_OP_SET': 173,
'FOR_ITER_GEN': 174,
'FOR_ITER_LIST': 175,
'FOR_ITER_RANGE': 176,
'FOR_ITER_TUPLE': 177,
'JUMP_BACKWARD_JIT': 178,
'JUMP_BACKWARD_NO_JIT': 179,
'LOAD_ATTR_CLASS': 180,
'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 181,
'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 182,
'LOAD_ATTR_INSTANCE_VALUE': 183,
'LOAD_ATTR_METHOD_LAZY_DICT': 184,
'LOAD_ATTR_METHOD_NO_DICT': 185,
'LOAD_ATTR_METHOD_WITH_VALUES': 186,
'LOAD_ATTR_MODULE': 187,
'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 188,
'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 189,
'LOAD_ATTR_PROPERTY': 190,
'LOAD_ATTR_SLOT': 191,
'LOAD_ATTR_WITH_HINT': 192,
'LOAD_GLOBAL_BUILTIN': 193,
'LOAD_GLOBAL_MODULE': 194,
'LOAD_SUPER_ATTR_ATTR': 195,
'LOAD_SUPER_ATTR_METHOD': 196,
'RESUME_CHECK': 197,
'SEND_GEN': 198,
'STORE_ATTR_INSTANCE_VALUE': 199,
'STORE_ATTR_SLOT': 200,
'STORE_ATTR_WITH_HINT': 201,
'STORE_SUBSCR_DICT': 202,
'STORE_SUBSCR_LIST_INT': 203,
'TO_BOOL_ALWAYS_TRUE': 204,
'TO_BOOL_BOOL': 205,
'TO_BOOL_INT': 206,
'TO_BOOL_LIST': 207,
'TO_BOOL_NONE': 208,
'TO_BOOL_STR': 209,
'UNPACK_SEQUENCE_LIST': 210,
'UNPACK_SEQUENCE_TUPLE': 211,
'UNPACK_SEQUENCE_TWO_TUPLE': 212,
}
opmap = {

3
Lib/opcode.py
View File

@ -91,6 +91,9 @@ _cache_format = {
"counter": 1,
"func_version": 2,
},
"CALL_FUNCTION_EX": {
"counter": 1,
},
"STORE_SUBSCR": {
"counter": 1,
},

18
Lib/test/test_capi/test_opt.py
View File

@ -400,6 +400,24 @@ class TestUops(unittest.TestCase):
self.assertIn("_PUSH_FRAME", uops)
self.assertIn("_BINARY_OP_ADD_INT", uops)
def test_call_py_ex(self):
def testfunc(n):
def ex_py(*args, **kwargs):
return 1
for _ in range(n):
args = (1, 2, 3)
kwargs = {}
ex_py(*args, **kwargs)
testfunc(TIER2_THRESHOLD)
ex = get_first_executor(testfunc)
self.assertIsNotNone(ex)
uops = get_opnames(ex)
self.assertIn("_PUSH_FRAME", uops)
self.assertIn("_PY_FRAME_EX", uops)
def test_branch_taken(self):
def testfunc(n):
for i in range(n):

33
Lib/test/test_opcache.py
View File

@ -575,6 +575,39 @@ class TestCallCache(TestBase):
for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
f()
@requires_jit_disabled
@requires_specialization_ft
def test_specialize_call_function_ex_py(self):
def ex_py(*args, **kwargs):
return 1
def instantiate():
args = (1, 2, 3)
kwargs = {}
return ex_py(*args, **kwargs)
# Trigger specialization
for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
instantiate()
self.assert_specialized(instantiate, "CALL_EX_PY")
@requires_jit_disabled
@requires_specialization_ft
def test_specialize_call_function_ex_py_fail(self):
def ex_py(*args, **kwargs):
return 1
def instantiate():
args = (1, 2, 3)
kwargs = {}
return ex_py(*args, **kwargs)
_testinternalcapi.set_vectorcall_nop(ex_py)
# Trigger specialization
for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
instantiate()
self.assert_no_opcode(instantiate, "CALL_EX_PY")
self.assert_specialized(instantiate, "CALL_EX_NON_PY_GENERAL")
def make_deferred_ref_count_obj():
"""Create an object that uses deferred reference counting.

1
Misc/NEWS.d/next/Core_and_Builtins/2026-01-03-14-47-49.gh-issue-142982.0lAtc7.rst Normal file
View File

@ -0,0 +1 @@
Specialize :opcode:`CALL_FUNCTION_EX` for Python and non-Python callables.

89
Python/bytecodes.c
View File

@ -4794,6 +4794,11 @@ dummy_func(
_CALL_KW_NON_PY +
_CHECK_PERIODIC_AT_END;
family(CALL_FUNCTION_EX, INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX) = {
CALL_EX_PY,
CALL_EX_NON_PY_GENERAL,
};
op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs -- func, unused, callargs, kwargs)) {
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (!PyTuple_CheckExact(callargs_o)) {
@ -4872,8 +4877,8 @@ dummy_func(
if (new_frame == NULL) {
ERROR_NO_POP();
}
assert(INSTRUCTION_SIZE == 1);
frame->return_offset = 1;
assert(INSTRUCTION_SIZE == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX);
frame->return_offset = INSTRUCTION_SIZE;
DISPATCH_INLINED(new_frame);
}
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
@ -4890,12 +4895,92 @@ dummy_func(
result = PyStackRef_FromPyObjectSteal(result_o);
}
specializing op(_SPECIALIZE_CALL_FUNCTION_EX, (counter/1, func, unused, unused, unused -- func, unused, unused, unused)) {
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_Py_Specialize_CallFunctionEx(func, next_instr);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(CALL_FUNCTION_EX);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION_FT */
}
macro(CALL_FUNCTION_EX) =
_SPECIALIZE_CALL_FUNCTION_EX +
_MAKE_CALLARGS_A_TUPLE +
_DO_CALL_FUNCTION_EX +
_CHECK_PERIODIC_AT_END;
op(_CHECK_IS_PY_CALLABLE_EX, (func_st, unused, unused, unused -- func_st, unused, unused, unused)) {
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
EXIT_IF(Py_TYPE(func) != &PyFunction_Type);
EXIT_IF(((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall);
}
op(_PY_FRAME_EX, (func_st, null, callargs_st, kwargs_st -- ex_frame)) {
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st);
assert(PyTuple_CheckExact(callargs));
assert(Py_TYPE(func) == &PyFunction_Type);
assert(((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall);
PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(
tstate, func_st, locals,
nargs, callargs, kwargs, frame);
INPUTS_DEAD();
SYNC_SP();
if (new_frame == NULL) {
ERROR_NO_POP();
}
ex_frame = PyStackRef_Wrap(new_frame);
}
macro(CALL_EX_PY) =
unused/1 +
_CHECK_PEP_523 +
_MAKE_CALLARGS_A_TUPLE +
_CHECK_IS_PY_CALLABLE_EX +
_PY_FRAME_EX +
_SAVE_RETURN_OFFSET +
_PUSH_FRAME;
op(_CHECK_IS_NOT_PY_CALLABLE_EX, (func_st, unused, unused, unused -- func_st, unused, unused, unused)) {
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
EXIT_IF(Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall);
}
op(_CALL_FUNCTION_EX_NON_PY_GENERAL, (func_st, null, callargs_st, kwargs_st -- result)) {
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
(void)null;
assert(PyTuple_CheckExact(callargs));
PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
PyObject *result_o = PyObject_Call(func, callargs, kwargs);
PyStackRef_XCLOSE(kwargs_st);
PyStackRef_CLOSE(callargs_st);
DEAD(null);
PyStackRef_CLOSE(func_st);
ERROR_IF(result_o == NULL);
result = PyStackRef_FromPyObjectSteal(result_o);
}
macro(CALL_EX_NON_PY_GENERAL) =
unused/1 +
_CHECK_IS_NOT_PY_CALLABLE_EX +
_MAKE_CALLARGS_A_TUPLE +
_CALL_FUNCTION_EX_NON_PY_GENERAL +
_CHECK_PERIODIC_AT_END;
macro(INSTRUMENTED_CALL_FUNCTION_EX) =
unused/1 +
_MAKE_CALLARGS_A_TUPLE +
_DO_CALL_FUNCTION_EX +
_CHECK_PERIODIC_AT_END;

5
Python/ceval.c
View File

@ -329,9 +329,6 @@ static void monitor_throw(PyThreadState *tstate,
_Py_CODEUNIT *instr);
static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);
static _PyInterpreterFrame *
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func,
PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous);
#ifdef HAVE_ERRNO_H
#include <errno.h>
@ -2435,7 +2432,7 @@ fail:
/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict.
Steals references to func, callargs and kwargs.
*/
static _PyInterpreterFrame *
_PyInterpreterFrame *
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func,
PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous)
{

317
Python/executor_cases.c.h generated
View File

@ -15694,6 +15694,323 @@
/* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _CHECK_IS_PY_CALLABLE_EX_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
func_st = stack_pointer[-4];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) != &PyFunction_Type) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = stack_pointer[-1];
_tos_cache1 = stack_pointer[-2];
_tos_cache0 = stack_pointer[-3];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_PY_CALLABLE_EX_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
func_st = stack_pointer[-3];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) != &PyFunction_Type) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_0;
_tos_cache1 = stack_pointer[-1];
_tos_cache0 = stack_pointer[-2];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_PY_CALLABLE_EX_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
func_st = stack_pointer[-2];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) != &PyFunction_Type) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_1;
_tos_cache1 = _stack_item_0;
_tos_cache0 = stack_pointer[-1];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_PY_CALLABLE_EX_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
func_st = stack_pointer[-1];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) != &PyFunction_Type) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _PY_FRAME_EX_r31: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef kwargs_st;
_PyStackRef callargs_st;
_PyStackRef func_st;
_PyStackRef ex_frame;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
kwargs_st = _stack_item_2;
callargs_st = _stack_item_1;
func_st = stack_pointer[-1];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st);
assert(PyTuple_CheckExact(callargs));
assert(Py_TYPE(func) == &PyFunction_Type);
assert(((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall);
PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
stack_pointer[0] = _stack_item_0;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(
tstate, func_st, locals,
nargs, callargs, kwargs, frame);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
if (new_frame == NULL) {
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
ex_frame = PyStackRef_Wrap(new_frame);
_tos_cache0 = ex_frame;
_tos_cache1 = PyStackRef_ZERO_BITS;
_tos_cache2 = PyStackRef_ZERO_BITS;
SET_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_NOT_PY_CALLABLE_EX_r03: {
CHECK_CURRENT_CACHED_VALUES(0);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
func_st = stack_pointer[-4];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = stack_pointer[-1];
_tos_cache1 = stack_pointer[-2];
_tos_cache0 = stack_pointer[-3];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_NOT_PY_CALLABLE_EX_r13: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
func_st = stack_pointer[-3];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(1);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_0;
_tos_cache1 = stack_pointer[-1];
_tos_cache0 = stack_pointer[-2];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_NOT_PY_CALLABLE_EX_r23: {
CHECK_CURRENT_CACHED_VALUES(2);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
func_st = stack_pointer[-2];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(2);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_1;
_tos_cache1 = _stack_item_0;
_tos_cache0 = stack_pointer[-1];
SET_CURRENT_CACHED_VALUES(3);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CHECK_IS_NOT_PY_CALLABLE_EX_r33: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef func_st;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
func_st = stack_pointer[-1];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
UOP_STAT_INC(uopcode, miss);
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
JUMP_TO_JUMP_TARGET();
}
_tos_cache2 = _stack_item_2;
_tos_cache1 = _stack_item_1;
_tos_cache0 = _stack_item_0;
SET_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _CALL_FUNCTION_EX_NON_PY_GENERAL_r31: {
CHECK_CURRENT_CACHED_VALUES(3);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
_PyStackRef kwargs_st;
_PyStackRef callargs_st;
_PyStackRef null;
_PyStackRef func_st;
_PyStackRef result;
_PyStackRef _stack_item_0 = _tos_cache0;
_PyStackRef _stack_item_1 = _tos_cache1;
_PyStackRef _stack_item_2 = _tos_cache2;
kwargs_st = _stack_item_2;
callargs_st = _stack_item_1;
null = _stack_item_0;
func_st = stack_pointer[-1];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
(void)null;
assert(PyTuple_CheckExact(callargs));
PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
stack_pointer[0] = null;
stack_pointer[1] = callargs_st;
stack_pointer[2] = kwargs_st;
stack_pointer += 3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *result_o = PyObject_Call(func, callargs, kwargs);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_XCLOSE(kwargs_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(callargs_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(func_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (result_o == NULL) {
SET_CURRENT_CACHED_VALUES(0);
JUMP_TO_ERROR();
}
result = PyStackRef_FromPyObjectSteal(result_o);
_tos_cache0 = result;
_tos_cache1 = PyStackRef_ZERO_BITS;
_tos_cache2 = PyStackRef_ZERO_BITS;
SET_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());
break;
}
case _MAKE_FUNCTION_r11: {
CHECK_CURRENT_CACHED_VALUES(1);
assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());

264
Python/generated_cases.c.h generated
View File

@ -2499,16 +2499,245 @@
DISPATCH();
}
TARGET(CALL_FUNCTION_EX) {
TARGET(CALL_EX_NON_PY_GENERAL) {
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_FUNCTION_EX;
int opcode = CALL_EX_NON_PY_GENERAL;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 1;
next_instr += 2;
INSTRUCTION_STATS(CALL_EX_NON_PY_GENERAL);
static_assert(INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX == 1, "incorrect cache size");
_PyStackRef func_st;
_PyStackRef func;
_PyStackRef callargs;
_PyStackRef null;
_PyStackRef callargs_st;
_PyStackRef kwargs_st;
_PyStackRef result;
/* Skip 1 cache entry */
// _CHECK_IS_NOT_PY_CALLABLE_EX
{
func_st = stack_pointer[-4];
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
UPDATE_MISS_STATS(CALL_FUNCTION_EX);
assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX));
JUMP_TO_PREDICTED(CALL_FUNCTION_EX);
}
}
// _MAKE_CALLARGS_A_TUPLE
{
callargs = stack_pointer[-2];
func = func_st;
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (!PyTuple_CheckExact(callargs_o)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *tuple_o = PySequence_Tuple(callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (tuple_o == NULL) {
JUMP_TO_LABEL(error);
}
_PyStackRef temp = callargs;
callargs = PyStackRef_FromPyObjectSteal(tuple_o);
stack_pointer[-2] = callargs;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(temp);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
}
// _CALL_FUNCTION_EX_NON_PY_GENERAL
{
kwargs_st = stack_pointer[-1];
callargs_st = callargs;
null = stack_pointer[-3];
func_st = func;
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
(void)null;
assert(PyTuple_CheckExact(callargs));
PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
stack_pointer[-2] = callargs_st;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *result_o = PyObject_Call(func, callargs, kwargs);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_XCLOSE(kwargs_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(callargs_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -2;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(func_st);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (result_o == NULL) {
JUMP_TO_LABEL(error);
}
result = PyStackRef_FromPyObjectSteal(result_o);
}
// _CHECK_PERIODIC_AT_END
{
stack_pointer[0] = result;
stack_pointer += 1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = check_periodics(tstate);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err != 0) {
JUMP_TO_LABEL(error);
}
}
DISPATCH();
}
TARGET(CALL_EX_PY) {
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_EX_PY;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CALL_EX_PY);
static_assert(INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX == 1, "incorrect cache size");
_PyStackRef func;
_PyStackRef callargs;
_PyStackRef func_st;
_PyStackRef callargs_st;
_PyStackRef kwargs_st;
_PyStackRef ex_frame;
_PyStackRef new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
if (tstate->interp->eval_frame) {
UPDATE_MISS_STATS(CALL_FUNCTION_EX);
assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX));
JUMP_TO_PREDICTED(CALL_FUNCTION_EX);
}
}
// _MAKE_CALLARGS_A_TUPLE
{
callargs = stack_pointer[-2];
func = stack_pointer[-4];
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (!PyTuple_CheckExact(callargs_o)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (err < 0) {
JUMP_TO_LABEL(error);
}
_PyFrame_SetStackPointer(frame, stack_pointer);
PyObject *tuple_o = PySequence_Tuple(callargs_o);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (tuple_o == NULL) {
JUMP_TO_LABEL(error);
}
_PyStackRef temp = callargs;
callargs = PyStackRef_FromPyObjectSteal(tuple_o);
stack_pointer[-2] = callargs;
_PyFrame_SetStackPointer(frame, stack_pointer);
PyStackRef_CLOSE(temp);
stack_pointer = _PyFrame_GetStackPointer(frame);
}
}
// _CHECK_IS_PY_CALLABLE_EX
{
func_st = func;
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
if (Py_TYPE(func) != &PyFunction_Type) {
UPDATE_MISS_STATS(CALL_FUNCTION_EX);
assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX));
JUMP_TO_PREDICTED(CALL_FUNCTION_EX);
}
if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) {
UPDATE_MISS_STATS(CALL_FUNCTION_EX);
assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX));
JUMP_TO_PREDICTED(CALL_FUNCTION_EX);
}
}
// _PY_FRAME_EX
{
kwargs_st = stack_pointer[-1];
callargs_st = callargs;
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st);
assert(PyTuple_CheckExact(callargs));
assert(Py_TYPE(func) == &PyFunction_Type);
assert(((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall);
PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st);
assert(kwargs == NULL || PyDict_CheckExact(kwargs));
Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags;
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
_PyFrame_SetStackPointer(frame, stack_pointer);
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(
tstate, func_st, locals,
nargs, callargs, kwargs, frame);
stack_pointer = _PyFrame_GetStackPointer(frame);
stack_pointer += -1;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
if (new_frame == NULL) {
JUMP_TO_LABEL(error);
}
ex_frame = PyStackRef_Wrap(new_frame);
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
new_frame = ex_frame;
assert(tstate->interp->eval_frame == NULL);
_PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame);
_PyFrame_SetStackPointer(frame, stack_pointer);
assert(temp->previous == frame || temp->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = temp;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(CALL_FUNCTION_EX) {
#if _Py_TAIL_CALL_INTERP
int opcode = CALL_FUNCTION_EX;
(void)(opcode);
#endif
frame->instr_ptr = next_instr;
next_instr += 2;
INSTRUCTION_STATS(CALL_FUNCTION_EX);
PREDICTED_CALL_FUNCTION_EX:;
_Py_CODEUNIT* const this_instr = next_instr - 2;
(void)this_instr;
opcode = CALL_FUNCTION_EX;
_PyStackRef func;
_PyStackRef callargs;
@ -2517,10 +2746,26 @@
_PyStackRef callargs_st;
_PyStackRef kwargs_st;
_PyStackRef result;
// _SPECIALIZE_CALL_FUNCTION_EX
{
func = stack_pointer[-4];
uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter;
#if ENABLE_SPECIALIZATION_FT
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr;
_PyFrame_SetStackPointer(frame, stack_pointer);
_Py_Specialize_CallFunctionEx(func, next_instr);
stack_pointer = _PyFrame_GetStackPointer(frame);
DISPATCH_SAME_OPARG();
}
OPCODE_DEFERRED_INC(CALL_FUNCTION_EX);
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION_FT */
}
// _MAKE_CALLARGS_A_TUPLE
{
callargs = stack_pointer[-2];
func = stack_pointer[-4];
PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs);
if (!PyTuple_CheckExact(callargs_o)) {
_PyFrame_SetStackPointer(frame, stack_pointer);
@ -2618,8 +2863,8 @@
if (new_frame == NULL) {
JUMP_TO_LABEL(error);
}
assert( 1u == 1);
frame->return_offset = 1;
assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX);
frame->return_offset = 2u ;
DISPATCH_INLINED(new_frame);
}
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);
@ -6247,7 +6492,7 @@
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 1;
next_instr += 2;
INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX);
opcode = INSTRUMENTED_CALL_FUNCTION_EX;
_PyStackRef func;
@ -6257,6 +6502,7 @@
_PyStackRef callargs_st;
_PyStackRef kwargs_st;
_PyStackRef result;
/* Skip 1 cache entry */
// _MAKE_CALLARGS_A_TUPLE
{
callargs = stack_pointer[-2];
@ -6358,8 +6604,8 @@
if (new_frame == NULL) {
JUMP_TO_LABEL(error);
}
assert( 1u == 1);
frame->return_offset = 1;
assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX);
frame->return_offset = 2u ;
DISPATCH_INLINED(new_frame);
}
PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st);

18
Python/opcode_targets.h generated
View File

@ -151,6 +151,8 @@ static void *opcode_targets_table[256] = {
&&TARGET_CALL_BUILTIN_FAST,
&&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS,
&&TARGET_CALL_BUILTIN_O,
&&TARGET_CALL_EX_NON_PY_GENERAL,
&&TARGET_CALL_EX_PY,
&&TARGET_CALL_ISINSTANCE,
&&TARGET_CALL_KW_BOUND_METHOD,
&&TARGET_CALL_KW_NON_PY,
@ -231,8 +233,6 @@ static void *opcode_targets_table[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_INSTRUMENTED_END_FOR,
&&TARGET_INSTRUMENTED_POP_ITER,
&&TARGET_INSTRUMENTED_END_SEND,
@ -470,8 +470,8 @@ static void *opcode_tracing_targets_table[256] = {
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&_unknown_opcode,
&&_unknown_opcode,
&&TARGET_TRACE_RECORD,
&&TARGET_TRACE_RECORD,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
@ -565,6 +565,8 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PAR
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS);
static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS);
@ -804,6 +806,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {
[CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST,
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS,
[CALL_BUILTIN_O] = _TAIL_CALL_CALL_BUILTIN_O,
[CALL_EX_NON_PY_GENERAL] = _TAIL_CALL_CALL_EX_NON_PY_GENERAL,
[CALL_EX_PY] = _TAIL_CALL_CALL_EX_PY,
[CALL_FUNCTION_EX] = _TAIL_CALL_CALL_FUNCTION_EX,
[CALL_INTRINSIC_1] = _TAIL_CALL_CALL_INTRINSIC_1,
[CALL_INTRINSIC_2] = _TAIL_CALL_CALL_INTRINSIC_2,
@ -1003,8 +1007,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[126] = _TAIL_CALL_UNKNOWN_OPCODE,
[127] = _TAIL_CALL_UNKNOWN_OPCODE,
[211] = _TAIL_CALL_UNKNOWN_OPCODE,
[212] = _TAIL_CALL_UNKNOWN_OPCODE,
[213] = _TAIL_CALL_UNKNOWN_OPCODE,
[214] = _TAIL_CALL_UNKNOWN_OPCODE,
[215] = _TAIL_CALL_UNKNOWN_OPCODE,
@ -1062,6 +1064,8 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {
[CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD,
[CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD,
[CALL_BUILTIN_O] = _TAIL_CALL_TRACE_RECORD,
[CALL_EX_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD,
[CALL_EX_PY] = _TAIL_CALL_TRACE_RECORD,
[CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD,
[CALL_INTRINSIC_1] = _TAIL_CALL_TRACE_RECORD,
[CALL_INTRINSIC_2] = _TAIL_CALL_TRACE_RECORD,
@ -1261,8 +1265,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {
[125] = _TAIL_CALL_UNKNOWN_OPCODE,
[126] = _TAIL_CALL_UNKNOWN_OPCODE,
[127] = _TAIL_CALL_UNKNOWN_OPCODE,
[211] = _TAIL_CALL_UNKNOWN_OPCODE,
[212] = _TAIL_CALL_UNKNOWN_OPCODE,
[213] = _TAIL_CALL_UNKNOWN_OPCODE,
[214] = _TAIL_CALL_UNKNOWN_OPCODE,
[215] = _TAIL_CALL_UNKNOWN_OPCODE,

14
Python/optimizer_bytecodes.c
View File

@ -837,6 +837,17 @@ dummy_func(void) {
new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0));
}
op(_PY_FRAME_EX, (func_st, null, callargs_st, kwargs_st -- ex_frame)) {
assert((this_instr + 2)->opcode == _PUSH_FRAME);
PyCodeObject *co = get_code_with_logging((this_instr + 2));
if (co == NULL) {
ctx->done = true;
break;
}
ex_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0));
}
op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
(void)type_version;
(void)args;
@ -1000,8 +1011,7 @@ dummy_func(void) {
ctx->frame->func = func;
}
// Fixed calls don't need IP guards.
if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET ||
(this_instr-1)->opcode == _CREATE_INIT_FRAME) {
if ((this_instr-1)->opcode == _CREATE_INIT_FRAME) {
assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME);
REPLACE_OP(this_instr+1, _NOP, 0, 0);
}

37
Python/optimizer_cases.c.h generated
View File

@ -2640,8 +2640,7 @@
PyFunctionObject *func = (PyFunctionObject *)operand;
ctx->frame->func = func;
}
if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET ||
(this_instr-1)->opcode == _CREATE_INIT_FRAME) {
if ((this_instr-1)->opcode == _CREATE_INIT_FRAME) {
assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME);
REPLACE_OP(this_instr+1, _NOP, 0, 0);
}
@ -3112,6 +3111,40 @@
/* _DO_CALL_FUNCTION_EX is not a viable micro-op for tier 2 */
case _CHECK_IS_PY_CALLABLE_EX: {
break;
}
case _PY_FRAME_EX: {
JitOptRef ex_frame;
assert((this_instr + 2)->opcode == _PUSH_FRAME);
PyCodeObject *co = get_code_with_logging((this_instr + 2));
if (co == NULL) {
ctx->done = true;
break;
}
ex_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0));
CHECK_STACK_BOUNDS(-3);
stack_pointer[-4] = ex_frame;
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _CHECK_IS_NOT_PY_CALLABLE_EX: {
break;
}
case _CALL_FUNCTION_EX_NON_PY_GENERAL: {
JitOptRef result;
result = sym_new_not_null(ctx);
CHECK_STACK_BOUNDS(-3);
stack_pointer[-4] = result;
stack_pointer += -3;
ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);
break;
}
case _MAKE_FUNCTION: {
JitOptRef func;
func = sym_new_not_null(ctx);

22
Python/specialize.c
View File

@ -2587,6 +2587,28 @@ failure:
unspecialize(instr);
}
Py_NO_INLINE void
_Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr)
{
PyObject *func = PyStackRef_AsPyObjectBorrow(func_st);
assert(ENABLE_SPECIALIZATION_FT);
assert(_PyOpcode_Caches[CALL_FUNCTION_EX] == INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX);
if (Py_TYPE(func) == &PyFunction_Type &&
((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) {
if (_PyInterpreterState_GET()->eval_frame) {
goto failure;
}
specialize(instr, CALL_EX_PY);
return;
}
specialize(instr, CALL_EX_NON_PY_GENERAL);
return;
failure:
unspecialize(instr);
}
#ifdef Py_STATS
static int
to_bool_fail_kind(PyObject *value)