From 839fcddd81f33502cf017048afffad1b17bd6daf Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 18 Nov 2026 21:14:56 +0100 Subject: [PATCH 1/3] gh-161170: Encode an awaited flag in the oparg of CALL, CALL_KW and CALL_FUNCTION_EX The oparg of CALL and CALL_KW is now the number of arguments shifted left by one, and the low bit of all three calls is reserved for marking a call whose result is awaited right away. The compiler doesn't set it yet, so there is no behavior change. _INIT_CALL_PY_EXACT_ARGS is replicated for the opargs 0..9 so that calls with up to 4 arguments, awaited or not, still get a constant argument count in tier 2. Co-Authored-By: Claude Opus 5.5 --- Doc/library/dis.rst | 22 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 42 +- Include/internal/pycore_uop_ids.h | 14 +- Include/internal/pycore_uop_metadata.h | 107 ++++- Lib/dis.py | 11 + Lib/test/test_compiler_codegen.py | 6 +- Lib/test/test_dis.py | 76 ++-- Modules/_testinternalcapi/test_cases.c.h | 374 ++++++++-------- Python/bytecodes.c | 162 +++---- Python/codegen.c | 32 +- Python/executor_cases.c.h | 507 ++++++++++++++-------- Python/generated_cases.c.h | 374 ++++++++-------- Python/optimizer_bytecodes.c | 106 ++--- Python/optimizer_cases.c.h | 268 ++++++------ Python/record_functions.c.h | 6 +- Python/specialize.c | 8 +- 17 files changed, 1189 insertions(+), 929 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 16c2dab5b8adba1..96b4003ef93d34e 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1615,7 +1615,8 @@ iterations of the loop. * ``self`` or ``NULL`` * The remaining positional arguments - ``argc`` is the total of the positional arguments, excluding ``self``. + ``argc >> 1`` is the total of the positional arguments, excluding ``self``. + The low bit of ``argc`` is set when the result is awaited right away. ``CALL`` pops all arguments and the callable object off the stack, calls the callable object with those arguments, and pushes the return value @@ -1629,6 +1630,10 @@ iterations of the loop. .. versionchanged:: 3.13 Calls with keyword arguments are now handled by :opcode:`CALL_KW`. + .. versionchanged:: next + The number of arguments is now ``argc >> 1``; the low bit marks an + awaited call. + .. opcode:: CALL_KW (argc) @@ -1641,7 +1646,8 @@ iterations of the loop. * The named arguments * A :class:`tuple` of keyword argument names - ``argc`` is the total of the positional and named arguments, excluding ``self``. + ``argc >> 1`` is the total of the positional and named arguments, excluding + ``self``. The low bit of ``argc`` is set when the result is awaited right away. The length of the tuple of keyword argument names is the number of named arguments. ``CALL_KW`` pops all arguments, the keyword names, and the callable object @@ -1650,21 +1656,29 @@ iterations of the loop. .. versionadded:: 3.13 + .. versionchanged:: next + The number of arguments is now ``argc >> 1``; the low bit marks an + awaited call. + .. opcode:: CALL_FUNCTION_EX (flags) Calls a callable object with variable set of positional and keyword - arguments. If the lowest bit of *flags* is set, the top of the stack - contains a mapping object containing additional keyword arguments. + arguments. The top of the stack contains a mapping object with additional + keyword arguments, or ``NULL``. Before the callable is called, the mapping object and iterable object are each "unpacked" and their contents passed in as keyword and positional arguments respectively. ``CALL_FUNCTION_EX`` pops all arguments and the callable object off the stack, calls the callable object with those arguments, and pushes the return value returned by the callable object. + The lowest bit of *flags* is set when the result is awaited right away. .. versionadded:: 3.6 + .. versionchanged:: next + The lowest bit of *flags* marks an awaited call. + .. opcode:: PUSH_NULL diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 15d3ad6f3a336b7..ddc005a9b0f2f28 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -304,6 +304,7 @@ Known values: Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) Python 3.16a1 3706 (Add INTRINSIC_MATCH_CLASS_ISINSTANCE) + Python 3.16a1 3707 (Lazily materialize coroutine objects for awaited calls) Python 3.17 will start with 3750 @@ -313,7 +314,7 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3706 +#define PYC_MAGIC_NUMBER 3707 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 04b826ed09144e7..9f96aa4ba98f37a 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -93,21 +93,21 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CACHE: return 0; case CALL: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_ALLOC_AND_ENTER_INIT: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BOUND_METHOD_EXACT_ARGS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BOUND_METHOD_GENERAL: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BUILTIN_CLASS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BUILTIN_FAST: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BUILTIN_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BUILTIN_O: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_EX_NON_PY_GENERAL: return 4; case CALL_EX_PY: @@ -121,31 +121,31 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CALL_ISINSTANCE: return 4; case CALL_KW: - return 3 + oparg; + return 3 + (oparg >> 1); case CALL_KW_BOUND_METHOD: - return 3 + oparg; + return 3 + (oparg >> 1); case CALL_KW_NON_PY: - return 3 + oparg; + return 3 + (oparg >> 1); case CALL_KW_PY: - return 3 + oparg; + return 3 + (oparg >> 1); case CALL_LEN: return 3; case CALL_LIST_APPEND: return 3; case CALL_METHOD_DESCRIPTOR_FAST: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_METHOD_DESCRIPTOR_NOARGS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_METHOD_DESCRIPTOR_O: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_NON_PY_GENERAL: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_PY_EXACT_ARGS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_PY_GENERAL: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_STR_1: return 3; case CALL_TUPLE_1: @@ -235,11 +235,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case IMPORT_NAME: return 2; case INSTRUMENTED_CALL: - return 2 + oparg; + return 2 + (oparg >> 1); case INSTRUMENTED_CALL_FUNCTION_EX: return 4; case INSTRUMENTED_CALL_KW: - return 3 + oparg; + return 3 + (oparg >> 1); case INSTRUMENTED_END_ASYNC_FOR: return 2; case INSTRUMENTED_END_FOR: diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ba6e0252024a6df..3f582b7dcb82453 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -219,6 +219,11 @@ enum { _INIT_CALL_PY_EXACT_ARGS_2, _INIT_CALL_PY_EXACT_ARGS_3, _INIT_CALL_PY_EXACT_ARGS_4, + _INIT_CALL_PY_EXACT_ARGS_5, + _INIT_CALL_PY_EXACT_ARGS_6, + _INIT_CALL_PY_EXACT_ARGS_7, + _INIT_CALL_PY_EXACT_ARGS_8, + _INIT_CALL_PY_EXACT_ARGS_9, _INSERT_NULL, _IS_NONE, _IS_OP, @@ -366,7 +371,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 654 +#define MAX_UOP_ID 659 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -1013,6 +1018,11 @@ enum { _INIT_CALL_PY_EXACT_ARGS_2_r01, _INIT_CALL_PY_EXACT_ARGS_3_r01, _INIT_CALL_PY_EXACT_ARGS_4_r01, + _INIT_CALL_PY_EXACT_ARGS_5_r01, + _INIT_CALL_PY_EXACT_ARGS_6_r01, + _INIT_CALL_PY_EXACT_ARGS_7_r01, + _INIT_CALL_PY_EXACT_ARGS_8_r01, + _INIT_CALL_PY_EXACT_ARGS_9_r01, _INSERT_NULL_r10, _INSTRUMENTED_FOR_ITER_r23, _INSTRUMENTED_INSTRUCTION_r00, @@ -1444,7 +1454,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1655 +#define MAX_UOP_REGS_ID 1665 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0cd82c53aa932ae..f6ffadf77282b40 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -311,6 +311,11 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_5] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_6] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_7] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_8] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_9] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GUARD_NOS_NULL] = HAS_EXIT_FLAG, @@ -439,7 +444,7 @@ const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { [_LOAD_FAST_BORROW] = { 0, 8 }, [_LOAD_SMALL_INT] = { 0, 4 }, [_SWAP_FAST] = { 0, 8 }, - [_INIT_CALL_PY_EXACT_ARGS] = { 0, 5 }, + [_INIT_CALL_PY_EXACT_ARGS] = { 0, 10 }, [_COPY] = { 1, 4 }, [_SWAP] = { 2, 4 }, [_GUARD_BIT_IS_SET_POP] = { 4, 8 }, @@ -2931,6 +2936,51 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_INIT_CALL_PY_EXACT_ARGS_5] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_5_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_6] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_6_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_7] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_7_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_8] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_8_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_9] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_9_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_INIT_CALL_PY_EXACT_ARGS] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -4568,6 +4618,11 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_2_r01] = _INIT_CALL_PY_EXACT_ARGS_2, [_INIT_CALL_PY_EXACT_ARGS_3_r01] = _INIT_CALL_PY_EXACT_ARGS_3, [_INIT_CALL_PY_EXACT_ARGS_4_r01] = _INIT_CALL_PY_EXACT_ARGS_4, + [_INIT_CALL_PY_EXACT_ARGS_5_r01] = _INIT_CALL_PY_EXACT_ARGS_5, + [_INIT_CALL_PY_EXACT_ARGS_6_r01] = _INIT_CALL_PY_EXACT_ARGS_6, + [_INIT_CALL_PY_EXACT_ARGS_7_r01] = _INIT_CALL_PY_EXACT_ARGS_7, + [_INIT_CALL_PY_EXACT_ARGS_8_r01] = _INIT_CALL_PY_EXACT_ARGS_8, + [_INIT_CALL_PY_EXACT_ARGS_9_r01] = _INIT_CALL_PY_EXACT_ARGS_9, [_INIT_CALL_PY_EXACT_ARGS_r01] = _INIT_CALL_PY_EXACT_ARGS, [_PUSH_FRAME_r10] = _PUSH_FRAME, [_GUARD_NOS_NULL_r02] = _GUARD_NOS_NULL, @@ -5658,6 +5713,16 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3_r01] = "_INIT_CALL_PY_EXACT_ARGS_3_r01", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", [_INIT_CALL_PY_EXACT_ARGS_4_r01] = "_INIT_CALL_PY_EXACT_ARGS_4_r01", + [_INIT_CALL_PY_EXACT_ARGS_5] = "_INIT_CALL_PY_EXACT_ARGS_5", + [_INIT_CALL_PY_EXACT_ARGS_5_r01] = "_INIT_CALL_PY_EXACT_ARGS_5_r01", + [_INIT_CALL_PY_EXACT_ARGS_6] = "_INIT_CALL_PY_EXACT_ARGS_6", + [_INIT_CALL_PY_EXACT_ARGS_6_r01] = "_INIT_CALL_PY_EXACT_ARGS_6_r01", + [_INIT_CALL_PY_EXACT_ARGS_7] = "_INIT_CALL_PY_EXACT_ARGS_7", + [_INIT_CALL_PY_EXACT_ARGS_7_r01] = "_INIT_CALL_PY_EXACT_ARGS_7_r01", + [_INIT_CALL_PY_EXACT_ARGS_8] = "_INIT_CALL_PY_EXACT_ARGS_8", + [_INIT_CALL_PY_EXACT_ARGS_8_r01] = "_INIT_CALL_PY_EXACT_ARGS_8_r01", + [_INIT_CALL_PY_EXACT_ARGS_9] = "_INIT_CALL_PY_EXACT_ARGS_9", + [_INIT_CALL_PY_EXACT_ARGS_9_r01] = "_INIT_CALL_PY_EXACT_ARGS_9_r01", [_INSERT_NULL] = "_INSERT_NULL", [_INSERT_NULL_r10] = "_INSERT_NULL_r10", [_IS_NONE] = "_IS_NONE", @@ -6287,7 +6352,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _POP_TOP_UNICODE: return 1; case _POP_TOP_OPARG: - return oparg; + return (oparg >> 1); case _PUSH_NULL: return 0; case _END_FOR: @@ -6715,7 +6780,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _MAYBE_EXPAND_METHOD: return 0; case _PY_FRAME_GENERAL: - return 2 + oparg; + return 2 + (oparg >> 1); case _CHECK_FUNCTION_VERSION: return 0; case _CHECK_FUNCTION_VERSION_INLINE: @@ -6727,7 +6792,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_IS_NOT_PY_CALLABLE: return 0; case _CALL_NON_PY_GENERAL: - return 2 + oparg; + return 2 + (oparg >> 1); case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: return 0; case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: @@ -6741,17 +6806,27 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_RECURSION_REMAINING: return 0; case _INIT_CALL_PY_EXACT_ARGS_0: - return 2 + oparg; + return 2 + (oparg >> 1); case _INIT_CALL_PY_EXACT_ARGS_1: - return 2 + oparg; + return 2 + (oparg >> 1); case _INIT_CALL_PY_EXACT_ARGS_2: - return 2 + oparg; + return 2 + (oparg >> 1); case _INIT_CALL_PY_EXACT_ARGS_3: - return 2 + oparg; + return 2 + (oparg >> 1); case _INIT_CALL_PY_EXACT_ARGS_4: - return 2 + oparg; + return 2 + (oparg >> 1); + case _INIT_CALL_PY_EXACT_ARGS_5: + return 2 + (oparg >> 1); + case _INIT_CALL_PY_EXACT_ARGS_6: + return 2 + (oparg >> 1); + case _INIT_CALL_PY_EXACT_ARGS_7: + return 2 + (oparg >> 1); + case _INIT_CALL_PY_EXACT_ARGS_8: + return 2 + (oparg >> 1); + case _INIT_CALL_PY_EXACT_ARGS_9: + return 2 + (oparg >> 1); case _INIT_CALL_PY_EXACT_ARGS: - return 2 + oparg; + return 2 + (oparg >> 1); case _PUSH_FRAME: return 1; case _GUARD_NOS_NULL: @@ -6775,7 +6850,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _ALLOCATE_OBJECT: return 0; case _CREATE_INIT_FRAME: - return 2 + oparg; + return 2 + (oparg >> 1); case _EXIT_INIT_CHECK: return 1; case _GUARD_CALLABLE_BUILTIN_CLASS: @@ -6785,7 +6860,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_BUILTIN_O: return 0; case _CALL_BUILTIN_O: - return 2 + oparg; + return 2 + (oparg >> 1); case _GUARD_CALLABLE_BUILTIN_FAST: return 0; case _CALL_BUILTIN_FAST: @@ -6809,7 +6884,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O: return 0; case _CALL_METHOD_DESCRIPTOR_O: - return 2 + oparg; + return 2 + (oparg >> 1); case _CHECK_RECURSION_LIMIT: return 0; case _CALL_METHOD_DESCRIPTOR_O_INLINE: @@ -6823,7 +6898,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS: return 0; case _CALL_METHOD_DESCRIPTOR_NOARGS: - return 2 + oparg; + return 2 + (oparg >> 1); case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE: return 1 + oparg; case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST: @@ -6835,7 +6910,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: - return 3 + oparg; + return 3 + (oparg >> 1); case _CHECK_FUNCTION_VERSION_KW: return 0; case _CHECK_METHOD_VERSION_KW: @@ -6845,7 +6920,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_IS_NOT_PY_CALLABLE_KW: return 0; case _CALL_KW_NON_PY: - return 3 + oparg; + return 3 + (oparg >> 1); case _MAKE_CALLARGS_A_TUPLE: return 0; case _CHECK_IS_PY_CALLABLE_EX: diff --git a/Lib/dis.py b/Lib/dis.py index 318729091fa098c..54df95d6bb0689e 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -44,6 +44,9 @@ SEND = opmap['SEND'] LOAD_ATTR = opmap['LOAD_ATTR'] LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] +CALL = opmap['CALL'] +CALL_KW = opmap['CALL_KW'] +CALL_FUNCTION_EX = opmap['CALL_FUNCTION_EX'] CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] @@ -642,6 +645,14 @@ def get_argval_argrepr(self, op, arg, offset): if arg & (1<> 1 + argrepr = str(argval) + if arg & 1: + argrepr = f"{argrepr} + awaited" + elif deop == CALL_FUNCTION_EX: + if arg & 1: + argrepr = "awaited" elif deop == CALL_INTRINSIC_1: argrepr = _intrinsic_1_descs[arg] elif deop == CALL_INTRINSIC_2: diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 87509245f6a866c..350bf4dd3b0ee93 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -56,7 +56,7 @@ def test_for_loop(self): ('LOAD_NAME', 2, 2), ('PUSH_NULL', None, 2), ('LOAD_NAME', 1, 2), - ('CALL', 1, 2), + ('CALL', 2, 2), ('POP_TOP', None), ('JUMP', loop_lbl), exit_lbl, @@ -210,7 +210,7 @@ def test_frozenset_optimization(self): ('LOAD_CONST', 2), ('LOAD_CONST', 3), ('BUILD_SET', 3), - ('CALL', 1), + ('CALL', 2), ('POP_TOP', None), ('LOAD_CONST', 0), ('RETURN_VALUE', None) @@ -225,7 +225,7 @@ def test_comp_without_target_optimization(self): ('LOAD_NAME', 0), ('PUSH_NULL', None), ('LOAD_CONST', 0), - ('CALL', 1), + ('CALL', 2), ('LOAD_FAST_AND_CLEAR', 0), ('SWAP', 2), ('SETUP_FINALLY', 20), diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 709fa374704f1f5..351e1eff4cb5b9f 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -116,7 +116,7 @@ def _f(a): %3d LOAD_GLOBAL 1 (print + NULL) LOAD_FAST_BORROW 0 (a) - CALL 1 + CALL 2 (1) POP_TOP %3d LOAD_SMALL_INT 1 @@ -130,7 +130,7 @@ def _f(a): %3d 4 LOAD_GLOBAL 1 (print + NULL) 14 LOAD_FAST_BORROW 0 (a) - 16 CALL 1 + 16 CALL 2 (1) 24 POP_TOP %3d 26 LOAD_SMALL_INT 1 @@ -144,7 +144,7 @@ def _f(a): %-14s LOAD_GLOBAL 1 (print + NULL) %-14s LOAD_FAST_BORROW 0 (a) -%-14s CALL 1 +%-14s CALL 2 (1) %-14s POP_TOP %-14s LOAD_SMALL_INT 1 @@ -155,7 +155,7 @@ def _f(a): RESUME 0 LOAD_GLOBAL 1 LOAD_FAST_BORROW 0 - CALL 1 + CALL 2 (1) POP_TOP LOAD_SMALL_INT 1 RETURN_VALUE @@ -174,7 +174,7 @@ def bug708901(): %3d LOAD_SMALL_INT 10 -%3d CALL 2 +%3d CALL 4 (2) GET_ITER 0 L1: FOR_ITER 3 (to L2) STORE_FAST 0 (res) @@ -205,12 +205,12 @@ def bug1333982(x=[]): LOAD_CONST 1 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION LOAD_FAST_BORROW 0 (x) - CALL 0 + CALL 0 (0) %3d LOAD_SMALL_INT 1 %3d BINARY_OP 0 (+) - CALL 0 + CALL 0 (0) RAISE_VARARGS 1 """ % (bug1333982.__code__.co_firstlineno, bug1333982.__code__.co_firstlineno + 1, @@ -281,7 +281,7 @@ def wrap_func_w_kwargs(): LOAD_SMALL_INT 2 LOAD_SMALL_INT 5 LOAD_CONST 1 (('c',)) - CALL_KW 3 + CALL_KW 6 (3) POP_TOP LOAD_COMMON_CONSTANT 7 (None) RETURN_VALUE @@ -408,7 +408,7 @@ def wrap_func_w_kwargs(): LOAD_NAME 3 (fun) PUSH_NULL LOAD_SMALL_INT 0 - CALL 1 + CALL 2 (1) STORE_SUBSCR LOAD_COMMON_CONSTANT 7 (None) RETURN_VALUE @@ -548,7 +548,7 @@ def _with(c): SWAP 2 SWAP 3 LOAD_SPECIAL 0 (__enter__) - CALL 0 + CALL 0 (0) L1: POP_TOP %4d LOAD_SMALL_INT 1 @@ -557,7 +557,7 @@ def _with(c): %4d L2: LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) - CALL 3 + CALL 6 (3) POP_TOP %4d LOAD_SMALL_INT 2 @@ -613,7 +613,7 @@ async def _asyncwith(c): SWAP 2 SWAP 3 LOAD_SPECIAL 2 (__aenter__) - CALL 0 + CALL 0 (0) GET_AWAITABLE 1 PUSH_NULL LOAD_COMMON_CONSTANT 7 (None) @@ -630,7 +630,7 @@ async def _asyncwith(c): %4d L7: LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) - CALL 3 + CALL 6 (3) GET_AWAITABLE 2 PUSH_NULL LOAD_COMMON_CONSTANT 7 (None) @@ -725,7 +725,7 @@ def _tryfinallyconst(b): %4d L2: LOAD_FAST_BORROW 1 (b) PUSH_NULL - CALL 0 + CALL 0 (0) POP_TOP RETURN_VALUE @@ -733,7 +733,7 @@ def _tryfinallyconst(b): %4d LOAD_FAST 1 (b) PUSH_NULL - CALL 0 + CALL 0 (0) POP_TOP RERAISE 0 @@ -759,7 +759,7 @@ def _tryfinallyconst(b): %4d LOAD_FAST_BORROW 0 (b) PUSH_NULL - CALL 0 + CALL 0 (0) POP_TOP LOAD_SMALL_INT 1 RETURN_VALUE @@ -768,7 +768,7 @@ def _tryfinallyconst(b): %4d LOAD_FAST 0 (b) PUSH_NULL - CALL 0 + CALL 0 (0) POP_TOP RERAISE 0 @@ -842,7 +842,7 @@ def foo(x): MAKE_FUNCTION SET_FUNCTION_ATTRIBUTE 8 (closure) LOAD_DEREF 1 (y) - CALL 0 + CALL 0 (0) PUSH_NULL L1: FOR_ITER 3 (to L2) LIST_APPEND 3 @@ -857,8 +857,8 @@ def foo(x): MAKE_FUNCTION SET_FUNCTION_ATTRIBUTE 8 (closure) LOAD_DEREF 1 (y) - CALL 0 - CALL 1 + CALL 0 (0) + CALL 2 (1) RETURN_VALUE """ % (dis_nested_0, __file__, @@ -941,7 +941,7 @@ def loop_test(): %3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST_BORROW 0 (i) - CALL_PY_GENERAL 1 + CALL_PY_GENERAL 2 (1) POP_TOP JUMP_BACKWARD_{: <6} 16 (to L1) @@ -1356,7 +1356,7 @@ def test_call_specialize(self): 1 LOAD_NAME 0 (str) PUSH_NULL LOAD_SMALL_INT 1 - CALL_STR_1 1 + CALL_STR_1 2 (1) RETURN_VALUE """ co = compile("str(1)", "", "eval") @@ -1494,7 +1494,7 @@ def loop(n): {line+1} LOAD_GLOBAL_BUILTIN 1 (range + NULL) LOAD_FAST_BORROW 0 (n) - CALL_BUILTIN_CLASS 1 + CALL_BUILTIN_CLASS 2 (1) GET_ITER 0 L1: FOR_ITER_RANGE 3 (to L2) STORE_FAST 1 (i) @@ -1832,7 +1832,7 @@ def _prepare_test_cases(): make_inst(opname='BUILD_LIST', arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7), make_inst(opname='BUILD_MAP', arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7), make_inst(opname='LOAD_CONST', arg=2, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7), - make_inst(opname='CALL', arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=14, argval=7, argrepr='7', offset=48, start_offset=48, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7), make_inst(opname='LOAD_FAST_BORROW', arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8), make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8), @@ -1857,7 +1857,7 @@ def _prepare_test_cases(): make_inst(opname='LOAD_DEREF', arg=4, argval='b', argrepr='b', offset=40, start_offset=40, starts_line=False, line_number=5), make_inst(opname='LOAD_DEREF', arg=0, argval='c', argrepr='c', offset=42, start_offset=42, starts_line=False, line_number=5), make_inst(opname='LOAD_DEREF', arg=1, argval='d', argrepr='d', offset=44, start_offset=44, starts_line=False, line_number=5), - make_inst(opname='CALL', arg=4, argval=4, argrepr='', offset=46, start_offset=46, starts_line=False, line_number=5, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=8, argval=4, argrepr='4', offset=46, start_offset=46, starts_line=False, line_number=5, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=54, start_offset=54, starts_line=False, line_number=5), make_inst(opname='LOAD_FAST_BORROW', arg=2, argval='inner', argrepr='inner', offset=56, start_offset=56, starts_line=True, line_number=6), make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=58, start_offset=58, starts_line=False, line_number=6), @@ -1872,7 +1872,7 @@ def _prepare_test_cases(): make_inst(opname='LOAD_DEREF', arg=4, argval='c', argrepr='c', offset=20, start_offset=20, starts_line=False, line_number=4), make_inst(opname='LOAD_DEREF', arg=5, argval='d', argrepr='d', offset=22, start_offset=22, starts_line=False, line_number=4), make_inst(opname='LOAD_FAST_BORROW_LOAD_FAST_BORROW', arg=1, argval=('e', 'f'), argrepr='e, f', offset=24, start_offset=24, starts_line=False, line_number=4), - make_inst(opname='CALL', arg=6, argval=6, argrepr='', offset=26, start_offset=26, starts_line=False, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=12, argval=6, argrepr='6', offset=26, start_offset=26, starts_line=False, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=34, start_offset=34, starts_line=False, line_number=4), make_inst(opname='LOAD_COMMON_CONSTANT', arg=7, argval=None, argrepr='None', offset=36, start_offset=36, starts_line=False, line_number=4), make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=38, start_offset=38, starts_line=False, line_number=4), @@ -1882,13 +1882,13 @@ def _prepare_test_cases(): make_inst(opname='RESUME', arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='LOAD_GLOBAL', arg=1, argval='range', argrepr='range + NULL', offset=4, start_offset=4, starts_line=True, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_SMALL_INT', arg=10, argval=10, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=16, start_offset=16, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='GET_ITER', arg=0, argval=0, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='FOR_ITER', arg=33, argval=98, argrepr='to L4', offset=28, start_offset=28, starts_line=False, line_number=3, label=1, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=32, start_offset=32, starts_line=False, line_number=3), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=34, start_offset=34, starts_line=True, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=44, start_offset=44, starts_line=False, line_number=4), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=46, start_offset=46, starts_line=False, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=46, start_offset=46, starts_line=False, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=54, start_offset=54, starts_line=False, line_number=4), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=56, start_offset=56, starts_line=True, line_number=5), make_inst(opname='LOAD_SMALL_INT', arg=4, argval=4, argrepr='', offset=58, start_offset=58, starts_line=False, line_number=5), @@ -1909,7 +1909,7 @@ def _prepare_test_cases(): make_inst(opname='POP_ITER', arg=None, argval=None, argrepr='', offset=100, start_offset=100, starts_line=False, line_number=3), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=102, start_offset=102, starts_line=True, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=1, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=112, start_offset=112, starts_line=False, line_number=10), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=114, start_offset=114, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=114, start_offset=114, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=122, start_offset=122, starts_line=False, line_number=10), make_inst(opname='LOAD_FAST_CHECK', arg=0, argval='i', argrepr='i', offset=124, start_offset=124, starts_line=True, line_number=11, label=5), make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=126, start_offset=126, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), @@ -1917,7 +1917,7 @@ def _prepare_test_cases(): make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=138, start_offset=138, starts_line=False, line_number=11), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=140, start_offset=140, starts_line=True, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=False, line_number=12), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=152, start_offset=152, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=152, start_offset=152, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=160, start_offset=160, starts_line=False, line_number=12), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=162, start_offset=162, starts_line=True, line_number=13), make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=164, start_offset=164, starts_line=False, line_number=13), @@ -1938,7 +1938,7 @@ def _prepare_test_cases(): make_inst(opname='JUMP_FORWARD', arg=11, argval=240, argrepr='to L9', offset=216, start_offset=216, starts_line=True, line_number=17, label=7), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=218, start_offset=218, starts_line=True, line_number=19, label=8, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=2, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=228, start_offset=228, starts_line=False, line_number=19), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=230, start_offset=230, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=19), make_inst(opname='NOP', arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=True, line_number=20, label=9), make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=242, start_offset=242, starts_line=True, line_number=21), @@ -1951,20 +1951,20 @@ def _prepare_test_cases(): make_inst(opname='SWAP', arg=2, argval=2, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=25), make_inst(opname='SWAP', arg=3, argval=3, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=25), make_inst(opname='LOAD_SPECIAL', arg=0, argval=0, argrepr='__enter__', offset=270, start_offset=270, starts_line=False, line_number=25), - make_inst(opname='CALL', arg=0, argval=0, argrepr='', offset=272, start_offset=272, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=0, argval=0, argrepr='0', offset=272, start_offset=272, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='STORE_FAST', arg=1, argval='dodgy', argrepr='dodgy', offset=280, start_offset=280, starts_line=False, line_number=25), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=282, start_offset=282, starts_line=True, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=3, argval='Never reach this', argrepr="'Never reach this'", offset=292, start_offset=292, starts_line=False, line_number=26), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=294, start_offset=294, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=294, start_offset=294, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=302, start_offset=302, starts_line=False, line_number=26), make_inst(opname='LOAD_COMMON_CONSTANT', arg=7, argval=None, argrepr='None', offset=304, start_offset=304, starts_line=True, line_number=25), make_inst(opname='LOAD_COMMON_CONSTANT', arg=7, argval=None, argrepr='None', offset=306, start_offset=306, starts_line=False, line_number=25), make_inst(opname='LOAD_COMMON_CONSTANT', arg=7, argval=None, argrepr='None', offset=308, start_offset=308, starts_line=False, line_number=25), - make_inst(opname='CALL', arg=3, argval=3, argrepr='', offset=310, start_offset=310, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=6, argval=3, argrepr='3', offset=310, start_offset=310, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=False, line_number=25), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=320, start_offset=320, starts_line=True, line_number=28, label=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=330, start_offset=330, starts_line=False, line_number=28), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=332, start_offset=332, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=28), make_inst(opname='LOAD_COMMON_CONSTANT', arg=7, argval=None, argrepr='None', offset=342, start_offset=342, starts_line=False, line_number=28), make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=28), @@ -1991,7 +1991,7 @@ def _prepare_test_cases(): make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=404, start_offset=404, starts_line=False, line_number=22), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=406, start_offset=406, starts_line=True, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=416, start_offset=416, starts_line=False, line_number=23), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=418, start_offset=418, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=418, start_offset=418, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=23), make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=23), make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=56, argval=320, argrepr='to L10', offset=430, start_offset=430, starts_line=False, line_number=23), @@ -2002,7 +2002,7 @@ def _prepare_test_cases(): make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=440, start_offset=440, starts_line=False, line_number=None), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=442, start_offset=442, starts_line=True, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=452, start_offset=452, starts_line=False, line_number=28), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=454, start_offset=454, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=454, start_offset=454, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=462, start_offset=462, starts_line=False, line_number=28), make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=28), make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=466, start_offset=466, starts_line=True, line_number=None), @@ -2618,7 +2618,7 @@ def test_show_cache(self): 1 LOAD_NAME 0 (print) PUSH_NULL - CALL 0 + CALL 0 (0) CACHE 0 (counter: 0) CACHE 0 (func_version: 0) CACHE 0 diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 8e7ba1f69553dda..91a736ce871c538 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -1831,8 +1831,8 @@ _PyStackRef res; // _SPECIALIZE_CALL { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -1840,7 +1840,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -1858,8 +1858,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -1868,9 +1868,9 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -1882,8 +1882,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -1891,7 +1891,7 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); @@ -1899,8 +1899,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( @@ -1914,7 +1914,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -1922,8 +1922,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -1965,8 +1965,8 @@ } // _CHECK_OBJECT { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { @@ -2022,8 +2022,8 @@ self_or_null = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2031,7 +2031,7 @@ } // _CREATE_INIT_FRAME { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; self = self_or_null; init = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -2045,9 +2045,9 @@ assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, (oparg >> 1)+1, NULL, shim); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2110,8 +2110,8 @@ } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -2132,8 +2132,8 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2162,7 +2162,7 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2189,13 +2189,13 @@ } // _INIT_CALL_PY_EXACT_ARGS { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); @@ -2213,7 +2213,7 @@ { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2260,8 +2260,8 @@ } // _CHECK_METHOD_VERSION { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -2296,8 +2296,8 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2314,9 +2314,9 @@ } // _PY_FRAME_GENERAL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -2331,7 +2331,7 @@ args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -2388,7 +2388,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_CLASS { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2404,9 +2404,9 @@ } // _CALL_BUILTIN_CLASS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2425,7 +2425,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2433,16 +2433,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2481,7 +2481,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_FAST { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2497,9 +2497,9 @@ } // _CALL_BUILTIN_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2519,7 +2519,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2527,16 +2527,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2575,7 +2575,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2591,9 +2591,9 @@ } // _CALL_BUILTIN_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2609,7 +2609,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2617,16 +2617,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2668,8 +2668,8 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_O { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2682,7 +2682,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -2702,7 +2702,7 @@ } // _CALL_BUILTIN_O { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { args--; @@ -2726,9 +2726,9 @@ // _POP_TOP { value = s; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer += -oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3394,8 +3394,8 @@ _PyStackRef res; // _SPECIALIZE_CALL_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -3403,7 +3403,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -3421,8 +3421,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -3432,10 +3432,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3448,8 +3448,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -3457,7 +3457,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3470,8 +3470,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( @@ -3485,14 +3485,14 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3525,8 +3525,8 @@ } // _CHECK_METHOD_VERSION_KW { - null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -3561,8 +3561,8 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); @@ -3572,9 +3572,9 @@ // _PY_FRAME_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3598,7 +3598,7 @@ _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -3657,7 +3657,7 @@ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE_KW { - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); @@ -3673,12 +3673,12 @@ // _CALL_KW_NON_PY { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3693,7 +3693,7 @@ kwnames); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -3701,8 +3701,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3742,7 +3742,7 @@ } // _CHECK_FUNCTION_VERSION_KW { - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -3768,10 +3768,10 @@ // _PY_FRAME_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3795,7 +3795,7 @@ _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -3973,7 +3973,7 @@ { arg = stack_pointer[-1]; self = nos; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); @@ -4050,9 +4050,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4066,7 +4066,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4087,7 +4087,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4115,7 +4115,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -4125,13 +4125,13 @@ { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4170,9 +4170,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4186,7 +4186,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4208,7 +4208,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4236,7 +4236,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -4246,13 +4246,13 @@ { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4294,9 +4294,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4310,7 +4310,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4339,7 +4339,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + assert((oparg >> 1) == 1 || !PyStackRef_IsNull(self_or_null)); if (!PyStackRef_IsNull(self_or_null)) { args--; } @@ -4363,9 +4363,9 @@ // _POP_TOP { value = s; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer += -oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4418,9 +4418,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_O { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4434,7 +4434,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4488,10 +4488,10 @@ // _POP_TOP { value = a; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer[-oparg] = s; - stack_pointer += 1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer[-(oparg >> 1)] = s; + stack_pointer += 1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4551,7 +4551,7 @@ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -4566,12 +4566,12 @@ } // _CALL_NON_PY_GENERAL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4586,7 +4586,7 @@ PyStackRef_NULL); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -4594,8 +4594,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4634,7 +4634,7 @@ } // _CHECK_FUNCTION_VERSION { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -4651,12 +4651,12 @@ } // _CHECK_FUNCTION_EXACT_ARGS { - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4683,13 +4683,13 @@ } // _INIT_CALL_PY_EXACT_ARGS { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); @@ -4707,7 +4707,7 @@ { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4753,7 +4753,7 @@ } // _CHECK_FUNCTION_VERSION { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -4778,10 +4778,10 @@ } // _PY_FRAME_GENERAL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -4796,7 +4796,7 @@ args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -4876,7 +4876,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4954,7 +4954,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -5032,7 +5032,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -7111,8 +7111,8 @@ /* Skip 3 cache entries */ // _MAYBE_EXPAND_METHOD { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7120,8 +7120,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7130,7 +7130,7 @@ } // _MONITOR_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; maybe_self = self_or_null; func = callable; int is_meth = !PyStackRef_IsNull(maybe_self); @@ -7139,14 +7139,14 @@ if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); } - else if (oparg) { + else if ((oparg >> 1)) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { arg0 = &_PyInstrumentation_MISSING; } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; + stack_pointer[-2 - (oparg >> 1)] = func; + stack_pointer[-1 - (oparg >> 1)] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7160,11 +7160,11 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7183,7 +7183,7 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); @@ -7204,7 +7204,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -7212,8 +7212,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7428,8 +7428,8 @@ /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7437,8 +7437,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7447,7 +7447,7 @@ } // _MONITOR_CALL_KW { - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; int is_meth = !PyStackRef_IsNull(self_or_null); PyObject *arg; if (is_meth) { @@ -7460,8 +7460,8 @@ arg = &_PyInstrumentation_MISSING; } PyObject *function = PyStackRef_AsPyObjectBorrow(callable); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7475,10 +7475,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7498,7 +7498,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7524,14 +7524,14 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 42ad83daa1ea461..0a8147e19fcd3dc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -383,8 +383,8 @@ dummy_func( PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); } - op(_POP_TOP_OPARG, (args[oparg] -- )) { - _PyStackRef_CloseStack(args, oparg); + op(_POP_TOP_OPARG, (args[oparg >> 1] -- )) { + _PyStackRef_CloseStack(args, (oparg >> 1)); DEAD(args); } @@ -4316,11 +4316,11 @@ dummy_func( CALL_NON_PY_GENERAL, }; - specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -4328,7 +4328,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -4341,11 +4341,11 @@ dummy_func( } // When calling Python, inline the call using DISPATCH_INLINED(). - op(_DO_CALL, (callable, self_or_null, args[oparg] -- res)) { + op(_DO_CALL, (callable, self_or_null, args[oparg >> 1] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg >> 1 counts all of the args, but *not* self: + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4391,14 +4391,14 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) { + op(_MONITOR_CALL, (func, maybe_self, args[oparg >> 1] -- func, maybe_self, args[oparg >> 1])) { int is_meth = !PyStackRef_IsNull(maybe_self); PyObject *function = PyStackRef_AsPyObjectBorrow(func); PyObject *arg0; if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); } - else if (oparg) { + else if ((oparg >> 1)) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { @@ -4415,11 +4415,11 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC_AT_END; macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC_AT_END; - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg >> 1] -- new_frame)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg >> 1 counts all of the args, but *not* self: + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -4440,7 +4440,7 @@ dummy_func( new_frame = PyStackRef_Wrap(temp); } - op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; @@ -4463,7 +4463,7 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg >> 1] -- callable, null, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); @@ -4473,7 +4473,7 @@ dummy_func( EXIT_IF(!PyStackRef_IsNull(null)); } - op(_EXPAND_METHOD, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_EXPAND_METHOD, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyStackRef_IsNull(self_or_null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); @@ -4496,17 +4496,17 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(PyFunction_Check(callable_o)); EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); } - op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg >> 1] -- res)) { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4532,12 +4532,12 @@ dummy_func( _CALL_NON_PY_GENERAL + _CHECK_PERIODIC_AT_END; - op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg >> 1] -- callable, null, unused[oparg >> 1])) { EXIT_IF(!PyStackRef_IsNull(null)); EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type); } - op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { assert(PyStackRef_IsNull(self_or_null)); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); @@ -4551,15 +4551,15 @@ dummy_func( DEOPT_IF(IS_PEP523_HOOKED(tstate)); } - op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - EXIT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))); + EXIT_IF(code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))); } - op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -4570,13 +4570,13 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { + replicate(10) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg >> 1] -- new_frame)) { int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } INPUTS_DEAD(); @@ -4644,7 +4644,7 @@ dummy_func( op(_CALL_TYPE_1, (callable, null, arg -- res, a)) { PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; INPUTS_DEAD(); @@ -4667,7 +4667,7 @@ dummy_func( op(_CALL_STR_1, (callable, null, arg -- res, a)) { PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); PyObject *res_o = PyObject_Str(arg_o); if (res_o == NULL) { @@ -4695,7 +4695,7 @@ dummy_func( op(_CALL_TUPLE_1, (callable, null, arg -- res, a)) { PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); PyObject *res_o = PySequence_Tuple(arg_o); if (res_o == NULL) { @@ -4715,7 +4715,7 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_CHECK_OBJECT, (type_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_OBJECT, (type_version/2, callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyStackRef_IsNull(self_or_null)); EXIT_IF(!PyType_Check(callable_o)); @@ -4723,7 +4723,7 @@ dummy_func( EXIT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version); } - op(_ALLOCATE_OBJECT, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_ALLOCATE_OBJECT, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyStackRef_IsNull(self_or_null)); assert(PyType_Check(callable_o)); @@ -4747,7 +4747,7 @@ dummy_func( PyStackRef_CLOSE(temp); } - op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { + op(_CREATE_INIT_FRAME, (init, self, args[oparg >> 1] -- init_frame)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); @@ -4755,7 +4755,7 @@ dummy_func( /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, (oparg >> 1)+1, NULL, shim); DEAD(init); DEAD(self); DEAD(args); @@ -4792,15 +4792,15 @@ dummy_func( DEAD(should_be_none); } - op(_GUARD_CALLABLE_BUILTIN_CLASS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_CLASS, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; EXIT_IF(tp->tp_vectorcall == NULL); } - op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - int total_args = oparg; + op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4829,19 +4829,19 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) != METH_O); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } EXIT_IF(total_args != 1); } - op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) { + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg >> 1] -- res, c, s)) { /* Builtin METH_O functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4874,16 +4874,16 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) != METH_FASTCALL); } - op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4913,16 +4913,16 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)); } - op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5030,7 +5030,7 @@ dummy_func( } op(_CALL_LIST_APPEND, (callable, self, arg -- none, c, s)) { - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); DEOPT_IF(!LOCK_OBJECT(self_o)); @@ -5050,13 +5050,13 @@ dummy_func( none = PyStackRef_None; } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != METH_O); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -5066,7 +5066,7 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); } - op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg >> 1] -- res, c, s, a)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -5126,13 +5126,13 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5143,11 +5143,11 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); } - op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5201,13 +5201,13 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != METH_NOARGS); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -5217,11 +5217,11 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); } - op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res, c, s)) { + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg >> 1] -- res, c, s)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + assert((oparg >> 1) == 1 || !PyStackRef_IsNull(self_or_null)); if (!PyStackRef_IsNull(self_or_null)) { args--; } @@ -5270,14 +5270,14 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != METH_FASTCALL); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -5287,11 +5287,11 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); } - op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5352,7 +5352,7 @@ dummy_func( CALL_KW_NON_PY, }; - op(_MONITOR_CALL_KW, (callable, self_or_null, args[oparg], unused -- callable, self_or_null, args[oparg], unused)) { + op(_MONITOR_CALL_KW, (callable, self_or_null, args[oparg >> 1], unused -- callable, self_or_null, args[oparg >> 1], unused)) { int is_meth = !PyStackRef_IsNull(self_or_null); PyObject *arg; if (is_meth) { @@ -5371,7 +5371,7 @@ dummy_func( ERROR_IF(err); } - op(_MAYBE_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + op(_MAYBE_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg >> 1], unused -- callable, self_or_null, unused[oparg >> 1], unused)) { if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -5383,12 +5383,12 @@ dummy_func( } } - op(_DO_CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { + op(_DO_CALL_KW, (callable, self_or_null, args[oparg >> 1], kwnames -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg >> 1 counts all of the args, but *not* self: + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5438,11 +5438,11 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg >> 1], kwnames -- new_frame)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg >> 1 counts all of the args, but *not* self: + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5468,7 +5468,7 @@ dummy_func( new_frame = PyStackRef_Wrap(temp); } - op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { + op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, unused, unused[oparg >> 1], unused -- callable, unused, unused[oparg >> 1], unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; @@ -5485,7 +5485,7 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg], unused -- callable, null, unused[oparg], unused)) { + op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg >> 1], unused -- callable, null, unused[oparg >> 1], unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); @@ -5495,7 +5495,7 @@ dummy_func( EXIT_IF(!PyStackRef_IsNull(null)); } - op(_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + op(_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg >> 1], unused -- callable, self_or_null, unused[oparg >> 1], unused)) { assert(PyStackRef_IsNull(self_or_null)); _PyStackRef callable_s = callable; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -5517,11 +5517,11 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, unused[oparg >> 1], unused -- callable, self_or_null, unused[oparg >> 1], unused)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -5542,18 +5542,18 @@ dummy_func( _MONITOR_CALL_KW + _DO_CALL_KW; - op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { + op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg >> 1], unused -- callable, unused, unused[oparg >> 1], unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(PyFunction_Check(callable_o)); EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); } - op(_CALL_KW_NON_PY, (callable, self_or_null, args[oparg], kwnames -- res)) { + op(_CALL_KW_NON_PY, (callable, self_or_null, args[oparg >> 1], kwnames -- res)) { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -6392,15 +6392,15 @@ dummy_func( RECORD_VALUE(PyStackRef_AsPyObjectBorrow(value)); } - tier2 op(_RECORD_CALLABLE, (func, self, args[oparg] -- func, self, args[oparg])) { + tier2 op(_RECORD_CALLABLE, (func, self, args[oparg >> 1] -- func, self, args[oparg >> 1])) { RECORD_VALUE(PyStackRef_AsPyObjectBorrow(func)); } - tier2 op(_RECORD_CALLABLE_KW, (func, self, args[oparg], kwnames -- func, self, args[oparg], kwnames)) { + tier2 op(_RECORD_CALLABLE_KW, (func, self, args[oparg >> 1], kwnames -- func, self, args[oparg >> 1], kwnames)) { RECORD_VALUE(PyStackRef_AsPyObjectBorrow(func)); } - tier2 op(_RECORD_BOUND_METHOD, (callable, self, args[oparg] -- callable, self, args[oparg])) { + tier2 op(_RECORD_BOUND_METHOD, (callable, self, args[oparg >> 1] -- callable, self, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) == &PyMethod_Type) { RECORD_VALUE(callable_o); diff --git a/Python/codegen.c b/Python/codegen.c index a975705576a167c..2a651037e707fb5 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -276,6 +276,10 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) return _PyInstructionSequence_Addop(seq, opcode, oparg_, loc); } +/* The oparg of CALL and CALL_KW: the number of arguments shifted left by one, + * with the low bit set when the result is awaited right away. */ +#define CALL_OPARG(NARGS, AWAITED) (((NARGS) << 1) | (AWAITED)) + #define ADDOP_I(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) @@ -492,7 +496,7 @@ codegen_call_exit_with_nones(compiler *c, location loc) ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); - ADDOP_I(c, loc, CALL, 3); + ADDOP_I(c, loc, CALL, CALL_OPARG(3, 0)); return SUCCESS; } @@ -1056,7 +1060,7 @@ codegen_apply_decorators(compiler *c, asdl_expr_seq* decos) for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) { location loc = LOC((expr_ty)asdl_seq_GET(decos, i)); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } return SUCCESS; } @@ -1584,11 +1588,11 @@ codegen_function(compiler *c, stmt_ty s, int is_async) RETURN_IF_ERROR(ret); if (num_typeparam_args > 0) { ADDOP_I(c, loc, SWAP, num_typeparam_args + 1); - ADDOP_I(c, loc, CALL, num_typeparam_args - 1); + ADDOP_I(c, loc, CALL, CALL_OPARG(num_typeparam_args - 1, 0)); } else { ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } } @@ -1770,7 +1774,7 @@ codegen_class(compiler *c, stmt_ty s) Py_DECREF(co); RETURN_IF_ERROR(ret); ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } else { RETURN_IF_ERROR(codegen_call_helper(c, loc, 2, s->v.ClassDef.bases, @@ -1859,7 +1863,7 @@ codegen_typealias(compiler *c, stmt_ty s) Py_DECREF(co); RETURN_IF_ERROR(ret); ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } RETURN_IF_ERROR(codegen_nameop(c, loc, name, Store)); return SUCCESS; @@ -4217,11 +4221,11 @@ maybe_optimize_method_call(compiler *c, expr_ty e) RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, kwds, kwdsl)); loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL_KW, argsl + kwdsl); + ADDOP_I(c, loc, CALL_KW, CALL_OPARG(argsl + kwdsl, 0)); } else { loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL, argsl); + ADDOP_I(c, loc, CALL, CALL_OPARG(argsl, 0)); } return 1; } @@ -4327,7 +4331,7 @@ codegen_joined_str(compiler *c, expr_ty e) VISIT(c, expr, asdl_seq_GET(e->v.JoinedStr.values, i)); ADDOP_I(c, loc, LIST_APPEND, 1); } - ADDOP_I(c, loc, CALL, 1); + ADDOP_I(c, loc, CALL, CALL_OPARG(1, 0)); } else { VISIT_SEQ(c, expr, e->v.JoinedStr.values); @@ -4497,10 +4501,10 @@ codegen_call_helper_impl(compiler *c, location loc, VISIT_SEQ(c, keyword, keywords); RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, keywords, nkwelts)); - ADDOP_I(c, loc, CALL_KW, n + nelts + nkwelts); + ADDOP_I(c, loc, CALL_KW, CALL_OPARG(n + nelts + nkwelts, 0)); } else { - ADDOP_I(c, loc, CALL, n + nelts); + ADDOP_I(c, loc, CALL, CALL_OPARG(n + nelts, 0)); } return SUCCESS; @@ -5163,7 +5167,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, Py_CLEAR(co); VISIT(c, expr, outermost->iter); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); if (is_async_comprehension && type != COMP_GENEXP) { ADDOP_I(c, loc, GET_AWAITABLE, 0); @@ -5310,7 +5314,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, SWAP, 2); ADDOP_I(c, loc, SWAP, 3); ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); ADDOP_I(c, loc, GET_AWAITABLE, 1); ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -5422,7 +5426,7 @@ codegen_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, SWAP, 2); ADDOP_I(c, loc, SWAP, 3); ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___ENTER__); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); ADDOP_JUMP(c, loc, SETUP_WITH, final); /* SETUP_WITH pushes a finally block. */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bc85ccd9ddb4fc1..c38c7df7948a520 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2508,16 +2508,16 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -oparg; + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -16860,8 +16860,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -16869,8 +16869,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -16880,8 +16880,8 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -16898,11 +16898,11 @@ _PyStackRef callable; _PyStackRef new_frame; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -16917,7 +16917,7 @@ args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -16937,7 +16937,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -17048,8 +17048,8 @@ _PyStackRef null; _PyStackRef callable; oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -17084,8 +17084,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyStackRef_IsNull(self_or_null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); @@ -17093,8 +17093,8 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -17112,7 +17112,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -17137,13 +17137,13 @@ _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -17158,7 +17158,7 @@ PyStackRef_NULL); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -17168,7 +17168,7 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17180,8 +17180,8 @@ _PyStackRef null; _PyStackRef callable; oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -17203,16 +17203,16 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; assert(PyStackRef_IsNull(self_or_null)); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -17301,13 +17301,13 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -17322,7 +17322,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -17415,21 +17415,21 @@ _PyStackRef new_frame; oparg = 0; assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17444,21 +17444,21 @@ _PyStackRef new_frame; oparg = 1; assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17473,21 +17473,21 @@ _PyStackRef new_frame; oparg = 2; assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17502,21 +17502,21 @@ _PyStackRef new_frame; oparg = 3; assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17531,21 +17531,166 @@ _PyStackRef new_frame; oparg = 4; assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < (oparg >> 1); i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - (oparg >> 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_5_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_6_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < (oparg >> 1); i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - (oparg >> 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_7_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < (oparg >> 1); i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - (oparg >> 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_8_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 8; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < (oparg >> 1); i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - (oparg >> 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_9_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 9; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < (oparg >> 1); i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17559,21 +17704,21 @@ _PyStackRef callable; _PyStackRef new_frame; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17884,7 +18029,7 @@ oparg = CURRENT_OPARG(); arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -17907,7 +18052,7 @@ oparg = CURRENT_OPARG(); arg = _stack_item_0; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -17931,7 +18076,7 @@ oparg = CURRENT_OPARG(); arg = _stack_item_1; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -17956,7 +18101,7 @@ oparg = CURRENT_OPARG(); arg = _stack_item_2; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -18073,7 +18218,7 @@ oparg = CURRENT_OPARG(); arg = _stack_item_2; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); stack_pointer[0] = _stack_item_0; stack_pointer[1] = _stack_item_1; @@ -18206,7 +18351,7 @@ oparg = CURRENT_OPARG(); arg = _stack_item_2; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); stack_pointer[0] = _stack_item_0; stack_pointer[1] = _stack_item_1; @@ -18239,8 +18384,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { @@ -18270,8 +18415,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyStackRef_IsNull(self_or_null)); assert(PyType_Check(callable_o)); @@ -18299,8 +18444,8 @@ self_or_null = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -18321,9 +18466,9 @@ _PyStackRef init; _PyStackRef init_frame; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - init = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self = stack_pointer[-1 - (oparg >> 1)]; + init = stack_pointer[-2 - (oparg >> 1)]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -18335,9 +18480,9 @@ assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, (oparg >> 1)+1, NULL, shim); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -18390,7 +18535,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyType_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -18415,10 +18560,10 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -18438,7 +18583,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -18457,8 +18602,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -18471,7 +18616,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -18495,9 +18640,9 @@ _PyStackRef c; _PyStackRef s; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { args--; @@ -18522,7 +18667,7 @@ _tos_cache1 = c; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -18533,7 +18678,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -18558,10 +18703,10 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -18582,7 +18727,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -18600,7 +18745,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -18625,10 +18770,10 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -18645,7 +18790,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -19072,7 +19217,7 @@ arg = stack_pointer[-1]; self = stack_pointer[-2]; callable = stack_pointer[-3]; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); @@ -19129,7 +19274,7 @@ arg = _stack_item_0; self = stack_pointer[-1]; callable = stack_pointer[-2]; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); @@ -19188,7 +19333,7 @@ arg = _stack_item_1; self = _stack_item_0; callable = stack_pointer[-1]; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); @@ -19249,7 +19394,7 @@ arg = _stack_item_2; self = _stack_item_1; callable = _stack_item_0; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); @@ -19299,9 +19444,9 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -19315,7 +19460,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -19347,9 +19492,9 @@ _PyStackRef s; _PyStackRef a; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; _PyStackRef *arguments = args; @@ -19378,8 +19523,8 @@ _tos_cache1 = s; _tos_cache0 = c; SET_CURRENT_CACHED_VALUES(3); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -19505,9 +19650,9 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -19521,7 +19666,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -19550,12 +19695,12 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -19584,7 +19729,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -19650,9 +19795,9 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -19666,7 +19811,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -19697,12 +19842,12 @@ _PyStackRef c; _PyStackRef s; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + assert((oparg >> 1) == 1 || !PyStackRef_IsNull(self_or_null)); if (!PyStackRef_IsNull(self_or_null)) { args--; } @@ -19727,7 +19872,7 @@ _tos_cache1 = c; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -19780,9 +19925,9 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -19796,7 +19941,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -19824,12 +19969,12 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -19858,7 +20003,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -19927,8 +20072,8 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -19936,8 +20081,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; stack_pointer[0] = _stack_item_0; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -19951,8 +20096,8 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -19970,11 +20115,11 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); kwnames = _stack_item_0; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -20001,7 +20146,7 @@ _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -20022,7 +20167,7 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -20051,8 +20196,8 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -20093,8 +20238,8 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; assert(PyStackRef_IsNull(self_or_null)); _PyStackRef callable_s = callable; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -20102,8 +20247,8 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; stack_pointer[0] = _stack_item_0; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -20127,7 +20272,7 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -20158,13 +20303,13 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); kwnames = _stack_item_0; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -20182,7 +20327,7 @@ kwnames); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -20192,7 +20337,7 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ee9b64107bb9f24..2dd50328e818e62 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1831,8 +1831,8 @@ _PyStackRef res; // _SPECIALIZE_CALL { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -1840,7 +1840,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -1858,8 +1858,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -1868,9 +1868,9 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -1882,8 +1882,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -1891,7 +1891,7 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); @@ -1899,8 +1899,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( @@ -1914,7 +1914,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -1922,8 +1922,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -1965,8 +1965,8 @@ } // _CHECK_OBJECT { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { @@ -2022,8 +2022,8 @@ self_or_null = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2031,7 +2031,7 @@ } // _CREATE_INIT_FRAME { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; self = self_or_null; init = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -2045,9 +2045,9 @@ assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, (oparg >> 1)+1, NULL, shim); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2110,8 +2110,8 @@ } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -2132,8 +2132,8 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2162,7 +2162,7 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2189,13 +2189,13 @@ } // _INIT_CALL_PY_EXACT_ARGS { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); @@ -2213,7 +2213,7 @@ { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2260,8 +2260,8 @@ } // _CHECK_METHOD_VERSION { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -2296,8 +2296,8 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2314,9 +2314,9 @@ } // _PY_FRAME_GENERAL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -2331,7 +2331,7 @@ args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -2388,7 +2388,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_CLASS { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2404,9 +2404,9 @@ } // _CALL_BUILTIN_CLASS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2425,7 +2425,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2433,16 +2433,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2481,7 +2481,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_FAST { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2497,9 +2497,9 @@ } // _CALL_BUILTIN_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2519,7 +2519,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2527,16 +2527,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2575,7 +2575,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2591,9 +2591,9 @@ } // _CALL_BUILTIN_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2609,7 +2609,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2617,16 +2617,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2668,8 +2668,8 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_O { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2682,7 +2682,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -2702,7 +2702,7 @@ } // _CALL_BUILTIN_O { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { args--; @@ -2726,9 +2726,9 @@ // _POP_TOP { value = s; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer += -oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3394,8 +3394,8 @@ _PyStackRef res; // _SPECIALIZE_CALL_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -3403,7 +3403,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -3421,8 +3421,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -3432,10 +3432,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3448,8 +3448,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -3457,7 +3457,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3470,8 +3470,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( @@ -3485,14 +3485,14 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3525,8 +3525,8 @@ } // _CHECK_METHOD_VERSION_KW { - null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -3561,8 +3561,8 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); @@ -3572,9 +3572,9 @@ // _PY_FRAME_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3598,7 +3598,7 @@ _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -3657,7 +3657,7 @@ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE_KW { - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); @@ -3673,12 +3673,12 @@ // _CALL_KW_NON_PY { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3693,7 +3693,7 @@ kwnames); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -3701,8 +3701,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3742,7 +3742,7 @@ } // _CHECK_FUNCTION_VERSION_KW { - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -3768,10 +3768,10 @@ // _PY_FRAME_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3795,7 +3795,7 @@ _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -3973,7 +3973,7 @@ { arg = stack_pointer[-1]; self = nos; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); @@ -4050,9 +4050,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4066,7 +4066,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4087,7 +4087,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4115,7 +4115,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -4125,13 +4125,13 @@ { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4170,9 +4170,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4186,7 +4186,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4208,7 +4208,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4236,7 +4236,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -4246,13 +4246,13 @@ { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4294,9 +4294,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4310,7 +4310,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4339,7 +4339,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + assert((oparg >> 1) == 1 || !PyStackRef_IsNull(self_or_null)); if (!PyStackRef_IsNull(self_or_null)) { args--; } @@ -4363,9 +4363,9 @@ // _POP_TOP { value = s; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer += -oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4418,9 +4418,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_O { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4434,7 +4434,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4488,10 +4488,10 @@ // _POP_TOP { value = a; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer[-oparg] = s; - stack_pointer += 1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer[-(oparg >> 1)] = s; + stack_pointer += 1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4551,7 +4551,7 @@ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -4566,12 +4566,12 @@ } // _CALL_NON_PY_GENERAL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4586,7 +4586,7 @@ PyStackRef_NULL); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -4594,8 +4594,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4634,7 +4634,7 @@ } // _CHECK_FUNCTION_VERSION { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -4651,12 +4651,12 @@ } // _CHECK_FUNCTION_EXACT_ARGS { - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4683,13 +4683,13 @@ } // _INIT_CALL_PY_EXACT_ARGS { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); @@ -4707,7 +4707,7 @@ { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4753,7 +4753,7 @@ } // _CHECK_FUNCTION_VERSION { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -4778,10 +4778,10 @@ } // _PY_FRAME_GENERAL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -4796,7 +4796,7 @@ args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -4876,7 +4876,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4954,7 +4954,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -5032,7 +5032,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -7111,8 +7111,8 @@ /* Skip 3 cache entries */ // _MAYBE_EXPAND_METHOD { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7120,8 +7120,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7130,7 +7130,7 @@ } // _MONITOR_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; maybe_self = self_or_null; func = callable; int is_meth = !PyStackRef_IsNull(maybe_self); @@ -7139,14 +7139,14 @@ if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); } - else if (oparg) { + else if ((oparg >> 1)) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { arg0 = &_PyInstrumentation_MISSING; } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; + stack_pointer[-2 - (oparg >> 1)] = func; + stack_pointer[-1 - (oparg >> 1)] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7160,11 +7160,11 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7183,7 +7183,7 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); @@ -7204,7 +7204,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -7212,8 +7212,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7428,8 +7428,8 @@ /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7437,8 +7437,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7447,7 +7447,7 @@ } // _MONITOR_CALL_KW { - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; int is_meth = !PyStackRef_IsNull(self_or_null); PyObject *arg; if (is_meth) { @@ -7460,8 +7460,8 @@ arg = &_PyInstrumentation_MISSING; } PyObject *function = PyStackRef_AsPyObjectBorrow(callable); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7475,10 +7475,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7498,7 +7498,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7524,14 +7524,14 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6284885c428832b..c6420a9c1bb4af6 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -902,8 +902,8 @@ dummy_func(void) { value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } - op(_POP_TOP_OPARG, (args[oparg] --)) { - for (int i = oparg-1; i >= 0; i--) { + op(_POP_TOP_OPARG, (args[oparg >> 1] --)) { + for (int i = (oparg >> 1)-1; i >= 0; i--) { optimize_pop_top(ctx, this_instr, args[i]); } } @@ -1139,7 +1139,7 @@ dummy_func(void) { new_frame = PyJitRef_WrapInvalid(f); } - op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *bound_method = sym_get_probable_value(callable); if (bound_method != NULL && Py_TYPE(bound_method) == &PyMethod_Type) { PyMethodObject *method = (PyMethodObject *)bound_method; @@ -1154,7 +1154,7 @@ dummy_func(void) { } } - op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *func = sym_get_probable_value(callable); if (func == NULL || !PyFunction_Check(func) || ((PyFunctionObject *)func)->func_version != func_version) { ctx->contradiction = true; @@ -1166,7 +1166,7 @@ dummy_func(void) { _Py_BloomFilter_Add(dependencies, func); } - op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { + op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, unused, unused[oparg >> 1], unused -- callable, unused, unused[oparg >> 1], unused)) { PyObject *func = sym_get_probable_value(callable); if (func == NULL || !PyFunction_Check(func) || ((PyFunctionObject *)func)->func_version != func_version) { ctx->contradiction = true; @@ -1178,7 +1178,7 @@ dummy_func(void) { _Py_BloomFilter_Add(dependencies, func); } - op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg >> 1] -- callable, null, unused[oparg >> 1])) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); @@ -1201,7 +1201,7 @@ dummy_func(void) { sym_set_type(callable, &PyMethod_Type); } - op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg], unused -- callable, null, unused[oparg], unused)) { + op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg >> 1], unused -- callable, null, unused[oparg >> 1], unused)) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); @@ -1224,26 +1224,26 @@ dummy_func(void) { sym_set_type(callable, &PyMethod_Type); } - op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { assert(sym_matches_type(callable, &PyFunction_Type)); if (sym_is_const(ctx, callable)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + sym_is_not_null(self_or_null)) { + if (co->co_argcount == (oparg >> 1) + sym_is_not_null(self_or_null)) { ADD_OP(_NOP, 0 ,0); } } } } - op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg >> 1] -- callable, null, unused[oparg >> 1])) { sym_set_null(null); sym_set_type(callable, &PyMethod_Type); } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { - int argcount = oparg; + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg >> 1] -- new_frame)) { + int argcount = (oparg >> 1); assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { @@ -1259,7 +1259,7 @@ dummy_func(void) { } } - op(_EXPAND_METHOD, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_EXPAND_METHOD, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); callable = sym_new_const(ctx, method->im_func); @@ -1271,7 +1271,7 @@ dummy_func(void) { } } - op(_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + op(_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg >> 1], unused -- callable, self_or_null, unused[oparg >> 1], unused)) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); callable = sym_new_const(ctx, method->im_func); @@ -1283,17 +1283,17 @@ dummy_func(void) { } } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { (void)args; callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); } - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg >> 1] -- new_frame)) { new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg >> 1], kwnames -- new_frame)) { new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); } @@ -1301,7 +1301,7 @@ dummy_func(void) { ex_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, func_st, NULL, 0)); } - op(_CHECK_OBJECT, (type_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_OBJECT, (type_version/2, callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *probable_callable = sym_get_probable_value(callable); assert(probable_callable != NULL); PyObject *const_callable = sym_get_const(ctx, callable); @@ -1332,12 +1332,12 @@ dummy_func(void) { } } - op(_ALLOCATE_OBJECT, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_ALLOCATE_OBJECT, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { self_or_null = sym_new_not_null(ctx); } - op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - ctx->frame->stack_pointer = stack_pointer - oparg - 2; + op(_CREATE_INIT_FRAME, (init, self, args[oparg >> 1] -- init_frame)) { + ctx->frame->stack_pointer = stack_pointer - (oparg >> 1) - 2; _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, NULL, 0); if (shim == NULL) { break; @@ -1349,7 +1349,7 @@ dummy_func(void) { ctx->frame = shim; ctx->curr_frame_depth++; assert((this_instr + 1)->opcode == _PUSH_FRAME); - init_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, init, args-1, oparg+1)); + init_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, init, args-1, (oparg >> 1)+1)); } op(_RETURN_VALUE, (retval -- res)) { @@ -1539,7 +1539,7 @@ dummy_func(void) { gen_frame = PyJitRef_WrapInvalid(new_frame); } - op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyCodeObject *co = sym_get_probable_func_code(callable); if (co == NULL) { ctx->done = true; @@ -1552,7 +1552,7 @@ dummy_func(void) { (void)framesize; } - op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyTypeObject *type = sym_get_type(callable); if (type && type != &PyFunction_Type && type != &PyMethod_Type) { ADD_OP(_NOP, 0, 0); @@ -1566,7 +1566,7 @@ dummy_func(void) { } } - op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { + op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg >> 1], unused -- callable, unused, unused[oparg >> 1], unused)) { PyTypeObject *type = sym_get_type(callable); if (type && type != &PyFunction_Type && type != &PyMethod_Type) { ADD_OP(_NOP, 0, 0); @@ -1691,7 +1691,7 @@ dummy_func(void) { none = sym_new_const(ctx, Py_None); } - op(_GUARD_CALLABLE_BUILTIN_CLASS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_CLASS, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyType_Type)) { PyTypeObject *tp = (PyTypeObject *)callable_o; @@ -1704,11 +1704,11 @@ dummy_func(void) { } } - op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -1723,7 +1723,7 @@ dummy_func(void) { } } - op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == @@ -1736,7 +1736,7 @@ dummy_func(void) { } } - op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == @@ -1749,11 +1749,11 @@ dummy_func(void) { } } - op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { callable = sym_new_not_null(ctx); } - op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) { + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg >> 1] -- res, c, s)) { res = sym_new_not_null(ctx); c = callable; if (sym_is_not_null(self_or_null)) { @@ -1768,19 +1768,19 @@ dummy_func(void) { } } - op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { callable = sym_new_not_null(ctx); } - op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { callable = sym_new_not_null(ctx); } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -1804,11 +1804,11 @@ dummy_func(void) { } } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -1833,11 +1833,11 @@ dummy_func(void) { } } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -1869,13 +1869,13 @@ dummy_func(void) { ctx->frame->is_c_recursion_checked = true; } - op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res, c, s)) { + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg >> 1] -- res, c, s)) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, (oparg >> 1) + 1, (uintptr_t)cfunc); } res = sym_new_not_null(ctx); c = callable; @@ -1891,33 +1891,33 @@ dummy_func(void) { } } - op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, (oparg >> 1), (uintptr_t)cfunc); } callable = sym_new_not_null(ctx); } - op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, (oparg >> 1), (uintptr_t)cfunc); } callable = sym_new_not_null(ctx); } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -1942,13 +1942,13 @@ dummy_func(void) { } } - op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg >> 1] -- res, c, s, a)) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, (oparg >> 1) + 1, (uintptr_t)cfunc); } res = sym_new_not_null(ctx); c = callable; @@ -2649,15 +2649,15 @@ dummy_func(void) { sym_set_recorded_value(value, (PyObject *)this_instr->operand0); } - op(_RECORD_CALLABLE, (func, self, args[oparg] -- func, self, args[oparg])) { + op(_RECORD_CALLABLE, (func, self, args[oparg >> 1] -- func, self, args[oparg >> 1])) { sym_set_recorded_value(func, (PyObject *)this_instr->operand0); } - op(_RECORD_CALLABLE_KW, (func, self, args[oparg], kwnames -- func, self, args[oparg], kwnames)) { + op(_RECORD_CALLABLE_KW, (func, self, args[oparg >> 1], kwnames -- func, self, args[oparg >> 1], kwnames)) { sym_set_recorded_value(func, (PyObject *)this_instr->operand0); } - op(_RECORD_BOUND_METHOD, (callable, self, args[oparg] -- callable, self, args[oparg])) { + op(_RECORD_BOUND_METHOD, (callable, self, args[oparg >> 1] -- callable, self, args[oparg >> 1])) { sym_set_recorded_value(callable, (PyObject *)this_instr->operand0); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9cd20c495db4418..d446e9d9828358a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -167,12 +167,12 @@ case _POP_TOP_OPARG: { JitOptRef *args; - args = &stack_pointer[-oparg]; - for (int i = oparg-1; i >= 0; i--) { + args = &stack_pointer[-(oparg >> 1)]; + for (int i = (oparg >> 1)-1; i >= 0; i--) { optimize_pop_top(ctx, this_instr, args[i]); } - CHECK_STACK_BOUNDS(-oparg); - stack_pointer += -oparg; + CHECK_STACK_BOUNDS(-(oparg >> 1)); + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4047,14 +4047,14 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; (void)args; callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; break; } @@ -4065,18 +4065,18 @@ case _PY_FRAME_GENERAL: { JitOptRef callable; JitOptRef new_frame; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); - CHECK_STACK_BOUNDS(-1 - oparg); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + CHECK_STACK_BOUNDS(-1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = new_frame; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)this_instr->operand0; PyObject *func = sym_get_probable_value(callable); if (func == NULL || !PyFunction_Check(func) || ((PyFunctionObject *)func)->func_version != func_version) { @@ -4095,7 +4095,7 @@ case _CHECK_METHOD_VERSION: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); @@ -4122,8 +4122,8 @@ case _EXPAND_METHOD: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); callable = sym_new_const(ctx, method->im_func); @@ -4133,14 +4133,14 @@ callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; break; } case _CHECK_IS_NOT_PY_CALLABLE: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyTypeObject *type = sym_get_type(callable); if (type && type != &PyFunction_Type && type != &PyMethod_Type) { ADD_OP(_NOP, 0, 0); @@ -4151,9 +4151,9 @@ case _CALL_NON_PY_GENERAL: { JitOptRef res; res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-1 - oparg); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + CHECK_STACK_BOUNDS(-1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4161,8 +4161,8 @@ case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { JitOptRef null; JitOptRef callable; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; sym_set_null(null); sym_set_type(callable, &PyMethod_Type); break; @@ -4171,8 +4171,8 @@ case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *bound_method = sym_get_probable_value(callable); if (bound_method != NULL && Py_TYPE(bound_method) == &PyMethod_Type) { PyMethodObject *method = (PyMethodObject *)bound_method; @@ -4185,8 +4185,8 @@ callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; break; } @@ -4200,14 +4200,14 @@ case _CHECK_FUNCTION_EXACT_ARGS: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; assert(sym_matches_type(callable, &PyFunction_Type)); if (sym_is_const(ctx, callable)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + sym_is_not_null(self_or_null)) { + if (co->co_argcount == (oparg >> 1) + sym_is_not_null(self_or_null)) { ADD_OP(_NOP, 0 ,0); } } @@ -4217,7 +4217,7 @@ case _CHECK_STACK_SPACE: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyCodeObject *co = sym_get_probable_func_code(callable); if (co == NULL) { ctx->done = true; @@ -4236,10 +4236,10 @@ JitOptRef self_or_null; JitOptRef callable; JitOptRef new_frame; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int argcount = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int argcount = (oparg >> 1); assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { @@ -4251,9 +4251,9 @@ } else { new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); } - CHECK_STACK_BOUNDS(-1 - oparg); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + CHECK_STACK_BOUNDS(-1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = new_frame; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4408,8 +4408,8 @@ case _CHECK_OBJECT: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t type_version = (uint32_t)this_instr->operand0; PyObject *probable_callable = sym_get_probable_value(callable); assert(probable_callable != NULL); @@ -4431,21 +4431,21 @@ assert(init != NULL); assert(PyFunction_Check(init)); callable = sym_new_const(ctx, init); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; watch_type((PyTypeObject *)callable_o, dependencies); } else { callable = sym_new_not_null(ctx); } - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } case _ALLOCATE_OBJECT: { JitOptRef self_or_null; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; self_or_null = sym_new_not_null(ctx); - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; break; } @@ -4454,10 +4454,10 @@ JitOptRef self; JitOptRef init; JitOptRef init_frame; - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - init = stack_pointer[-2 - oparg]; - ctx->frame->stack_pointer = stack_pointer - oparg - 2; + args = &stack_pointer[-(oparg >> 1)]; + self = stack_pointer[-1 - (oparg >> 1)]; + init = stack_pointer[-2 - (oparg >> 1)]; + ctx->frame->stack_pointer = stack_pointer - (oparg >> 1) - 2; _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, NULL, 0); if (shim == NULL) { break; @@ -4468,10 +4468,10 @@ ctx->frame = shim; ctx->curr_frame_depth++; assert((this_instr + 1)->opcode == _PUSH_FRAME); - init_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, init, args-1, oparg+1)); - CHECK_STACK_BOUNDS(-1 - oparg); - stack_pointer[-2 - oparg] = init_frame; - stack_pointer += -1 - oparg; + init_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, init, args-1, (oparg >> 1)+1)); + CHECK_STACK_BOUNDS(-1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = init_frame; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4485,7 +4485,7 @@ case _GUARD_CALLABLE_BUILTIN_CLASS: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyType_Type)) { PyTypeObject *tp = (PyTypeObject *)callable_o; @@ -4501,21 +4501,21 @@ case _CALL_BUILTIN_CLASS: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; callable = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } case _GUARD_CALLABLE_BUILTIN_O: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -4538,9 +4538,9 @@ JitOptRef res; JitOptRef c; JitOptRef s; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; res = sym_new_not_null(ctx); c = callable; if (sym_is_not_null(self_or_null)) { @@ -4553,18 +4553,18 @@ else { s = sym_new_unknown(ctx); } - CHECK_STACK_BOUNDS(1 - oparg); - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer[-oparg] = s; - stack_pointer += 1 - oparg; + CHECK_STACK_BOUNDS(1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer[-(oparg >> 1)] = s; + stack_pointer += 1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_BUILTIN_FAST: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == @@ -4580,15 +4580,15 @@ case _CALL_BUILTIN_FAST: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; callable = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == @@ -4604,9 +4604,9 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; callable = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } @@ -4761,13 +4761,13 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -4800,15 +4800,15 @@ JitOptRef c; JitOptRef s; JitOptRef a; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, (oparg >> 1) + 1, (uintptr_t)cfunc); } res = sym_new_not_null(ctx); c = callable; @@ -4821,12 +4821,12 @@ s = sym_new_unknown(ctx); a = sym_new_unknown(ctx); } - CHECK_STACK_BOUNDS(2 - oparg); - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer[-oparg] = s; - stack_pointer[1 - oparg] = a; - stack_pointer += 2 - oparg; + CHECK_STACK_BOUNDS(2 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer[-(oparg >> 1)] = s; + stack_pointer[1 - (oparg >> 1)] = a; + stack_pointer += 2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4862,13 +4862,13 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -4897,17 +4897,17 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, (oparg >> 1), (uintptr_t)cfunc); } callable = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } @@ -4919,13 +4919,13 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -4958,15 +4958,15 @@ JitOptRef res; JitOptRef c; JitOptRef s; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, (oparg >> 1) + 1, (uintptr_t)cfunc); } res = sym_new_not_null(ctx); c = callable; @@ -4980,11 +4980,11 @@ else { s = sym_new_unknown(ctx); } - CHECK_STACK_BOUNDS(1 - oparg); - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer[-oparg] = s; - stack_pointer += 1 - oparg; + CHECK_STACK_BOUNDS(1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer[-(oparg >> 1)] = s; + stack_pointer += 1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5009,13 +5009,13 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -5044,17 +5044,17 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, (oparg >> 1), (uintptr_t)cfunc); } callable = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } @@ -5073,18 +5073,18 @@ case _PY_FRAME_KW: { JitOptRef callable; JitOptRef new_frame; - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); - CHECK_STACK_BOUNDS(-2 - oparg); - stack_pointer[-3 - oparg] = new_frame; - stack_pointer += -2 - oparg; + CHECK_STACK_BOUNDS(-2 - (oparg >> 1)); + stack_pointer[-3 - (oparg >> 1)] = new_frame; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION_KW: { JitOptRef callable; - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = (uint32_t)this_instr->operand0; PyObject *func = sym_get_probable_value(callable); if (func == NULL || !PyFunction_Check(func) || ((PyFunctionObject *)func)->func_version != func_version) { @@ -5099,7 +5099,7 @@ case _CHECK_METHOD_VERSION_KW: { JitOptRef callable; - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); @@ -5126,8 +5126,8 @@ case _EXPAND_METHOD_KW: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); callable = sym_new_const(ctx, method->im_func); @@ -5137,14 +5137,14 @@ callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; break; } case _CHECK_IS_NOT_PY_CALLABLE_KW: { JitOptRef callable; - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; PyTypeObject *type = sym_get_type(callable); if (type && type != &PyFunction_Type && type != &PyMethod_Type) { ADD_OP(_NOP, 0, 0); @@ -5155,9 +5155,9 @@ case _CALL_KW_NON_PY: { JitOptRef res; res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-2 - oparg); - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + CHECK_STACK_BOUNDS(-2 - (oparg >> 1)); + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5824,21 +5824,21 @@ case _RECORD_CALLABLE: { JitOptRef func; - func = stack_pointer[-2 - oparg]; + func = stack_pointer[-2 - (oparg >> 1)]; sym_set_recorded_value(func, (PyObject *)this_instr->operand0); break; } case _RECORD_CALLABLE_KW: { JitOptRef func; - func = stack_pointer[-3 - oparg]; + func = stack_pointer[-3 - (oparg >> 1)]; sym_set_recorded_value(func, (PyObject *)this_instr->operand0); break; } case _RECORD_BOUND_METHOD: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; sym_set_recorded_value(callable, (PyObject *)this_instr->operand0); break; } diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 98abe3d0505e202..75b5ba65f972f15 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -71,21 +71,21 @@ void _PyOpcode_RecordFunction_4OS(_PyInterpreterFrame *frame, _PyStackRef *stack void _PyOpcode_RecordFunction_CALLABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { _PyStackRef func; - func = stack_pointer[-2 - oparg]; + func = stack_pointer[-2 - (oparg >> 1)]; *recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(func); Py_INCREF(*recorded_value); } void _PyOpcode_RecordFunction_CALLABLE_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { _PyStackRef func; - func = stack_pointer[-3 - oparg]; + func = stack_pointer[-3 - (oparg >> 1)]; *recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(func); Py_INCREF(*recorded_value); } void _PyOpcode_RecordFunction_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { _PyStackRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) == &PyMethod_Type) { *recorded_value = (PyObject *)callable_o; diff --git a/Python/specialize.c b/Python/specialize.c index 067bae59f6073b7..cb84fb4b81866da 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1660,7 +1660,7 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) assert(PyType_Check(callable)); PyTypeObject *tp = _PyType_CAST(callable); if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { - int oparg = instr->op.arg; + int oparg = instr->op.arg >> 1; if (nargs == 1 && oparg == 1) { if (tp == &PyUnicode_Type) { specialize(instr, CALL_STR_1); @@ -1727,7 +1727,7 @@ specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, } PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *list_append = interp->callable_cache.list_append; - int oparg = instr->op.arg; + int oparg = instr->op.arg >> 1; if ((PyObject *)descr == list_append && oparg == 1) { assert(self_or_null != NULL); if (PyList_CheckExact(self_or_null)) { @@ -1839,7 +1839,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } /* len(o) */ PyInterpreterState *interp = _PyInterpreterState_GET(); - if (callable == interp->callable_cache.len && instr->op.arg == 1) { + if (callable == interp->callable_cache.len && (instr->op.arg >> 1) == 1) { specialize(instr, CALL_LEN); return 0; } @@ -1850,7 +1850,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) if (nargs == 2) { /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); - if (callable == interp->callable_cache.isinstance && instr->op.arg == 2) { + if (callable == interp->callable_cache.isinstance && (instr->op.arg >> 1) == 2) { specialize(instr, CALL_ISINSTANCE); return 0; } From 35215db3c38a1ae7f7c5c3e575435c6a1409e8b3 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 18 Nov 2026 21:32:08 +0100 Subject: [PATCH 2/3] gh-161170: Compile awaited calls with an extra slot for GET_AWAITABLE GET_AWAITABLE now also pushes the NULL and None that start the following SEND loop, so the compiler no longer emits them. `await ` sets the awaited flag of the call and pushes an extra NULL below its callable. GET_AWAITABLE with the low bit of its oparg set pops that slot as well. Nothing writes to it yet, so there is no behavior change; it will report how a coroutine body that ran without a coroutine object ended. Co-Authored-By: Claude Opus 5.5 --- Doc/library/dis.rst | 19 +++-- Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_uop_ids.h | 2 +- Include/internal/pycore_uop_metadata.h | 8 +-- Lib/dis.py | 4 ++ Lib/test/test_dis.py | 14 ++-- Modules/_testinternalcapi/test_cases.c.h | 17 ++++- Python/bytecodes.c | 11 ++- Python/codegen.c | 86 +++++++++++++---------- Python/executor_cases.c.h | 21 ++++-- Python/generated_cases.c.h | 17 ++++- Python/optimizer_bytecodes.c | 7 ++ Python/optimizer_cases.c.h | 13 +++- 13 files changed, 150 insertions(+), 73 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 96b4003ef93d34e..ef16d9c7d0c53ff 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -820,19 +820,28 @@ not have to be) the original ``STACK[-2]``. Implements ``STACK[-1] = get_awaitable(STACK[-1])``, where ``get_awaitable(o)`` returns ``o`` if ``o`` is a coroutine object or a generator object with the :data:`~inspect.CO_ITERABLE_COROUTINE` flag, or resolves - ``o.__await__``. + ``o.__await__``. Then pushes ``NULL`` and ``None``, which start the + :opcode:`SEND` loop that follows. - If the ``where`` operand is nonzero, it indicates where the instruction - occurs: + If ``where >> 1`` is nonzero, it indicates where the instruction occurs: - * ``1``: After a call to ``__aenter__`` - * ``2``: After a call to ``__aexit__`` + * ``1``: After a call to ``__aenter__`` + * ``2``: After a call to ``__aexit__`` + + If the low bit of ``where`` is set, the awaited object is the result of an + awaited :opcode:`CALL`, :opcode:`CALL_KW` or :opcode:`CALL_FUNCTION_EX`, + and the instruction also pops the extra slot that was pushed below that + call's callable. .. versionadded:: 3.5 .. versionchanged:: 3.11 Previously, this instruction did not have an oparg. + .. versionchanged:: next + Pushes ``NULL`` and ``None`` for the following :opcode:`SEND`; + the low bit of ``where`` marks an awaited call. + .. opcode:: GET_AITER diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 9f96aa4ba98f37a..398583b79ba96e6 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -221,7 +221,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case GET_ANEXT: return 1; case GET_AWAITABLE: - return 1; + return 1 + (oparg & 1); case GET_ITER: return 1; case GET_ITER_SELF: @@ -716,7 +716,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case GET_ANEXT: return 2; case GET_AWAITABLE: - return 1; + return 3; case GET_ITER: return 2; case GET_ITER_SELF: diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 3f582b7dcb82453..1c6ee2b27608262 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -695,7 +695,7 @@ enum { _FOR_ITER_VIRTUAL_TIER_TWO_r23, _GET_AITER_r11, _GET_ANEXT_r12, - _GET_AWAITABLE_r11, + _GET_AWAITABLE_r13, _GET_ITER_r12, _GET_ITER_TRAD_r12, _GET_LEN_r12, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f6ffadf77282b40..17c359876174196 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -1644,7 +1644,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 1, _GET_AWAITABLE_r11 }, + { 3, 1, _GET_AWAITABLE_r13 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -4334,7 +4334,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_RETURN_VALUE_r11] = _RETURN_VALUE, [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, - [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_GET_AWAITABLE_r13] = _GET_AWAITABLE, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, @@ -5293,7 +5293,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GET_ANEXT] = "_GET_ANEXT", [_GET_ANEXT_r12] = "_GET_ANEXT_r12", [_GET_AWAITABLE] = "_GET_AWAITABLE", - [_GET_AWAITABLE_r11] = "_GET_AWAITABLE_r11", + [_GET_AWAITABLE_r13] = "_GET_AWAITABLE_r13", [_GET_ITER] = "_GET_ITER", [_GET_ITER_r12] = "_GET_ITER_r12", [_GET_ITER_TRAD] = "_GET_ITER_TRAD", @@ -6528,7 +6528,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _GET_ANEXT: return 0; case _GET_AWAITABLE: - return 1; + return 1 + (oparg & 1); case _SEND_GEN_FRAME: return 1; case _GUARD_TOS_IS_NONE: diff --git a/Lib/dis.py b/Lib/dis.py index 54df95d6bb0689e..a1ce9a1620fb135 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -47,6 +47,7 @@ CALL = opmap['CALL'] CALL_KW = opmap['CALL_KW'] CALL_FUNCTION_EX = opmap['CALL_FUNCTION_EX'] +GET_AWAITABLE = opmap['GET_AWAITABLE'] CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] @@ -653,6 +654,9 @@ def get_argval_argrepr(self, op, arg, offset): elif deop == CALL_FUNCTION_EX: if arg & 1: argrepr = "awaited" + elif deop == GET_AWAITABLE: + if arg & 1: + argrepr = "after call" elif deop == CALL_INTRINSIC_1: argrepr = _intrinsic_1_descs[arg] elif deop == CALL_INTRINSIC_2: diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 351e1eff4cb5b9f..5a3de22fd6bb2fc 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -614,9 +614,7 @@ async def _asyncwith(c): SWAP 3 LOAD_SPECIAL 2 (__aenter__) CALL 0 (0) - GET_AWAITABLE 1 - PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) + GET_AWAITABLE 2 L2: SEND 4 (to L5) L3: YIELD_VALUE 1 L4: RESUME 3 @@ -631,9 +629,7 @@ async def _asyncwith(c): LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) CALL 6 (3) - GET_AWAITABLE 2 - PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) + GET_AWAITABLE 4 L8: SEND 4 (to L11) L9: YIELD_VALUE 1 L10: RESUME 3 @@ -647,14 +643,12 @@ async def _asyncwith(c): RETURN_VALUE %4d L12: CLEANUP_THROW - L13: JUMP_BACKWARD_NO_INTERRUPT 28 (to L5) + L13: JUMP_BACKWARD_NO_INTERRUPT 26 (to L5) L14: CLEANUP_THROW L15: JUMP_BACKWARD_NO_INTERRUPT 10 (to L11) L16: PUSH_EXC_INFO WITH_EXCEPT_START - GET_AWAITABLE 2 - PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) + GET_AWAITABLE 4 L17: SEND 5 (to L21) L18: YIELD_VALUE 1 L19: RESUME 3 diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 91a736ce871c538..05a81652a9f7e23 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -6822,12 +6822,17 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef *aux; _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; + _PyStackRef none; iterable = stack_pointer[-1]; + aux = &stack_pointer[-1 - (oparg & 1)]; + assert((oparg & 1) == 0 || PyStackRef_IsNull(aux[0])); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -6836,11 +6841,17 @@ PyStackRef_CLOSE(iterable); _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { + stack_pointer += -(oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; + null = PyStackRef_NULL; + none = PyStackRef_None; + stack_pointer[-(oparg & 1)] = iter; + stack_pointer[1 - (oparg & 1)] = null; + stack_pointer[2 - (oparg & 1)] = none; + stack_pointer += 3 - (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0a8147e19fcd3dc..b735c653cf43733 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1649,11 +1649,18 @@ dummy_func( awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); } - inst(GET_AWAITABLE, (iterable -- iter)) { - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + // Also pushes the NULL and None that the following SEND loop starts + // with. If the low bit of oparg is set, `iterable` is the result of + // an awaited call and `aux` is the slot pushed below that call. + inst(GET_AWAITABLE, (aux[oparg & 1], iterable -- iter, null, none)) { + assert((oparg & 1) == 0 || PyStackRef_IsNull(aux[0])); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); PyStackRef_CLOSE(iterable); + DEAD(aux); ERROR_IF(iter_o == NULL); iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; } family(SEND, INLINE_CACHE_ENTRIES_SEND) = { diff --git a/Python/codegen.c b/Python/codegen.c index 2a651037e707fb5..647d6a18336fe81 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -224,7 +224,8 @@ static int codegen_call_helper_impl(compiler *c, location loc, int n, /* Args already pushed */ asdl_expr_seq *args, PyObject *injected_arg, - asdl_keyword_seq *keywords); + asdl_keyword_seq *keywords, + int awaited); static int codegen_call_helper(compiler *c, location loc, int n, asdl_expr_seq *args, asdl_keyword_seq *keywords); @@ -280,6 +281,12 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) * with the low bit set when the result is awaited right away. */ #define CALL_OPARG(NARGS, AWAITED) (((NARGS) << 1) | (AWAITED)) +/* The oparg of GET_AWAITABLE: where the awaitable comes from (for error + * messages) shifted left by one, with the low bit set when it is the + * result of an awaited call, preceded on the stack by an extra slot + * pushed for that call (see codegen_await()). */ +#define GET_AWAITABLE_OPARG(WHERE, AFTER_CALL) (((WHERE) << 1) | (AFTER_CALL)) + #define ADDOP_I(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) @@ -621,9 +628,7 @@ codegen_unwind_fblock(compiler *c, location *ploc, } RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc)); if (info->fb_type == COMPILE_FBLOCK_ASYNC_WITH) { - ADDOP_I(c, *ploc, GET_AWAITABLE, 2); - ADDOP(c, *ploc, PUSH_NULL); - ADDOP_LOAD_CONST(c, *ploc, Py_None); + ADDOP_I(c, *ploc, GET_AWAITABLE, GET_AWAITABLE_OPARG(2, 0)); ADD_YIELD_FROM(c, *ploc, 1); } ADDOP(c, *ploc, POP_TOP); @@ -1762,7 +1767,7 @@ codegen_class(compiler *c, stmt_ty s) RETURN_IF_ERROR_IN_SCOPE(c, codegen_call_helper_impl(c, loc, 2, s->v.ClassDef.bases, &_Py_STR(generic_base), - s->v.ClassDef.keywords)); + s->v.ClassDef.keywords, 0)); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); @@ -4156,7 +4161,7 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) // Return 1 if the method call was optimized, 0 if not, and -1 on error. static int -maybe_optimize_method_call(compiler *c, expr_ty e) +maybe_optimize_method_call(compiler *c, expr_ty e, int awaited) { Py_ssize_t argsl, i, kwdsl; expr_ty meth = e->v.Call.func; @@ -4221,11 +4226,11 @@ maybe_optimize_method_call(compiler *c, expr_ty e) RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, kwds, kwdsl)); loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL_KW, CALL_OPARG(argsl + kwdsl, 0)); + ADDOP_I(c, loc, CALL_KW, CALL_OPARG(argsl + kwdsl, awaited)); } else { loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL, CALL_OPARG(argsl, 0)); + ADDOP_I(c, loc, CALL, CALL_OPARG(argsl, awaited)); } return 1; } @@ -4250,10 +4255,10 @@ codegen_validate_keywords(compiler *c, asdl_keyword_seq *keywords) } static int -codegen_call(compiler *c, expr_ty e) +codegen_call(compiler *c, expr_ty e, int awaited) { RETURN_IF_ERROR(codegen_validate_keywords(c, e->v.Call.keywords)); - int ret = maybe_optimize_method_call(c, e); + int ret = maybe_optimize_method_call(c, e, awaited); if (ret < 0) { return ERROR; } @@ -4267,13 +4272,34 @@ codegen_call(compiler *c, expr_ty e) location loc = LOC(e->v.Call.func); ADDOP(c, loc, PUSH_NULL); loc = LOC(e); - ret = codegen_call_helper(c, loc, 0, - e->v.Call.args, - e->v.Call.keywords); + ret = codegen_call_helper_impl(c, loc, 0, e->v.Call.args, NULL, + e->v.Call.keywords, awaited); USE_LABEL(c, skip_normal_call); return ret; } +/* `await ` marks the call as awaited and pushes an extra slot + * below the callable: if the call runs a coroutine body without creating + * the coroutine object, the body reports there how it ended, and + * GET_AWAITABLE consumes it. */ +static int +codegen_await(compiler *c, expr_ty e) +{ + location loc = LOC(e); + expr_ty value = e->v.Await.value; + int after_call = value->kind == Call_kind; + if (after_call) { + ADDOP(c, LOC(value), PUSH_NULL); + RETURN_IF_ERROR(codegen_call(c, value, 1)); + } + else { + VISIT(c, expr, value); + } + ADDOP_I(c, loc, GET_AWAITABLE, GET_AWAITABLE_OPARG(0, after_call)); + ADD_YIELD_FROM(c, loc, 1); + return SUCCESS; +} + static int codegen_template_str(compiler *c, expr_ty e) { @@ -4462,7 +4488,8 @@ codegen_call_helper_impl(compiler *c, location loc, int n, /* Args already pushed */ asdl_expr_seq *args, PyObject *injected_arg, - asdl_keyword_seq *keywords) + asdl_keyword_seq *keywords, + int awaited) { Py_ssize_t i, nseen, nelts, nkwelts; @@ -4501,10 +4528,10 @@ codegen_call_helper_impl(compiler *c, location loc, VISIT_SEQ(c, keyword, keywords); RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, keywords, nkwelts)); - ADDOP_I(c, loc, CALL_KW, CALL_OPARG(n + nelts + nkwelts, 0)); + ADDOP_I(c, loc, CALL_KW, CALL_OPARG(n + nelts + nkwelts, awaited)); } else { - ADDOP_I(c, loc, CALL, CALL_OPARG(n + nelts, 0)); + ADDOP_I(c, loc, CALL, CALL_OPARG(n + nelts, awaited)); } return SUCCESS; @@ -4570,7 +4597,7 @@ codegen_call_helper(compiler *c, location loc, asdl_expr_seq *args, asdl_keyword_seq *keywords) { - return codegen_call_helper_impl(c, loc, n, args, NULL, keywords); + return codegen_call_helper_impl(c, loc, n, args, NULL, keywords, 0); } /* List and set comprehensions work by being inlined at the location where @@ -5170,9 +5197,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); if (is_async_comprehension && type != COMP_GENEXP) { - ADDOP_I(c, loc, GET_AWAITABLE, 0); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_I(c, loc, GET_AWAITABLE, GET_AWAITABLE_OPARG(0, 0)); ADD_YIELD_FROM(c, loc, 1); } @@ -5315,9 +5340,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, SWAP, 3); ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); - ADDOP_I(c, loc, GET_AWAITABLE, 1); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_I(c, loc, GET_AWAITABLE, GET_AWAITABLE_OPARG(1, 0)); ADD_YIELD_FROM(c, loc, 1); ADDOP_JUMP(c, loc, SETUP_WITH, final); @@ -5352,9 +5375,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) * call __exit__(None, None, None) */ RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); - ADDOP_I(c, loc, GET_AWAITABLE, 2); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_I(c, loc, GET_AWAITABLE, GET_AWAITABLE_OPARG(2, 0)); ADD_YIELD_FROM(c, loc, 1); ADDOP(c, loc, POP_TOP); @@ -5367,9 +5388,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); ADDOP(c, loc, PUSH_EXC_INFO); ADDOP(c, loc, WITH_EXCEPT_START); - ADDOP_I(c, loc, GET_AWAITABLE, 2); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_I(c, loc, GET_AWAITABLE, GET_AWAITABLE_OPARG(2, 0)); ADD_YIELD_FROM(c, loc, 1); RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); @@ -5554,16 +5573,11 @@ codegen_visit_expr_impl(compiler *c, expr_ty e, bool result_is_unused) ADD_YIELD_FROM(c, loc, 0); break; case Await_kind: - VISIT(c, expr, e->v.Await.value); - ADDOP_I(c, loc, GET_AWAITABLE, 0); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 1); - break; + return codegen_await(c, e); case Compare_kind: return codegen_compare(c, e); case Call_kind: - return codegen_call(c, e); + return codegen_call(c, e, 0); case Constant_kind: ADDOP_LOAD_CONST(c, loc, e->v.Constant.value); break; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c38c7df7948a520..d69a3bc052ae7bc 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8906,20 +8906,25 @@ break; } - case _GET_AWAITABLE_r11: { + case _GET_AWAITABLE_r13: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; + _PyStackRef *aux; _PyStackRef iter; + _PyStackRef null; + _PyStackRef none; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); iterable = _stack_item_0; + aux = &stack_pointer[-(oparg & 1)]; + assert((oparg & 1) == 0 || PyStackRef_IsNull(aux[0])); stack_pointer[0] = iterable; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -8928,14 +8933,20 @@ PyStackRef_CLOSE(iterable); _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { + stack_pointer += -(oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; + _tos_cache2 = none; + _tos_cache1 = null; _tos_cache0 = iter; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -(oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2dd50328e818e62..0f8863b4fe84920 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6822,12 +6822,17 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef *aux; _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; + _PyStackRef none; iterable = stack_pointer[-1]; + aux = &stack_pointer[-1 - (oparg & 1)]; + assert((oparg & 1) == 0 || PyStackRef_IsNull(aux[0])); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -6836,11 +6841,17 @@ PyStackRef_CLOSE(iterable); _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { + stack_pointer += -(oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; + null = PyStackRef_NULL; + none = PyStackRef_None; + stack_pointer[-(oparg & 1)] = iter; + stack_pointer[1 - (oparg & 1)] = null; + stack_pointer[2 - (oparg & 1)] = none; + stack_pointer += 3 - (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index c6420a9c1bb4af6..28cc0981b94f8c7 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -206,6 +206,13 @@ dummy_func(void) { res = PyJitRef_Borrow(sym_new_null(ctx)); } + op(_GET_AWAITABLE, (aux[oparg & 1], iterable -- iter, null, none)) { + // After an awaited call, any of the outputs may be NULL. + iter = sym_new_unknown(ctx); + null = sym_new_unknown(ctx); + none = sym_new_unknown(ctx); + } + op(_GUARD_TOS_INT, (value -- value)) { if (sym_is_compact_int(value)) { ADD_OP(_NOP, 0, 0); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index d446e9d9828358a..54c72787952885b 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1888,8 +1888,17 @@ case _GET_AWAITABLE: { JitOptRef iter; - iter = sym_new_not_null(ctx); - stack_pointer[-1] = iter; + JitOptRef null; + JitOptRef none; + iter = sym_new_unknown(ctx); + null = sym_new_unknown(ctx); + none = sym_new_unknown(ctx); + CHECK_STACK_BOUNDS(2 - (oparg & 1)); + stack_pointer[-1 - (oparg & 1)] = iter; + stack_pointer[-(oparg & 1)] = null; + stack_pointer[1 - (oparg & 1)] = none; + stack_pointer += 2 - (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } From 9d2657d9160ccd8c657d36c1482394650bef4984 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 18 Nov 2026 21:44:48 +0100 Subject: [PATCH 3/3] gh-161170: Lazily materialize coroutine objects for awaited calls `await f()` on an `async def` no longer allocates a coroutine object unless the body actually suspends. Python-function CALL paths with the awaited flag mark the new frame of a coroutine function (_MAYBE_LAZY_CORO). RETURN_GENERATOR then keeps running the body in that frame, with its own exception state item on the data stack. A body that returns or raises without suspending never creates a coroutine object; YIELD_VALUE creates one when the body first suspends and moves the frame into it. The body reports how it ended through the slot that the compiler pushes below the awaited call: a tagged int when it returned, or the coroutine when it suspended. GET_AWAITABLE turns that into the SEND loop's input: a NULL receiver makes SEND jump to END_SEND, and None in place of the NULL makes it go on to YIELD_VALUE with the value that the body has already yielded. All forms of SEND check for this, since an await site may have been specialized for another awaitable. Exceptions leaving the body are raised from the caller's SEND, as before. Calls from C, instrumented calls, coroutine origin tracking, JIT trace recording and tier 2 keep creating the coroutine eagerly. frame.f_generator and PyFrame_GetGenerator() return None/NULL for such a coroutine until it first suspends; asyncio.graph classifies frames by co_flags instead. Co-Authored-By: Claude Opus 5.5 --- Doc/c-api/frame.rst | 4 + Doc/library/dis.rst | 16 +- Doc/library/inspect.rst | 4 +- Doc/reference/datamodel.rst | 14 +- Doc/whatsnew/3.16.rst | 41 ++ Include/internal/pycore_ceval.h | 6 + Include/internal/pycore_interpframe.h | 4 + Include/internal/pycore_interpframe_structs.h | 3 +- Include/internal/pycore_opcode_metadata.h | 38 +- Include/internal/pycore_uop_ids.h | 26 +- Include/internal/pycore_uop_metadata.h | 44 +- Include/opcode_ids.h | 94 +-- InternalDocs/generators.md | 24 +- Lib/_opcode_metadata.py | 94 +-- Lib/asyncio/graph.py | 63 +- Lib/test/test_asyncio/test_graph.py | 47 +- Lib/test/test_coroutines.py | 591 ++++++++++++++++++ Lib/test/test_dis.py | 36 ++ Lib/test/test_opcache.py | 7 + ...-10-27-12-00-00.gh-issue-161170.lazyCo.rst | 3 + Modules/_testinternalcapi/test_cases.c.h | 371 +++++++++-- Modules/_testinternalcapi/test_targets.h | 4 +- Objects/genobject.c | 81 ++- Programs/test_frozenmain.h | 20 +- Python/bytecodes.c | 241 +++++-- Python/ceval.c | 52 ++ Python/ceval_macros.h | 21 + Python/codegen.c | 2 +- Python/executor_cases.c.h | 385 ++++++++++-- Python/generated_cases.c.h | 371 +++++++++-- Python/intrinsics.c | 6 +- Python/opcode_targets.h | 4 +- Python/optimizer.c | 6 + Python/optimizer_bytecodes.c | 4 +- Python/optimizer_cases.c.h | 20 +- Python/record_functions.c.h | 2 +- Python/specialize.c | 10 +- 37 files changed, 2311 insertions(+), 448 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index a086c60905402ea..bfd39507461d03c 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -85,6 +85,10 @@ See also :ref:`Reflection `. .. versionadded:: 3.11 + .. versionchanged:: 3.16 + May return ``NULL`` for a coroutine frame before its first suspension; + see :attr:`frame.f_generator`. + .. c:function:: PyObject* PyFrame_GetGlobals(PyFrameObject *frame) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index ef16d9c7d0c53ff..52eb8cdd9a6e516 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -831,7 +831,12 @@ not have to be) the original ``STACK[-2]``. If the low bit of ``where`` is set, the awaited object is the result of an awaited :opcode:`CALL`, :opcode:`CALL_KW` or :opcode:`CALL_FUNCTION_EX`, and the instruction also pops the extra slot that was pushed below that - call's callable. + call's callable. If the call ran a coroutine function without creating the + coroutine object, that slot tells how the body ended, and + :opcode:`GET_AWAITABLE` pushes the result or the yielded value in place of + ``None``, with a ``NULL`` receiver if the body returned, or with the new + coroutine and ``None`` in place of ``NULL`` if it suspended (see + :opcode:`SEND`). .. versionadded:: 3.5 @@ -1863,12 +1868,21 @@ iterations of the loop. push the exception's ``value`` attribute, and increment the bytecode counter by *delta*. + After an awaited call (see :opcode:`GET_AWAITABLE`), the receiver may be + ``NULL``: then ``STACK[-1]`` is already the result and the bytecode counter + is incremented by *delta*. If ``STACK[-2]`` is ``None``, ``STACK[-1]`` has + already been yielded by the receiver: ``STACK[-2]`` is set to ``NULL`` and + execution continues with the next instruction. + .. versionadded:: 3.11 .. versionchanged:: 3.15 The receiver is now at ``STACK[-3]``, followed by the value pushed with it by :opcode:`GET_ITER`. + .. versionchanged:: next + Handles the results of awaited calls. + .. opcode:: HAVE_ARGUMENT diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 51c465e8f00b5ee..328bfea8a372abd 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -151,7 +151,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | coroutine object that | | | | owns this frame, or | | | | ``None`` if the frame is | -| | | of a regular function | +| | | of a regular function or | +| | | the generator is not | +| | | materialized yet | +-----------------+-------------------+---------------------------+ | | f_trace | tracing function for this | | | | frame, or ``None`` | diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 02663ed465f424e..07fa9cf5441e643 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1736,10 +1736,22 @@ Special read-only attributes * - .. attribute:: frame.f_generator - The :term:`generator` or :term:`coroutine` object that owns this frame, - or ``None`` if the frame is a normal function. + or ``None`` if the frame is a normal function or if the generator + is not materialized yet. + + .. impl-detail:: + + A coroutine that is awaited as soon as it is called + (``await coro_func()``) may run without a coroutine object until + it first suspends, so this attribute is ``None`` before that. + Use :attr:`~codeobject.co_flags` of :attr:`frame.f_code` to tell + coroutine frames apart from normal function frames. .. versionadded:: 3.14 + .. versionchanged:: 3.16 + May be ``None`` for a coroutine frame before its first suspension. + .. index:: single: f_trace (frame attribute) single: f_trace_lines (frame attribute) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 33328b2ee005219..c32cffe73c87d58 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1147,6 +1147,19 @@ datetime and time (Contributed by Paul Ganssle in :gh:`158252`, based on work by Stan Ulbrych.) +Interpreter +----------- + +* A coroutine that is awaited as soon as it is called + (``await coro_func()``) now runs without creating a coroutine object + unless it suspends, making awaits that complete immediately about 40% + faster in a micro-benchmark (``await noop()``: 48 ns to 27 ns on x86-64). + This does not apply while coroutine origin tracking is enabled or inside + code compiled by the experimental JIT. + Until it first suspends, :attr:`frame.f_generator` is ``None`` for such a + coroutine; see :ref:`whatsnew316-porting`. + (Contributed by Andrew Svetlov in :gh:`161170`.) + re -- @@ -1405,12 +1418,23 @@ New deprecations .. include:: ../deprecations/pending-removal-in-future.rst +.. _whatsnew316-porting: + Porting to Python 3.16 ====================== This section lists previously described changes and other bugfixes that may require changes to your code. +* :attr:`frame.f_generator` and :c:func:`PyFrame_GetGenerator` may now + return ``None``/``NULL`` for a coroutine frame before its first + suspension, because a coroutine that is awaited as soon as it is called + (``await coro_func()``) runs without a coroutine object until then. + Code that used ``frame.f_generator is None`` to detect frames of normal + functions should check ``frame.f_code.co_flags`` for + :data:`~inspect.CO_COROUTINE` instead. + (Contributed by Andrew Svetlov in :gh:`161170`.) + * In :mod:`tkinter`, the *name* parameter of the :meth:`~tkinter.Misc.wait_variable`, :meth:`~tkinter.Misc.setvar` and :meth:`~tkinter.Misc.getvar` methods and the *value* parameter of @@ -1488,6 +1512,23 @@ that may require changes to your code. (Contributed by An Long in :gh:`86768`.) +CPython bytecode changes +======================== + +* The oparg of :opcode:`CALL` and :opcode:`CALL_KW` is now the number of + arguments shifted left by one. The low bit of their oparg, and of the + oparg of :opcode:`CALL_FUNCTION_EX`, marks a call whose result is awaited + right away. + (Contributed by Andrew Svetlov in :gh:`161170`.) + +* :opcode:`GET_AWAITABLE` now also pushes the ``NULL`` and ``None`` that + start the :opcode:`SEND` loop following it, so the compiler no longer + emits :opcode:`PUSH_NULL` and :opcode:`LOAD_COMMON_CONSTANT` after it. + After an awaited call, the low bit of its oparg is set and it also pops + a slot that the compiler pushes below the call's callable. + (Contributed by Andrew Svetlov in :gh:`161170`.) + + Build changes ============= diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a0fea389b6502c0..b9c6377219b3030 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -267,6 +267,12 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyObject **p_builtins); PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); +// Lazily created coroutines (see RETURN_GENERATOR in bytecodes.c). +// Exported for _testinternalcapi and external JIT support. +PyAPI_FUNC(PyObject *) _PyCoro_MaterializeLazy(_PyInterpreterFrame *frame); +PyAPI_FUNC(_PyInterpreterFrame *) _PyEval_MoveLazyCoroFrame(PyThreadState *tstate, + PyGenObject *gen, + _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index a85fa9bd32853c8..ea280a324e758a8 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -152,6 +152,8 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * dest->stackpointer = dest->localsplus + stacktop; // visited is GC bookkeeping for the current stack walk, not frame state. dest->visited = 0; + // A lazily started coroutine stops being lazy once its frame is copied. + dest->lazy_coro = 0; #ifdef Py_DEBUG dest->stackpointer_valid = src->stackpointer_valid; dest->lltrace = src->lltrace; @@ -208,6 +210,7 @@ _PyFrame_Initialize( frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->lazy_coro = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; @@ -441,6 +444,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int #endif frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->lazy_coro = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 0d6a37cc2c1a17a..4937e905081fae2 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -44,9 +44,10 @@ struct _PyInterpreterFrame { char owner; uint8_t visited:1; uint8_t external:1; /* Set when the frame was pushed by something other than the VM */ + uint8_t lazy_coro:1; #ifdef Py_DEBUG uint8_t stackpointer_valid:1; - uint8_t lltrace:5; + uint8_t lltrace:4; #endif /* Locals and stack */ _PyStackRef localsplus[1]; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 398583b79ba96e6..8c29b794e4b9930 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1137,7 +1137,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_EX_NON_PY_GENERAL] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_EX_PY] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1199,7 +1199,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, @@ -1290,10 +1290,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESUME_CHECK_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [SEND_ASYNC_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, - [SEND_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_ASYNC_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [SEND_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1330,7 +1330,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1345,7 +1345,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { }; #endif -#define MAX_UOP_PER_EXPANSION 11 +#define MAX_UOP_PER_EXPANSION 12 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1382,20 +1382,20 @@ _PyOpcode_macro_expansion[256] = { [BUILD_TEMPLATE] = { .nuops = 1, .uops = { { _BUILD_TEMPLATE, OPARG_SIMPLE, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, OPARG_SIMPLE, 0 } } }, [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_OBJECT, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _ALLOCATE_OBJECT, OPARG_SIMPLE, 3 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 11, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BOUND_METHOD_GENERAL] = { .nuops = 8, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 12, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_GENERAL] = { .nuops = 9, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BUILTIN_CLASS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_BUILTIN_FAST] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_BUILTIN_O] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_EX_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CALL_FUNCTION_EX_NON_PY_GENERAL, OPARG_SIMPLE, 1 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 1 } } }, - [CALL_EX_PY] = { .nuops = 7, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [CALL_EX_PY] = { .nuops = 8, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [CALL_INTRINSIC_1] = { .nuops = 2, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 3, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, - [CALL_KW_BOUND_METHOD] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_KW_PY] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_PY] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CALL_LIST_APPEND] = { .nuops = 6, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 5, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, @@ -1403,8 +1403,8 @@ _PyOpcode_macro_expansion[256] = { [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 9, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_PY_GENERAL] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 10, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_PY_GENERAL] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_STR_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_TUPLE_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_TYPE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, @@ -1507,9 +1507,9 @@ _PyOpcode_macro_expansion[256] = { [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 1 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, OPARG_SIMPLE, 0 } } }, [RETURN_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, - [SEND_ASYNC_GEN] = { .nuops = 2, .uops = { { _GUARD_3OS_ASYNC_GEN_ASEND, OPARG_SIMPLE, 1 }, { _SEND_ASYNC_GEN, OPARG_REPLACED, 1 } } }, - [SEND_GEN] = { .nuops = 4, .uops = { { _RECORD_3OS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, - [SEND_VIRTUAL] = { .nuops = 3, .uops = { { _GUARD_TOS_IS_NONE, OPARG_SIMPLE, 1 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _SEND_VIRTUAL, OPARG_REPLACED, 1 } } }, + [SEND_ASYNC_GEN] = { .nuops = 3, .uops = { { _SEND_LAZY_CORO, OPARG_REPLACED, 1 }, { _GUARD_3OS_ASYNC_GEN_ASEND, OPARG_SIMPLE, 1 }, { _SEND_ASYNC_GEN, OPARG_REPLACED, 1 } } }, + [SEND_GEN] = { .nuops = 5, .uops = { { _RECORD_3OS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _SEND_LAZY_CORO, OPARG_REPLACED, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [SEND_VIRTUAL] = { .nuops = 4, .uops = { { _SEND_LAZY_CORO, OPARG_REPLACED, 1 }, { _GUARD_TOS_IS_NONE, OPARG_SIMPLE, 1 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _SEND_VIRTUAL, OPARG_REPLACED, 1 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, OPARG_SIMPLE, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, OPARG_SIMPLE, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, OPARG_SIMPLE, 0 } } }, @@ -1545,7 +1545,7 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TUPLE, OPARG_SIMPLE, 1 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TWO_TUPLE, OPARG_SIMPLE, 1 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, OPARG_SIMPLE, 0 } } }, - [YIELD_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, + [YIELD_VALUE] = { .nuops = 3, .uops = { { _CHECK_LAZY_CORO, OPARG_REPLACED, 0 }, { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, }; #endif // NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 1c6ee2b27608262..9e4dd360c4ffa8c 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -82,6 +82,7 @@ enum { _CHECK_IS_NOT_PY_CALLABLE_EX, _CHECK_IS_NOT_PY_CALLABLE_KW, _CHECK_IS_PY_CALLABLE_EX, + _CHECK_LAZY_CORO, _CHECK_MANAGED_OBJECT_HAS_VALUES, _CHECK_METHOD_VERSION, _CHECK_METHOD_VERSION_KW, @@ -191,6 +192,8 @@ enum { _GUARD_NOT_EXHAUSTED_LIST, _GUARD_NOT_EXHAUSTED_RANGE, _GUARD_NOT_EXHAUSTED_TUPLE, + _GUARD_NOT_LAZY_CORO, + _GUARD_SEND_NOT_LAZY_CORO, _GUARD_THIRD_NULL, _GUARD_TOS_ANY_DICT, _GUARD_TOS_ANY_SET, @@ -292,6 +295,7 @@ enum { _MATCH_CLASS, _MAYBE_EXPAND_METHOD, _MAYBE_EXPAND_METHOD_KW, + _MAYBE_LAZY_CORO, _MONITOR_CALL, _MONITOR_CALL_KW, _MONITOR_JUMP_BACKWARD, @@ -328,6 +332,7 @@ enum { _SEND_ASYNC_GEN, _SEND_ASYNC_GEN_TIER_TWO, _SEND_GEN_FRAME, + _SEND_LAZY_CORO, _SEND_VIRTUAL, _SEND_VIRTUAL_TIER_TWO, _SET_UPDATE, @@ -371,7 +376,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 659 +#define MAX_UOP_ID 664 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -595,6 +600,10 @@ enum { _CHECK_IS_PY_CALLABLE_EX_r13, _CHECK_IS_PY_CALLABLE_EX_r23, _CHECK_IS_PY_CALLABLE_EX_r33, + _CHECK_LAZY_CORO_r00, + _CHECK_LAZY_CORO_r11, + _CHECK_LAZY_CORO_r22, + _CHECK_LAZY_CORO_r33, _CHECK_MANAGED_OBJECT_HAS_VALUES_r01, _CHECK_MANAGED_OBJECT_HAS_VALUES_r11, _CHECK_MANAGED_OBJECT_HAS_VALUES_r22, @@ -925,6 +934,14 @@ enum { _GUARD_NOT_EXHAUSTED_TUPLE_r12, _GUARD_NOT_EXHAUSTED_TUPLE_r22, _GUARD_NOT_EXHAUSTED_TUPLE_r33, + _GUARD_NOT_LAZY_CORO_r00, + _GUARD_NOT_LAZY_CORO_r11, + _GUARD_NOT_LAZY_CORO_r22, + _GUARD_NOT_LAZY_CORO_r33, + _GUARD_SEND_NOT_LAZY_CORO_r03, + _GUARD_SEND_NOT_LAZY_CORO_r13, + _GUARD_SEND_NOT_LAZY_CORO_r23, + _GUARD_SEND_NOT_LAZY_CORO_r33, _GUARD_THIRD_NULL_r03, _GUARD_THIRD_NULL_r13, _GUARD_THIRD_NULL_r23, @@ -1247,6 +1264,10 @@ enum { _MATCH_SEQUENCE_r23, _MAYBE_EXPAND_METHOD_r00, _MAYBE_EXPAND_METHOD_KW_r11, + _MAYBE_LAZY_CORO_r01, + _MAYBE_LAZY_CORO_r11, + _MAYBE_LAZY_CORO_r22, + _MAYBE_LAZY_CORO_r33, _MONITOR_CALL_r00, _MONITOR_CALL_KW_r11, _MONITOR_JUMP_BACKWARD_r00, @@ -1320,6 +1341,7 @@ enum { _SEND_ASYNC_GEN_r33, _SEND_ASYNC_GEN_TIER_TWO_r33, _SEND_GEN_FRAME_r33, + _SEND_LAZY_CORO_r33, _SEND_VIRTUAL_r33, _SEND_VIRTUAL_TIER_TWO_r03, _SEND_VIRTUAL_TIER_TWO_r13, @@ -1454,7 +1476,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1665 +#define MAX_UOP_REGS_ID 1687 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 17c359876174196..953b193f809e19e 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -168,13 +168,15 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_SEND_NOT_LAZY_CORO] = HAS_EXIT_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_GUARD_TOS_IS_NONE] = HAS_EXIT_FLAG, [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_SEND_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_GUARD_3OS_ASYNC_GEN_ASEND] = HAS_EXIT_FLAG, [_SEND_ASYNC_GEN_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_GUARD_NOT_LAZY_CORO] = HAS_EXIT_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -1649,6 +1651,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_SEND_NOT_LAZY_CORO] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_SEND_NOT_LAZY_CORO_r03 }, + { 3, 1, _GUARD_SEND_NOT_LAZY_CORO_r13 }, + { 3, 2, _GUARD_SEND_NOT_LAZY_CORO_r23 }, + { 3, 3, _GUARD_SEND_NOT_LAZY_CORO_r33 }, + }, + }, [_SEND_GEN_FRAME] = { .best = { 3, 3, 3, 3 }, .entries = { @@ -1703,6 +1714,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _SEND_ASYNC_GEN_TIER_TWO_r33 }, }, }, + [_GUARD_NOT_LAZY_CORO] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOT_LAZY_CORO_r00 }, + { 1, 1, _GUARD_NOT_LAZY_CORO_r11 }, + { 2, 2, _GUARD_NOT_LAZY_CORO_r22 }, + { 3, 3, _GUARD_NOT_LAZY_CORO_r33 }, + }, + }, [_YIELD_VALUE] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -4335,6 +4355,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, [_GET_AWAITABLE_r13] = _GET_AWAITABLE, + [_GUARD_SEND_NOT_LAZY_CORO_r03] = _GUARD_SEND_NOT_LAZY_CORO, + [_GUARD_SEND_NOT_LAZY_CORO_r13] = _GUARD_SEND_NOT_LAZY_CORO, + [_GUARD_SEND_NOT_LAZY_CORO_r23] = _GUARD_SEND_NOT_LAZY_CORO, + [_GUARD_SEND_NOT_LAZY_CORO_r33] = _GUARD_SEND_NOT_LAZY_CORO, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, @@ -4353,6 +4377,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = _GUARD_3OS_ASYNC_GEN_ASEND, [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = _GUARD_3OS_ASYNC_GEN_ASEND, [_SEND_ASYNC_GEN_TIER_TWO_r33] = _SEND_ASYNC_GEN_TIER_TWO, + [_GUARD_NOT_LAZY_CORO_r00] = _GUARD_NOT_LAZY_CORO, + [_GUARD_NOT_LAZY_CORO_r11] = _GUARD_NOT_LAZY_CORO, + [_GUARD_NOT_LAZY_CORO_r22] = _GUARD_NOT_LAZY_CORO, + [_GUARD_NOT_LAZY_CORO_r33] = _GUARD_NOT_LAZY_CORO, [_YIELD_VALUE_r11] = _YIELD_VALUE, [_POP_EXCEPT_r10] = _POP_EXCEPT, [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, @@ -5590,6 +5618,16 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_NOT_EXHAUSTED_TUPLE_r12] = "_GUARD_NOT_EXHAUSTED_TUPLE_r12", [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = "_GUARD_NOT_EXHAUSTED_TUPLE_r22", [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = "_GUARD_NOT_EXHAUSTED_TUPLE_r33", + [_GUARD_NOT_LAZY_CORO] = "_GUARD_NOT_LAZY_CORO", + [_GUARD_NOT_LAZY_CORO_r00] = "_GUARD_NOT_LAZY_CORO_r00", + [_GUARD_NOT_LAZY_CORO_r11] = "_GUARD_NOT_LAZY_CORO_r11", + [_GUARD_NOT_LAZY_CORO_r22] = "_GUARD_NOT_LAZY_CORO_r22", + [_GUARD_NOT_LAZY_CORO_r33] = "_GUARD_NOT_LAZY_CORO_r33", + [_GUARD_SEND_NOT_LAZY_CORO] = "_GUARD_SEND_NOT_LAZY_CORO", + [_GUARD_SEND_NOT_LAZY_CORO_r03] = "_GUARD_SEND_NOT_LAZY_CORO_r03", + [_GUARD_SEND_NOT_LAZY_CORO_r13] = "_GUARD_SEND_NOT_LAZY_CORO_r13", + [_GUARD_SEND_NOT_LAZY_CORO_r23] = "_GUARD_SEND_NOT_LAZY_CORO_r23", + [_GUARD_SEND_NOT_LAZY_CORO_r33] = "_GUARD_SEND_NOT_LAZY_CORO_r33", [_GUARD_THIRD_NULL] = "_GUARD_THIRD_NULL", [_GUARD_THIRD_NULL_r03] = "_GUARD_THIRD_NULL_r03", [_GUARD_THIRD_NULL_r13] = "_GUARD_THIRD_NULL_r13", @@ -6529,6 +6567,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GET_AWAITABLE: return 1 + (oparg & 1); + case _GUARD_SEND_NOT_LAZY_CORO: + return 0; case _SEND_GEN_FRAME: return 1; case _GUARD_TOS_IS_NONE: @@ -6541,6 +6581,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _SEND_ASYNC_GEN_TIER_TWO: return 3; + case _GUARD_NOT_LAZY_CORO: + return 0; case _YIELD_VALUE: return 1; case _POP_EXCEPT: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 11342ae451b9f6c..dbd1786a5e6808f 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -14,53 +14,53 @@ extern "C" { #define BINARY_SLICE 1 #define BUILD_TEMPLATE 2 #define BINARY_OP_INPLACE_ADD_UNICODE 3 -#define CALL_FUNCTION_EX 4 -#define CHECK_EG_MATCH 5 -#define CHECK_EXC_MATCH 6 -#define CLEANUP_THROW 7 -#define DELETE_SUBSCR 8 -#define END_FOR 9 -#define END_SEND 10 -#define EXIT_INIT_CHECK 11 -#define FORMAT_SIMPLE 12 -#define FORMAT_WITH_SPEC 13 -#define GET_AITER 14 -#define GET_ANEXT 15 -#define GET_LEN 16 +#define CHECK_EG_MATCH 4 +#define CHECK_EXC_MATCH 5 +#define CLEANUP_THROW 6 +#define DELETE_SUBSCR 7 +#define END_FOR 8 +#define END_SEND 9 +#define EXIT_INIT_CHECK 10 +#define FORMAT_SIMPLE 11 +#define FORMAT_WITH_SPEC 12 +#define GET_AITER 13 +#define GET_ANEXT 14 +#define GET_LEN 15 +#define INTERPRETER_EXIT 16 #define RESERVED 17 -#define INTERPRETER_EXIT 18 -#define LOAD_BUILD_CLASS 19 -#define LOAD_LOCALS 20 -#define MAKE_FUNCTION 21 -#define MATCH_KEYS 22 -#define MATCH_MAPPING 23 -#define MATCH_SEQUENCE 24 -#define NOP 25 -#define NOT_TAKEN 26 -#define POP_EXCEPT 27 -#define POP_ITER 28 -#define POP_TOP 29 -#define PUSH_EXC_INFO 30 -#define PUSH_NULL 31 -#define RETURN_GENERATOR 32 -#define RETURN_VALUE 33 -#define SETUP_ANNOTATIONS 34 -#define STORE_SLICE 35 -#define STORE_SUBSCR 36 -#define TO_BOOL 37 -#define UNARY_INVERT 38 -#define UNARY_NEGATIVE 39 -#define UNARY_NOT 40 -#define WITH_EXCEPT_START 41 -#define BINARY_OP 42 -#define BUILD_INTERPOLATION 43 -#define BUILD_LIST 44 -#define BUILD_MAP 45 -#define BUILD_SET 46 -#define BUILD_SLICE 47 -#define BUILD_STRING 48 -#define BUILD_TUPLE 49 -#define CALL 50 +#define LOAD_BUILD_CLASS 18 +#define LOAD_LOCALS 19 +#define MAKE_FUNCTION 20 +#define MATCH_KEYS 21 +#define MATCH_MAPPING 22 +#define MATCH_SEQUENCE 23 +#define NOP 24 +#define NOT_TAKEN 25 +#define POP_EXCEPT 26 +#define POP_ITER 27 +#define POP_TOP 28 +#define PUSH_EXC_INFO 29 +#define PUSH_NULL 30 +#define RETURN_GENERATOR 31 +#define RETURN_VALUE 32 +#define SETUP_ANNOTATIONS 33 +#define STORE_SLICE 34 +#define STORE_SUBSCR 35 +#define TO_BOOL 36 +#define UNARY_INVERT 37 +#define UNARY_NEGATIVE 38 +#define UNARY_NOT 39 +#define WITH_EXCEPT_START 40 +#define BINARY_OP 41 +#define BUILD_INTERPOLATION 42 +#define BUILD_LIST 43 +#define BUILD_MAP 44 +#define BUILD_SET 45 +#define BUILD_SLICE 46 +#define BUILD_STRING 47 +#define BUILD_TUPLE 48 +#define CALL 49 +#define CALL_FUNCTION_EX 50 #define CALL_INTRINSIC_1 51 #define CALL_INTRINSIC_2 52 #define CALL_KW 53 @@ -253,7 +253,7 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 41 +#define HAVE_ARGUMENT 40 #define MIN_SPECIALIZED_OPCODE 129 #define MIN_INSTRUMENTED_OPCODE 233 diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index fa652cd231cfa95..593c73ced32bb3e 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -45,7 +45,29 @@ The generator's frame is initialized as a copy of the frame in which `RETURN_GENERATOR` is executing, but its `owner` field is overwritten to indicate that it is owned by a generator. Finally, `RETURN_GENERATOR` pushes the new generator object to the stack and returns to the caller of the generator function (at -which time its frame is destroyed). When the generator is next resumed by +which time its frame is destroyed). + +There is one exception: a coroutine function called by an awaited call +(`await f()`, compiled as `CALL`, `CALL_KW` or `CALL_FUNCTION_EX` with the low +bit of the oparg set) gets the `lazy_coro` flag on its frame (see +`_MAYBE_LAZY_CORO`). `RETURN_GENERATOR` then keeps running the body in that +frame, with its own `_PyErr_StackItem` pushed on the data stack right after the +frame, so that a body that returns or raises without suspending never creates a +coroutine object. If the body reaches `YIELD_VALUE`, the coroutine object is +created then and the frame is moved into it (see `_PyCoro_MaterializeLazy()` +and `_PyEval_MoveLazyCoroFrame()`). + +The compiler pushes an extra `NULL` below the callable of an awaited call, and +the `GET_AWAITABLE` that follows the call pops it. A lazily run body reports +there how it ended: a tagged int if it returned, or the new coroutine object if +it suspended. `GET_AWAITABLE` turns this into the input of the `SEND` loop: a +`NULL` receiver, which makes `SEND` jump to `END_SEND` with the return value, or +`None` in place of the `NULL` below the value, which makes `SEND` go on to +`YIELD_VALUE` with the value that the body has already yielded. An exception +that leaves the body is raised from the caller's `SEND`, as it would be if the +coroutine object had been created. + +When the generator is next resumed by [`gen_send_ex2()`](../Objects/genobject.c), `_PyEval_EvalFrame()` is called to continue executing the generator function, in the frame that is embedded in the generator object. diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index df92eae151d2488..74bd09aea6f7437 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -235,52 +235,52 @@ TRACE_RECORD=255, BINARY_SLICE=1, BUILD_TEMPLATE=2, - CALL_FUNCTION_EX=4, - CHECK_EG_MATCH=5, - CHECK_EXC_MATCH=6, - CLEANUP_THROW=7, - DELETE_SUBSCR=8, - END_FOR=9, - END_SEND=10, - EXIT_INIT_CHECK=11, - FORMAT_SIMPLE=12, - FORMAT_WITH_SPEC=13, - GET_AITER=14, - GET_ANEXT=15, - GET_LEN=16, - INTERPRETER_EXIT=18, - LOAD_BUILD_CLASS=19, - LOAD_LOCALS=20, - MAKE_FUNCTION=21, - MATCH_KEYS=22, - MATCH_MAPPING=23, - MATCH_SEQUENCE=24, - NOP=25, - NOT_TAKEN=26, - POP_EXCEPT=27, - POP_ITER=28, - POP_TOP=29, - PUSH_EXC_INFO=30, - PUSH_NULL=31, - RETURN_GENERATOR=32, - RETURN_VALUE=33, - SETUP_ANNOTATIONS=34, - STORE_SLICE=35, - STORE_SUBSCR=36, - TO_BOOL=37, - UNARY_INVERT=38, - UNARY_NEGATIVE=39, - UNARY_NOT=40, - WITH_EXCEPT_START=41, - BINARY_OP=42, - BUILD_INTERPOLATION=43, - BUILD_LIST=44, - BUILD_MAP=45, - BUILD_SET=46, - BUILD_SLICE=47, - BUILD_STRING=48, - BUILD_TUPLE=49, - CALL=50, + CHECK_EG_MATCH=4, + CHECK_EXC_MATCH=5, + CLEANUP_THROW=6, + DELETE_SUBSCR=7, + END_FOR=8, + END_SEND=9, + EXIT_INIT_CHECK=10, + FORMAT_SIMPLE=11, + FORMAT_WITH_SPEC=12, + GET_AITER=13, + GET_ANEXT=14, + GET_LEN=15, + INTERPRETER_EXIT=16, + LOAD_BUILD_CLASS=18, + LOAD_LOCALS=19, + MAKE_FUNCTION=20, + MATCH_KEYS=21, + MATCH_MAPPING=22, + MATCH_SEQUENCE=23, + NOP=24, + NOT_TAKEN=25, + POP_EXCEPT=26, + POP_ITER=27, + POP_TOP=28, + PUSH_EXC_INFO=29, + PUSH_NULL=30, + RETURN_GENERATOR=31, + RETURN_VALUE=32, + SETUP_ANNOTATIONS=33, + STORE_SLICE=34, + STORE_SUBSCR=35, + TO_BOOL=36, + UNARY_INVERT=37, + UNARY_NEGATIVE=38, + UNARY_NOT=39, + WITH_EXCEPT_START=40, + BINARY_OP=41, + BUILD_INTERPOLATION=42, + BUILD_LIST=43, + BUILD_MAP=44, + BUILD_SET=45, + BUILD_SLICE=46, + BUILD_STRING=47, + BUILD_TUPLE=48, + CALL=49, + CALL_FUNCTION_EX=50, CALL_INTRINSIC_1=51, CALL_INTRINSIC_2=52, CALL_KW=53, @@ -380,5 +380,5 @@ STORE_FAST_MAYBE_NULL=266, ) -HAVE_ARGUMENT = 41 +HAVE_ARGUMENT = 40 MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index d76d2f6383343d5..d7714c9c681a198 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -1,6 +1,7 @@ """Introspection utils for tasks call graphs.""" import dataclasses +import inspect import io import operator import sys @@ -38,6 +39,26 @@ class FutureCallGraph: awaited_by: tuple["FutureCallGraph", ...] +# A coroutine that is awaited right away runs without a coroutine +# object until it first suspends, so ``f.f_generator`` may be None +# for a coroutine frame; classify frames by their code instead. +_CO_ASYNC_FLAGS = inspect.CO_COROUTINE | inspect.CO_ASYNC_GENERATOR +_CO_GEN_FLAGS = _CO_ASYNC_FLAGS | inspect.CO_GENERATOR +_FRAME_TAGS = { + inspect.CO_COROUTINE: 'async ', + inspect.CO_ASYNC_GENERATOR: 'async generator ', + inspect.CO_GENERATOR: 'generator ', +} + + +def _is_async_frame(f): + return bool(f.f_code.co_flags & _CO_ASYNC_FLAGS) + + +def _is_sync_frame(f): + return not (f.f_code.co_flags & _CO_GEN_FLAGS) + + def _build_graph_for_future( future: futures.Future, *, @@ -166,12 +187,10 @@ def capture_call_graph( try: while f is not None: # gh-156988: sync gen should not clear the call chain - is_async = isinstance( - f.f_generator, (types.CoroutineType, types.AsyncGeneratorType)) call_stack.append(FrameCallGraphEntry(f)) - if is_async: - if f.f_back is not None and f.f_back.f_generator is None: + if _is_async_frame(f): + if f.f_back is not None and _is_sync_frame(f.f_back): # We've reached the bottom of the coroutine stack, which # must be the Task that runs it. break @@ -228,36 +247,12 @@ def add_line(line: str) -> None: ) for ste in st.call_stack: f = ste.frame - - if f.f_generator is None: - f = ste.frame - add_line( - f' | File {f.f_code.co_filename!r},' - f' line {f.f_lineno}, in' - f' {f.f_code.co_qualname}()' - ) - else: - c = f.f_generator - - try: - f = c.cr_frame - code = c.cr_code - tag = 'async' - except AttributeError: - try: - f = c.ag_frame - code = c.ag_code - tag = 'async generator' - except AttributeError: - f = c.gi_frame - code = c.gi_code - tag = 'generator' - - add_line( - f' | File {f.f_code.co_filename!r},' - f' line {f.f_lineno}, in' - f' {tag} {code.co_qualname}()' - ) + tag = _FRAME_TAGS.get(f.f_code.co_flags & _CO_GEN_FLAGS, '') + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag}{f.f_code.co_qualname}()' + ) if st.awaited_by: add_line( diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 1f1c76e4aef69f8..71fd41d27ca8122 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -1,4 +1,5 @@ import asyncio +import inspect import io import sys import types @@ -20,19 +21,23 @@ def walk(s): if isinstance(s.future, asyncio.Task) else 'F' ] + def tag(frame): + # A coroutine awaited right after it is called may run + # without a coroutine object until it first suspends, so + # classify frames by their code rather than f_generator. + flags = frame.f_code.co_flags + if flags & inspect.CO_COROUTINE: + return "a" + if flags & inspect.CO_ASYNC_GENERATOR: + return "ag" + if flags & inspect.CO_GENERATOR: + return "g" + return "s" + ret.append( [ - ( - f"s {entry.frame.f_code.co_name}" - if entry.frame.f_generator is None else - ( - f"a {entry.frame.f_generator.cr_code.co_name}" - if hasattr(entry.frame.f_generator, 'cr_code') else - f"ag {entry.frame.f_generator.ag_code.co_name}" - if hasattr(entry.frame.f_generator, 'ag_code') else - f"g {entry.frame.f_generator.gi_code.co_name}" - ) - ) for entry in s.call_stack + f"{tag(entry.frame)} {entry.frame.f_code.co_name}" + for entry in s.call_stack # Skip the pure Python Future.__await__ generator frame if entry.frame.f_code is not future_await_code ] @@ -986,6 +991,26 @@ async def child(): self.assertEqual(stack[0][2], [['T', ['a test_stack_eager_task'], []]]) + async def test_capture_call_graph_through_generator_based_coroutine(self): + # A native coroutine driven by a generator-based coroutine + # (yield from) keeps the frames of its callers. + stack = None + + async def native(): + nonlocal stack + graph = asyncio.capture_call_graph() + stack = [entry.frame.f_code.co_name for entry in graph.call_stack] + + @types.coroutine + def legacy(): + return (yield from native()) + + async def main(): + await legacy() + + await main() + self.assertEqual(stack[:3], ['native', 'legacy', 'main']) + @unittest.skipIf( not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index c5900da09501b36..03d62ea10d412c8 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1,6 +1,8 @@ import ast import contextlib import copy +import builtins +import gc import inspect import pickle import sys @@ -2594,6 +2596,595 @@ def test_unawaited_warning_during_shutdown(self): assert_python_ok("-c", code) +@support.cpython_only +class LazyCoroutineTest(unittest.TestCase): + """A coroutine that is awaited right away (``await f()``) runs in + the calling thread's frame and only gets a coroutine object when + it first suspends. This must not be observable, except that no + coroutine object exists before the first suspension.""" + + @staticmethod + def coroutines_for(code): + return [o for o in gc.get_objects() + if type(o) is types.CoroutineType and o.cr_code is code] + + def test_no_object_before_first_suspension(self): + seen = [] + + async def inner(): + seen.append(len(self.coroutines_for(inner.__code__))) + await AsyncYield(1) + seen.append(len(self.coroutines_for(inner.__code__))) + return 42 + + async def outer(): + return await inner() + + self.assertEqual(run_async(outer()), ([1], 42)) + self.assertEqual(seen, [0, 1]) + + def test_return_without_suspending(self): + async def inner(x): + return x + 1 + + async def outer(): + total = 0 + for i in range(100): + total += await inner(i) + return total + + self.assertEqual(run_async(outer()), ([], sum(range(1, 101)))) + + def test_nested_chain(self): + async def chain(n): + if n == 0: + await AsyncYield("bottom") + return 0 + return 1 + await chain(n - 1) + + self.assertEqual(run_async(chain(20)), (["bottom"], 20)) + + def test_exception_before_first_suspension(self): + async def inner(): + raise ValueError("boom") + + async def outer(): + try: + await inner() + except ValueError as exc: + tb = exc.__traceback__ + names = [] + while tb is not None: + names.append(tb.tb_frame.f_code.co_name) + tb = tb.tb_next + return names + + self.assertEqual(run_async(outer()), ([], ["outer", "inner"])) + + def test_stopiteration_before_first_suspension(self): + async def inner(): + raise StopIteration + + async def outer(): + await inner() + + with self.assertRaisesRegex(RuntimeError, + "coroutine raised StopIteration"): + run_async(outer()) + + def test_frame_object(self): + async def inner(): + x = 1 + frame = sys._getframe() + self.assertIsNone(frame.f_generator) + self.assertEqual(frame.f_locals["x"], 1) + self.assertEqual(frame.f_back.f_code.co_name, "outer") + await AsyncYield(1) + coro = frame.f_generator + self.assertIsInstance(coro, types.CoroutineType) + self.assertIs(coro.cr_frame, frame) + self.assertIs(coro.cr_code, inner.__code__) + self.assertEqual(frame.f_locals["x"], 1) + self.assertEqual(frame.f_back.f_code.co_name, "outer") + x = 2 + return frame + + async def outer(): + return await inner() + + _, frame = run_async(outer()) + self.assertIsNone(frame.f_generator) # finished + self.assertEqual(frame.f_locals["x"], 2) + + def test_exc_info_suspend_inside_handler(self): + async def inner(): + try: + raise KeyError + except KeyError: + await AsyncYield(1) + self.assertIsInstance(sys.exc_info()[1], KeyError) + await AsyncYield(2) + self.assertIsInstance(sys.exc_info()[1], KeyError) + # The caller's context is visible after the handler. + self.assertIsInstance(sys.exc_info()[1], TypeError) + await AsyncYield(3) + self.assertIsInstance(sys.exc_info()[1], TypeError) + return "ok" + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exc_info()[1], TypeError) + self.assertIsNone(sys.exc_info()[1]) + return result + + self.assertEqual(run_async(outer()), ([1, 2, 3], "ok")) + + def test_exc_info_handler_without_suspending(self): + async def inner(): + try: + raise KeyError + except KeyError: + self.assertIsInstance(sys.exc_info()[1], KeyError) + self.assertIsInstance(sys.exc_info()[1], TypeError) + return "ok" + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exc_info()[1], TypeError) + self.assertIsNone(sys.exc_info()[1]) + return result + + self.assertEqual(run_async(outer()), ([], "ok")) + + def test_throw_into_materialized_coroutine(self): + async def inner(): + try: + await AsyncYield(1) + except ZeroDivisionError: + return "caught" + return "not caught" + + async def outer(): + return await inner() + + coro = outer() + self.assertEqual(coro.send(None), 1) + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertEqual(cm.exception.value, "caught") + + def test_close_materialized_coroutine(self): + closed = [] + + async def inner(): + try: + await AsyncYield(1) + finally: + closed.append(True) + + async def outer(): + await inner() + + coro = outer() + self.assertEqual(coro.send(None), 1) + coro.close() + self.assertEqual(closed, [True]) + + def test_async_with(self): + log = [] + + class CM: + async def __aenter__(self): + log.append("enter") + return self + + async def __aexit__(self, *args): + log.append("exit") + return False + + async def outer(): + async with CM() as cm: + log.append(type(cm).__name__) + await AsyncYield(1) + return "done" + + self.assertEqual(run_async(outer()), ([1], "done")) + self.assertEqual(log, ["enter", "CM", "exit"]) + + def test_async_comprehension(self): + async def inner(x): + return x * 2 + + async def outer(): + return [await inner(x) for x in range(5)] + + self.assertEqual(run_async(outer()), ([], [0, 2, 4, 6, 8])) + + def test_origin_tracking(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + + async def inner(): + await AsyncYield(1) + return sys._getframe().f_generator.cr_origin + + async def outer(): + coro = inner() + eager = (await coro)[0] + lazy = (await inner())[0] + return eager, lazy + + sys.set_coroutine_origin_tracking_depth(2) + try: + _, (eager, lazy) = run_async(outer()) + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + # Origin tracking disables the lazy path: the origin is the + # call, not the await, and is captured at creation. + self.assertEqual(eager[0], __file__) + self.assertEqual(eager[2], "outer") + self.assertEqual(lazy[0], __file__) + self.assertEqual(lazy[2], "outer") + self.assertEqual(lazy[1], eager[1] + 2) + + def test_origin_tracking_multiline_await(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + + async def inner(): + await AsyncYield(1) + return sys._getframe().f_generator.cr_origin + + async def outer(): + origin = await ( + inner() + ) + return origin[0][1] - outer.__code__.co_firstlineno + + sys.set_coroutine_origin_tracking_depth(1) + try: + _, line = run_async(outer()) + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + # The line of the call, not of the await. + self.assertEqual(line, 2) + + def test_settrace_events(self): + async def inner(): + await AsyncYield(1) + return 1 + + async def eager(): + coro = inner() + return await coro + + async def lazy(): + return await inner() + + def trace(coro_fn): + events = [] + + def tracer(frame, event, arg): + if frame.f_code is inner.__code__: + events.append((event, frame.f_lineno - frame.f_code.co_firstlineno)) + return tracer + + sys.settrace(tracer) + try: + result = run_async(coro_fn()) + finally: + sys.settrace(None) + return events, result + + self.assertEqual(trace(lazy), trace(eager)) + + def test_monitoring_events(self): + async def inner(): + await AsyncYield(1) + return 1 + + async def eager(): + coro = inner() + return await coro + + async def lazy(): + return await inner() + + E = sys.monitoring.events + # No LINE events: they instrument the caller's SEND, which + # makes the await take the regular path. + names = {E.PY_START: "start", E.PY_RESUME: "resume", + E.PY_RETURN: "return", E.PY_YIELD: "yield", + E.STOP_ITERATION: "stop_iteration"} + codes = {inner.__code__: "inner", eager.__code__: "outer", + lazy.__code__: "outer"} + + def monitor(coro_fn): + events = [] + tool = sys.monitoring.DEBUGGER_ID + sys.monitoring.use_tool_id(tool, "test") + try: + for event, name in names.items(): + def callback(code, offset, *args, name=name): + if code in codes: + events.append( + (codes[code], name, type(args[-1]) if args else None)) + sys.monitoring.register_callback(tool, event, callback) + sys.monitoring.set_events(tool, sum(names)) + result = run_async(coro_fn()) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + return events, result + + self.assertEqual(monitor(lazy), monitor(eager)) + self.assertIn(("outer", "stop_iteration", StopIteration), + monitor(lazy)[0]) + + def test_stop_iteration_event_without_suspending(self): + async def inner(): + return 42 + + async def outer(): + return await inner() + + E = sys.monitoring.events + events = [] + tool = sys.monitoring.DEBUGGER_ID + sys.monitoring.use_tool_id(tool, "test") + try: + def callback(code, offset, exc): + if code is outer.__code__: + events.append(exc.value) + sys.monitoring.register_callback(tool, E.STOP_ITERATION, callback) + sys.monitoring.set_events(tool, E.STOP_ITERATION) + self.assertEqual(run_async(outer()), ([], 42)) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + self.assertEqual(events, [42]) + + def test_exc_state_after_suspending_inside_handler(self): + # The coroutine's own exception state must not keep the + # caller's exception once its handler is done. + async def inner(): + try: + raise KeyError + except KeyError: + await AsyncYield(1) + await AsyncYield(2) + return sys.exc_info()[1] + + async def outer(): + try: + raise TypeError + except TypeError: + return await inner() + + coro = outer() + coro.send(None) + inner_coro = coro.cr_await + inner_coro.send(None) + self.assertNotIn(TypeError, + map(type, gc.get_referents(inner_coro))) + # Resumed directly, not through outer: no handled exception. + with self.assertRaises(StopIteration) as cm: + inner_coro.send(None) + self.assertIsNone(cm.exception.value) + coro.close() + + def test_throw_into_coroutine_suspended_inside_handler(self): + async def inner(): + try: + raise KeyError + except KeyError: + try: + await AsyncYield(1) + except ZeroDivisionError: + pass + return sys.exception() + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exception(), TypeError) + return result + + coro = outer() + self.assertEqual(coro.send(None), 1) + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertIsNone(cm.exception.value) + + def test_import_hook_locals(self): + seen = [] + orig_import = builtins.__import__ + + def hook(name, globals=None, locals=None, fromlist=(), level=0): + seen.append(locals) + return orig_import(name, globals, locals, fromlist, level) + + async def inner(): + import sys as _sys + return _sys + + async def outer(): + try: + raise TypeError + except TypeError: + return await inner() + + with support.swap_attr(builtins, "__import__", hook): + self.assertEqual(run_async(outer()), ([], sys)) + self.assertEqual(seen, [None]) + + def test_traceback_frame_keeps_no_caller_exception(self): + async def inner(): + raise ValueError + + async def outer(): + try: + raise TypeError + except TypeError: + try: + await inner() + except ValueError as exc: + return exc.__traceback__.tb_next.tb_frame + + _, frame = run_async(outer()) + self.assertIs(frame.f_code, inner.__code__) + self.assertNotIn(TypeError, map(type, gc.get_referents(frame))) + + def test_code_replaced_before_first_suspension(self): + # The coroutine object is created at the first suspension, so it + # must be sized for the code the frame runs, not func.__code__. + async def small(): + big, small # same free variables as big() + await AsyncYield(1) + + async def big(): + a = b = c = d = e = f = g = h = i = j = k = m = n = o = p = 1 + big.__code__ = small.__code__ + await AsyncYield(1) + return (a, b, c, d, e, f, g, h, i, j, k, m, n, o, p) + + async def outer(): + return await big() + + self.assertEqual(run_async(outer()), ([1], (1,) * 15)) + + def test_finished_frame_has_no_f_back(self): + async def inner(): + return sys._getframe() + + async def outer(): + return await inner() + + _, frame = run_async(outer()) + self.assertIs(frame.f_code, inner.__code__) + self.assertIsNone(frame.f_back) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_memory_error_at_first_suspension(self): + @types.coroutine + def suspend(): + yield 1 + + awaitable = suspend() + + async def inner(): + _testcapi.set_nomemory(0, 1) + try: + await awaitable + finally: + _testcapi.remove_mem_hooks() + return "resumed" + + async def outer(): + return await inner() + + coro = outer() + try: + with self.assertRaises(MemoryError): + coro.send(None) + finally: + _testcapi.remove_mem_hooks() + awaitable.close() + + def test_call_forms(self): + class C: + async def meth(self, x, *, y=0): + if y: + await AsyncYield(y) + return ("meth", x, y) + + async def func(*args, **kwargs): + if kwargs.get("suspend"): + await AsyncYield("f") + return (args, kwargs) + + obj = C() + bound = obj.meth + + async def outer(): + return [ + await obj.meth(1), + await obj.meth(2, y=3), + await bound(4), + await bound(5, y=6), + await func(7), + await func(*[8, 9]), + await func(**{"a": 10}), + await func(11, suspend=True), + ] + + for _ in range(3): # also with specialized calls + self.assertEqual(run_async(outer()), ( + [3, 6, "f"], + [("meth", 1, 0), ("meth", 2, 3), ("meth", 4, 0), + ("meth", 5, 6), ((7,), {}), ((8, 9), {}), + ((), {"a": 10}), ((11,), {"suspend": True})], + )) + + def test_send_specialized_for_other_awaitables(self): + # The SEND of an await site may have been specialized for another + # kind of awaitable before a coroutine function is awaited there. + async def agen(): + yield 1 + + class Iterator: + async def __anext__(self): + return 2 + + class SuspendingIterator: + async def __anext__(self): + await AsyncYield("s") + return 3 + + async def outer(it): + return await type(it).__anext__(it) + + def check(it, expected): + self.assertEqual(run_async(outer(it)), expected) + + for _ in range(3): + for _ in range(20): + check(agen(), ([], 1)) + check(Iterator(), ([], 2)) + check(SuspendingIterator(), (["s"], 3)) + + def test_traceback_like_eager_coroutine(self): + async def inner(): + raise ValueError + + async def outer(): + await inner() + + def format_tb(): + try: + run_async(outer()) + except ValueError: + return traceback.format_exc() + + lazy = format_tb() + # Origin tracking makes awaited calls create the coroutine eagerly. + old_depth = sys.get_coroutine_origin_tracking_depth() + sys.set_coroutine_origin_tracking_depth(1) + try: + eager = format_tb() + finally: + sys.set_coroutine_origin_tracking_depth(old_depth) + # The whole `await inner()` line, not just `inner()` with carets. + self.assertIn(" await inner()\n File", lazy) + self.assertEqual(lazy, eager) + + @support.cpython_only @unittest.skipIf(_testcapi is None, "requires _testcapi") class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 5a3de22fd6bb2fc..7813c5081edcc7b 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -597,6 +597,39 @@ def _with(c): _with.__code__.co_firstlineno + 3, ) +async def _await_call(f): + return await f(1) + +dis_await_call = """\ +%4d RETURN_GENERATOR + POP_TOP + L1: RESUME 0 + +%4d PUSH_NULL + LOAD_FAST_BORROW 0 (f) + PUSH_NULL + LOAD_SMALL_INT 1 + CALL 3 (1 + awaited) + GET_AWAITABLE 1 (after call) + L2: SEND 4 (to L5) + L3: YIELD_VALUE 1 + L4: RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 6 (to L2) + L5: END_SEND + RETURN_VALUE + L6: CLEANUP_THROW + L7: JUMP_BACKWARD_NO_INTERRUPT 4 (to L5) + + -- L8: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + RERAISE 1 +ExceptionTable: + L1 to L3 -> L8 [0] lasti + L3 to L4 -> L6 [3] + L4 to L7 -> L8 [0] lasti +""" % (_await_call.__code__.co_firstlineno, + _await_call.__code__.co_firstlineno + 1, + ) + async def _asyncwith(c): async with c: x = 1 @@ -1238,6 +1271,9 @@ def test_disassemble_with(self): def test_disassemble_asyncwith(self): self.do_disassembly_test(_asyncwith, dis_asyncwith) + def test_disassemble_await_call(self): + self.do_disassembly_test(_await_call, dis_await_call) + def test_disassemble_try_finally(self): self.do_disassembly_test(_tryfinally, dis_tryfinally) self.do_disassembly_test(_tryfinallyconst, dis_tryfinallyconst) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5f7c15f93a8332a..f2570f4e220ae17 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1630,8 +1630,15 @@ def run_async(coro): except StopIteration: break + class Suspend: + def __await__(self): + yield + class CM: async def __aenter__(self): + # A coroutine that is awaited right away and never + # suspends doesn't execute the caller's SEND at all. + await Suspend() return self async def __aexit__(self, *exc): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst new file mode 100644 index 000000000000000..e7ba18578dfe11d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst @@ -0,0 +1,3 @@ +Speed up ``await`` of a coroutine that completes without suspending by about +40% by running it without creating a coroutine object. Until such a coroutine +first suspends, :attr:`frame.f_generator` is ``None``. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 05a81652a9f7e23..6a280796d64b54a 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -1896,6 +1896,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } @@ -2200,6 +2203,12 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2338,6 +2347,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2972,6 +2987,13 @@ } ex_frame = PyStackRef_Wrap(new_frame); } + // _MAYBE_LAZY_CORO + { + new_frame = ex_frame; + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2983,7 +3005,6 @@ } // _PUSH_FRAME { - new_frame = ex_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3150,6 +3171,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -3466,6 +3490,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -3605,6 +3632,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -3802,6 +3835,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -4694,6 +4733,12 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -4803,6 +4848,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -5197,9 +5248,6 @@ null_in = stack_pointer[-3]; sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6159,7 +6207,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -6826,32 +6874,46 @@ _PyStackRef iterable; _PyStackRef iter; _PyStackRef null; - _PyStackRef none; + _PyStackRef v; iterable = stack_pointer[-1]; aux = &stack_pointer[-1 - (oparg & 1)]; - assert((oparg & 1) == 0 || PyStackRef_IsNull(aux[0])); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); - _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(iterable); - _PyFrame_StackPointerInvalidate(frame); - if (iter_o == NULL) { - stack_pointer += -(oparg & 1); + if ((oparg & 1) && !PyStackRef_IsNull(aux[0])) { + if (PyStackRef_IsTaggedInt(aux[0])) { + iter = PyStackRef_NULL; + null = PyStackRef_NULL; + } + else { + assert(PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(aux[0]))); + iter = aux[0]; + null = PyStackRef_None; + } + v = iterable; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_LABEL(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(iterable); + _PyFrame_StackPointerInvalidate(frame); + if (iter_o == NULL) { + stack_pointer += -(oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + v = PyStackRef_None; + stack_pointer += 1; } - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - none = PyStackRef_None; - stack_pointer[-(oparg & 1)] = iter; - stack_pointer[1 - (oparg & 1)] = null; - stack_pointer[2 - (oparg & 1)] = none; - stack_pointer += 3 - (oparg & 1); + stack_pointer[-1 - (oparg & 1)] = iter; + stack_pointer[-(oparg & 1)] = null; + stack_pointer[1 - (oparg & 1)] = v; + stack_pointer += 2 - (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7199,6 +7261,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } @@ -7366,6 +7431,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -7518,6 +7586,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -7660,8 +7731,10 @@ value = stack_pointer[-1]; index_or_null = stack_pointer[-2]; receiver = stack_pointer[-3]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (PyStackRef_IsNull(receiver) || + PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver)) || + PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(receiver))) + { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -7677,7 +7750,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -8294,6 +8367,9 @@ _PyStackRef val; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _YIELD_VALUE_EVENT { val = stack_pointer[-1]; @@ -8326,8 +8402,31 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + stack_pointer[-1] = retval; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -8342,6 +8441,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -8351,12 +8453,21 @@ Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -11453,32 +11564,57 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + if (exc_info != NULL) { + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11540,6 +11676,7 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; + _PyStackRef null; _PyStackRef null_or_index; _PyStackRef v; _PyStackRef retval; @@ -11561,10 +11698,23 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + if (PyStackRef_IsNone(null)) { + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _SEND { v = stack_pointer[-1]; - null_or_index = stack_pointer[-2]; + null_or_index = null; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && @@ -11637,6 +11787,8 @@ next_instr += 2; INSTRUCTION_STATS(SEND_ASYNC_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef null; _PyStackRef iter; _PyStackRef null_in; _PyStackRef v; @@ -11644,9 +11796,23 @@ _PyStackRef null_out; _PyStackRef retval; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + if (PyStackRef_IsNone(null)) { + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _GUARD_3OS_ASYNC_GEN_ASEND { - iter = stack_pointer[-3]; + iter = receiver; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (!PyAsyncGenASend_CheckExact(iter_o)) { UPDATE_MISS_STATS(SEND); @@ -11657,7 +11823,7 @@ // _SEND_ASYNC_GEN { v = stack_pointer[-1]; - null_in = stack_pointer[-2]; + null_in = null; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); assert(PyAsyncGenASend_CheckExact(iter_o)); PyObject *val = PyStackRef_AsPyObjectBorrow(v); @@ -11702,10 +11868,25 @@ INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; + _PyStackRef null; _PyStackRef v; _PyStackRef gen_frame; _PyStackRef new_frame; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + if (PyStackRef_IsNone(null)) { + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { @@ -11717,7 +11898,6 @@ // _SEND_GEN_FRAME { v = stack_pointer[-1]; - receiver = stack_pointer[-3]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); @@ -11775,6 +11955,8 @@ next_instr += 2; INSTRUCTION_STATS(SEND_VIRTUAL); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef null; _PyStackRef val; _PyStackRef nos; _PyStackRef iter; @@ -11782,6 +11964,20 @@ _PyStackRef none; _PyStackRef next; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + if (PyStackRef_IsNone(null)) { + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _GUARD_TOS_IS_NONE { val = stack_pointer[-1]; @@ -11793,7 +11989,7 @@ } // _GUARD_NOS_NOT_NULL { - nos = stack_pointer[-2]; + nos = null; if (PyStackRef_IsNull(nos)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); @@ -11804,7 +12000,7 @@ { none = val; null_or_index = nos; - iter = stack_pointer[-3]; + iter = receiver; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -13585,6 +13781,9 @@ opcode = YIELD_VALUE; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _MAKE_HEAP_SAFE { value = stack_pointer[-1]; @@ -13594,8 +13793,31 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + stack_pointer[-1] = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -13610,6 +13832,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -13619,12 +13844,21 @@ Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -13777,6 +14011,9 @@ JUMP_TO_LABEL(error); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; + if (dying->lazy_coro) { + frame->instr_ptr += frame->return_offset + 1; + } _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4b..e250145e1693607 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -4,7 +4,6 @@ static void *opcode_targets_table[256] = { &&TARGET_BINARY_SLICE, &&TARGET_BUILD_TEMPLATE, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_CHECK_EG_MATCH, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, @@ -17,8 +16,8 @@ static void *opcode_targets_table[256] = { &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_GET_LEN, - &&TARGET_RESERVED, &&TARGET_INTERPRETER_EXIT, + &&TARGET_RESERVED, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, &&TARGET_MAKE_FUNCTION, @@ -51,6 +50,7 @@ static void *opcode_targets_table[256] = { &&TARGET_BUILD_STRING, &&TARGET_BUILD_TUPLE, &&TARGET_CALL, + &&TARGET_CALL_FUNCTION_EX, &&TARGET_CALL_INTRINSIC_1, &&TARGET_CALL_INTRINSIC_2, &&TARGET_CALL_KW, diff --git a/Objects/genobject.c b/Objects/genobject.c index 82a1bb1ff420682..f5982cbdb3bb199 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1104,9 +1104,8 @@ PyTypeObject PyGen_Type = { }; static PyObject * -make_gen(PyTypeObject *type, PyFunctionObject *func) +make_gen(PyTypeObject *type, PyFunctionObject *func, PyCodeObject *code) { - PyCodeObject *code = (PyCodeObject *)func->func_code; int slots = _PyFrame_NumSlotsForCodeObject(code); PyGenObject *gen = PyObject_GC_NewVar(PyGenObject, type, slots); if (gen == NULL) { @@ -1127,18 +1126,44 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) return (PyObject *)gen; } +static PyObject * +make_coro(PyFunctionObject *func, PyCodeObject *code, + _PyInterpreterFrame *caller) +{ + PyObject *coro = make_gen(&PyCoro_Type, func, code); + if (!coro) { + return NULL; + } + PyThreadState *tstate = _PyThreadState_GET(); + int origin_depth = tstate->coroutine_origin_tracking_depth; + + if (origin_depth == 0) { + ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; + } else { + _PyInterpreterFrame *frame = _PyFrame_GetFirstComplete(caller); + PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); + ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; + if (!cr_origin) { + Py_DECREF(coro); + return NULL; + } + } + return coro; +} + PyObject * _Py_MakeCoro(PyFunctionObject *func) { - int coro_flags = ((PyCodeObject *)func->func_code)->co_flags & + PyCodeObject *code = (PyCodeObject *)func->func_code; + int coro_flags = code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); assert(coro_flags); if (coro_flags == CO_GENERATOR) { - return make_gen(&PyGen_Type, func); + return make_gen(&PyGen_Type, func, code); } if (coro_flags == CO_ASYNC_GENERATOR) { PyAsyncGenObject *ag; - ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); + ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func, code); if (ag == NULL) { return NULL; } @@ -1150,28 +1175,32 @@ _Py_MakeCoro(PyFunctionObject *func) } assert (coro_flags == CO_COROUTINE); - PyObject *coro = make_gen(&PyCoro_Type, func); - if (!coro) { - return NULL; - } PyThreadState *tstate = _PyThreadState_GET(); - int origin_depth = tstate->coroutine_origin_tracking_depth; - - if (origin_depth == 0) { - ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; - } else { - _PyInterpreterFrame *frame = tstate->current_frame; - assert(frame); - assert(_PyFrame_IsIncomplete(frame)); - frame = _PyFrame_GetFirstComplete(frame->previous); - PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); - ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; - if (!cr_origin) { - Py_DECREF(coro); - return NULL; - } - } - return coro; + _PyInterpreterFrame *frame = tstate->current_frame; + assert(frame); + assert(_PyFrame_IsIncomplete(frame)); + return make_coro(func, code, frame->previous); +} + +/* Create the coroutine object for a coroutine whose body has been + * executing directly in `frame`, a frame owned by the thread, because + * its caller awaited the result immediately (see RETURN_GENERATOR in + * bytecodes.c). Called when the body suspends for the first time. + * The frame itself is moved into the coroutine by the caller. + * The body may have replaced func.__code__ in the meantime, so the + * coroutine is sized for the code the frame is actually running. */ +PyObject * +_PyCoro_MaterializeLazy(_PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(frame->lazy_coro); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = + (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyCodeObject *code = _PyFrame_GetCode(frame); + assert((code->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) == CO_COROUTINE); + return make_coro(func, code, frame->previous); } static PyObject * diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index ebcb38b18dec90c..d8adaf26ec18b39 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -3,17 +3,17 @@ unsigned char M_test_frozenmain[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, 0,0,0,0,0,115,188,0,0,0,128,0,0,0,90,0, 77,7,69,0,112,0,90,0,77,7,69,4,112,1,89,2, - 31,0,78,1,50,1,0,0,0,0,0,0,29,0,89,2, - 31,0,78,2,89,0,76,6,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,50,2,0,0,0,0, - 0,0,29,0,89,1,76,8,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,31,0,50,0,0,0, - 0,0,0,0,78,3,42,26,0,0,0,0,0,0,0,0, + 30,0,78,1,49,2,0,0,0,0,0,0,28,0,89,2, + 30,0,78,2,89,0,76,6,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,49,4,0,0,0,0, + 0,0,28,0,89,1,76,8,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,30,0,49,0,0,0, + 0,0,0,0,78,3,41,26,0,0,0,0,0,0,0,0, 0,0,112,5,78,6,67,0,0,0,65,24,0,0,112,6, - 89,2,31,0,78,4,89,6,12,0,78,5,89,5,89,6, - 42,26,0,0,0,0,0,0,0,0,0,0,12,0,48,4, - 50,1,0,0,0,0,0,0,29,0,71,26,0,0,9,0, - 28,0,77,7,33,0,41,7,105,0,0,0,0,122,18,70, + 89,2,30,0,78,4,89,6,11,0,78,5,89,5,89,6, + 41,26,0,0,0,0,0,0,0,0,0,0,11,0,47,4, + 49,2,0,0,0,0,0,0,28,0,71,26,0,0,8,0, + 27,0,77,7,32,0,41,7,105,0,0,0,0,122,18,70, 114,111,122,101,110,32,72,101,108,108,111,32,87,111,114,108, 100,122,8,115,121,115,46,97,114,103,118,218,6,99,111,110, 102,105,103,122,7,99,111,110,102,105,103,32,122,2,58,32, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b735c653cf43733..f81d5fbca731483 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -432,12 +432,17 @@ dummy_func( (void)index_or_null; DEAD(value); DEAD(index_or_null); - PyStackRef_CLOSE(receiver); + // NULL if an awaited call returned without creating a coroutine. + PyStackRef_XCLOSE(receiver); } tier1 inst(INSTRUMENTED_END_SEND, (receiver, index_or_null, value -- val)) { - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + /* A NULL receiver is a coroutine that returned before it was + * created (see GET_AWAITABLE). */ + if (PyStackRef_IsNull(receiver) || + PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver)) || + PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(receiver))) + { int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); if (err) { ERROR_NO_POP(); @@ -447,7 +452,7 @@ dummy_func( (void)index_or_null; DEAD(index_or_null); DEAD(value); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); } macro(UNARY_NEGATIVE) = _UNARY_NEGATIVE + POP_TOP; @@ -1649,18 +1654,39 @@ dummy_func( awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); } - // Also pushes the NULL and None that the following SEND loop starts - // with. If the low bit of oparg is set, `iterable` is the result of - // an awaited call and `aux` is the slot pushed below that call. - inst(GET_AWAITABLE, (aux[oparg & 1], iterable -- iter, null, none)) { - assert((oparg & 1) == 0 || PyStackRef_IsNull(aux[0])); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); - PyStackRef_CLOSE(iterable); - DEAD(aux); - ERROR_IF(iter_o == NULL); - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - none = PyStackRef_None; + // Also pushes the receiver, NULL and value that the following SEND + // loop starts with. If the low bit of oparg is set, `iterable` is the + // result of an awaited call and `aux` is the slot pushed below that + // call. A coroutine body that the call ran without creating the + // coroutine object (see RETURN_GENERATOR) left in it: + // - a tagged int if the body returned `iterable`: SEND gets a NULL + // receiver and jumps to END_SEND; + // - the coroutine if the body suspended, yielding `iterable`: SEND + // gets None instead of NULL and goes on to YIELD_VALUE. + inst(GET_AWAITABLE, (aux[oparg & 1], iterable -- iter, null, v)) { + if ((oparg & 1) && !PyStackRef_IsNull(aux[0])) { + if (PyStackRef_IsTaggedInt(aux[0])) { + iter = PyStackRef_NULL; + null = PyStackRef_NULL; + } + else { + assert(PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(aux[0]))); + iter = aux[0]; + null = PyStackRef_None; + } + v = iterable; + DEAD(iterable); + DEAD(aux); + } + else { + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); + PyStackRef_CLOSE(iterable); + DEAD(aux); + ERROR_IF(iter_o == NULL); + iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + v = PyStackRef_None; + } } family(SEND, INLINE_CACHE_ENTRIES_SEND) = { @@ -1727,7 +1753,29 @@ dummy_func( } } - macro(SEND) = _SPECIALIZE_SEND + _SEND; + // See GET_AWAITABLE: a NULL receiver means that an awaited call + // returned `v` without creating a coroutine object, and None in + // place of NULL that the coroutine it created has already yielded `v`. + // Every form of SEND starts with this, since an await site may have + // been specialized for whatever its awaitable was before. + replaced op(_SEND_LAZY_CORO, (receiver, null, v -- receiver, null, v)) { + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + if (PyStackRef_IsNone(null)) { + null = PyStackRef_NULL; + SYNC_SP(); + DISPATCH(); + } + } + + tier2 op(_GUARD_SEND_NOT_LAZY_CORO, (receiver, null, v -- receiver, null, v)) { + EXIT_IF(PyStackRef_IsNull(receiver)); + EXIT_IF(PyStackRef_IsNone(null)); + } + + macro(SEND) = _SPECIALIZE_SEND + _SEND_LAZY_CORO + _SEND; op(_SEND_GEN_FRAME, (receiver, null, v -- receiver, null, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); @@ -1748,6 +1796,7 @@ dummy_func( macro(SEND_GEN) = unused/1 + _RECORD_3OS_GEN_FUNC + + _SEND_LAZY_CORO + _CHECK_PEP_523 + _SEND_GEN_FRAME + _PUSH_FRAME; @@ -1802,6 +1851,7 @@ dummy_func( macro(SEND_VIRTUAL) = unused/1 + + _SEND_LAZY_CORO + _GUARD_TOS_IS_NONE + _GUARD_NOS_NOT_NULL + _SEND_VIRTUAL; @@ -1834,6 +1884,7 @@ dummy_func( macro(SEND_ASYNC_GEN) = unused/1 + + _SEND_LAZY_CORO + _GUARD_3OS_ASYNC_GEN_ASEND + _SEND_ASYNC_GEN; @@ -1856,13 +1907,44 @@ dummy_func( EXIT_IF(what == PYGEN_RETURN); } + // A lazily started coroutine (see RETURN_GENERATOR) is materialized + // by tier one (see _YIELD_VALUE), which also resumes the caller at a + // different place. Tier two replaces this with _GUARD_NOT_LAZY_CORO. + replaced op(_CHECK_LAZY_CORO, (--)) { + } + + tier2 op(_GUARD_NOT_LAZY_CORO, (--)) { + EXIT_IF(frame->lazy_coro); + } + op(_YIELD_VALUE, (retval -- value)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. + // The one exception is a lazily started coroutine (see + // RETURN_GENERATOR) that cannot be allocated: the await then + // raises MemoryError through CLEANUP_THROW. assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + if (gen == NULL) { + ERROR_NO_POP(); + } + /* Move the body's exception state (see RETURN_GENERATOR) + * into the coroutine, where SEND_GEN would have put it. */ + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -1875,6 +1957,9 @@ dummy_func( _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -1885,16 +1970,29 @@ dummy_func( Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif RELOAD_STACK(); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + /* The caller is right after the awaited call: hand the + * coroutine to its GET_AWAITABLE through the slot below + * the call. */ + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } macro(YIELD_VALUE) = + _CHECK_LAZY_CORO + _MAKE_HEAP_SAFE + _YIELD_VALUE; @@ -1914,6 +2012,7 @@ dummy_func( } macro(INSTRUMENTED_YIELD_VALUE) = + _CHECK_LAZY_CORO + _YIELD_VALUE_EVENT + _MAKE_HEAP_SAFE + _YIELD_VALUE; @@ -1969,9 +2068,9 @@ dummy_func( tier1 inst(CLEANUP_THROW, (sub_iter, null_in, last_sent_val, exc_value_st -- none, null_out, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif + /* Reached on throw()/close(), or when a lazily started + * coroutine could not be created at its first suspension + * (see _YIELD_VALUE). */ assert(exc_value && PyExceptionInstance_Check(exc_value)); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); if (matches) { @@ -4379,6 +4478,9 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = INSTRUCTION_SIZE; DISPATCH_INLINED(new_frame); } @@ -4467,6 +4569,7 @@ dummy_func( _CHECK_FUNCTION_VERSION + _CHECK_RECURSION_REMAINING + _PY_FRAME_GENERAL + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -4500,6 +4603,7 @@ dummy_func( flush + // so that self is in the argument array _CHECK_RECURSION_REMAINING + _PY_FRAME_GENERAL + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -4620,6 +4724,7 @@ dummy_func( _CHECK_STACK_SPACE + _CHECK_RECURSION_REMAINING + _INIT_CALL_PY_EXACT_ARGS + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -4632,6 +4737,7 @@ dummy_func( _CHECK_STACK_SPACE + _CHECK_RECURSION_REMAINING + _INIT_CALL_PY_EXACT_ARGS + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -5424,6 +5530,9 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert(INSTRUCTION_SIZE == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = INSTRUCTION_SIZE; DISPATCH_INLINED(new_frame); @@ -5489,6 +5598,7 @@ dummy_func( _CHECK_FUNCTION_VERSION_KW + _CHECK_RECURSION_REMAINING + _PY_FRAME_KW + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -5521,6 +5631,7 @@ dummy_func( _EXPAND_METHOD_KW + flush + // so that self is in the argument array _PY_FRAME_KW + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -5669,6 +5780,9 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert(INSTRUCTION_SIZE == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = INSTRUCTION_SIZE; DISPATCH_INLINED(new_frame); @@ -5741,6 +5855,7 @@ dummy_func( _MAKE_CALLARGS_A_TUPLE + _CHECK_IS_PY_CALLABLE_EX + _PY_FRAME_EX + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -5813,25 +5928,53 @@ dummy_func( inst(RETURN_GENERATOR, (-- res)) { assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - ERROR_IF(gen == NULL); - assert(STACK_LEVEL() <= 2); - SAVE_STACK(); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - RELOAD_STACK(); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + /* The body gets its own exception state, as SEND_GEN + * would have given it. It lives on the data stack right + * after this frame; clear_thread_frame() and _YIELD_VALUE + * pop it. */ + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + if (exc_info == NULL) { + /* Out of memory: take the regular path. */ + _PyErr_Clear(tstate); + } + } + #endif + if (exc_info != NULL) { + /* An awaited call (see _MAYBE_LAZY_CORO): keep running the + * body in this frame, and only create the coroutine object + * if the body suspends (see _YIELD_VALUE). */ + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + /* The following POP_TOP discards this. */ + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + ERROR_IF(gen == NULL); + assert(STACK_LEVEL() <= 2); + SAVE_STACK(); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + RELOAD_STACK(); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + } } inst(BUILD_SLICE, (args[oparg] -- slice)) { @@ -6108,6 +6251,12 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, framesize)); } + replaced op(_MAYBE_LAZY_CORO, (new_frame -- new_frame)) { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } + op(_SAVE_RETURN_OFFSET, (--)) { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); @@ -6386,7 +6535,8 @@ dummy_func( tier2 op(_RECORD_3OS_GEN_FUNC, (gen, nos, tos -- gen, nos, tos)) { PyObject *obj = PyStackRef_AsPyObjectBorrow(gen); - if (PyGen_Check(obj)) { + // `gen` is NULL after an awaited call (see GET_AWAITABLE). + if (obj != NULL && PyGen_Check(obj)) { PyGenObject *gen_obj = (PyGenObject *)obj; _PyStackRef func = gen_obj->gi_iframe.f_funcobj; if (!PyStackRef_IsNull(func)) { @@ -6527,6 +6677,11 @@ dummy_func( // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; + if (dying->lazy_coro) { + /* Raise from the caller's SEND, as if the coroutine had been + * created (see GET_AWAITABLE). */ + frame->instr_ptr += frame->return_offset + 1; + } _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { diff --git a/Python/ceval.c b/Python/ceval.c index ccc0edca2fc7940..bbd73b1fff96db5 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2005,10 +2005,38 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, return -1; } +/* Pop the exception state that RETURN_GENERATOR gave a lazily started + * coroutine body, restoring the caller's. Returns the body's exception, + * if any (a new reference). */ +static PyObject * +pop_lazy_coro_exc_state(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + assert(frame->lazy_coro); + _PyErr_StackItem *exc_info = tstate->exc_info; + tstate->exc_info = exc_info->previous_item; + PyObject *exc_value = exc_info->exc_value; + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + frame->lazy_coro = 0; + return exc_value; +} + static void clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { assert(frame->owner == FRAME_OWNED_BY_THREAD); + PyObject *lazy_exc = NULL; + if (frame->lazy_coro) { + lazy_exc = pop_lazy_coro_exc_state(tstate, frame); + /* The body finished without creating its coroutine object: tell + * the caller's GET_AWAITABLE through the slot that the compiler + * pushed below the awaited call. */ + _PyInterpreterFrame *caller = frame->previous; + assert(PyStackRef_IsNull(caller->stackpointer[-1])); + caller->stackpointer[-1] = PyStackRef_TagInt(0); + // Like clear_gen_frame(): a frame object that outlives the + // coroutine must not keep the caller's frame alive via f_back. + frame->previous = NULL; + } // Make sure that this is, indeed, the top frame. We can't check this in // _PyThreadState_PopFrame, since f_code is already cleared at that point: assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == @@ -2017,6 +2045,7 @@ clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) _PyFrame_ClearExceptCode(frame); PyStackRef_CLEAR(frame->f_executable); _PyThreadState_PopFrame(tstate, frame); + Py_XDECREF(lazy_exc); } static void @@ -3686,6 +3715,29 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name } } +/* YIELD_VALUE support: move the frame of a lazily started coroutine + * from the thread's data stack into `gen`. Returns the new frame. */ +_PyInterpreterFrame * +_PyEval_MoveLazyCoroFrame(PyThreadState *tstate, PyGenObject *gen, + _PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(frame->lazy_coro); + assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == + tstate->datastack_top); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_Copy(frame, gen_frame); + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + PyFrameObject *f = gen_frame->frame_obj; + if (f != NULL) { + Py_BEGIN_CRITICAL_SECTION(f); + f->f_frame = gen_frame; + Py_END_CRITICAL_SECTION(); + } + _PyThreadState_PopFrame(tstate, frame); + return gen_frame; +} + PyObject * _PyEval_GetAwaitable(PyObject *iterable, int oparg) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 36921e35533433d..56d546dadd16804 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -156,6 +156,27 @@ # define LEAVE_TRACING() tracing_mode = 0 #endif +/* An awaited call (the low bit of oparg is set) of a coroutine function lets + * the body run in the pushed frame without creating the coroutine object + * until it first suspends (see RETURN_GENERATOR). Not while the JIT records + * a trace, which expects the regular path, nor while coroutine origin + * tracking is on, since the origin is captured when the object is created. */ +#if _Py_TIER2 +# define LAZY_CORO_ALLOWED() (!IS_JIT_TRACING()) +#else +# define LAZY_CORO_ALLOWED() 1 +#endif +#define MAYBE_START_LAZY_CORO(FRAME) \ + do { \ + _PyInterpreterFrame *lazy_frame_ = (FRAME); \ + if ((_PyFrame_GetCode(lazy_frame_)->co_flags & CO_COROUTINE) && \ + tstate->coroutine_origin_tracking_depth == 0 && \ + LAZY_CORO_ALLOWED()) \ + { \ + lazy_frame_->lazy_coro = 1; \ + } \ + } while (0) + #if _Py_TIER2 #define STOP_TRACING() \ do { \ diff --git a/Python/codegen.c b/Python/codegen.c index 647d6a18336fe81..fe27ce040d14a64 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -4587,7 +4587,7 @@ codegen_call_helper_impl(compiler *c, location loc, if (nkwelts == 0) { ADDOP(c, loc, PUSH_NULL); } - ADDOP(c, loc, CALL_FUNCTION_EX); + ADDOP_I(c, loc, CALL_FUNCTION_EX, awaited); return SUCCESS; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d69a3bc052ae7bc..fb126663f365abb 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2622,7 +2622,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = val; _tos_cache1 = PyStackRef_ZERO_BITS; @@ -8913,35 +8913,48 @@ _PyStackRef *aux; _PyStackRef iter; _PyStackRef null; - _PyStackRef none; + _PyStackRef v; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); iterable = _stack_item_0; aux = &stack_pointer[-(oparg & 1)]; - assert((oparg & 1) == 0 || PyStackRef_IsNull(aux[0])); - stack_pointer[0] = iterable; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); - _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(iterable); - _PyFrame_StackPointerInvalidate(frame); - if (iter_o == NULL) { - stack_pointer += -(oparg & 1); + if ((oparg & 1) && !PyStackRef_IsNull(aux[0])) { + if (PyStackRef_IsTaggedInt(aux[0])) { + iter = PyStackRef_NULL; + null = PyStackRef_NULL; + } + else { + assert(PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(aux[0]))); + iter = aux[0]; + null = PyStackRef_None; + } + v = iterable; + } + else { + stack_pointer[0] = iterable; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(iterable); + _PyFrame_StackPointerInvalidate(frame); + if (iter_o == NULL) { + stack_pointer += -(oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + v = PyStackRef_None; } - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - none = PyStackRef_None; - _tos_cache2 = none; + _tos_cache2 = v; _tos_cache1 = null; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(3); @@ -8951,6 +8964,132 @@ break; } + /* _SEND_LAZY_CORO is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_SEND_NOT_LAZY_CORO_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null; + _PyStackRef receiver; + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (PyStackRef_IsNone(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = null; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_SEND_NOT_LAZY_CORO_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null; + _PyStackRef receiver; + _PyStackRef _stack_item_0 = _tos_cache0; + null = stack_pointer[-1]; + receiver = stack_pointer[-2]; + if (PyStackRef_IsNull(receiver)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (PyStackRef_IsNone(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = null; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_SEND_NOT_LAZY_CORO_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null; + _PyStackRef receiver; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null = _stack_item_0; + receiver = stack_pointer[-1]; + if (PyStackRef_IsNull(receiver)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (PyStackRef_IsNone(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = null; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_SEND_NOT_LAZY_CORO_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null; + _PyStackRef receiver; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null = _stack_item_1; + receiver = _stack_item_0; + if (PyStackRef_IsNull(receiver)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = null; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (PyStackRef_IsNone(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = null; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = null; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _SEND_GEN_FRAME_r33: { CHECK_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -9500,6 +9639,78 @@ break; } + /* _CHECK_LAZY_CORO is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_LAZY_CORO_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOT_LAZY_CORO_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOT_LAZY_CORO_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOT_LAZY_CORO_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _YIELD_VALUE_r11: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -9509,11 +9720,40 @@ oparg = CURRENT_OPARG(); retval = _stack_item_0; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + stack_pointer[0] = retval; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + stack_pointer += 1; + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); @@ -9523,6 +9763,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -9532,12 +9775,21 @@ Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); _tos_cache0 = value; @@ -20876,33 +21128,58 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + if (exc_info != NULL) { + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); _tos_cache0 = res; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -23007,6 +23284,8 @@ break; } + /* _MAYBE_LAZY_CORO is not a viable micro-op for tier 2 because it is replaced */ + case _SAVE_RETURN_OFFSET_r00: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0f8863b4fe84920..94f8e178f69dc29 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1896,6 +1896,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } @@ -2200,6 +2203,12 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2338,6 +2347,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2972,6 +2987,13 @@ } ex_frame = PyStackRef_Wrap(new_frame); } + // _MAYBE_LAZY_CORO + { + new_frame = ex_frame; + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2983,7 +3005,6 @@ } // _PUSH_FRAME { - new_frame = ex_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3150,6 +3171,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -3466,6 +3490,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -3605,6 +3632,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -3802,6 +3835,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -4694,6 +4733,12 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -4803,6 +4848,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -5197,9 +5248,6 @@ null_in = stack_pointer[-3]; sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6159,7 +6207,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -6826,32 +6874,46 @@ _PyStackRef iterable; _PyStackRef iter; _PyStackRef null; - _PyStackRef none; + _PyStackRef v; iterable = stack_pointer[-1]; aux = &stack_pointer[-1 - (oparg & 1)]; - assert((oparg & 1) == 0 || PyStackRef_IsNull(aux[0])); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); - _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(iterable); - _PyFrame_StackPointerInvalidate(frame); - if (iter_o == NULL) { - stack_pointer += -(oparg & 1); + if ((oparg & 1) && !PyStackRef_IsNull(aux[0])) { + if (PyStackRef_IsTaggedInt(aux[0])) { + iter = PyStackRef_NULL; + null = PyStackRef_NULL; + } + else { + assert(PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(aux[0]))); + iter = aux[0]; + null = PyStackRef_None; + } + v = iterable; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg >> 1); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - JUMP_TO_LABEL(error); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(iterable); + _PyFrame_StackPointerInvalidate(frame); + if (iter_o == NULL) { + stack_pointer += -(oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + v = PyStackRef_None; + stack_pointer += 1; } - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - none = PyStackRef_None; - stack_pointer[-(oparg & 1)] = iter; - stack_pointer[1 - (oparg & 1)] = null; - stack_pointer[2 - (oparg & 1)] = none; - stack_pointer += 3 - (oparg & 1); + stack_pointer[-1 - (oparg & 1)] = iter; + stack_pointer[-(oparg & 1)] = null; + stack_pointer[1 - (oparg & 1)] = v; + stack_pointer += 2 - (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7199,6 +7261,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } @@ -7366,6 +7431,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -7518,6 +7586,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -7660,8 +7731,10 @@ value = stack_pointer[-1]; index_or_null = stack_pointer[-2]; receiver = stack_pointer[-3]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (PyStackRef_IsNull(receiver) || + PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver)) || + PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(receiver))) + { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -7677,7 +7750,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -8293,6 +8366,9 @@ _PyStackRef val; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _YIELD_VALUE_EVENT { val = stack_pointer[-1]; @@ -8325,8 +8401,31 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + stack_pointer[-1] = retval; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -8341,6 +8440,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -8350,12 +8452,21 @@ Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -11450,32 +11561,57 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + if (exc_info != NULL) { + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11537,6 +11673,7 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; + _PyStackRef null; _PyStackRef null_or_index; _PyStackRef v; _PyStackRef retval; @@ -11558,10 +11695,23 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + if (PyStackRef_IsNone(null)) { + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _SEND { v = stack_pointer[-1]; - null_or_index = stack_pointer[-2]; + null_or_index = null; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && @@ -11634,6 +11784,8 @@ next_instr += 2; INSTRUCTION_STATS(SEND_ASYNC_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef null; _PyStackRef iter; _PyStackRef null_in; _PyStackRef v; @@ -11641,9 +11793,23 @@ _PyStackRef null_out; _PyStackRef retval; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + if (PyStackRef_IsNone(null)) { + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _GUARD_3OS_ASYNC_GEN_ASEND { - iter = stack_pointer[-3]; + iter = receiver; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (!PyAsyncGenASend_CheckExact(iter_o)) { UPDATE_MISS_STATS(SEND); @@ -11654,7 +11820,7 @@ // _SEND_ASYNC_GEN { v = stack_pointer[-1]; - null_in = stack_pointer[-2]; + null_in = null; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); assert(PyAsyncGenASend_CheckExact(iter_o)); PyObject *val = PyStackRef_AsPyObjectBorrow(v); @@ -11699,10 +11865,25 @@ INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; + _PyStackRef null; _PyStackRef v; _PyStackRef gen_frame; _PyStackRef new_frame; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + if (PyStackRef_IsNone(null)) { + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { @@ -11714,7 +11895,6 @@ // _SEND_GEN_FRAME { v = stack_pointer[-1]; - receiver = stack_pointer[-3]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); @@ -11772,6 +11952,8 @@ next_instr += 2; INSTRUCTION_STATS(SEND_VIRTUAL); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef null; _PyStackRef val; _PyStackRef nos; _PyStackRef iter; @@ -11779,6 +11961,20 @@ _PyStackRef none; _PyStackRef next; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + if (PyStackRef_IsNone(null)) { + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _GUARD_TOS_IS_NONE { val = stack_pointer[-1]; @@ -11790,7 +11986,7 @@ } // _GUARD_NOS_NOT_NULL { - nos = stack_pointer[-2]; + nos = null; if (PyStackRef_IsNull(nos)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); @@ -11801,7 +11997,7 @@ { none = val; null_or_index = nos; - iter = stack_pointer[-3]; + iter = receiver; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -13582,6 +13778,9 @@ opcode = YIELD_VALUE; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _MAKE_HEAP_SAFE { value = stack_pointer[-1]; @@ -13591,8 +13790,31 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + stack_pointer[-1] = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -13607,6 +13829,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -13616,12 +13841,21 @@ Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -13774,6 +14008,9 @@ JUMP_TO_LABEL(error); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; + if (dying->lazy_coro) { + frame->instr_ptr += frame->return_offset + 1; + } _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { diff --git a/Python/intrinsics.c b/Python/intrinsics.c index dfa81baa654e123..e3b0fd406a612a4 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -144,7 +144,11 @@ static PyObject * stopiteration_error(PyThreadState* tstate, PyObject *exc) { _PyInterpreterFrame *frame = tstate->current_frame; - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + /* A coroutine that is awaited immediately runs in a thread-owned + * frame until it first suspends (see RETURN_GENERATOR). */ + assert(frame->owner == FRAME_OWNED_BY_GENERATOR || + (frame->owner == FRAME_OWNED_BY_THREAD && + (_PyFrame_GetCode(frame)->co_flags & CO_COROUTINE))); assert(PyExceptionInstance_Check(exc)); const char *msg = NULL; if (PyErr_GivenExceptionMatches(exc, PyExc_StopIteration)) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4b..e250145e1693607 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -4,7 +4,6 @@ static void *opcode_targets_table[256] = { &&TARGET_BINARY_SLICE, &&TARGET_BUILD_TEMPLATE, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_CHECK_EG_MATCH, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, @@ -17,8 +16,8 @@ static void *opcode_targets_table[256] = { &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_GET_LEN, - &&TARGET_RESERVED, &&TARGET_INTERPRETER_EXIT, + &&TARGET_RESERVED, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, &&TARGET_MAKE_FUNCTION, @@ -51,6 +50,7 @@ static void *opcode_targets_table[256] = { &&TARGET_BUILD_STRING, &&TARGET_BUILD_TUPLE, &&TARGET_CALL, + &&TARGET_CALL_FUNCTION_EX, &&TARGET_CALL_INTRINSIC_1, &&TARGET_CALL_INTRINSIC_2, &&TARGET_CALL_KW, diff --git a/Python/optimizer.c b/Python/optimizer.c index e05adb344c8d06d..bc161d7a85200f5 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -502,6 +502,9 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_LOAD_BYTECODE] = _NOP, [_SEND_VIRTUAL] = _SEND_VIRTUAL_TIER_TWO, [_SEND_ASYNC_GEN] = _SEND_ASYNC_GEN_TIER_TWO, + [_SEND_LAZY_CORO] = _GUARD_SEND_NOT_LAZY_CORO, + [_MAYBE_LAZY_CORO] = _NOP, + [_CHECK_LAZY_CORO] = _GUARD_NOT_LAZY_CORO, }; static const uint8_t @@ -966,6 +969,9 @@ _PyJit_translate_single_bytecode_to_trace( target = next_inst; } } + else if (uop == _NOP || uop == _GUARD_NOT_LAZY_CORO) { + // _MAYBE_LAZY_CORO and _CHECK_LAZY_CORO don't jump. + } else { int extended_arg = orig_oparg > 255; uint32_t jump_target = next_inst + orig_oparg + extended_arg; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 28cc0981b94f8c7..530c8ab71150787 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -206,11 +206,11 @@ dummy_func(void) { res = PyJitRef_Borrow(sym_new_null(ctx)); } - op(_GET_AWAITABLE, (aux[oparg & 1], iterable -- iter, null, none)) { + op(_GET_AWAITABLE, (aux[oparg & 1], iterable -- iter, null, v)) { // After an awaited call, any of the outputs may be NULL. iter = sym_new_unknown(ctx); null = sym_new_unknown(ctx); - none = sym_new_unknown(ctx); + v = sym_new_unknown(ctx); } op(_GUARD_TOS_INT, (value -- value)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 54c72787952885b..842af3a56607e3d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1889,19 +1889,25 @@ case _GET_AWAITABLE: { JitOptRef iter; JitOptRef null; - JitOptRef none; + JitOptRef v; iter = sym_new_unknown(ctx); null = sym_new_unknown(ctx); - none = sym_new_unknown(ctx); + v = sym_new_unknown(ctx); CHECK_STACK_BOUNDS(2 - (oparg & 1)); stack_pointer[-1 - (oparg & 1)] = iter; stack_pointer[-(oparg & 1)] = null; - stack_pointer[1 - (oparg & 1)] = none; + stack_pointer[1 - (oparg & 1)] = v; stack_pointer += 2 - (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } + /* _SEND_LAZY_CORO is not a viable micro-op for tier 2 */ + + case _GUARD_SEND_NOT_LAZY_CORO: { + break; + } + case _SEND_GEN_FRAME: { JitOptRef v; JitOptRef receiver; @@ -1972,6 +1978,12 @@ break; } + /* _CHECK_LAZY_CORO is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_LAZY_CORO: { + break; + } + case _YIELD_VALUE: { JitOptRef retval; JitOptRef value; @@ -5590,6 +5602,8 @@ break; } + /* _MAYBE_LAZY_CORO is not a viable micro-op for tier 2 */ + case _SAVE_RETURN_OFFSET: { break; } diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 75b5ba65f972f15..c0a33c6e1b201f2 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -52,7 +52,7 @@ void _PyOpcode_RecordFunction_3OS_GEN_FUNC(_PyInterpreterFrame *frame, _PyStackR _PyStackRef gen; gen = stack_pointer[-3]; PyObject *obj = PyStackRef_AsPyObjectBorrow(gen); - if (PyGen_Check(obj)) { + if (obj != NULL && PyGen_Check(obj)) { PyGenObject *gen_obj = (PyGenObject *)obj; _PyStackRef func = gen_obj->gi_iframe.f_funcobj; if (!PyStackRef_IsNull(func)) { diff --git a/Python/specialize.c b/Python/specialize.c index cb84fb4b81866da..ce8cf19ff11f735 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2738,11 +2738,12 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT Py_NO_INLINE void _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) { - PyObject *receiver = PyStackRef_AsPyObjectBorrow(receiver_st); - assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND); - PyTypeObject *tp = Py_TYPE(receiver); + /* NULL is a coroutine that an awaited call ran to completion without + * creating it (see GET_AWAITABLE); SEND_GEN handles that too. */ + PyTypeObject *tp = PyStackRef_IsNull(receiver_st) ? &PyCoro_Type + : PyStackRef_TYPE(receiver_st); if (tp == &PyGen_Type || tp == &PyCoro_Type) { /* Don't specialize if PEP 523 is active */ if (!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) { @@ -2761,7 +2762,8 @@ _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) return; } SPECIALIZATION_FAIL(SEND, - _PySpecialization_ClassifyIterator(receiver)); + _PySpecialization_ClassifyIterator( + PyStackRef_AsPyObjectBorrow(receiver_st))); failure: unspecialize(instr); }