From 839fcddd81f33502cf017048afffad1b17bd6daf Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 18 Nov 2026 21:14:56 +0100 Subject: [PATCH 1/3] gh-161170: Encode an awaited flag in the oparg of CALL, CALL_KW and CALL_FUNCTION_EX The oparg of CALL and CALL_KW is now the number of arguments shifted left by one, and the low bit of all three calls is reserved for marking a call whose result is awaited right away. The compiler doesn't set it yet, so there is no behavior change. _INIT_CALL_PY_EXACT_ARGS is replicated for the opargs 0..9 so that calls with up to 4 arguments, awaited or not, still get a constant argument count in tier 2. Co-Authored-By: Claude Opus 5.5 --- Doc/library/dis.rst | 22 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 42 +- Include/internal/pycore_uop_ids.h | 14 +- Include/internal/pycore_uop_metadata.h | 107 ++++- Lib/dis.py | 11 + Lib/test/test_compiler_codegen.py | 6 +- Lib/test/test_dis.py | 76 ++-- Modules/_testinternalcapi/test_cases.c.h | 374 ++++++++-------- Python/bytecodes.c | 162 +++---- Python/codegen.c | 32 +- Python/executor_cases.c.h | 507 ++++++++++++++-------- Python/generated_cases.c.h | 374 ++++++++-------- Python/optimizer_bytecodes.c | 106 ++--- Python/optimizer_cases.c.h | 268 ++++++------ Python/record_functions.c.h | 6 +- Python/specialize.c | 8 +- 17 files changed, 1189 insertions(+), 929 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 16c2dab5b8adba..96b4003ef93d34 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1615,7 +1615,8 @@ iterations of the loop. * ``self`` or ``NULL`` * The remaining positional arguments - ``argc`` is the total of the positional arguments, excluding ``self``. + ``argc >> 1`` is the total of the positional arguments, excluding ``self``. + The low bit of ``argc`` is set when the result is awaited right away. ``CALL`` pops all arguments and the callable object off the stack, calls the callable object with those arguments, and pushes the return value @@ -1629,6 +1630,10 @@ iterations of the loop. .. versionchanged:: 3.13 Calls with keyword arguments are now handled by :opcode:`CALL_KW`. + .. versionchanged:: next + The number of arguments is now ``argc >> 1``; the low bit marks an + awaited call. + .. opcode:: CALL_KW (argc) @@ -1641,7 +1646,8 @@ iterations of the loop. * The named arguments * A :class:`tuple` of keyword argument names - ``argc`` is the total of the positional and named arguments, excluding ``self``. + ``argc >> 1`` is the total of the positional and named arguments, excluding + ``self``. The low bit of ``argc`` is set when the result is awaited right away. The length of the tuple of keyword argument names is the number of named arguments. ``CALL_KW`` pops all arguments, the keyword names, and the callable object @@ -1650,21 +1656,29 @@ iterations of the loop. .. versionadded:: 3.13 + .. versionchanged:: next + The number of arguments is now ``argc >> 1``; the low bit marks an + awaited call. + .. opcode:: CALL_FUNCTION_EX (flags) Calls a callable object with variable set of positional and keyword - arguments. If the lowest bit of *flags* is set, the top of the stack - contains a mapping object containing additional keyword arguments. + arguments. The top of the stack contains a mapping object with additional + keyword arguments, or ``NULL``. Before the callable is called, the mapping object and iterable object are each "unpacked" and their contents passed in as keyword and positional arguments respectively. ``CALL_FUNCTION_EX`` pops all arguments and the callable object off the stack, calls the callable object with those arguments, and pushes the return value returned by the callable object. + The lowest bit of *flags* is set when the result is awaited right away. .. versionadded:: 3.6 + .. versionchanged:: next + The lowest bit of *flags* marks an awaited call. + .. opcode:: PUSH_NULL diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 15d3ad6f3a336b..ddc005a9b0f2f2 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -304,6 +304,7 @@ Known values: Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) Python 3.16a1 3706 (Add INTRINSIC_MATCH_CLASS_ISINSTANCE) + Python 3.16a1 3707 (Lazily materialize coroutine objects for awaited calls) Python 3.17 will start with 3750 @@ -313,7 +314,7 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3706 +#define PYC_MAGIC_NUMBER 3707 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 04b826ed09144e..9f96aa4ba98f37 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -93,21 +93,21 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CACHE: return 0; case CALL: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_ALLOC_AND_ENTER_INIT: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BOUND_METHOD_EXACT_ARGS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BOUND_METHOD_GENERAL: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BUILTIN_CLASS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BUILTIN_FAST: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BUILTIN_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_BUILTIN_O: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_EX_NON_PY_GENERAL: return 4; case CALL_EX_PY: @@ -121,31 +121,31 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CALL_ISINSTANCE: return 4; case CALL_KW: - return 3 + oparg; + return 3 + (oparg >> 1); case CALL_KW_BOUND_METHOD: - return 3 + oparg; + return 3 + (oparg >> 1); case CALL_KW_NON_PY: - return 3 + oparg; + return 3 + (oparg >> 1); case CALL_KW_PY: - return 3 + oparg; + return 3 + (oparg >> 1); case CALL_LEN: return 3; case CALL_LIST_APPEND: return 3; case CALL_METHOD_DESCRIPTOR_FAST: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_METHOD_DESCRIPTOR_NOARGS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_METHOD_DESCRIPTOR_O: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_NON_PY_GENERAL: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_PY_EXACT_ARGS: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_PY_GENERAL: - return 2 + oparg; + return 2 + (oparg >> 1); case CALL_STR_1: return 3; case CALL_TUPLE_1: @@ -235,11 +235,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case IMPORT_NAME: return 2; case INSTRUMENTED_CALL: - return 2 + oparg; + return 2 + (oparg >> 1); case INSTRUMENTED_CALL_FUNCTION_EX: return 4; case INSTRUMENTED_CALL_KW: - return 3 + oparg; + return 3 + (oparg >> 1); case INSTRUMENTED_END_ASYNC_FOR: return 2; case INSTRUMENTED_END_FOR: diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ba6e0252024a6d..3f582b7dcb8245 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -219,6 +219,11 @@ enum { _INIT_CALL_PY_EXACT_ARGS_2, _INIT_CALL_PY_EXACT_ARGS_3, _INIT_CALL_PY_EXACT_ARGS_4, + _INIT_CALL_PY_EXACT_ARGS_5, + _INIT_CALL_PY_EXACT_ARGS_6, + _INIT_CALL_PY_EXACT_ARGS_7, + _INIT_CALL_PY_EXACT_ARGS_8, + _INIT_CALL_PY_EXACT_ARGS_9, _INSERT_NULL, _IS_NONE, _IS_OP, @@ -366,7 +371,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 654 +#define MAX_UOP_ID 659 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -1013,6 +1018,11 @@ enum { _INIT_CALL_PY_EXACT_ARGS_2_r01, _INIT_CALL_PY_EXACT_ARGS_3_r01, _INIT_CALL_PY_EXACT_ARGS_4_r01, + _INIT_CALL_PY_EXACT_ARGS_5_r01, + _INIT_CALL_PY_EXACT_ARGS_6_r01, + _INIT_CALL_PY_EXACT_ARGS_7_r01, + _INIT_CALL_PY_EXACT_ARGS_8_r01, + _INIT_CALL_PY_EXACT_ARGS_9_r01, _INSERT_NULL_r10, _INSTRUMENTED_FOR_ITER_r23, _INSTRUMENTED_INSTRUCTION_r00, @@ -1444,7 +1454,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1655 +#define MAX_UOP_REGS_ID 1665 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0cd82c53aa932a..f6ffadf77282b4 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -311,6 +311,11 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_2] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_5] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_6] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_7] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_8] = HAS_PURE_FLAG, + [_INIT_CALL_PY_EXACT_ARGS_9] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GUARD_NOS_NULL] = HAS_EXIT_FLAG, @@ -439,7 +444,7 @@ const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { [_LOAD_FAST_BORROW] = { 0, 8 }, [_LOAD_SMALL_INT] = { 0, 4 }, [_SWAP_FAST] = { 0, 8 }, - [_INIT_CALL_PY_EXACT_ARGS] = { 0, 5 }, + [_INIT_CALL_PY_EXACT_ARGS] = { 0, 10 }, [_COPY] = { 1, 4 }, [_SWAP] = { 2, 4 }, [_GUARD_BIT_IS_SET_POP] = { 4, 8 }, @@ -2931,6 +2936,51 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_INIT_CALL_PY_EXACT_ARGS_5] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_5_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_6] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_6_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_7] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_7_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_8] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_8_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_9] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_9_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_INIT_CALL_PY_EXACT_ARGS] = { .best = { 0, 0, 0, 0 }, .entries = { @@ -4568,6 +4618,11 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_2_r01] = _INIT_CALL_PY_EXACT_ARGS_2, [_INIT_CALL_PY_EXACT_ARGS_3_r01] = _INIT_CALL_PY_EXACT_ARGS_3, [_INIT_CALL_PY_EXACT_ARGS_4_r01] = _INIT_CALL_PY_EXACT_ARGS_4, + [_INIT_CALL_PY_EXACT_ARGS_5_r01] = _INIT_CALL_PY_EXACT_ARGS_5, + [_INIT_CALL_PY_EXACT_ARGS_6_r01] = _INIT_CALL_PY_EXACT_ARGS_6, + [_INIT_CALL_PY_EXACT_ARGS_7_r01] = _INIT_CALL_PY_EXACT_ARGS_7, + [_INIT_CALL_PY_EXACT_ARGS_8_r01] = _INIT_CALL_PY_EXACT_ARGS_8, + [_INIT_CALL_PY_EXACT_ARGS_9_r01] = _INIT_CALL_PY_EXACT_ARGS_9, [_INIT_CALL_PY_EXACT_ARGS_r01] = _INIT_CALL_PY_EXACT_ARGS, [_PUSH_FRAME_r10] = _PUSH_FRAME, [_GUARD_NOS_NULL_r02] = _GUARD_NOS_NULL, @@ -5658,6 +5713,16 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3_r01] = "_INIT_CALL_PY_EXACT_ARGS_3_r01", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", [_INIT_CALL_PY_EXACT_ARGS_4_r01] = "_INIT_CALL_PY_EXACT_ARGS_4_r01", + [_INIT_CALL_PY_EXACT_ARGS_5] = "_INIT_CALL_PY_EXACT_ARGS_5", + [_INIT_CALL_PY_EXACT_ARGS_5_r01] = "_INIT_CALL_PY_EXACT_ARGS_5_r01", + [_INIT_CALL_PY_EXACT_ARGS_6] = "_INIT_CALL_PY_EXACT_ARGS_6", + [_INIT_CALL_PY_EXACT_ARGS_6_r01] = "_INIT_CALL_PY_EXACT_ARGS_6_r01", + [_INIT_CALL_PY_EXACT_ARGS_7] = "_INIT_CALL_PY_EXACT_ARGS_7", + [_INIT_CALL_PY_EXACT_ARGS_7_r01] = "_INIT_CALL_PY_EXACT_ARGS_7_r01", + [_INIT_CALL_PY_EXACT_ARGS_8] = "_INIT_CALL_PY_EXACT_ARGS_8", + [_INIT_CALL_PY_EXACT_ARGS_8_r01] = "_INIT_CALL_PY_EXACT_ARGS_8_r01", + [_INIT_CALL_PY_EXACT_ARGS_9] = "_INIT_CALL_PY_EXACT_ARGS_9", + [_INIT_CALL_PY_EXACT_ARGS_9_r01] = "_INIT_CALL_PY_EXACT_ARGS_9_r01", [_INSERT_NULL] = "_INSERT_NULL", [_INSERT_NULL_r10] = "_INSERT_NULL_r10", [_IS_NONE] = "_IS_NONE", @@ -6287,7 +6352,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _POP_TOP_UNICODE: return 1; case _POP_TOP_OPARG: - return oparg; + return (oparg >> 1); case _PUSH_NULL: return 0; case _END_FOR: @@ -6715,7 +6780,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _MAYBE_EXPAND_METHOD: return 0; case _PY_FRAME_GENERAL: - return 2 + oparg; + return 2 + (oparg >> 1); case _CHECK_FUNCTION_VERSION: return 0; case _CHECK_FUNCTION_VERSION_INLINE: @@ -6727,7 +6792,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_IS_NOT_PY_CALLABLE: return 0; case _CALL_NON_PY_GENERAL: - return 2 + oparg; + return 2 + (oparg >> 1); case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: return 0; case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: @@ -6741,17 +6806,27 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_RECURSION_REMAINING: return 0; case _INIT_CALL_PY_EXACT_ARGS_0: - return 2 + oparg; + return 2 + (oparg >> 1); case _INIT_CALL_PY_EXACT_ARGS_1: - return 2 + oparg; + return 2 + (oparg >> 1); case _INIT_CALL_PY_EXACT_ARGS_2: - return 2 + oparg; + return 2 + (oparg >> 1); case _INIT_CALL_PY_EXACT_ARGS_3: - return 2 + oparg; + return 2 + (oparg >> 1); case _INIT_CALL_PY_EXACT_ARGS_4: - return 2 + oparg; + return 2 + (oparg >> 1); + case _INIT_CALL_PY_EXACT_ARGS_5: + return 2 + (oparg >> 1); + case _INIT_CALL_PY_EXACT_ARGS_6: + return 2 + (oparg >> 1); + case _INIT_CALL_PY_EXACT_ARGS_7: + return 2 + (oparg >> 1); + case _INIT_CALL_PY_EXACT_ARGS_8: + return 2 + (oparg >> 1); + case _INIT_CALL_PY_EXACT_ARGS_9: + return 2 + (oparg >> 1); case _INIT_CALL_PY_EXACT_ARGS: - return 2 + oparg; + return 2 + (oparg >> 1); case _PUSH_FRAME: return 1; case _GUARD_NOS_NULL: @@ -6775,7 +6850,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _ALLOCATE_OBJECT: return 0; case _CREATE_INIT_FRAME: - return 2 + oparg; + return 2 + (oparg >> 1); case _EXIT_INIT_CHECK: return 1; case _GUARD_CALLABLE_BUILTIN_CLASS: @@ -6785,7 +6860,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_BUILTIN_O: return 0; case _CALL_BUILTIN_O: - return 2 + oparg; + return 2 + (oparg >> 1); case _GUARD_CALLABLE_BUILTIN_FAST: return 0; case _CALL_BUILTIN_FAST: @@ -6809,7 +6884,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O: return 0; case _CALL_METHOD_DESCRIPTOR_O: - return 2 + oparg; + return 2 + (oparg >> 1); case _CHECK_RECURSION_LIMIT: return 0; case _CALL_METHOD_DESCRIPTOR_O_INLINE: @@ -6823,7 +6898,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS: return 0; case _CALL_METHOD_DESCRIPTOR_NOARGS: - return 2 + oparg; + return 2 + (oparg >> 1); case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE: return 1 + oparg; case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST: @@ -6835,7 +6910,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: - return 3 + oparg; + return 3 + (oparg >> 1); case _CHECK_FUNCTION_VERSION_KW: return 0; case _CHECK_METHOD_VERSION_KW: @@ -6845,7 +6920,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_IS_NOT_PY_CALLABLE_KW: return 0; case _CALL_KW_NON_PY: - return 3 + oparg; + return 3 + (oparg >> 1); case _MAKE_CALLARGS_A_TUPLE: return 0; case _CHECK_IS_PY_CALLABLE_EX: diff --git a/Lib/dis.py b/Lib/dis.py index 318729091fa098..54df95d6bb0689 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -44,6 +44,9 @@ SEND = opmap['SEND'] LOAD_ATTR = opmap['LOAD_ATTR'] LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] +CALL = opmap['CALL'] +CALL_KW = opmap['CALL_KW'] +CALL_FUNCTION_EX = opmap['CALL_FUNCTION_EX'] CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] @@ -642,6 +645,14 @@ def get_argval_argrepr(self, op, arg, offset): if arg & (1<> 1 + argrepr = str(argval) + if arg & 1: + argrepr = f"{argrepr} + awaited" + elif deop == CALL_FUNCTION_EX: + if arg & 1: + argrepr = "awaited" elif deop == CALL_INTRINSIC_1: argrepr = _intrinsic_1_descs[arg] elif deop == CALL_INTRINSIC_2: diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 87509245f6a866..350bf4dd3b0ee9 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -56,7 +56,7 @@ def test_for_loop(self): ('LOAD_NAME', 2, 2), ('PUSH_NULL', None, 2), ('LOAD_NAME', 1, 2), - ('CALL', 1, 2), + ('CALL', 2, 2), ('POP_TOP', None), ('JUMP', loop_lbl), exit_lbl, @@ -210,7 +210,7 @@ def test_frozenset_optimization(self): ('LOAD_CONST', 2), ('LOAD_CONST', 3), ('BUILD_SET', 3), - ('CALL', 1), + ('CALL', 2), ('POP_TOP', None), ('LOAD_CONST', 0), ('RETURN_VALUE', None) @@ -225,7 +225,7 @@ def test_comp_without_target_optimization(self): ('LOAD_NAME', 0), ('PUSH_NULL', None), ('LOAD_CONST', 0), - ('CALL', 1), + ('CALL', 2), ('LOAD_FAST_AND_CLEAR', 0), ('SWAP', 2), ('SETUP_FINALLY', 20), diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 709fa374704f1f..351e1eff4cb5b9 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -116,7 +116,7 @@ def _f(a): %3d LOAD_GLOBAL 1 (print + NULL) LOAD_FAST_BORROW 0 (a) - CALL 1 + CALL 2 (1) POP_TOP %3d LOAD_SMALL_INT 1 @@ -130,7 +130,7 @@ def _f(a): %3d 4 LOAD_GLOBAL 1 (print + NULL) 14 LOAD_FAST_BORROW 0 (a) - 16 CALL 1 + 16 CALL 2 (1) 24 POP_TOP %3d 26 LOAD_SMALL_INT 1 @@ -144,7 +144,7 @@ def _f(a): %-14s LOAD_GLOBAL 1 (print + NULL) %-14s LOAD_FAST_BORROW 0 (a) -%-14s CALL 1 +%-14s CALL 2 (1) %-14s POP_TOP %-14s LOAD_SMALL_INT 1 @@ -155,7 +155,7 @@ def _f(a): RESUME 0 LOAD_GLOBAL 1 LOAD_FAST_BORROW 0 - CALL 1 + CALL 2 (1) POP_TOP LOAD_SMALL_INT 1 RETURN_VALUE @@ -174,7 +174,7 @@ def bug708901(): %3d LOAD_SMALL_INT 10 -%3d CALL 2 +%3d CALL 4 (2) GET_ITER 0 L1: FOR_ITER 3 (to L2) STORE_FAST 0 (res) @@ -205,12 +205,12 @@ def bug1333982(x=[]): LOAD_CONST 1 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION LOAD_FAST_BORROW 0 (x) - CALL 0 + CALL 0 (0) %3d LOAD_SMALL_INT 1 %3d BINARY_OP 0 (+) - CALL 0 + CALL 0 (0) RAISE_VARARGS 1 """ % (bug1333982.__code__.co_firstlineno, bug1333982.__code__.co_firstlineno + 1, @@ -281,7 +281,7 @@ def wrap_func_w_kwargs(): LOAD_SMALL_INT 2 LOAD_SMALL_INT 5 LOAD_CONST 1 (('c',)) - CALL_KW 3 + CALL_KW 6 (3) POP_TOP LOAD_COMMON_CONSTANT 7 (None) RETURN_VALUE @@ -408,7 +408,7 @@ def wrap_func_w_kwargs(): LOAD_NAME 3 (fun) PUSH_NULL LOAD_SMALL_INT 0 - CALL 1 + CALL 2 (1) STORE_SUBSCR LOAD_COMMON_CONSTANT 7 (None) RETURN_VALUE @@ -548,7 +548,7 @@ def _with(c): SWAP 2 SWAP 3 LOAD_SPECIAL 0 (__enter__) - CALL 0 + CALL 0 (0) L1: POP_TOP %4d LOAD_SMALL_INT 1 @@ -557,7 +557,7 @@ def _with(c): %4d L2: LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) - CALL 3 + CALL 6 (3) POP_TOP %4d LOAD_SMALL_INT 2 @@ -613,7 +613,7 @@ async def _asyncwith(c): SWAP 2 SWAP 3 LOAD_SPECIAL 2 (__aenter__) - CALL 0 + CALL 0 (0) GET_AWAITABLE 1 PUSH_NULL LOAD_COMMON_CONSTANT 7 (None) @@ -630,7 +630,7 @@ async def _asyncwith(c): %4d L7: LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) - CALL 3 + CALL 6 (3) GET_AWAITABLE 2 PUSH_NULL LOAD_COMMON_CONSTANT 7 (None) @@ -725,7 +725,7 @@ def _tryfinallyconst(b): %4d L2: LOAD_FAST_BORROW 1 (b) PUSH_NULL - CALL 0 + CALL 0 (0) POP_TOP RETURN_VALUE @@ -733,7 +733,7 @@ def _tryfinallyconst(b): %4d LOAD_FAST 1 (b) PUSH_NULL - CALL 0 + CALL 0 (0) POP_TOP RERAISE 0 @@ -759,7 +759,7 @@ def _tryfinallyconst(b): %4d LOAD_FAST_BORROW 0 (b) PUSH_NULL - CALL 0 + CALL 0 (0) POP_TOP LOAD_SMALL_INT 1 RETURN_VALUE @@ -768,7 +768,7 @@ def _tryfinallyconst(b): %4d LOAD_FAST 0 (b) PUSH_NULL - CALL 0 + CALL 0 (0) POP_TOP RERAISE 0 @@ -842,7 +842,7 @@ def foo(x): MAKE_FUNCTION SET_FUNCTION_ATTRIBUTE 8 (closure) LOAD_DEREF 1 (y) - CALL 0 + CALL 0 (0) PUSH_NULL L1: FOR_ITER 3 (to L2) LIST_APPEND 3 @@ -857,8 +857,8 @@ def foo(x): MAKE_FUNCTION SET_FUNCTION_ATTRIBUTE 8 (closure) LOAD_DEREF 1 (y) - CALL 0 - CALL 1 + CALL 0 (0) + CALL 2 (1) RETURN_VALUE """ % (dis_nested_0, __file__, @@ -941,7 +941,7 @@ def loop_test(): %3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST_BORROW 0 (i) - CALL_PY_GENERAL 1 + CALL_PY_GENERAL 2 (1) POP_TOP JUMP_BACKWARD_{: <6} 16 (to L1) @@ -1356,7 +1356,7 @@ def test_call_specialize(self): 1 LOAD_NAME 0 (str) PUSH_NULL LOAD_SMALL_INT 1 - CALL_STR_1 1 + CALL_STR_1 2 (1) RETURN_VALUE """ co = compile("str(1)", "", "eval") @@ -1494,7 +1494,7 @@ def loop(n): {line+1} LOAD_GLOBAL_BUILTIN 1 (range + NULL) LOAD_FAST_BORROW 0 (n) - CALL_BUILTIN_CLASS 1 + CALL_BUILTIN_CLASS 2 (1) GET_ITER 0 L1: FOR_ITER_RANGE 3 (to L2) STORE_FAST 1 (i) @@ -1832,7 +1832,7 @@ def _prepare_test_cases(): make_inst(opname='BUILD_LIST', arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=False, line_number=7), make_inst(opname='BUILD_MAP', arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=False, line_number=7), make_inst(opname='LOAD_CONST', arg=2, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=False, line_number=7), - make_inst(opname='CALL', arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=14, argval=7, argrepr='7', offset=48, start_offset=48, starts_line=False, line_number=7, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=False, line_number=7), make_inst(opname='LOAD_FAST_BORROW', arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=True, line_number=8), make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=False, line_number=8), @@ -1857,7 +1857,7 @@ def _prepare_test_cases(): make_inst(opname='LOAD_DEREF', arg=4, argval='b', argrepr='b', offset=40, start_offset=40, starts_line=False, line_number=5), make_inst(opname='LOAD_DEREF', arg=0, argval='c', argrepr='c', offset=42, start_offset=42, starts_line=False, line_number=5), make_inst(opname='LOAD_DEREF', arg=1, argval='d', argrepr='d', offset=44, start_offset=44, starts_line=False, line_number=5), - make_inst(opname='CALL', arg=4, argval=4, argrepr='', offset=46, start_offset=46, starts_line=False, line_number=5, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=8, argval=4, argrepr='4', offset=46, start_offset=46, starts_line=False, line_number=5, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=54, start_offset=54, starts_line=False, line_number=5), make_inst(opname='LOAD_FAST_BORROW', arg=2, argval='inner', argrepr='inner', offset=56, start_offset=56, starts_line=True, line_number=6), make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=58, start_offset=58, starts_line=False, line_number=6), @@ -1872,7 +1872,7 @@ def _prepare_test_cases(): make_inst(opname='LOAD_DEREF', arg=4, argval='c', argrepr='c', offset=20, start_offset=20, starts_line=False, line_number=4), make_inst(opname='LOAD_DEREF', arg=5, argval='d', argrepr='d', offset=22, start_offset=22, starts_line=False, line_number=4), make_inst(opname='LOAD_FAST_BORROW_LOAD_FAST_BORROW', arg=1, argval=('e', 'f'), argrepr='e, f', offset=24, start_offset=24, starts_line=False, line_number=4), - make_inst(opname='CALL', arg=6, argval=6, argrepr='', offset=26, start_offset=26, starts_line=False, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=12, argval=6, argrepr='6', offset=26, start_offset=26, starts_line=False, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=34, start_offset=34, starts_line=False, line_number=4), make_inst(opname='LOAD_COMMON_CONSTANT', arg=7, argval=None, argrepr='None', offset=36, start_offset=36, starts_line=False, line_number=4), make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=38, start_offset=38, starts_line=False, line_number=4), @@ -1882,13 +1882,13 @@ def _prepare_test_cases(): make_inst(opname='RESUME', arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=True, line_number=1, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='LOAD_GLOBAL', arg=1, argval='range', argrepr='range + NULL', offset=4, start_offset=4, starts_line=True, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_SMALL_INT', arg=10, argval=10, argrepr='', offset=14, start_offset=14, starts_line=False, line_number=3), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=16, start_offset=16, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=16, start_offset=16, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='GET_ITER', arg=0, argval=0, argrepr='', offset=24, start_offset=24, starts_line=False, line_number=3, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='FOR_ITER', arg=33, argval=98, argrepr='to L4', offset=28, start_offset=28, starts_line=False, line_number=3, label=1, cache_info=[('counter', 1, b'\x00\x00')]), make_inst(opname='STORE_FAST', arg=0, argval='i', argrepr='i', offset=32, start_offset=32, starts_line=False, line_number=3), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=34, start_offset=34, starts_line=True, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=44, start_offset=44, starts_line=False, line_number=4), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=46, start_offset=46, starts_line=False, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=46, start_offset=46, starts_line=False, line_number=4, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=54, start_offset=54, starts_line=False, line_number=4), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=56, start_offset=56, starts_line=True, line_number=5), make_inst(opname='LOAD_SMALL_INT', arg=4, argval=4, argrepr='', offset=58, start_offset=58, starts_line=False, line_number=5), @@ -1909,7 +1909,7 @@ def _prepare_test_cases(): make_inst(opname='POP_ITER', arg=None, argval=None, argrepr='', offset=100, start_offset=100, starts_line=False, line_number=3), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=102, start_offset=102, starts_line=True, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=1, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=112, start_offset=112, starts_line=False, line_number=10), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=114, start_offset=114, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=114, start_offset=114, starts_line=False, line_number=10, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=122, start_offset=122, starts_line=False, line_number=10), make_inst(opname='LOAD_FAST_CHECK', arg=0, argval='i', argrepr='i', offset=124, start_offset=124, starts_line=True, line_number=11, label=5), make_inst(opname='TO_BOOL', arg=None, argval=None, argrepr='', offset=126, start_offset=126, starts_line=False, line_number=11, cache_info=[('counter', 1, b'\x00\x00'), ('version', 2, b'\x00\x00\x00\x00')]), @@ -1917,7 +1917,7 @@ def _prepare_test_cases(): make_inst(opname='NOT_TAKEN', arg=None, argval=None, argrepr='', offset=138, start_offset=138, starts_line=False, line_number=11), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=140, start_offset=140, starts_line=True, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=False, line_number=12), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=152, start_offset=152, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=152, start_offset=152, starts_line=False, line_number=12, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=160, start_offset=160, starts_line=False, line_number=12), make_inst(opname='LOAD_FAST_BORROW', arg=0, argval='i', argrepr='i', offset=162, start_offset=162, starts_line=True, line_number=13), make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=164, start_offset=164, starts_line=False, line_number=13), @@ -1938,7 +1938,7 @@ def _prepare_test_cases(): make_inst(opname='JUMP_FORWARD', arg=11, argval=240, argrepr='to L9', offset=216, start_offset=216, starts_line=True, line_number=17, label=7), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=218, start_offset=218, starts_line=True, line_number=19, label=8, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=2, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=228, start_offset=228, starts_line=False, line_number=19), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=230, start_offset=230, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=230, start_offset=230, starts_line=False, line_number=19, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=238, start_offset=238, starts_line=False, line_number=19), make_inst(opname='NOP', arg=None, argval=None, argrepr='', offset=240, start_offset=240, starts_line=True, line_number=20, label=9), make_inst(opname='LOAD_SMALL_INT', arg=1, argval=1, argrepr='', offset=242, start_offset=242, starts_line=True, line_number=21), @@ -1951,20 +1951,20 @@ def _prepare_test_cases(): make_inst(opname='SWAP', arg=2, argval=2, argrepr='', offset=266, start_offset=266, starts_line=False, line_number=25), make_inst(opname='SWAP', arg=3, argval=3, argrepr='', offset=268, start_offset=268, starts_line=False, line_number=25), make_inst(opname='LOAD_SPECIAL', arg=0, argval=0, argrepr='__enter__', offset=270, start_offset=270, starts_line=False, line_number=25), - make_inst(opname='CALL', arg=0, argval=0, argrepr='', offset=272, start_offset=272, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=0, argval=0, argrepr='0', offset=272, start_offset=272, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='STORE_FAST', arg=1, argval='dodgy', argrepr='dodgy', offset=280, start_offset=280, starts_line=False, line_number=25), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=282, start_offset=282, starts_line=True, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=3, argval='Never reach this', argrepr="'Never reach this'", offset=292, start_offset=292, starts_line=False, line_number=26), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=294, start_offset=294, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=294, start_offset=294, starts_line=False, line_number=26, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=302, start_offset=302, starts_line=False, line_number=26), make_inst(opname='LOAD_COMMON_CONSTANT', arg=7, argval=None, argrepr='None', offset=304, start_offset=304, starts_line=True, line_number=25), make_inst(opname='LOAD_COMMON_CONSTANT', arg=7, argval=None, argrepr='None', offset=306, start_offset=306, starts_line=False, line_number=25), make_inst(opname='LOAD_COMMON_CONSTANT', arg=7, argval=None, argrepr='None', offset=308, start_offset=308, starts_line=False, line_number=25), - make_inst(opname='CALL', arg=3, argval=3, argrepr='', offset=310, start_offset=310, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=6, argval=3, argrepr='3', offset=310, start_offset=310, starts_line=False, line_number=25, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=False, line_number=25), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=320, start_offset=320, starts_line=True, line_number=28, label=10, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=330, start_offset=330, starts_line=False, line_number=28), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=332, start_offset=332, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=332, start_offset=332, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=340, start_offset=340, starts_line=False, line_number=28), make_inst(opname='LOAD_COMMON_CONSTANT', arg=7, argval=None, argrepr='None', offset=342, start_offset=342, starts_line=False, line_number=28), make_inst(opname='RETURN_VALUE', arg=None, argval=None, argrepr='', offset=344, start_offset=344, starts_line=False, line_number=28), @@ -1991,7 +1991,7 @@ def _prepare_test_cases(): make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=404, start_offset=404, starts_line=False, line_number=22), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=406, start_offset=406, starts_line=True, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=4, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=416, start_offset=416, starts_line=False, line_number=23), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=418, start_offset=418, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=418, start_offset=418, starts_line=False, line_number=23, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=426, start_offset=426, starts_line=False, line_number=23), make_inst(opname='POP_EXCEPT', arg=None, argval=None, argrepr='', offset=428, start_offset=428, starts_line=False, line_number=23), make_inst(opname='JUMP_BACKWARD_NO_INTERRUPT', arg=56, argval=320, argrepr='to L10', offset=430, start_offset=430, starts_line=False, line_number=23), @@ -2002,7 +2002,7 @@ def _prepare_test_cases(): make_inst(opname='PUSH_EXC_INFO', arg=None, argval=None, argrepr='', offset=440, start_offset=440, starts_line=False, line_number=None), make_inst(opname='LOAD_GLOBAL', arg=3, argval='print', argrepr='print + NULL', offset=442, start_offset=442, starts_line=True, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('index', 1, b'\x00\x00'), ('module_keys_version', 1, b'\x00\x00'), ('builtin_keys_version', 1, b'\x00\x00')]), make_inst(opname='LOAD_CONST', arg=5, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=452, start_offset=452, starts_line=False, line_number=28), - make_inst(opname='CALL', arg=1, argval=1, argrepr='', offset=454, start_offset=454, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), + make_inst(opname='CALL', arg=2, argval=1, argrepr='1', offset=454, start_offset=454, starts_line=False, line_number=28, cache_info=[('counter', 1, b'\x00\x00'), ('func_version', 2, b'\x00\x00\x00\x00')]), make_inst(opname='POP_TOP', arg=None, argval=None, argrepr='', offset=462, start_offset=462, starts_line=False, line_number=28), make_inst(opname='RERAISE', arg=0, argval=0, argrepr='', offset=464, start_offset=464, starts_line=False, line_number=28), make_inst(opname='COPY', arg=3, argval=3, argrepr='', offset=466, start_offset=466, starts_line=True, line_number=None), @@ -2618,7 +2618,7 @@ def test_show_cache(self): 1 LOAD_NAME 0 (print) PUSH_NULL - CALL 0 + CALL 0 (0) CACHE 0 (counter: 0) CACHE 0 (func_version: 0) CACHE 0 diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 8e7ba1f69553dd..91a736ce871c53 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -1831,8 +1831,8 @@ _PyStackRef res; // _SPECIALIZE_CALL { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -1840,7 +1840,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -1858,8 +1858,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -1868,9 +1868,9 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -1882,8 +1882,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -1891,7 +1891,7 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); @@ -1899,8 +1899,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( @@ -1914,7 +1914,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -1922,8 +1922,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -1965,8 +1965,8 @@ } // _CHECK_OBJECT { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { @@ -2022,8 +2022,8 @@ self_or_null = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2031,7 +2031,7 @@ } // _CREATE_INIT_FRAME { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; self = self_or_null; init = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -2045,9 +2045,9 @@ assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, (oparg >> 1)+1, NULL, shim); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2110,8 +2110,8 @@ } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -2132,8 +2132,8 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2162,7 +2162,7 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2189,13 +2189,13 @@ } // _INIT_CALL_PY_EXACT_ARGS { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); @@ -2213,7 +2213,7 @@ { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2260,8 +2260,8 @@ } // _CHECK_METHOD_VERSION { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -2296,8 +2296,8 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2314,9 +2314,9 @@ } // _PY_FRAME_GENERAL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -2331,7 +2331,7 @@ args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -2388,7 +2388,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_CLASS { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2404,9 +2404,9 @@ } // _CALL_BUILTIN_CLASS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2425,7 +2425,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2433,16 +2433,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2481,7 +2481,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_FAST { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2497,9 +2497,9 @@ } // _CALL_BUILTIN_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2519,7 +2519,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2527,16 +2527,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2575,7 +2575,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2591,9 +2591,9 @@ } // _CALL_BUILTIN_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2609,7 +2609,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2617,16 +2617,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2668,8 +2668,8 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_O { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2682,7 +2682,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -2702,7 +2702,7 @@ } // _CALL_BUILTIN_O { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { args--; @@ -2726,9 +2726,9 @@ // _POP_TOP { value = s; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer += -oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3394,8 +3394,8 @@ _PyStackRef res; // _SPECIALIZE_CALL_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -3403,7 +3403,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -3421,8 +3421,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -3432,10 +3432,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3448,8 +3448,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -3457,7 +3457,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3470,8 +3470,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( @@ -3485,14 +3485,14 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3525,8 +3525,8 @@ } // _CHECK_METHOD_VERSION_KW { - null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -3561,8 +3561,8 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); @@ -3572,9 +3572,9 @@ // _PY_FRAME_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3598,7 +3598,7 @@ _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -3657,7 +3657,7 @@ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE_KW { - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); @@ -3673,12 +3673,12 @@ // _CALL_KW_NON_PY { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3693,7 +3693,7 @@ kwnames); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -3701,8 +3701,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3742,7 +3742,7 @@ } // _CHECK_FUNCTION_VERSION_KW { - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -3768,10 +3768,10 @@ // _PY_FRAME_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3795,7 +3795,7 @@ _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -3973,7 +3973,7 @@ { arg = stack_pointer[-1]; self = nos; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); @@ -4050,9 +4050,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4066,7 +4066,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4087,7 +4087,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4115,7 +4115,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -4125,13 +4125,13 @@ { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4170,9 +4170,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4186,7 +4186,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4208,7 +4208,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4236,7 +4236,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -4246,13 +4246,13 @@ { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4294,9 +4294,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4310,7 +4310,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4339,7 +4339,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + assert((oparg >> 1) == 1 || !PyStackRef_IsNull(self_or_null)); if (!PyStackRef_IsNull(self_or_null)) { args--; } @@ -4363,9 +4363,9 @@ // _POP_TOP { value = s; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer += -oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4418,9 +4418,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_O { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4434,7 +4434,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4488,10 +4488,10 @@ // _POP_TOP { value = a; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer[-oparg] = s; - stack_pointer += 1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer[-(oparg >> 1)] = s; + stack_pointer += 1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4551,7 +4551,7 @@ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -4566,12 +4566,12 @@ } // _CALL_NON_PY_GENERAL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4586,7 +4586,7 @@ PyStackRef_NULL); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -4594,8 +4594,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4634,7 +4634,7 @@ } // _CHECK_FUNCTION_VERSION { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -4651,12 +4651,12 @@ } // _CHECK_FUNCTION_EXACT_ARGS { - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4683,13 +4683,13 @@ } // _INIT_CALL_PY_EXACT_ARGS { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); @@ -4707,7 +4707,7 @@ { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4753,7 +4753,7 @@ } // _CHECK_FUNCTION_VERSION { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -4778,10 +4778,10 @@ } // _PY_FRAME_GENERAL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -4796,7 +4796,7 @@ args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -4876,7 +4876,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4954,7 +4954,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -5032,7 +5032,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -7111,8 +7111,8 @@ /* Skip 3 cache entries */ // _MAYBE_EXPAND_METHOD { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7120,8 +7120,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7130,7 +7130,7 @@ } // _MONITOR_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; maybe_self = self_or_null; func = callable; int is_meth = !PyStackRef_IsNull(maybe_self); @@ -7139,14 +7139,14 @@ if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); } - else if (oparg) { + else if ((oparg >> 1)) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { arg0 = &_PyInstrumentation_MISSING; } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; + stack_pointer[-2 - (oparg >> 1)] = func; + stack_pointer[-1 - (oparg >> 1)] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7160,11 +7160,11 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7183,7 +7183,7 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); @@ -7204,7 +7204,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -7212,8 +7212,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7428,8 +7428,8 @@ /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7437,8 +7437,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7447,7 +7447,7 @@ } // _MONITOR_CALL_KW { - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; int is_meth = !PyStackRef_IsNull(self_or_null); PyObject *arg; if (is_meth) { @@ -7460,8 +7460,8 @@ arg = &_PyInstrumentation_MISSING; } PyObject *function = PyStackRef_AsPyObjectBorrow(callable); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7475,10 +7475,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7498,7 +7498,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7524,14 +7524,14 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 42ad83daa1ea46..0a8147e19fcd3d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -383,8 +383,8 @@ dummy_func( PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); } - op(_POP_TOP_OPARG, (args[oparg] -- )) { - _PyStackRef_CloseStack(args, oparg); + op(_POP_TOP_OPARG, (args[oparg >> 1] -- )) { + _PyStackRef_CloseStack(args, (oparg >> 1)); DEAD(args); } @@ -4316,11 +4316,11 @@ dummy_func( CALL_NON_PY_GENERAL, }; - specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -4328,7 +4328,7 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -4341,11 +4341,11 @@ dummy_func( } // When calling Python, inline the call using DISPATCH_INLINED(). - op(_DO_CALL, (callable, self_or_null, args[oparg] -- res)) { + op(_DO_CALL, (callable, self_or_null, args[oparg >> 1] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg >> 1 counts all of the args, but *not* self: + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4391,14 +4391,14 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_MONITOR_CALL, (func, maybe_self, args[oparg] -- func, maybe_self, args[oparg])) { + op(_MONITOR_CALL, (func, maybe_self, args[oparg >> 1] -- func, maybe_self, args[oparg >> 1])) { int is_meth = !PyStackRef_IsNull(maybe_self); PyObject *function = PyStackRef_AsPyObjectBorrow(func); PyObject *arg0; if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); } - else if (oparg) { + else if ((oparg >> 1)) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { @@ -4415,11 +4415,11 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC_AT_END; macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC_AT_END; - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg >> 1] -- new_frame)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg >> 1 counts all of the args, but *not* self: + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -4440,7 +4440,7 @@ dummy_func( new_frame = PyStackRef_Wrap(temp); } - op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; @@ -4463,7 +4463,7 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg >> 1] -- callable, null, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); @@ -4473,7 +4473,7 @@ dummy_func( EXIT_IF(!PyStackRef_IsNull(null)); } - op(_EXPAND_METHOD, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_EXPAND_METHOD, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyStackRef_IsNull(self_or_null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); @@ -4496,17 +4496,17 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(PyFunction_Check(callable_o)); EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); } - op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg >> 1] -- res)) { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4532,12 +4532,12 @@ dummy_func( _CALL_NON_PY_GENERAL + _CHECK_PERIODIC_AT_END; - op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg >> 1] -- callable, null, unused[oparg >> 1])) { EXIT_IF(!PyStackRef_IsNull(null)); EXIT_IF(Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type); } - op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { assert(PyStackRef_IsNull(self_or_null)); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); @@ -4551,15 +4551,15 @@ dummy_func( DEOPT_IF(IS_PEP523_HOOKED(tstate)); } - op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - EXIT_IF(code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))); + EXIT_IF(code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))); } - op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -4570,13 +4570,13 @@ dummy_func( DEOPT_IF(tstate->py_recursion_remaining <= 1); } - replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { + replicate(10) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg >> 1] -- new_frame)) { int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } INPUTS_DEAD(); @@ -4644,7 +4644,7 @@ dummy_func( op(_CALL_TYPE_1, (callable, null, arg -- res, a)) { PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; INPUTS_DEAD(); @@ -4667,7 +4667,7 @@ dummy_func( op(_CALL_STR_1, (callable, null, arg -- res, a)) { PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); PyObject *res_o = PyObject_Str(arg_o); if (res_o == NULL) { @@ -4695,7 +4695,7 @@ dummy_func( op(_CALL_TUPLE_1, (callable, null, arg -- res, a)) { PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); PyObject *res_o = PySequence_Tuple(arg_o); if (res_o == NULL) { @@ -4715,7 +4715,7 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_CHECK_OBJECT, (type_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_OBJECT, (type_version/2, callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyStackRef_IsNull(self_or_null)); EXIT_IF(!PyType_Check(callable_o)); @@ -4723,7 +4723,7 @@ dummy_func( EXIT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version); } - op(_ALLOCATE_OBJECT, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_ALLOCATE_OBJECT, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyStackRef_IsNull(self_or_null)); assert(PyType_Check(callable_o)); @@ -4747,7 +4747,7 @@ dummy_func( PyStackRef_CLOSE(temp); } - op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { + op(_CREATE_INIT_FRAME, (init, self, args[oparg >> 1] -- init_frame)) { _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); @@ -4755,7 +4755,7 @@ dummy_func( /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, (oparg >> 1)+1, NULL, shim); DEAD(init); DEAD(self); DEAD(args); @@ -4792,15 +4792,15 @@ dummy_func( DEAD(should_be_none); } - op(_GUARD_CALLABLE_BUILTIN_CLASS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_CLASS, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; EXIT_IF(tp->tp_vectorcall == NULL); } - op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { - int total_args = oparg; + op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4829,19 +4829,19 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) != METH_O); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } EXIT_IF(total_args != 1); } - op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) { + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg >> 1] -- res, c, s)) { /* Builtin METH_O functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4874,16 +4874,16 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) != METH_FASTCALL); } - op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4913,16 +4913,16 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyCFunction_CheckExact(callable_o)); EXIT_IF((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)); } - op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5030,7 +5030,7 @@ dummy_func( } op(_CALL_LIST_APPEND, (callable, self, arg -- none, c, s)) { - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); DEOPT_IF(!LOCK_OBJECT(self_o)); @@ -5050,13 +5050,13 @@ dummy_func( none = PyStackRef_None; } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != METH_O); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -5066,7 +5066,7 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); } - op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg >> 1] -- res, c, s, a)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; @@ -5126,13 +5126,13 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != (METH_FASTCALL | METH_KEYWORDS)); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5143,11 +5143,11 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); } - op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5201,13 +5201,13 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != METH_NOARGS); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -5217,11 +5217,11 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); } - op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res, c, s)) { + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg >> 1] -- res, c, s)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + assert((oparg >> 1) == 1 || !PyStackRef_IsNull(self_or_null)); if (!PyStackRef_IsNull(self_or_null)) { args--; } @@ -5270,14 +5270,14 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; /* Builtin METH_FASTCALL methods, without keywords */ EXIT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); EXIT_IF((method->d_method->ml_flags & _Py_METH_CALL_FLAGS) != METH_FASTCALL); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -5287,11 +5287,11 @@ dummy_func( EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); } - op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5352,7 +5352,7 @@ dummy_func( CALL_KW_NON_PY, }; - op(_MONITOR_CALL_KW, (callable, self_or_null, args[oparg], unused -- callable, self_or_null, args[oparg], unused)) { + op(_MONITOR_CALL_KW, (callable, self_or_null, args[oparg >> 1], unused -- callable, self_or_null, args[oparg >> 1], unused)) { int is_meth = !PyStackRef_IsNull(self_or_null); PyObject *arg; if (is_meth) { @@ -5371,7 +5371,7 @@ dummy_func( ERROR_IF(err); } - op(_MAYBE_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + op(_MAYBE_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg >> 1], unused -- callable, self_or_null, unused[oparg >> 1], unused)) { if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -5383,12 +5383,12 @@ dummy_func( } } - op(_DO_CALL_KW, (callable, self_or_null, args[oparg], kwnames -- res)) { + op(_DO_CALL_KW, (callable, self_or_null, args[oparg >> 1], kwnames -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg >> 1 counts all of the args, but *not* self: + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5438,11 +5438,11 @@ dummy_func( res = PyStackRef_FromPyObjectSteal(res_o); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg >> 1], kwnames -- new_frame)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - // oparg counts all of the args, but *not* self: - int total_args = oparg; + // oparg >> 1 counts all of the args, but *not* self: + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -5468,7 +5468,7 @@ dummy_func( new_frame = PyStackRef_Wrap(temp); } - op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { + op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, unused, unused[oparg >> 1], unused -- callable, unused, unused[oparg >> 1], unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(!PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; @@ -5485,7 +5485,7 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg], unused -- callable, null, unused[oparg], unused)) { + op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg >> 1], unused -- callable, null, unused[oparg >> 1], unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(Py_TYPE(callable_o) != &PyMethod_Type); @@ -5495,7 +5495,7 @@ dummy_func( EXIT_IF(!PyStackRef_IsNull(null)); } - op(_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + op(_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg >> 1], unused -- callable, self_or_null, unused[oparg >> 1], unused)) { assert(PyStackRef_IsNull(self_or_null)); _PyStackRef callable_s = callable; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -5517,11 +5517,11 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, unused[oparg >> 1], unused -- callable, self_or_null, unused[oparg >> 1], unused)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -5542,18 +5542,18 @@ dummy_func( _MONITOR_CALL_KW + _DO_CALL_KW; - op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { + op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg >> 1], unused -- callable, unused, unused[oparg >> 1], unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(PyFunction_Check(callable_o)); EXIT_IF(Py_TYPE(callable_o) == &PyMethod_Type); } - op(_CALL_KW_NON_PY, (callable, self_or_null, args[oparg], kwnames -- res)) { + op(_CALL_KW_NON_PY, (callable, self_or_null, args[oparg >> 1], kwnames -- res)) { #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -6392,15 +6392,15 @@ dummy_func( RECORD_VALUE(PyStackRef_AsPyObjectBorrow(value)); } - tier2 op(_RECORD_CALLABLE, (func, self, args[oparg] -- func, self, args[oparg])) { + tier2 op(_RECORD_CALLABLE, (func, self, args[oparg >> 1] -- func, self, args[oparg >> 1])) { RECORD_VALUE(PyStackRef_AsPyObjectBorrow(func)); } - tier2 op(_RECORD_CALLABLE_KW, (func, self, args[oparg], kwnames -- func, self, args[oparg], kwnames)) { + tier2 op(_RECORD_CALLABLE_KW, (func, self, args[oparg >> 1], kwnames -- func, self, args[oparg >> 1], kwnames)) { RECORD_VALUE(PyStackRef_AsPyObjectBorrow(func)); } - tier2 op(_RECORD_BOUND_METHOD, (callable, self, args[oparg] -- callable, self, args[oparg])) { + tier2 op(_RECORD_BOUND_METHOD, (callable, self, args[oparg >> 1] -- callable, self, args[oparg >> 1])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) == &PyMethod_Type) { RECORD_VALUE(callable_o); diff --git a/Python/codegen.c b/Python/codegen.c index a975705576a167..2a651037e707fb 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -276,6 +276,10 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) return _PyInstructionSequence_Addop(seq, opcode, oparg_, loc); } +/* The oparg of CALL and CALL_KW: the number of arguments shifted left by one, + * with the low bit set when the result is awaited right away. */ +#define CALL_OPARG(NARGS, AWAITED) (((NARGS) << 1) | (AWAITED)) + #define ADDOP_I(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) @@ -492,7 +496,7 @@ codegen_call_exit_with_nones(compiler *c, location loc) ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); - ADDOP_I(c, loc, CALL, 3); + ADDOP_I(c, loc, CALL, CALL_OPARG(3, 0)); return SUCCESS; } @@ -1056,7 +1060,7 @@ codegen_apply_decorators(compiler *c, asdl_expr_seq* decos) for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) { location loc = LOC((expr_ty)asdl_seq_GET(decos, i)); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } return SUCCESS; } @@ -1584,11 +1588,11 @@ codegen_function(compiler *c, stmt_ty s, int is_async) RETURN_IF_ERROR(ret); if (num_typeparam_args > 0) { ADDOP_I(c, loc, SWAP, num_typeparam_args + 1); - ADDOP_I(c, loc, CALL, num_typeparam_args - 1); + ADDOP_I(c, loc, CALL, CALL_OPARG(num_typeparam_args - 1, 0)); } else { ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } } @@ -1770,7 +1774,7 @@ codegen_class(compiler *c, stmt_ty s) Py_DECREF(co); RETURN_IF_ERROR(ret); ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } else { RETURN_IF_ERROR(codegen_call_helper(c, loc, 2, s->v.ClassDef.bases, @@ -1859,7 +1863,7 @@ codegen_typealias(compiler *c, stmt_ty s) Py_DECREF(co); RETURN_IF_ERROR(ret); ADDOP(c, loc, PUSH_NULL); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); } RETURN_IF_ERROR(codegen_nameop(c, loc, name, Store)); return SUCCESS; @@ -4217,11 +4221,11 @@ maybe_optimize_method_call(compiler *c, expr_ty e) RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, kwds, kwdsl)); loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL_KW, argsl + kwdsl); + ADDOP_I(c, loc, CALL_KW, CALL_OPARG(argsl + kwdsl, 0)); } else { loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL, argsl); + ADDOP_I(c, loc, CALL, CALL_OPARG(argsl, 0)); } return 1; } @@ -4327,7 +4331,7 @@ codegen_joined_str(compiler *c, expr_ty e) VISIT(c, expr, asdl_seq_GET(e->v.JoinedStr.values, i)); ADDOP_I(c, loc, LIST_APPEND, 1); } - ADDOP_I(c, loc, CALL, 1); + ADDOP_I(c, loc, CALL, CALL_OPARG(1, 0)); } else { VISIT_SEQ(c, expr, e->v.JoinedStr.values); @@ -4497,10 +4501,10 @@ codegen_call_helper_impl(compiler *c, location loc, VISIT_SEQ(c, keyword, keywords); RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, keywords, nkwelts)); - ADDOP_I(c, loc, CALL_KW, n + nelts + nkwelts); + ADDOP_I(c, loc, CALL_KW, CALL_OPARG(n + nelts + nkwelts, 0)); } else { - ADDOP_I(c, loc, CALL, n + nelts); + ADDOP_I(c, loc, CALL, CALL_OPARG(n + nelts, 0)); } return SUCCESS; @@ -5163,7 +5167,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, Py_CLEAR(co); VISIT(c, expr, outermost->iter); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); if (is_async_comprehension && type != COMP_GENEXP) { ADDOP_I(c, loc, GET_AWAITABLE, 0); @@ -5310,7 +5314,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, SWAP, 2); ADDOP_I(c, loc, SWAP, 3); ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); ADDOP_I(c, loc, GET_AWAITABLE, 1); ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -5422,7 +5426,7 @@ codegen_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, SWAP, 2); ADDOP_I(c, loc, SWAP, 3); ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___ENTER__); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); ADDOP_JUMP(c, loc, SETUP_WITH, final); /* SETUP_WITH pushes a finally block. */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bc85ccd9ddb4fc..c38c7df7948a52 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2508,16 +2508,16 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - stack_pointer += -oparg; + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -16860,8 +16860,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -16869,8 +16869,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -16880,8 +16880,8 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -16898,11 +16898,11 @@ _PyStackRef callable; _PyStackRef new_frame; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -16917,7 +16917,7 @@ args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -16937,7 +16937,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -17048,8 +17048,8 @@ _PyStackRef null; _PyStackRef callable; oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -17084,8 +17084,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyStackRef_IsNull(self_or_null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); @@ -17093,8 +17093,8 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -17112,7 +17112,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -17137,13 +17137,13 @@ _PyStackRef callable; _PyStackRef res; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -17158,7 +17158,7 @@ PyStackRef_NULL); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -17168,7 +17168,7 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17180,8 +17180,8 @@ _PyStackRef null; _PyStackRef callable; oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (!PyStackRef_IsNull(null)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -17203,16 +17203,16 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; assert(PyStackRef_IsNull(self_or_null)); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -17301,13 +17301,13 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -17322,7 +17322,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -17415,21 +17415,21 @@ _PyStackRef new_frame; oparg = 0; assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17444,21 +17444,21 @@ _PyStackRef new_frame; oparg = 1; assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17473,21 +17473,21 @@ _PyStackRef new_frame; oparg = 2; assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17502,21 +17502,21 @@ _PyStackRef new_frame; oparg = 3; assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17531,21 +17531,166 @@ _PyStackRef new_frame; oparg = 4; assert(oparg == CURRENT_OPARG()); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < (oparg >> 1); i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - (oparg >> 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_5_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_6_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < (oparg >> 1); i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - (oparg >> 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_7_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < (oparg >> 1); i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - (oparg >> 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_8_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 8; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < (oparg >> 1); i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - (oparg >> 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS_9_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef *args; + _PyStackRef self_or_null; + _PyStackRef callable; + _PyStackRef new_frame; + oparg = 9; + assert(oparg == CURRENT_OPARG()); + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < (oparg >> 1); i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache0 = new_frame; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17559,21 +17704,21 @@ _PyStackRef callable; _PyStackRef new_frame; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -17884,7 +18029,7 @@ oparg = CURRENT_OPARG(); arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -17907,7 +18052,7 @@ oparg = CURRENT_OPARG(); arg = _stack_item_0; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -17931,7 +18076,7 @@ oparg = CURRENT_OPARG(); arg = _stack_item_1; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -17956,7 +18101,7 @@ oparg = CURRENT_OPARG(); arg = _stack_item_2; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -18073,7 +18218,7 @@ oparg = CURRENT_OPARG(); arg = _stack_item_2; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); stack_pointer[0] = _stack_item_0; stack_pointer[1] = _stack_item_1; @@ -18206,7 +18351,7 @@ oparg = CURRENT_OPARG(); arg = _stack_item_2; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); stack_pointer[0] = _stack_item_0; stack_pointer[1] = _stack_item_1; @@ -18239,8 +18384,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { @@ -18270,8 +18415,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyStackRef_IsNull(self_or_null)); assert(PyType_Check(callable_o)); @@ -18299,8 +18444,8 @@ self_or_null = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -18321,9 +18466,9 @@ _PyStackRef init; _PyStackRef init_frame; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - init = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self = stack_pointer[-1 - (oparg >> 1)]; + init = stack_pointer[-2 - (oparg >> 1)]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( @@ -18335,9 +18480,9 @@ assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, (oparg >> 1)+1, NULL, shim); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -18390,7 +18535,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyType_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -18415,10 +18560,10 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -18438,7 +18583,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -18457,8 +18602,8 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -18471,7 +18616,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -18495,9 +18640,9 @@ _PyStackRef c; _PyStackRef s; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { args--; @@ -18522,7 +18667,7 @@ _tos_cache1 = c; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -18533,7 +18678,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -18558,10 +18703,10 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -18582,7 +18727,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -18600,7 +18745,7 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -18625,10 +18770,10 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -18645,7 +18790,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -19072,7 +19217,7 @@ arg = stack_pointer[-1]; self = stack_pointer[-2]; callable = stack_pointer[-3]; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); @@ -19129,7 +19274,7 @@ arg = _stack_item_0; self = stack_pointer[-1]; callable = stack_pointer[-2]; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); @@ -19188,7 +19333,7 @@ arg = _stack_item_1; self = _stack_item_0; callable = stack_pointer[-1]; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); @@ -19249,7 +19394,7 @@ arg = _stack_item_2; self = _stack_item_1; callable = _stack_item_0; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); @@ -19299,9 +19444,9 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -19315,7 +19460,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -19347,9 +19492,9 @@ _PyStackRef s; _PyStackRef a; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; _PyStackRef *arguments = args; @@ -19378,8 +19523,8 @@ _tos_cache1 = s; _tos_cache0 = c; SET_CURRENT_CACHED_VALUES(3); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -19505,9 +19650,9 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -19521,7 +19666,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -19550,12 +19695,12 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -19584,7 +19729,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -19650,9 +19795,9 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -19666,7 +19811,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -19697,12 +19842,12 @@ _PyStackRef c; _PyStackRef s; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + assert((oparg >> 1) == 1 || !PyStackRef_IsNull(self_or_null)); if (!PyStackRef_IsNull(self_or_null)) { args--; } @@ -19727,7 +19872,7 @@ _tos_cache1 = c; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -19780,9 +19925,9 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -19796,7 +19941,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -19824,12 +19969,12 @@ _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -19858,7 +20003,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -19927,8 +20072,8 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -19936,8 +20081,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; stack_pointer[0] = _stack_item_0; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -19951,8 +20096,8 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -19970,11 +20115,11 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); kwnames = _stack_item_0; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -20001,7 +20146,7 @@ _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -20022,7 +20167,7 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -20051,8 +20196,8 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -20093,8 +20238,8 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; assert(PyStackRef_IsNull(self_or_null)); _PyStackRef callable_s = callable; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -20102,8 +20247,8 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; stack_pointer[0] = _stack_item_0; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -20127,7 +20272,7 @@ _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -20158,13 +20303,13 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); kwnames = _stack_item_0; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -20182,7 +20327,7 @@ kwnames); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -20192,7 +20337,7 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ee9b64107bb9f2..2dd50328e818e6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1831,8 +1831,8 @@ _PyStackRef res; // _SPECIALIZE_CALL { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -1840,7 +1840,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -1858,8 +1858,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -1868,9 +1868,9 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -1882,8 +1882,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -1891,7 +1891,7 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); @@ -1899,8 +1899,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( @@ -1914,7 +1914,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -1922,8 +1922,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -1965,8 +1965,8 @@ } // _CHECK_OBJECT { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { @@ -2022,8 +2022,8 @@ self_or_null = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2031,7 +2031,7 @@ } // _CREATE_INIT_FRAME { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; self = self_or_null; init = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -2045,9 +2045,9 @@ assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, (oparg >> 1)+1, NULL, shim); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2110,8 +2110,8 @@ } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -2132,8 +2132,8 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2162,7 +2162,7 @@ assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -2189,13 +2189,13 @@ } // _INIT_CALL_PY_EXACT_ARGS { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); @@ -2213,7 +2213,7 @@ { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2260,8 +2260,8 @@ } // _CHECK_METHOD_VERSION { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -2296,8 +2296,8 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2314,9 +2314,9 @@ } // _PY_FRAME_GENERAL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -2331,7 +2331,7 @@ args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -2388,7 +2388,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_CLASS { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2404,9 +2404,9 @@ } // _CALL_BUILTIN_CLASS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2425,7 +2425,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2433,16 +2433,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2481,7 +2481,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_FAST { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2497,9 +2497,9 @@ } // _CALL_BUILTIN_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2519,7 +2519,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2527,16 +2527,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2575,7 +2575,7 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2591,9 +2591,9 @@ } // _CALL_BUILTIN_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - int total_args = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -2609,7 +2609,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -2617,16 +2617,16 @@ } // _POP_TOP_OPARG { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -2668,8 +2668,8 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_O { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2682,7 +2682,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -2702,7 +2702,7 @@ } // _CALL_BUILTIN_O { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { args--; @@ -2726,9 +2726,9 @@ // _POP_TOP { value = s; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer += -oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3394,8 +3394,8 @@ _PyStackRef res; // _SPECIALIZE_CALL_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -3403,7 +3403,7 @@ next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_CallKw(callable, next_instr, (oparg >> 1) + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } @@ -3421,8 +3421,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -3432,10 +3432,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3448,8 +3448,8 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( @@ -3457,7 +3457,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3470,8 +3470,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( @@ -3485,14 +3485,14 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3525,8 +3525,8 @@ } // _CHECK_METHOD_VERSION_KW { - null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { @@ -3561,8 +3561,8 @@ self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); @@ -3572,9 +3572,9 @@ // _PY_FRAME_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3598,7 +3598,7 @@ _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -3657,7 +3657,7 @@ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE_KW { - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); @@ -3673,12 +3673,12 @@ // _CALL_KW_NON_PY { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3693,7 +3693,7 @@ kwnames); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -3701,8 +3701,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -3742,7 +3742,7 @@ } // _CHECK_FUNCTION_VERSION_KW { - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -3768,10 +3768,10 @@ // _PY_FRAME_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -3795,7 +3795,7 @@ _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -3973,7 +3973,7 @@ { arg = stack_pointer[-1]; self = nos; - assert(oparg == 1); + assert((oparg >> 1) == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); @@ -4050,9 +4050,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4066,7 +4066,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4087,7 +4087,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4115,7 +4115,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -4125,13 +4125,13 @@ { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4170,9 +4170,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4186,7 +4186,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4208,7 +4208,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4236,7 +4236,7 @@ } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -4246,13 +4246,13 @@ { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyStackRef_CloseStack(args, oparg); + _PyStackRef_CloseStack(args, (oparg >> 1)); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; - stack_pointer += -1 - oparg; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4294,9 +4294,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4310,7 +4310,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4339,7 +4339,7 @@ { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + assert((oparg >> 1) == 1 || !PyStackRef_IsNull(self_or_null)); if (!PyStackRef_IsNull(self_or_null)) { args--; } @@ -4363,9 +4363,9 @@ // _POP_TOP { value = s; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer += -oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4418,9 +4418,9 @@ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_O { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { @@ -4434,7 +4434,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { total_args++; } @@ -4488,10 +4488,10 @@ // _POP_TOP { value = a; - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer[-oparg] = s; - stack_pointer += 1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer[-(oparg >> 1)] = s; + stack_pointer += 1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4551,7 +4551,7 @@ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -4566,12 +4566,12 @@ } // _CALL_NON_PY_GENERAL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -4586,7 +4586,7 @@ PyStackRef_NULL); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -4594,8 +4594,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4634,7 +4634,7 @@ } // _CHECK_FUNCTION_VERSION { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -4651,12 +4651,12 @@ } // _CHECK_FUNCTION_EXACT_ARGS { - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + if (code->co_argcount != (oparg >> 1) + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -4683,13 +4683,13 @@ } // _INIT_CALL_PY_EXACT_ARGS { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, (oparg >> 1) + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { + for (int i = 0; i < (oparg >> 1); i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); @@ -4707,7 +4707,7 @@ { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4753,7 +4753,7 @@ } // _CHECK_FUNCTION_VERSION { - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { @@ -4778,10 +4778,10 @@ } // _PY_FRAME_GENERAL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; @@ -4796,7 +4796,7 @@ args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); @@ -4876,7 +4876,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -4954,7 +4954,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -5032,7 +5032,7 @@ { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); + assert((oparg >> 1) == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); @@ -7111,8 +7111,8 @@ /* Skip 3 cache entries */ // _MAYBE_EXPAND_METHOD { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7120,8 +7120,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7130,7 +7130,7 @@ } // _MONITOR_CALL { - args = &stack_pointer[-oparg]; + args = &stack_pointer[-(oparg >> 1)]; maybe_self = self_or_null; func = callable; int is_meth = !PyStackRef_IsNull(maybe_self); @@ -7139,14 +7139,14 @@ if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); } - else if (oparg) { + else if ((oparg >> 1)) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { arg0 = &_PyInstrumentation_MISSING; } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; + stack_pointer[-2 - (oparg >> 1)] = func; + stack_pointer[-1 - (oparg >> 1)] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7160,11 +7160,11 @@ } // _DO_CALL { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7183,7 +7183,7 @@ arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); @@ -7204,7 +7204,7 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } @@ -7212,8 +7212,8 @@ } // _CHECK_PERIODIC_AT_END { - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7428,8 +7428,8 @@ /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD_KW { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; @@ -7437,8 +7437,8 @@ PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); @@ -7447,7 +7447,7 @@ } // _MONITOR_CALL_KW { - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; int is_meth = !PyStackRef_IsNull(self_or_null); PyObject *arg; if (is_meth) { @@ -7460,8 +7460,8 @@ arg = &_PyInstrumentation_MISSING; } PyObject *function = PyStackRef_AsPyObjectBorrow(callable); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( @@ -7475,10 +7475,10 @@ // _DO_CALL_KW { kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; + args = &stack_pointer[-1 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; + int total_args = (oparg >> 1); _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; @@ -7498,7 +7498,7 @@ arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -7524,14 +7524,14 @@ tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { - stack_pointer += -3 - oparg; + stack_pointer += -3 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6284885c428832..c6420a9c1bb4af 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -902,8 +902,8 @@ dummy_func(void) { value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); } - op(_POP_TOP_OPARG, (args[oparg] --)) { - for (int i = oparg-1; i >= 0; i--) { + op(_POP_TOP_OPARG, (args[oparg >> 1] --)) { + for (int i = (oparg >> 1)-1; i >= 0; i--) { optimize_pop_top(ctx, this_instr, args[i]); } } @@ -1139,7 +1139,7 @@ dummy_func(void) { new_frame = PyJitRef_WrapInvalid(f); } - op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *bound_method = sym_get_probable_value(callable); if (bound_method != NULL && Py_TYPE(bound_method) == &PyMethod_Type) { PyMethodObject *method = (PyMethodObject *)bound_method; @@ -1154,7 +1154,7 @@ dummy_func(void) { } } - op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *func = sym_get_probable_value(callable); if (func == NULL || !PyFunction_Check(func) || ((PyFunctionObject *)func)->func_version != func_version) { ctx->contradiction = true; @@ -1166,7 +1166,7 @@ dummy_func(void) { _Py_BloomFilter_Add(dependencies, func); } - op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { + op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable, unused, unused[oparg >> 1], unused -- callable, unused, unused[oparg >> 1], unused)) { PyObject *func = sym_get_probable_value(callable); if (func == NULL || !PyFunction_Check(func) || ((PyFunctionObject *)func)->func_version != func_version) { ctx->contradiction = true; @@ -1178,7 +1178,7 @@ dummy_func(void) { _Py_BloomFilter_Add(dependencies, func); } - op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg >> 1] -- callable, null, unused[oparg >> 1])) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); @@ -1201,7 +1201,7 @@ dummy_func(void) { sym_set_type(callable, &PyMethod_Type); } - op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg], unused -- callable, null, unused[oparg], unused)) { + op(_CHECK_METHOD_VERSION_KW, (func_version/2, callable, null, unused[oparg >> 1], unused -- callable, null, unused[oparg >> 1], unused)) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); assert(PyMethod_Check(method)); @@ -1224,26 +1224,26 @@ dummy_func(void) { sym_set_type(callable, &PyMethod_Type); } - op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { assert(sym_matches_type(callable, &PyFunction_Type)); if (sym_is_const(ctx, callable)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + sym_is_not_null(self_or_null)) { + if (co->co_argcount == (oparg >> 1) + sym_is_not_null(self_or_null)) { ADD_OP(_NOP, 0 ,0); } } } } - op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg >> 1] -- callable, null, unused[oparg >> 1])) { sym_set_null(null); sym_set_type(callable, &PyMethod_Type); } - op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame)) { - int argcount = oparg; + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg >> 1] -- new_frame)) { + int argcount = (oparg >> 1); assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { @@ -1259,7 +1259,7 @@ dummy_func(void) { } } - op(_EXPAND_METHOD, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_EXPAND_METHOD, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); callable = sym_new_const(ctx, method->im_func); @@ -1271,7 +1271,7 @@ dummy_func(void) { } } - op(_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + op(_EXPAND_METHOD_KW, (callable, self_or_null, unused[oparg >> 1], unused -- callable, self_or_null, unused[oparg >> 1], unused)) { if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); callable = sym_new_const(ctx, method->im_func); @@ -1283,17 +1283,17 @@ dummy_func(void) { } } - op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_MAYBE_EXPAND_METHOD, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { (void)args; callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); } - op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg >> 1] -- new_frame)) { new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); } - op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { + op(_PY_FRAME_KW, (callable, self_or_null, args[oparg >> 1], kwnames -- new_frame)) { new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); } @@ -1301,7 +1301,7 @@ dummy_func(void) { ex_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, func_st, NULL, 0)); } - op(_CHECK_OBJECT, (type_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_CHECK_OBJECT, (type_version/2, callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { PyObject *probable_callable = sym_get_probable_value(callable); assert(probable_callable != NULL); PyObject *const_callable = sym_get_const(ctx, callable); @@ -1332,12 +1332,12 @@ dummy_func(void) { } } - op(_ALLOCATE_OBJECT, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + op(_ALLOCATE_OBJECT, (callable, self_or_null, unused[oparg >> 1] -- callable, self_or_null, unused[oparg >> 1])) { self_or_null = sym_new_not_null(ctx); } - op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - ctx->frame->stack_pointer = stack_pointer - oparg - 2; + op(_CREATE_INIT_FRAME, (init, self, args[oparg >> 1] -- init_frame)) { + ctx->frame->stack_pointer = stack_pointer - (oparg >> 1) - 2; _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, NULL, 0); if (shim == NULL) { break; @@ -1349,7 +1349,7 @@ dummy_func(void) { ctx->frame = shim; ctx->curr_frame_depth++; assert((this_instr + 1)->opcode == _PUSH_FRAME); - init_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, init, args-1, oparg+1)); + init_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, init, args-1, (oparg >> 1)+1)); } op(_RETURN_VALUE, (retval -- res)) { @@ -1539,7 +1539,7 @@ dummy_func(void) { gen_frame = PyJitRef_WrapInvalid(new_frame); } - op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyCodeObject *co = sym_get_probable_func_code(callable); if (co == NULL) { ctx->done = true; @@ -1552,7 +1552,7 @@ dummy_func(void) { (void)framesize; } - op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyTypeObject *type = sym_get_type(callable); if (type && type != &PyFunction_Type && type != &PyMethod_Type) { ADD_OP(_NOP, 0, 0); @@ -1566,7 +1566,7 @@ dummy_func(void) { } } - op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { + op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg >> 1], unused -- callable, unused, unused[oparg >> 1], unused)) { PyTypeObject *type = sym_get_type(callable); if (type && type != &PyFunction_Type && type != &PyMethod_Type) { ADD_OP(_NOP, 0, 0); @@ -1691,7 +1691,7 @@ dummy_func(void) { none = sym_new_const(ctx, Py_None); } - op(_GUARD_CALLABLE_BUILTIN_CLASS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_CLASS, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyType_Type)) { PyTypeObject *tp = (PyTypeObject *)callable_o; @@ -1704,11 +1704,11 @@ dummy_func(void) { } } - op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_O, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -1723,7 +1723,7 @@ dummy_func(void) { } } - op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_FAST, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == @@ -1736,7 +1736,7 @@ dummy_func(void) { } } - op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + op(_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, (callable, unused, unused[oparg >> 1] -- callable, unused, unused[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == @@ -1749,11 +1749,11 @@ dummy_func(void) { } } - op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { callable = sym_new_not_null(ctx); } - op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) { + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg >> 1] -- res, c, s)) { res = sym_new_not_null(ctx); c = callable; if (sym_is_not_null(self_or_null)) { @@ -1768,19 +1768,19 @@ dummy_func(void) { } } - op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { callable = sym_new_not_null(ctx); } - op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_BUILTIN_CLASS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { callable = sym_new_not_null(ctx); } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -1804,11 +1804,11 @@ dummy_func(void) { } } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -1833,11 +1833,11 @@ dummy_func(void) { } } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -1869,13 +1869,13 @@ dummy_func(void) { ctx->frame->is_c_recursion_checked = true; } - op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res, c, s)) { + op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg >> 1] -- res, c, s)) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, (oparg >> 1) + 1, (uintptr_t)cfunc); } res = sym_new_not_null(ctx); c = callable; @@ -1891,33 +1891,33 @@ dummy_func(void) { } } - op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, (oparg >> 1), (uintptr_t)cfunc); } callable = sym_new_not_null(ctx); } - op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, (oparg >> 1), (uintptr_t)cfunc); } callable = sym_new_not_null(ctx); } - op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + op(_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg >> 1] -- callable, self_or_null, args[oparg >> 1])) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -1942,13 +1942,13 @@ dummy_func(void) { } } - op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg >> 1] -- res, c, s, a)) { PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, (oparg >> 1) + 1, (uintptr_t)cfunc); } res = sym_new_not_null(ctx); c = callable; @@ -2649,15 +2649,15 @@ dummy_func(void) { sym_set_recorded_value(value, (PyObject *)this_instr->operand0); } - op(_RECORD_CALLABLE, (func, self, args[oparg] -- func, self, args[oparg])) { + op(_RECORD_CALLABLE, (func, self, args[oparg >> 1] -- func, self, args[oparg >> 1])) { sym_set_recorded_value(func, (PyObject *)this_instr->operand0); } - op(_RECORD_CALLABLE_KW, (func, self, args[oparg], kwnames -- func, self, args[oparg], kwnames)) { + op(_RECORD_CALLABLE_KW, (func, self, args[oparg >> 1], kwnames -- func, self, args[oparg >> 1], kwnames)) { sym_set_recorded_value(func, (PyObject *)this_instr->operand0); } - op(_RECORD_BOUND_METHOD, (callable, self, args[oparg] -- callable, self, args[oparg])) { + op(_RECORD_BOUND_METHOD, (callable, self, args[oparg >> 1] -- callable, self, args[oparg >> 1])) { sym_set_recorded_value(callable, (PyObject *)this_instr->operand0); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9cd20c495db441..d446e9d9828358 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -167,12 +167,12 @@ case _POP_TOP_OPARG: { JitOptRef *args; - args = &stack_pointer[-oparg]; - for (int i = oparg-1; i >= 0; i--) { + args = &stack_pointer[-(oparg >> 1)]; + for (int i = (oparg >> 1)-1; i >= 0; i--) { optimize_pop_top(ctx, this_instr, args[i]); } - CHECK_STACK_BOUNDS(-oparg); - stack_pointer += -oparg; + CHECK_STACK_BOUNDS(-(oparg >> 1)); + stack_pointer += -(oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4047,14 +4047,14 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; (void)args; callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; break; } @@ -4065,18 +4065,18 @@ case _PY_FRAME_GENERAL: { JitOptRef callable; JitOptRef new_frame; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); - CHECK_STACK_BOUNDS(-1 - oparg); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + CHECK_STACK_BOUNDS(-1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = new_frame; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)this_instr->operand0; PyObject *func = sym_get_probable_value(callable); if (func == NULL || !PyFunction_Check(func) || ((PyFunctionObject *)func)->func_version != func_version) { @@ -4095,7 +4095,7 @@ case _CHECK_METHOD_VERSION: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); @@ -4122,8 +4122,8 @@ case _EXPAND_METHOD: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); callable = sym_new_const(ctx, method->im_func); @@ -4133,14 +4133,14 @@ callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; break; } case _CHECK_IS_NOT_PY_CALLABLE: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyTypeObject *type = sym_get_type(callable); if (type && type != &PyFunction_Type && type != &PyMethod_Type) { ADD_OP(_NOP, 0, 0); @@ -4151,9 +4151,9 @@ case _CALL_NON_PY_GENERAL: { JitOptRef res; res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-1 - oparg); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + CHECK_STACK_BOUNDS(-1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4161,8 +4161,8 @@ case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { JitOptRef null; JitOptRef callable; - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; sym_set_null(null); sym_set_type(callable, &PyMethod_Type); break; @@ -4171,8 +4171,8 @@ case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *bound_method = sym_get_probable_value(callable); if (bound_method != NULL && Py_TYPE(bound_method) == &PyMethod_Type) { PyMethodObject *method = (PyMethodObject *)bound_method; @@ -4185,8 +4185,8 @@ callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-2 - (oparg >> 1)] = callable; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; break; } @@ -4200,14 +4200,14 @@ case _CHECK_FUNCTION_EXACT_ARGS: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; assert(sym_matches_type(callable, &PyFunction_Type)); if (sym_is_const(ctx, callable)) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { PyFunctionObject *func = (PyFunctionObject *)sym_get_const(ctx, callable); PyCodeObject *co = (PyCodeObject *)func->func_code; - if (co->co_argcount == oparg + sym_is_not_null(self_or_null)) { + if (co->co_argcount == (oparg >> 1) + sym_is_not_null(self_or_null)) { ADD_OP(_NOP, 0 ,0); } } @@ -4217,7 +4217,7 @@ case _CHECK_STACK_SPACE: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyCodeObject *co = sym_get_probable_func_code(callable); if (co == NULL) { ctx->done = true; @@ -4236,10 +4236,10 @@ JitOptRef self_or_null; JitOptRef callable; JitOptRef new_frame; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int argcount = oparg; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; + int argcount = (oparg >> 1); assert(!PyJitRef_IsNull(self_or_null)); assert(args != NULL); if (sym_is_not_null(self_or_null)) { @@ -4251,9 +4251,9 @@ } else { new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); } - CHECK_STACK_BOUNDS(-1 - oparg); - stack_pointer[-2 - oparg] = new_frame; - stack_pointer += -1 - oparg; + CHECK_STACK_BOUNDS(-1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = new_frame; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4408,8 +4408,8 @@ case _CHECK_OBJECT: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; uint32_t type_version = (uint32_t)this_instr->operand0; PyObject *probable_callable = sym_get_probable_value(callable); assert(probable_callable != NULL); @@ -4431,21 +4431,21 @@ assert(init != NULL); assert(PyFunction_Check(init)); callable = sym_new_const(ctx, init); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; watch_type((PyTypeObject *)callable_o, dependencies); } else { callable = sym_new_not_null(ctx); } - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } case _ALLOCATE_OBJECT: { JitOptRef self_or_null; - self_or_null = stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; self_or_null = sym_new_not_null(ctx); - stack_pointer[-1 - oparg] = self_or_null; + stack_pointer[-1 - (oparg >> 1)] = self_or_null; break; } @@ -4454,10 +4454,10 @@ JitOptRef self; JitOptRef init; JitOptRef init_frame; - args = &stack_pointer[-oparg]; - self = stack_pointer[-1 - oparg]; - init = stack_pointer[-2 - oparg]; - ctx->frame->stack_pointer = stack_pointer - oparg - 2; + args = &stack_pointer[-(oparg >> 1)]; + self = stack_pointer[-1 - (oparg >> 1)]; + init = stack_pointer[-2 - (oparg >> 1)]; + ctx->frame->stack_pointer = stack_pointer - (oparg >> 1) - 2; _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, NULL, 0); if (shim == NULL) { break; @@ -4468,10 +4468,10 @@ ctx->frame = shim; ctx->curr_frame_depth++; assert((this_instr + 1)->opcode == _PUSH_FRAME); - init_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, init, args-1, oparg+1)); - CHECK_STACK_BOUNDS(-1 - oparg); - stack_pointer[-2 - oparg] = init_frame; - stack_pointer += -1 - oparg; + init_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, init, args-1, (oparg >> 1)+1)); + CHECK_STACK_BOUNDS(-1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = init_frame; + stack_pointer += -1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4485,7 +4485,7 @@ case _GUARD_CALLABLE_BUILTIN_CLASS: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyType_Type)) { PyTypeObject *tp = (PyTypeObject *)callable_o; @@ -4501,21 +4501,21 @@ case _CALL_BUILTIN_CLASS: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; callable = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } case _GUARD_CALLABLE_BUILTIN_O: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -4538,9 +4538,9 @@ JitOptRef res; JitOptRef c; JitOptRef s; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; res = sym_new_not_null(ctx); c = callable; if (sym_is_not_null(self_or_null)) { @@ -4553,18 +4553,18 @@ else { s = sym_new_unknown(ctx); } - CHECK_STACK_BOUNDS(1 - oparg); - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer[-oparg] = s; - stack_pointer += 1 - oparg; + CHECK_STACK_BOUNDS(1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer[-(oparg >> 1)] = s; + stack_pointer += 1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_BUILTIN_FAST: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == @@ -4580,15 +4580,15 @@ case _CALL_BUILTIN_FAST: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; callable = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyCFunction_Type)) { if ((PyCFunction_GET_FLAGS(callable_o) & _Py_METH_CALL_FLAGS) == @@ -4604,9 +4604,9 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; callable = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } @@ -4761,13 +4761,13 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -4800,15 +4800,15 @@ JitOptRef c; JitOptRef s; JitOptRef a; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_O_INLINE, (oparg >> 1) + 1, (uintptr_t)cfunc); } res = sym_new_not_null(ctx); c = callable; @@ -4821,12 +4821,12 @@ s = sym_new_unknown(ctx); a = sym_new_unknown(ctx); } - CHECK_STACK_BOUNDS(2 - oparg); - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer[-oparg] = s; - stack_pointer[1 - oparg] = a; - stack_pointer += 2 - oparg; + CHECK_STACK_BOUNDS(2 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer[-(oparg >> 1)] = s; + stack_pointer[1 - (oparg >> 1)] = a; + stack_pointer += 2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4862,13 +4862,13 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -4897,17 +4897,17 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, oparg, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, (oparg >> 1), (uintptr_t)cfunc); } callable = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } @@ -4919,13 +4919,13 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -4958,15 +4958,15 @@ JitOptRef res; JitOptRef c; JitOptRef s; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, oparg + 1, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, (oparg >> 1) + 1, (uintptr_t)cfunc); } res = sym_new_not_null(ctx); c = callable; @@ -4980,11 +4980,11 @@ else { s = sym_new_unknown(ctx); } - CHECK_STACK_BOUNDS(1 - oparg); - stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = c; - stack_pointer[-oparg] = s; - stack_pointer += 1 - oparg; + CHECK_STACK_BOUNDS(1 - (oparg >> 1)); + stack_pointer[-2 - (oparg >> 1)] = res; + stack_pointer[-1 - (oparg >> 1)] = c; + stack_pointer[-(oparg >> 1)] = s; + stack_pointer += 1 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5009,13 +5009,13 @@ JitOptRef *args; JitOptRef self_or_null; JitOptRef callable; - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + args = &stack_pointer[-(oparg >> 1)]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && sym_matches_type(callable, &PyMethodDescr_Type) && (sym_is_not_null(self_or_null) || sym_is_null(self_or_null))) { - int total_args = oparg; + int total_args = (oparg >> 1); if (sym_is_not_null(self_or_null)) { total_args++; } @@ -5044,17 +5044,17 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - (oparg >> 1)]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = sym_get_const(ctx, callable); if (callable_o && Py_IS_TYPE(callable_o, &PyMethodDescr_Type) && sym_is_not_null(self_or_null)) { PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; PyCFunction cfunc = method->d_method->ml_meth; - ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, oparg, (uintptr_t)cfunc); + ADD_OP(_CALL_METHOD_DESCRIPTOR_FAST_INLINE, (oparg >> 1), (uintptr_t)cfunc); } callable = sym_new_not_null(ctx); - stack_pointer[-2 - oparg] = callable; + stack_pointer[-2 - (oparg >> 1)] = callable; break; } @@ -5073,18 +5073,18 @@ case _PY_FRAME_KW: { JitOptRef callable; JitOptRef new_frame; - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; new_frame = PyJitRef_WrapInvalid(frame_new_from_symbol(ctx, callable, NULL, 0)); - CHECK_STACK_BOUNDS(-2 - oparg); - stack_pointer[-3 - oparg] = new_frame; - stack_pointer += -2 - oparg; + CHECK_STACK_BOUNDS(-2 - (oparg >> 1)); + stack_pointer[-3 - (oparg >> 1)] = new_frame; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION_KW: { JitOptRef callable; - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = (uint32_t)this_instr->operand0; PyObject *func = sym_get_probable_value(callable); if (func == NULL || !PyFunction_Check(func) || ((PyFunctionObject *)func)->func_version != func_version) { @@ -5099,7 +5099,7 @@ case _CHECK_METHOD_VERSION_KW: { JitOptRef callable; - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; uint32_t func_version = (uint32_t)this_instr->operand0; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); @@ -5126,8 +5126,8 @@ case _EXPAND_METHOD_KW: { JitOptRef self_or_null; JitOptRef callable; - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; + self_or_null = stack_pointer[-2 - (oparg >> 1)]; + callable = stack_pointer[-3 - (oparg >> 1)]; if (sym_is_const(ctx, callable) && sym_matches_type(callable, &PyMethod_Type)) { PyMethodObject *method = (PyMethodObject *)sym_get_const(ctx, callable); callable = sym_new_const(ctx, method->im_func); @@ -5137,14 +5137,14 @@ callable = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-3 - (oparg >> 1)] = callable; + stack_pointer[-2 - (oparg >> 1)] = self_or_null; break; } case _CHECK_IS_NOT_PY_CALLABLE_KW: { JitOptRef callable; - callable = stack_pointer[-3 - oparg]; + callable = stack_pointer[-3 - (oparg >> 1)]; PyTypeObject *type = sym_get_type(callable); if (type && type != &PyFunction_Type && type != &PyMethod_Type) { ADD_OP(_NOP, 0, 0); @@ -5155,9 +5155,9 @@ case _CALL_KW_NON_PY: { JitOptRef res; res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-2 - oparg); - stack_pointer[-3 - oparg] = res; - stack_pointer += -2 - oparg; + CHECK_STACK_BOUNDS(-2 - (oparg >> 1)); + stack_pointer[-3 - (oparg >> 1)] = res; + stack_pointer += -2 - (oparg >> 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5824,21 +5824,21 @@ case _RECORD_CALLABLE: { JitOptRef func; - func = stack_pointer[-2 - oparg]; + func = stack_pointer[-2 - (oparg >> 1)]; sym_set_recorded_value(func, (PyObject *)this_instr->operand0); break; } case _RECORD_CALLABLE_KW: { JitOptRef func; - func = stack_pointer[-3 - oparg]; + func = stack_pointer[-3 - (oparg >> 1)]; sym_set_recorded_value(func, (PyObject *)this_instr->operand0); break; } case _RECORD_BOUND_METHOD: { JitOptRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; sym_set_recorded_value(callable, (PyObject *)this_instr->operand0); break; } diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 98abe3d0505e20..75b5ba65f972f1 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -71,21 +71,21 @@ void _PyOpcode_RecordFunction_4OS(_PyInterpreterFrame *frame, _PyStackRef *stack void _PyOpcode_RecordFunction_CALLABLE(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { _PyStackRef func; - func = stack_pointer[-2 - oparg]; + func = stack_pointer[-2 - (oparg >> 1)]; *recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(func); Py_INCREF(*recorded_value); } void _PyOpcode_RecordFunction_CALLABLE_KW(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { _PyStackRef func; - func = stack_pointer[-3 - oparg]; + func = stack_pointer[-3 - (oparg >> 1)]; *recorded_value = (PyObject *)PyStackRef_AsPyObjectBorrow(func); Py_INCREF(*recorded_value); } void _PyOpcode_RecordFunction_BOUND_METHOD(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value) { _PyStackRef callable; - callable = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - (oparg >> 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) == &PyMethod_Type) { *recorded_value = (PyObject *)callable_o; diff --git a/Python/specialize.c b/Python/specialize.c index 067bae59f6073b..cb84fb4b81866d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1660,7 +1660,7 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) assert(PyType_Check(callable)); PyTypeObject *tp = _PyType_CAST(callable); if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { - int oparg = instr->op.arg; + int oparg = instr->op.arg >> 1; if (nargs == 1 && oparg == 1) { if (tp == &PyUnicode_Type) { specialize(instr, CALL_STR_1); @@ -1727,7 +1727,7 @@ specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, } PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *list_append = interp->callable_cache.list_append; - int oparg = instr->op.arg; + int oparg = instr->op.arg >> 1; if ((PyObject *)descr == list_append && oparg == 1) { assert(self_or_null != NULL); if (PyList_CheckExact(self_or_null)) { @@ -1839,7 +1839,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } /* len(o) */ PyInterpreterState *interp = _PyInterpreterState_GET(); - if (callable == interp->callable_cache.len && instr->op.arg == 1) { + if (callable == interp->callable_cache.len && (instr->op.arg >> 1) == 1) { specialize(instr, CALL_LEN); return 0; } @@ -1850,7 +1850,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) if (nargs == 2) { /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); - if (callable == interp->callable_cache.isinstance && instr->op.arg == 2) { + if (callable == interp->callable_cache.isinstance && (instr->op.arg >> 1) == 2) { specialize(instr, CALL_ISINSTANCE); return 0; } From bef26b4807039696177f56cafc0367641a89c0c1 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 18 Nov 2026 22:54:37 +0100 Subject: [PATCH 2/3] gh-161170: Compile awaited calls with an extra slot for GET_AWAITABLE GET_AWAITABLE now also pushes the NULL and None that start the following SEND loop, so the compiler no longer emits them. GET_AWAITABLE also pops an extra slot. For `await ` the compiler sets the awaited flag of the call, pushes a NULL below its callable and sets GET_AWAITABLE_AFTER_CALL (4) in the oparg of GET_AWAITABLE; for other awaitables it pushes the NULL right after the awaitable. The low two bits of the oparg keep saying where the awaitable comes from. Nothing writes to the slot yet, so there is no behavior change; it will report how a coroutine body that ran without a coroutine object ended. Co-Authored-By: Claude Opus 5.5 --- Doc/library/dis.rst | 25 +++++--- Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_opcode_utils.h | 6 ++ Include/internal/pycore_uop_ids.h | 2 +- Include/internal/pycore_uop_metadata.h | 10 +-- Lib/dis.py | 4 ++ Lib/test/test_dis.py | 11 ++-- Modules/_testinternalcapi/test_cases.c.h | 23 +++++-- Python/bytecodes.c | 17 ++++- Python/codegen.c | 77 ++++++++++++++--------- Python/executor_cases.c.h | 35 +++++++---- Python/generated_cases.c.h | 23 +++++-- Python/optimizer_bytecodes.c | 7 +++ Python/optimizer_cases.c.h | 13 +++- 14 files changed, 174 insertions(+), 83 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 96b4003ef93d34..3580c03cc66356 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -817,22 +817,31 @@ not have to be) the original ``STACK[-2]``. .. opcode:: GET_AWAITABLE (where) - Implements ``STACK[-1] = get_awaitable(STACK[-1])``, where ``get_awaitable(o)`` - returns ``o`` if ``o`` is a coroutine object or a generator object with - the :data:`~inspect.CO_ITERABLE_COROUTINE` flag, or resolves - ``o.__await__``. + Pops two values: the awaitable and a ``NULL`` pushed after it, or, if + ``where & 4`` is set, a slot pushed below the callable of an awaited call + and the call's result. Pushes ``get_awaitable(awaitable)``, ``NULL`` and + ``None``, which start the :opcode:`SEND` loop that follows. + ``get_awaitable(o)`` returns ``o`` if ``o`` is a coroutine object or a + generator object with the :data:`~inspect.CO_ITERABLE_COROUTINE` flag, or + resolves ``o.__await__``. - If the ``where`` operand is nonzero, it indicates where the instruction - occurs: + If ``where & 3`` is nonzero, it indicates where the instruction occurs: - * ``1``: After a call to ``__aenter__`` - * ``2``: After a call to ``__aexit__`` + * ``1``: After a call to ``__aenter__`` + * ``2``: After a call to ``__aexit__`` + + If ``where & 4`` is set, the awaited object is the result of an + awaited :opcode:`CALL`, :opcode:`CALL_KW` or :opcode:`CALL_FUNCTION_EX`. .. versionadded:: 3.5 .. versionchanged:: 3.11 Previously, this instruction did not have an oparg. + .. versionchanged:: next + Pops an extra slot and pushes ``NULL`` and ``None`` for the following + :opcode:`SEND`; ``where & 4`` marks an awaited call. + .. opcode:: GET_AITER diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 9f96aa4ba98f37..c264224ffc4fbc 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -221,7 +221,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case GET_ANEXT: return 1; case GET_AWAITABLE: - return 1; + return 2; case GET_ITER: return 1; case GET_ITER_SELF: @@ -716,7 +716,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case GET_ANEXT: return 2; case GET_AWAITABLE: - return 1; + return 3; case GET_ITER: return 2; case GET_ITER_SELF: diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 7067b48ec22cb3..7b2a6d3299ffbd 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -94,6 +94,12 @@ extern "C" { #define RESUME_OPARG_LOCATION_MASK 0x7 #define RESUME_OPARG_DEPTH1_MASK 0x8 +/* Values used in the oparg for GET_AWAITABLE: where the awaitable comes from + * (0: await, 1: __aenter__, 2: __aexit__), and whether it is the result of + * an awaited call, preceded on the stack by a slot pushed for that call. */ +#define GET_AWAITABLE_WHERE_MASK 0x3 +#define GET_AWAITABLE_AFTER_CALL 0x4 + #define GET_ITER_YIELD_FROM 1 #define GET_ITER_YIELD_FROM_NO_CHECK 2 #define GET_ITER_YIELD_FROM_CORO_CHECK 3 diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 3f582b7dcb8245..8107f37d0fb013 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -695,7 +695,7 @@ enum { _FOR_ITER_VIRTUAL_TIER_TWO_r23, _GET_AITER_r11, _GET_ANEXT_r12, - _GET_AWAITABLE_r11, + _GET_AWAITABLE_r23, _GET_ITER_r12, _GET_ITER_TRAD_r12, _GET_LEN_r12, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f6ffadf77282b4..35a4931f05d562 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -1641,11 +1641,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_GET_AWAITABLE] = { - .best = { 1, 1, 1, 1 }, + .best = { 2, 2, 2, 2 }, .entries = { { -1, -1, -1 }, - { 1, 1, _GET_AWAITABLE_r11 }, { -1, -1, -1 }, + { 3, 2, _GET_AWAITABLE_r23 }, { -1, -1, -1 }, }, }, @@ -4334,7 +4334,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_RETURN_VALUE_r11] = _RETURN_VALUE, [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, - [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_GET_AWAITABLE_r23] = _GET_AWAITABLE, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, @@ -5293,7 +5293,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GET_ANEXT] = "_GET_ANEXT", [_GET_ANEXT_r12] = "_GET_ANEXT_r12", [_GET_AWAITABLE] = "_GET_AWAITABLE", - [_GET_AWAITABLE_r11] = "_GET_AWAITABLE_r11", + [_GET_AWAITABLE_r23] = "_GET_AWAITABLE_r23", [_GET_ITER] = "_GET_ITER", [_GET_ITER_r12] = "_GET_ITER_r12", [_GET_ITER_TRAD] = "_GET_ITER_TRAD", @@ -6528,7 +6528,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _GET_ANEXT: return 0; case _GET_AWAITABLE: - return 1; + return 2; case _SEND_GEN_FRAME: return 1; case _GUARD_TOS_IS_NONE: diff --git a/Lib/dis.py b/Lib/dis.py index 54df95d6bb0689..e22c128713b19e 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -47,6 +47,7 @@ CALL = opmap['CALL'] CALL_KW = opmap['CALL_KW'] CALL_FUNCTION_EX = opmap['CALL_FUNCTION_EX'] +GET_AWAITABLE = opmap['GET_AWAITABLE'] CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] @@ -653,6 +654,9 @@ def get_argval_argrepr(self, op, arg, offset): elif deop == CALL_FUNCTION_EX: if arg & 1: argrepr = "awaited" + elif deop == GET_AWAITABLE: + if arg & 4: + argrepr = "after call" elif deop == CALL_INTRINSIC_1: argrepr = _intrinsic_1_descs[arg] elif deop == CALL_INTRINSIC_2: diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 351e1eff4cb5b9..0419542f6fa2e6 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -614,9 +614,8 @@ async def _asyncwith(c): SWAP 3 LOAD_SPECIAL 2 (__aenter__) CALL 0 (0) - GET_AWAITABLE 1 PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) + GET_AWAITABLE 1 L2: SEND 4 (to L5) L3: YIELD_VALUE 1 L4: RESUME 3 @@ -631,9 +630,8 @@ async def _asyncwith(c): LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) CALL 6 (3) - GET_AWAITABLE 2 PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) + GET_AWAITABLE 2 L8: SEND 4 (to L11) L9: YIELD_VALUE 1 L10: RESUME 3 @@ -647,14 +645,13 @@ async def _asyncwith(c): RETURN_VALUE %4d L12: CLEANUP_THROW - L13: JUMP_BACKWARD_NO_INTERRUPT 28 (to L5) + L13: JUMP_BACKWARD_NO_INTERRUPT 27 (to L5) L14: CLEANUP_THROW L15: JUMP_BACKWARD_NO_INTERRUPT 10 (to L11) L16: PUSH_EXC_INFO WITH_EXCEPT_START - GET_AWAITABLE 2 PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) + GET_AWAITABLE 2 L17: SEND 5 (to L21) L18: YIELD_VALUE 1 L19: RESUME 3 diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 91a736ce871c53..c8f508f1ecf8d1 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -6822,25 +6822,36 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); - _PyStackRef iterable; + _PyStackRef bottom; + _PyStackRef top; _PyStackRef iter; - iterable = stack_pointer[-1]; + _PyStackRef null; + _PyStackRef none; + top = stack_pointer[-1]; + bottom = stack_pointer[-2]; + _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; + assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), + oparg & GET_AWAITABLE_WHERE_MASK); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -1; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(iterable); + PyStackRef_CLOSE(awaitable); _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; stack_pointer[0] = iter; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer[2] = none; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0a8147e19fcd3d..ab03bd08b51c0e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1649,11 +1649,22 @@ dummy_func( awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); } - inst(GET_AWAITABLE, (iterable -- iter)) { - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - PyStackRef_CLOSE(iterable); + // Also pushes the NULL and None that the following SEND loop starts + // with. The compiler pushes a NULL above the awaitable, or, with + // GET_AWAITABLE_AFTER_CALL, below the callable of an awaited call, + // so `top` is its result. + inst(GET_AWAITABLE, (bottom, top -- iter, null, none)) { + _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; + assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), + oparg & GET_AWAITABLE_WHERE_MASK); + DEAD(top); + DEAD(bottom); + PyStackRef_CLOSE(awaitable); ERROR_IF(iter_o == NULL); iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; } family(SEND, INLINE_CACHE_ENTRIES_SEND) = { diff --git a/Python/codegen.c b/Python/codegen.c index 2a651037e707fb..38a3e2d4336234 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -224,7 +224,8 @@ static int codegen_call_helper_impl(compiler *c, location loc, int n, /* Args already pushed */ asdl_expr_seq *args, PyObject *injected_arg, - asdl_keyword_seq *keywords); + asdl_keyword_seq *keywords, + int awaited); static int codegen_call_helper(compiler *c, location loc, int n, asdl_expr_seq *args, asdl_keyword_seq *keywords); @@ -280,6 +281,7 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) * with the low bit set when the result is awaited right away. */ #define CALL_OPARG(NARGS, AWAITED) (((NARGS) << 1) | (AWAITED)) + #define ADDOP_I(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) @@ -621,9 +623,8 @@ codegen_unwind_fblock(compiler *c, location *ploc, } RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc)); if (info->fb_type == COMPILE_FBLOCK_ASYNC_WITH) { - ADDOP_I(c, *ploc, GET_AWAITABLE, 2); ADDOP(c, *ploc, PUSH_NULL); - ADDOP_LOAD_CONST(c, *ploc, Py_None); + ADDOP_I(c, *ploc, GET_AWAITABLE, 2); ADD_YIELD_FROM(c, *ploc, 1); } ADDOP(c, *ploc, POP_TOP); @@ -1762,7 +1763,7 @@ codegen_class(compiler *c, stmt_ty s) RETURN_IF_ERROR_IN_SCOPE(c, codegen_call_helper_impl(c, loc, 2, s->v.ClassDef.bases, &_Py_STR(generic_base), - s->v.ClassDef.keywords)); + s->v.ClassDef.keywords, 0)); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); @@ -4156,7 +4157,7 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) // Return 1 if the method call was optimized, 0 if not, and -1 on error. static int -maybe_optimize_method_call(compiler *c, expr_ty e) +maybe_optimize_method_call(compiler *c, expr_ty e, int awaited) { Py_ssize_t argsl, i, kwdsl; expr_ty meth = e->v.Call.func; @@ -4221,11 +4222,11 @@ maybe_optimize_method_call(compiler *c, expr_ty e) RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, kwds, kwdsl)); loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL_KW, CALL_OPARG(argsl + kwdsl, 0)); + ADDOP_I(c, loc, CALL_KW, CALL_OPARG(argsl + kwdsl, awaited)); } else { loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL, CALL_OPARG(argsl, 0)); + ADDOP_I(c, loc, CALL, CALL_OPARG(argsl, awaited)); } return 1; } @@ -4250,10 +4251,10 @@ codegen_validate_keywords(compiler *c, asdl_keyword_seq *keywords) } static int -codegen_call(compiler *c, expr_ty e) +codegen_call(compiler *c, expr_ty e, int awaited) { RETURN_IF_ERROR(codegen_validate_keywords(c, e->v.Call.keywords)); - int ret = maybe_optimize_method_call(c, e); + int ret = maybe_optimize_method_call(c, e, awaited); if (ret < 0) { return ERROR; } @@ -4267,13 +4268,35 @@ codegen_call(compiler *c, expr_ty e) location loc = LOC(e->v.Call.func); ADDOP(c, loc, PUSH_NULL); loc = LOC(e); - ret = codegen_call_helper(c, loc, 0, - e->v.Call.args, - e->v.Call.keywords); + ret = codegen_call_helper_impl(c, loc, 0, e->v.Call.args, NULL, + e->v.Call.keywords, awaited); USE_LABEL(c, skip_normal_call); return ret; } +/* GET_AWAITABLE takes an extra slot. `await ` marks the call as awaited + * and pushes the slot below the callable: if the call runs a coroutine body + * without creating the coroutine object, the body reports there how it ended. + * Other awaitables get it pushed above them. */ +static int +codegen_await(compiler *c, expr_ty e) +{ + location loc = LOC(e); + expr_ty value = e->v.Await.value; + if (value->kind == Call_kind) { + ADDOP(c, LOC(value), PUSH_NULL); + RETURN_IF_ERROR(codegen_call(c, value, 1)); + ADDOP_I(c, loc, GET_AWAITABLE, GET_AWAITABLE_AFTER_CALL); + } + else { + VISIT(c, expr, value); + ADDOP(c, loc, PUSH_NULL); + ADDOP_I(c, loc, GET_AWAITABLE, 0); + } + ADD_YIELD_FROM(c, loc, 1); + return SUCCESS; +} + static int codegen_template_str(compiler *c, expr_ty e) { @@ -4462,7 +4485,8 @@ codegen_call_helper_impl(compiler *c, location loc, int n, /* Args already pushed */ asdl_expr_seq *args, PyObject *injected_arg, - asdl_keyword_seq *keywords) + asdl_keyword_seq *keywords, + int awaited) { Py_ssize_t i, nseen, nelts, nkwelts; @@ -4501,10 +4525,10 @@ codegen_call_helper_impl(compiler *c, location loc, VISIT_SEQ(c, keyword, keywords); RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, keywords, nkwelts)); - ADDOP_I(c, loc, CALL_KW, CALL_OPARG(n + nelts + nkwelts, 0)); + ADDOP_I(c, loc, CALL_KW, CALL_OPARG(n + nelts + nkwelts, awaited)); } else { - ADDOP_I(c, loc, CALL, CALL_OPARG(n + nelts, 0)); + ADDOP_I(c, loc, CALL, CALL_OPARG(n + nelts, awaited)); } return SUCCESS; @@ -4570,7 +4594,7 @@ codegen_call_helper(compiler *c, location loc, asdl_expr_seq *args, asdl_keyword_seq *keywords) { - return codegen_call_helper_impl(c, loc, n, args, NULL, keywords); + return codegen_call_helper_impl(c, loc, n, args, NULL, keywords, 0); } /* List and set comprehensions work by being inlined at the location where @@ -5170,9 +5194,8 @@ codegen_comprehension(compiler *c, expr_ty e, int type, ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); if (is_async_comprehension && type != COMP_GENEXP) { - ADDOP_I(c, loc, GET_AWAITABLE, 0); ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_I(c, loc, GET_AWAITABLE, 0); ADD_YIELD_FROM(c, loc, 1); } @@ -5315,9 +5338,8 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, SWAP, 3); ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); ADDOP_I(c, loc, CALL, CALL_OPARG(0, 0)); - ADDOP_I(c, loc, GET_AWAITABLE, 1); ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_I(c, loc, GET_AWAITABLE, 1); ADD_YIELD_FROM(c, loc, 1); ADDOP_JUMP(c, loc, SETUP_WITH, final); @@ -5352,9 +5374,8 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) * call __exit__(None, None, None) */ RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); - ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_I(c, loc, GET_AWAITABLE, 2); ADD_YIELD_FROM(c, loc, 1); ADDOP(c, loc, POP_TOP); @@ -5367,9 +5388,8 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_JUMP(c, loc, SETUP_CLEANUP, cleanup); ADDOP(c, loc, PUSH_EXC_INFO); ADDOP(c, loc, WITH_EXCEPT_START); - ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); + ADDOP_I(c, loc, GET_AWAITABLE, 2); ADD_YIELD_FROM(c, loc, 1); RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); @@ -5554,16 +5574,11 @@ codegen_visit_expr_impl(compiler *c, expr_ty e, bool result_is_unused) ADD_YIELD_FROM(c, loc, 0); break; case Await_kind: - VISIT(c, expr, e->v.Await.value); - ADDOP_I(c, loc, GET_AWAITABLE, 0); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); - ADD_YIELD_FROM(c, loc, 1); - break; + return codegen_await(c, e); case Compare_kind: return codegen_compare(c, e); case Call_kind: - return codegen_call(c, e); + return codegen_call(c, e, 0); case Constant_kind: ADDOP_LOAD_CONST(c, loc, e->v.Constant.value); break; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c38c7df7948a52..4314dd408600ac 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8906,36 +8906,47 @@ break; } - case _GET_AWAITABLE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); + case _GET_AWAITABLE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef iterable; + _PyStackRef top; + _PyStackRef bottom; _PyStackRef iter; + _PyStackRef null; + _PyStackRef none; _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); - iterable = _stack_item_0; - stack_pointer[0] = iterable; - stack_pointer += 1; + top = _stack_item_1; + bottom = _stack_item_0; + _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; + assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); + stack_pointer[0] = bottom; + stack_pointer[1] = top; + stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), + oparg & GET_AWAITABLE_WHERE_MASK); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -1; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(iterable); + PyStackRef_CLOSE(awaitable); _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; + _tos_cache2 = none; + _tos_cache1 = null; _tos_cache0 = iter; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2dd50328e818e6..ddd57d8782f833 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6822,25 +6822,36 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); - _PyStackRef iterable; + _PyStackRef bottom; + _PyStackRef top; _PyStackRef iter; - iterable = stack_pointer[-1]; + _PyStackRef null; + _PyStackRef none; + top = stack_pointer[-1]; + bottom = stack_pointer[-2]; + _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; + assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), + oparg & GET_AWAITABLE_WHERE_MASK); _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -1; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(iterable); + PyStackRef_CLOSE(awaitable); _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; stack_pointer[0] = iter; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer[2] = none; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index c6420a9c1bb4af..be4dcec949e916 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -206,6 +206,13 @@ dummy_func(void) { res = PyJitRef_Borrow(sym_new_null(ctx)); } + op(_GET_AWAITABLE, (bottom, top -- iter, null, none)) { + // After an awaited call, any of the outputs may be NULL. + iter = sym_new_unknown(ctx); + null = sym_new_unknown(ctx); + none = sym_new_unknown(ctx); + } + op(_GUARD_TOS_INT, (value -- value)) { if (sym_is_compact_int(value)) { ADD_OP(_NOP, 0, 0); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index d446e9d9828358..176e149d6a7ad3 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1888,8 +1888,17 @@ case _GET_AWAITABLE: { JitOptRef iter; - iter = sym_new_not_null(ctx); - stack_pointer[-1] = iter; + JitOptRef null; + JitOptRef none; + iter = sym_new_unknown(ctx); + null = sym_new_unknown(ctx); + none = sym_new_unknown(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = iter; + stack_pointer[-1] = null; + stack_pointer[0] = none; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } From b015d4ffb26852ea07f228cec03dbbd61f6139a8 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 18 Nov 2026 22:54:37 +0100 Subject: [PATCH 3/3] gh-161170: Lazily materialize coroutine objects for awaited calls `await f()` on an `async def` no longer allocates a coroutine object unless the body actually suspends. Python-function CALL paths with the awaited flag mark the new frame of a coroutine function (_MAYBE_LAZY_CORO). RETURN_GENERATOR then keeps running the body in that frame, with its own exception state item on the data stack. A body that returns or raises without suspending never creates a coroutine object; YIELD_VALUE creates one when the body first suspends and moves the frame into it. The body reports how it ended through the slot that the compiler pushes below the awaited call: a tagged int when it returned, or the coroutine when it suspended. GET_AWAITABLE turns that into the SEND loop's input: a NULL receiver makes SEND jump to END_SEND, and None in place of the NULL makes it go on to YIELD_VALUE with the value that the body has already yielded. All forms of SEND check for this, since an await site may have been specialized for another awaitable. Exceptions leaving the body are raised from the caller's SEND, as before. Calls from C, instrumented calls, coroutine origin tracking, JIT trace recording and tier 2 keep creating the coroutine eagerly. frame.f_generator and PyFrame_GetGenerator() return None/NULL for such a coroutine until it first suspends; asyncio.graph classifies frames by co_flags instead. Co-Authored-By: Claude Opus 5.5 --- Doc/c-api/frame.rst | 4 + Doc/library/dis.rst | 15 + Doc/library/inspect.rst | 4 +- Doc/reference/datamodel.rst | 14 +- Doc/whatsnew/3.16.rst | 42 ++ Include/internal/pycore_ceval.h | 6 + Include/internal/pycore_interpframe.h | 4 + Include/internal/pycore_interpframe_structs.h | 3 +- Include/internal/pycore_opcode_metadata.h | 38 +- Include/internal/pycore_uop_ids.h | 26 +- Include/internal/pycore_uop_metadata.h | 44 +- Include/opcode_ids.h | 94 +-- InternalDocs/generators.md | 25 +- Lib/_opcode_metadata.py | 94 +-- Lib/asyncio/graph.py | 63 +- Lib/test/test_asyncio/test_graph.py | 47 +- Lib/test/test_coroutines.py | 591 ++++++++++++++++++ Lib/test/test_dis.py | 36 ++ Lib/test/test_opcache.py | 7 + ...-10-27-12-00-00.gh-issue-161170.lazyCo.rst | 3 + Modules/_testinternalcapi/test_cases.c.h | 374 +++++++++-- Modules/_testinternalcapi/test_targets.h | 4 +- Objects/genobject.c | 81 ++- Programs/test_frozenmain.h | 20 +- Python/bytecodes.c | 249 ++++++-- Python/ceval.c | 52 ++ Python/ceval_macros.h | 21 + Python/codegen.c | 3 +- Python/executor_cases.c.h | 360 +++++++++-- Python/generated_cases.c.h | 374 +++++++++-- Python/intrinsics.c | 6 +- Python/opcode_targets.h | 4 +- Python/optimizer.c | 6 + Python/optimizer_bytecodes.c | 4 +- Python/optimizer_cases.c.h | 20 +- Python/record_functions.c.h | 2 +- Python/specialize.c | 10 +- 37 files changed, 2296 insertions(+), 454 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index a086c60905402e..bfd39507461d03 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -85,6 +85,10 @@ See also :ref:`Reflection `. .. versionadded:: 3.11 + .. versionchanged:: 3.16 + May return ``NULL`` for a coroutine frame before its first suspension; + see :attr:`frame.f_generator`. + .. c:function:: PyObject* PyFrame_GetGlobals(PyFrameObject *frame) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 3580c03cc66356..92fef8318afa1a 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -832,6 +832,12 @@ not have to be) the original ``STACK[-2]``. If ``where & 4`` is set, the awaited object is the result of an awaited :opcode:`CALL`, :opcode:`CALL_KW` or :opcode:`CALL_FUNCTION_EX`. + If the call ran a coroutine function without creating the coroutine + object, the slot below its callable tells how the body ended, and + :opcode:`GET_AWAITABLE` pushes the result or the yielded value in place of + ``None``, with a ``NULL`` receiver if the body returned, or with the new + coroutine and ``None`` in place of ``NULL`` if it suspended (see + :opcode:`SEND`). .. versionadded:: 3.5 @@ -1863,12 +1869,21 @@ iterations of the loop. push the exception's ``value`` attribute, and increment the bytecode counter by *delta*. + After an awaited call (see :opcode:`GET_AWAITABLE`), the receiver may be + ``NULL``: then ``STACK[-1]`` is already the result and the bytecode counter + is incremented by *delta*. If ``STACK[-2]`` is ``None``, ``STACK[-1]`` has + already been yielded by the receiver: ``STACK[-2]`` is set to ``NULL`` and + execution continues with the next instruction. + .. versionadded:: 3.11 .. versionchanged:: 3.15 The receiver is now at ``STACK[-3]``, followed by the value pushed with it by :opcode:`GET_ITER`. + .. versionchanged:: next + Handles the results of awaited calls. + .. opcode:: HAVE_ARGUMENT diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 51c465e8f00b5e..328bfea8a372ab 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -151,7 +151,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | coroutine object that | | | | owns this frame, or | | | | ``None`` if the frame is | -| | | of a regular function | +| | | of a regular function or | +| | | the generator is not | +| | | materialized yet | +-----------------+-------------------+---------------------------+ | | f_trace | tracing function for this | | | | frame, or ``None`` | diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 02663ed465f424..07fa9cf5441e64 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1736,10 +1736,22 @@ Special read-only attributes * - .. attribute:: frame.f_generator - The :term:`generator` or :term:`coroutine` object that owns this frame, - or ``None`` if the frame is a normal function. + or ``None`` if the frame is a normal function or if the generator + is not materialized yet. + + .. impl-detail:: + + A coroutine that is awaited as soon as it is called + (``await coro_func()``) may run without a coroutine object until + it first suspends, so this attribute is ``None`` before that. + Use :attr:`~codeobject.co_flags` of :attr:`frame.f_code` to tell + coroutine frames apart from normal function frames. .. versionadded:: 3.14 + .. versionchanged:: 3.16 + May be ``None`` for a coroutine frame before its first suspension. + .. index:: single: f_trace (frame attribute) single: f_trace_lines (frame attribute) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 33328b2ee00521..f1b4dc41be67c4 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1147,6 +1147,19 @@ datetime and time (Contributed by Paul Ganssle in :gh:`158252`, based on work by Stan Ulbrych.) +Interpreter +----------- + +* A coroutine that is awaited as soon as it is called + (``await coro_func()``) now runs without creating a coroutine object + unless it suspends, making awaits that complete immediately about 40% + faster in a micro-benchmark (``await noop()``: 48 ns to 27 ns on x86-64). + This does not apply while coroutine origin tracking is enabled or inside + code compiled by the experimental JIT. + Until it first suspends, :attr:`frame.f_generator` is ``None`` for such a + coroutine; see :ref:`whatsnew316-porting`. + (Contributed by Andrew Svetlov in :gh:`161170`.) + re -- @@ -1405,12 +1418,23 @@ New deprecations .. include:: ../deprecations/pending-removal-in-future.rst +.. _whatsnew316-porting: + Porting to Python 3.16 ====================== This section lists previously described changes and other bugfixes that may require changes to your code. +* :attr:`frame.f_generator` and :c:func:`PyFrame_GetGenerator` may now + return ``None``/``NULL`` for a coroutine frame before its first + suspension, because a coroutine that is awaited as soon as it is called + (``await coro_func()``) runs without a coroutine object until then. + Code that used ``frame.f_generator is None`` to detect frames of normal + functions should check ``frame.f_code.co_flags`` for + :data:`~inspect.CO_COROUTINE` instead. + (Contributed by Andrew Svetlov in :gh:`161170`.) + * In :mod:`tkinter`, the *name* parameter of the :meth:`~tkinter.Misc.wait_variable`, :meth:`~tkinter.Misc.setvar` and :meth:`~tkinter.Misc.getvar` methods and the *value* parameter of @@ -1488,6 +1512,24 @@ that may require changes to your code. (Contributed by An Long in :gh:`86768`.) +CPython bytecode changes +======================== + +* The oparg of :opcode:`CALL` and :opcode:`CALL_KW` is now the number of + arguments shifted left by one. The low bit of their oparg, and of the + oparg of :opcode:`CALL_FUNCTION_EX`, marks a call whose result is awaited + right away. + (Contributed by Andrew Svetlov in :gh:`161170`.) + +* :opcode:`GET_AWAITABLE` now also pushes the ``NULL`` and ``None`` that + start the :opcode:`SEND` loop following it, so the compiler no longer + emits :opcode:`PUSH_NULL` and :opcode:`LOAD_COMMON_CONSTANT` after it. + It also pops an extra slot: a :opcode:`PUSH_NULL` emitted after the + awaitable, or, after an awaited call (bit ``4`` of its oparg is set), + one emitted below the call's callable. + (Contributed by Andrew Svetlov in :gh:`161170`.) + + Build changes ============= diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a0fea389b6502c..b9c6377219b303 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -267,6 +267,12 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyObject **p_builtins); PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); +// Lazily created coroutines (see RETURN_GENERATOR in bytecodes.c). +// Exported for _testinternalcapi and external JIT support. +PyAPI_FUNC(PyObject *) _PyCoro_MaterializeLazy(_PyInterpreterFrame *frame); +PyAPI_FUNC(_PyInterpreterFrame *) _PyEval_MoveLazyCoroFrame(PyThreadState *tstate, + PyGenObject *gen, + _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index a85fa9bd32853c..ea280a324e758a 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -152,6 +152,8 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * dest->stackpointer = dest->localsplus + stacktop; // visited is GC bookkeeping for the current stack walk, not frame state. dest->visited = 0; + // A lazily started coroutine stops being lazy once its frame is copied. + dest->lazy_coro = 0; #ifdef Py_DEBUG dest->stackpointer_valid = src->stackpointer_valid; dest->lltrace = src->lltrace; @@ -208,6 +210,7 @@ _PyFrame_Initialize( frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->lazy_coro = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; @@ -441,6 +444,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int #endif frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->lazy_coro = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 0d6a37cc2c1a17..4937e905081fae 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -44,9 +44,10 @@ struct _PyInterpreterFrame { char owner; uint8_t visited:1; uint8_t external:1; /* Set when the frame was pushed by something other than the VM */ + uint8_t lazy_coro:1; #ifdef Py_DEBUG uint8_t stackpointer_valid:1; - uint8_t lltrace:5; + uint8_t lltrace:4; #endif /* Locals and stack */ _PyStackRef localsplus[1]; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index c264224ffc4fbc..7f95935b2d5c8c 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1137,7 +1137,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_EX_NON_PY_GENERAL] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_EX_PY] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1199,7 +1199,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, @@ -1290,10 +1290,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESUME_CHECK_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [SEND_ASYNC_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, - [SEND_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_ASYNC_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [SEND_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1330,7 +1330,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1345,7 +1345,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { }; #endif -#define MAX_UOP_PER_EXPANSION 11 +#define MAX_UOP_PER_EXPANSION 12 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1382,20 +1382,20 @@ _PyOpcode_macro_expansion[256] = { [BUILD_TEMPLATE] = { .nuops = 1, .uops = { { _BUILD_TEMPLATE, OPARG_SIMPLE, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, OPARG_SIMPLE, 0 } } }, [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_OBJECT, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _ALLOCATE_OBJECT, OPARG_SIMPLE, 3 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 11, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BOUND_METHOD_GENERAL] = { .nuops = 8, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 12, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_GENERAL] = { .nuops = 9, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BUILTIN_CLASS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_BUILTIN_FAST] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_BUILTIN_O] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_EX_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CALL_FUNCTION_EX_NON_PY_GENERAL, OPARG_SIMPLE, 1 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 1 } } }, - [CALL_EX_PY] = { .nuops = 7, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [CALL_EX_PY] = { .nuops = 8, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [CALL_INTRINSIC_1] = { .nuops = 2, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 3, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, - [CALL_KW_BOUND_METHOD] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_KW_PY] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_PY] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CALL_LIST_APPEND] = { .nuops = 6, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 5, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, @@ -1403,8 +1403,8 @@ _PyOpcode_macro_expansion[256] = { [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 9, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_PY_GENERAL] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 10, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_PY_GENERAL] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _MAYBE_LAZY_CORO, OPARG_REPLACED, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_STR_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_TUPLE_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_TYPE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, @@ -1507,9 +1507,9 @@ _PyOpcode_macro_expansion[256] = { [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 1 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, OPARG_SIMPLE, 0 } } }, [RETURN_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, - [SEND_ASYNC_GEN] = { .nuops = 2, .uops = { { _GUARD_3OS_ASYNC_GEN_ASEND, OPARG_SIMPLE, 1 }, { _SEND_ASYNC_GEN, OPARG_REPLACED, 1 } } }, - [SEND_GEN] = { .nuops = 4, .uops = { { _RECORD_3OS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, - [SEND_VIRTUAL] = { .nuops = 3, .uops = { { _GUARD_TOS_IS_NONE, OPARG_SIMPLE, 1 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _SEND_VIRTUAL, OPARG_REPLACED, 1 } } }, + [SEND_ASYNC_GEN] = { .nuops = 3, .uops = { { _SEND_LAZY_CORO, OPARG_REPLACED, 1 }, { _GUARD_3OS_ASYNC_GEN_ASEND, OPARG_SIMPLE, 1 }, { _SEND_ASYNC_GEN, OPARG_REPLACED, 1 } } }, + [SEND_GEN] = { .nuops = 5, .uops = { { _RECORD_3OS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _SEND_LAZY_CORO, OPARG_REPLACED, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [SEND_VIRTUAL] = { .nuops = 4, .uops = { { _SEND_LAZY_CORO, OPARG_REPLACED, 1 }, { _GUARD_TOS_IS_NONE, OPARG_SIMPLE, 1 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _SEND_VIRTUAL, OPARG_REPLACED, 1 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, OPARG_SIMPLE, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, OPARG_SIMPLE, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, OPARG_SIMPLE, 0 } } }, @@ -1545,7 +1545,7 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TUPLE, OPARG_SIMPLE, 1 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TWO_TUPLE, OPARG_SIMPLE, 1 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, OPARG_SIMPLE, 0 } } }, - [YIELD_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, + [YIELD_VALUE] = { .nuops = 3, .uops = { { _CHECK_LAZY_CORO, OPARG_REPLACED, 0 }, { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, }; #endif // NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 8107f37d0fb013..568b2e23bb1b57 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -82,6 +82,7 @@ enum { _CHECK_IS_NOT_PY_CALLABLE_EX, _CHECK_IS_NOT_PY_CALLABLE_KW, _CHECK_IS_PY_CALLABLE_EX, + _CHECK_LAZY_CORO, _CHECK_MANAGED_OBJECT_HAS_VALUES, _CHECK_METHOD_VERSION, _CHECK_METHOD_VERSION_KW, @@ -191,6 +192,8 @@ enum { _GUARD_NOT_EXHAUSTED_LIST, _GUARD_NOT_EXHAUSTED_RANGE, _GUARD_NOT_EXHAUSTED_TUPLE, + _GUARD_NOT_LAZY_CORO, + _GUARD_SEND_NOT_LAZY_CORO, _GUARD_THIRD_NULL, _GUARD_TOS_ANY_DICT, _GUARD_TOS_ANY_SET, @@ -292,6 +295,7 @@ enum { _MATCH_CLASS, _MAYBE_EXPAND_METHOD, _MAYBE_EXPAND_METHOD_KW, + _MAYBE_LAZY_CORO, _MONITOR_CALL, _MONITOR_CALL_KW, _MONITOR_JUMP_BACKWARD, @@ -328,6 +332,7 @@ enum { _SEND_ASYNC_GEN, _SEND_ASYNC_GEN_TIER_TWO, _SEND_GEN_FRAME, + _SEND_LAZY_CORO, _SEND_VIRTUAL, _SEND_VIRTUAL_TIER_TWO, _SET_UPDATE, @@ -371,7 +376,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 659 +#define MAX_UOP_ID 664 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -595,6 +600,10 @@ enum { _CHECK_IS_PY_CALLABLE_EX_r13, _CHECK_IS_PY_CALLABLE_EX_r23, _CHECK_IS_PY_CALLABLE_EX_r33, + _CHECK_LAZY_CORO_r00, + _CHECK_LAZY_CORO_r11, + _CHECK_LAZY_CORO_r22, + _CHECK_LAZY_CORO_r33, _CHECK_MANAGED_OBJECT_HAS_VALUES_r01, _CHECK_MANAGED_OBJECT_HAS_VALUES_r11, _CHECK_MANAGED_OBJECT_HAS_VALUES_r22, @@ -925,6 +934,14 @@ enum { _GUARD_NOT_EXHAUSTED_TUPLE_r12, _GUARD_NOT_EXHAUSTED_TUPLE_r22, _GUARD_NOT_EXHAUSTED_TUPLE_r33, + _GUARD_NOT_LAZY_CORO_r00, + _GUARD_NOT_LAZY_CORO_r11, + _GUARD_NOT_LAZY_CORO_r22, + _GUARD_NOT_LAZY_CORO_r33, + _GUARD_SEND_NOT_LAZY_CORO_r03, + _GUARD_SEND_NOT_LAZY_CORO_r13, + _GUARD_SEND_NOT_LAZY_CORO_r23, + _GUARD_SEND_NOT_LAZY_CORO_r33, _GUARD_THIRD_NULL_r03, _GUARD_THIRD_NULL_r13, _GUARD_THIRD_NULL_r23, @@ -1247,6 +1264,10 @@ enum { _MATCH_SEQUENCE_r23, _MAYBE_EXPAND_METHOD_r00, _MAYBE_EXPAND_METHOD_KW_r11, + _MAYBE_LAZY_CORO_r01, + _MAYBE_LAZY_CORO_r11, + _MAYBE_LAZY_CORO_r22, + _MAYBE_LAZY_CORO_r33, _MONITOR_CALL_r00, _MONITOR_CALL_KW_r11, _MONITOR_JUMP_BACKWARD_r00, @@ -1320,6 +1341,7 @@ enum { _SEND_ASYNC_GEN_r33, _SEND_ASYNC_GEN_TIER_TWO_r33, _SEND_GEN_FRAME_r33, + _SEND_LAZY_CORO_r33, _SEND_VIRTUAL_r33, _SEND_VIRTUAL_TIER_TWO_r03, _SEND_VIRTUAL_TIER_TWO_r13, @@ -1454,7 +1476,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1665 +#define MAX_UOP_REGS_ID 1687 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 35a4931f05d562..747f1eeda31999 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -168,13 +168,15 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_SEND_NOT_LAZY_CORO] = HAS_EXIT_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_GUARD_TOS_IS_NONE] = HAS_EXIT_FLAG, [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_SEND_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_GUARD_3OS_ASYNC_GEN_ASEND] = HAS_EXIT_FLAG, [_SEND_ASYNC_GEN_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_GUARD_NOT_LAZY_CORO] = HAS_EXIT_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -1649,6 +1651,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_SEND_NOT_LAZY_CORO] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_SEND_NOT_LAZY_CORO_r03 }, + { 3, 1, _GUARD_SEND_NOT_LAZY_CORO_r13 }, + { 3, 2, _GUARD_SEND_NOT_LAZY_CORO_r23 }, + { 3, 3, _GUARD_SEND_NOT_LAZY_CORO_r33 }, + }, + }, [_SEND_GEN_FRAME] = { .best = { 3, 3, 3, 3 }, .entries = { @@ -1703,6 +1714,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _SEND_ASYNC_GEN_TIER_TWO_r33 }, }, }, + [_GUARD_NOT_LAZY_CORO] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_NOT_LAZY_CORO_r00 }, + { 1, 1, _GUARD_NOT_LAZY_CORO_r11 }, + { 2, 2, _GUARD_NOT_LAZY_CORO_r22 }, + { 3, 3, _GUARD_NOT_LAZY_CORO_r33 }, + }, + }, [_YIELD_VALUE] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -4335,6 +4355,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, [_GET_AWAITABLE_r23] = _GET_AWAITABLE, + [_GUARD_SEND_NOT_LAZY_CORO_r03] = _GUARD_SEND_NOT_LAZY_CORO, + [_GUARD_SEND_NOT_LAZY_CORO_r13] = _GUARD_SEND_NOT_LAZY_CORO, + [_GUARD_SEND_NOT_LAZY_CORO_r23] = _GUARD_SEND_NOT_LAZY_CORO, + [_GUARD_SEND_NOT_LAZY_CORO_r33] = _GUARD_SEND_NOT_LAZY_CORO, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, @@ -4353,6 +4377,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = _GUARD_3OS_ASYNC_GEN_ASEND, [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = _GUARD_3OS_ASYNC_GEN_ASEND, [_SEND_ASYNC_GEN_TIER_TWO_r33] = _SEND_ASYNC_GEN_TIER_TWO, + [_GUARD_NOT_LAZY_CORO_r00] = _GUARD_NOT_LAZY_CORO, + [_GUARD_NOT_LAZY_CORO_r11] = _GUARD_NOT_LAZY_CORO, + [_GUARD_NOT_LAZY_CORO_r22] = _GUARD_NOT_LAZY_CORO, + [_GUARD_NOT_LAZY_CORO_r33] = _GUARD_NOT_LAZY_CORO, [_YIELD_VALUE_r11] = _YIELD_VALUE, [_POP_EXCEPT_r10] = _POP_EXCEPT, [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, @@ -5590,6 +5618,16 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_NOT_EXHAUSTED_TUPLE_r12] = "_GUARD_NOT_EXHAUSTED_TUPLE_r12", [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = "_GUARD_NOT_EXHAUSTED_TUPLE_r22", [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = "_GUARD_NOT_EXHAUSTED_TUPLE_r33", + [_GUARD_NOT_LAZY_CORO] = "_GUARD_NOT_LAZY_CORO", + [_GUARD_NOT_LAZY_CORO_r00] = "_GUARD_NOT_LAZY_CORO_r00", + [_GUARD_NOT_LAZY_CORO_r11] = "_GUARD_NOT_LAZY_CORO_r11", + [_GUARD_NOT_LAZY_CORO_r22] = "_GUARD_NOT_LAZY_CORO_r22", + [_GUARD_NOT_LAZY_CORO_r33] = "_GUARD_NOT_LAZY_CORO_r33", + [_GUARD_SEND_NOT_LAZY_CORO] = "_GUARD_SEND_NOT_LAZY_CORO", + [_GUARD_SEND_NOT_LAZY_CORO_r03] = "_GUARD_SEND_NOT_LAZY_CORO_r03", + [_GUARD_SEND_NOT_LAZY_CORO_r13] = "_GUARD_SEND_NOT_LAZY_CORO_r13", + [_GUARD_SEND_NOT_LAZY_CORO_r23] = "_GUARD_SEND_NOT_LAZY_CORO_r23", + [_GUARD_SEND_NOT_LAZY_CORO_r33] = "_GUARD_SEND_NOT_LAZY_CORO_r33", [_GUARD_THIRD_NULL] = "_GUARD_THIRD_NULL", [_GUARD_THIRD_NULL_r03] = "_GUARD_THIRD_NULL_r03", [_GUARD_THIRD_NULL_r13] = "_GUARD_THIRD_NULL_r13", @@ -6529,6 +6567,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GET_AWAITABLE: return 2; + case _GUARD_SEND_NOT_LAZY_CORO: + return 0; case _SEND_GEN_FRAME: return 1; case _GUARD_TOS_IS_NONE: @@ -6541,6 +6581,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _SEND_ASYNC_GEN_TIER_TWO: return 3; + case _GUARD_NOT_LAZY_CORO: + return 0; case _YIELD_VALUE: return 1; case _POP_EXCEPT: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 11342ae451b9f6..dbd1786a5e6808 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -14,53 +14,53 @@ extern "C" { #define BINARY_SLICE 1 #define BUILD_TEMPLATE 2 #define BINARY_OP_INPLACE_ADD_UNICODE 3 -#define CALL_FUNCTION_EX 4 -#define CHECK_EG_MATCH 5 -#define CHECK_EXC_MATCH 6 -#define CLEANUP_THROW 7 -#define DELETE_SUBSCR 8 -#define END_FOR 9 -#define END_SEND 10 -#define EXIT_INIT_CHECK 11 -#define FORMAT_SIMPLE 12 -#define FORMAT_WITH_SPEC 13 -#define GET_AITER 14 -#define GET_ANEXT 15 -#define GET_LEN 16 +#define CHECK_EG_MATCH 4 +#define CHECK_EXC_MATCH 5 +#define CLEANUP_THROW 6 +#define DELETE_SUBSCR 7 +#define END_FOR 8 +#define END_SEND 9 +#define EXIT_INIT_CHECK 10 +#define FORMAT_SIMPLE 11 +#define FORMAT_WITH_SPEC 12 +#define GET_AITER 13 +#define GET_ANEXT 14 +#define GET_LEN 15 +#define INTERPRETER_EXIT 16 #define RESERVED 17 -#define INTERPRETER_EXIT 18 -#define LOAD_BUILD_CLASS 19 -#define LOAD_LOCALS 20 -#define MAKE_FUNCTION 21 -#define MATCH_KEYS 22 -#define MATCH_MAPPING 23 -#define MATCH_SEQUENCE 24 -#define NOP 25 -#define NOT_TAKEN 26 -#define POP_EXCEPT 27 -#define POP_ITER 28 -#define POP_TOP 29 -#define PUSH_EXC_INFO 30 -#define PUSH_NULL 31 -#define RETURN_GENERATOR 32 -#define RETURN_VALUE 33 -#define SETUP_ANNOTATIONS 34 -#define STORE_SLICE 35 -#define STORE_SUBSCR 36 -#define TO_BOOL 37 -#define UNARY_INVERT 38 -#define UNARY_NEGATIVE 39 -#define UNARY_NOT 40 -#define WITH_EXCEPT_START 41 -#define BINARY_OP 42 -#define BUILD_INTERPOLATION 43 -#define BUILD_LIST 44 -#define BUILD_MAP 45 -#define BUILD_SET 46 -#define BUILD_SLICE 47 -#define BUILD_STRING 48 -#define BUILD_TUPLE 49 -#define CALL 50 +#define LOAD_BUILD_CLASS 18 +#define LOAD_LOCALS 19 +#define MAKE_FUNCTION 20 +#define MATCH_KEYS 21 +#define MATCH_MAPPING 22 +#define MATCH_SEQUENCE 23 +#define NOP 24 +#define NOT_TAKEN 25 +#define POP_EXCEPT 26 +#define POP_ITER 27 +#define POP_TOP 28 +#define PUSH_EXC_INFO 29 +#define PUSH_NULL 30 +#define RETURN_GENERATOR 31 +#define RETURN_VALUE 32 +#define SETUP_ANNOTATIONS 33 +#define STORE_SLICE 34 +#define STORE_SUBSCR 35 +#define TO_BOOL 36 +#define UNARY_INVERT 37 +#define UNARY_NEGATIVE 38 +#define UNARY_NOT 39 +#define WITH_EXCEPT_START 40 +#define BINARY_OP 41 +#define BUILD_INTERPOLATION 42 +#define BUILD_LIST 43 +#define BUILD_MAP 44 +#define BUILD_SET 45 +#define BUILD_SLICE 46 +#define BUILD_STRING 47 +#define BUILD_TUPLE 48 +#define CALL 49 +#define CALL_FUNCTION_EX 50 #define CALL_INTRINSIC_1 51 #define CALL_INTRINSIC_2 52 #define CALL_KW 53 @@ -253,7 +253,7 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 41 +#define HAVE_ARGUMENT 40 #define MIN_SPECIALIZED_OPCODE 129 #define MIN_INSTRUMENTED_OPCODE 233 diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index fa652cd231cfa9..99776fef7bdfde 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -45,7 +45,30 @@ The generator's frame is initialized as a copy of the frame in which `RETURN_GENERATOR` is executing, but its `owner` field is overwritten to indicate that it is owned by a generator. Finally, `RETURN_GENERATOR` pushes the new generator object to the stack and returns to the caller of the generator function (at -which time its frame is destroyed). When the generator is next resumed by +which time its frame is destroyed). + +There is one exception: a coroutine function called by an awaited call +(`await f()`, compiled as `CALL`, `CALL_KW` or `CALL_FUNCTION_EX` with the low +bit of the oparg set) gets the `lazy_coro` flag on its frame (see +`_MAYBE_LAZY_CORO`). `RETURN_GENERATOR` then keeps running the body in that +frame, with its own `_PyErr_StackItem` pushed on the data stack right after the +frame, so that a body that returns or raises without suspending never creates a +coroutine object. If the body reaches `YIELD_VALUE`, the coroutine object is +created then and the frame is moved into it (see `_PyCoro_MaterializeLazy()` +and `_PyEval_MoveLazyCoroFrame()`). + +The compiler pushes an extra `NULL` below the callable of an awaited call, and +the `GET_AWAITABLE` that follows the call pops it (other awaitables get the +`NULL` pushed above them, so `GET_AWAITABLE` always pops two values). A lazily run body reports +there how it ended: a tagged int if it returned, or the new coroutine object if +it suspended. `GET_AWAITABLE` turns this into the input of the `SEND` loop: a +`NULL` receiver, which makes `SEND` jump to `END_SEND` with the return value, or +`None` in place of the `NULL` below the value, which makes `SEND` go on to +`YIELD_VALUE` with the value that the body has already yielded. An exception +that leaves the body is raised from the caller's `SEND`, as it would be if the +coroutine object had been created. + +When the generator is next resumed by [`gen_send_ex2()`](../Objects/genobject.c), `_PyEval_EvalFrame()` is called to continue executing the generator function, in the frame that is embedded in the generator object. diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index df92eae151d248..74bd09aea6f743 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -235,52 +235,52 @@ TRACE_RECORD=255, BINARY_SLICE=1, BUILD_TEMPLATE=2, - CALL_FUNCTION_EX=4, - CHECK_EG_MATCH=5, - CHECK_EXC_MATCH=6, - CLEANUP_THROW=7, - DELETE_SUBSCR=8, - END_FOR=9, - END_SEND=10, - EXIT_INIT_CHECK=11, - FORMAT_SIMPLE=12, - FORMAT_WITH_SPEC=13, - GET_AITER=14, - GET_ANEXT=15, - GET_LEN=16, - INTERPRETER_EXIT=18, - LOAD_BUILD_CLASS=19, - LOAD_LOCALS=20, - MAKE_FUNCTION=21, - MATCH_KEYS=22, - MATCH_MAPPING=23, - MATCH_SEQUENCE=24, - NOP=25, - NOT_TAKEN=26, - POP_EXCEPT=27, - POP_ITER=28, - POP_TOP=29, - PUSH_EXC_INFO=30, - PUSH_NULL=31, - RETURN_GENERATOR=32, - RETURN_VALUE=33, - SETUP_ANNOTATIONS=34, - STORE_SLICE=35, - STORE_SUBSCR=36, - TO_BOOL=37, - UNARY_INVERT=38, - UNARY_NEGATIVE=39, - UNARY_NOT=40, - WITH_EXCEPT_START=41, - BINARY_OP=42, - BUILD_INTERPOLATION=43, - BUILD_LIST=44, - BUILD_MAP=45, - BUILD_SET=46, - BUILD_SLICE=47, - BUILD_STRING=48, - BUILD_TUPLE=49, - CALL=50, + CHECK_EG_MATCH=4, + CHECK_EXC_MATCH=5, + CLEANUP_THROW=6, + DELETE_SUBSCR=7, + END_FOR=8, + END_SEND=9, + EXIT_INIT_CHECK=10, + FORMAT_SIMPLE=11, + FORMAT_WITH_SPEC=12, + GET_AITER=13, + GET_ANEXT=14, + GET_LEN=15, + INTERPRETER_EXIT=16, + LOAD_BUILD_CLASS=18, + LOAD_LOCALS=19, + MAKE_FUNCTION=20, + MATCH_KEYS=21, + MATCH_MAPPING=22, + MATCH_SEQUENCE=23, + NOP=24, + NOT_TAKEN=25, + POP_EXCEPT=26, + POP_ITER=27, + POP_TOP=28, + PUSH_EXC_INFO=29, + PUSH_NULL=30, + RETURN_GENERATOR=31, + RETURN_VALUE=32, + SETUP_ANNOTATIONS=33, + STORE_SLICE=34, + STORE_SUBSCR=35, + TO_BOOL=36, + UNARY_INVERT=37, + UNARY_NEGATIVE=38, + UNARY_NOT=39, + WITH_EXCEPT_START=40, + BINARY_OP=41, + BUILD_INTERPOLATION=42, + BUILD_LIST=43, + BUILD_MAP=44, + BUILD_SET=45, + BUILD_SLICE=46, + BUILD_STRING=47, + BUILD_TUPLE=48, + CALL=49, + CALL_FUNCTION_EX=50, CALL_INTRINSIC_1=51, CALL_INTRINSIC_2=52, CALL_KW=53, @@ -380,5 +380,5 @@ STORE_FAST_MAYBE_NULL=266, ) -HAVE_ARGUMENT = 41 +HAVE_ARGUMENT = 40 MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index d76d2f6383343d..d7714c9c681a19 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -1,6 +1,7 @@ """Introspection utils for tasks call graphs.""" import dataclasses +import inspect import io import operator import sys @@ -38,6 +39,26 @@ class FutureCallGraph: awaited_by: tuple["FutureCallGraph", ...] +# A coroutine that is awaited right away runs without a coroutine +# object until it first suspends, so ``f.f_generator`` may be None +# for a coroutine frame; classify frames by their code instead. +_CO_ASYNC_FLAGS = inspect.CO_COROUTINE | inspect.CO_ASYNC_GENERATOR +_CO_GEN_FLAGS = _CO_ASYNC_FLAGS | inspect.CO_GENERATOR +_FRAME_TAGS = { + inspect.CO_COROUTINE: 'async ', + inspect.CO_ASYNC_GENERATOR: 'async generator ', + inspect.CO_GENERATOR: 'generator ', +} + + +def _is_async_frame(f): + return bool(f.f_code.co_flags & _CO_ASYNC_FLAGS) + + +def _is_sync_frame(f): + return not (f.f_code.co_flags & _CO_GEN_FLAGS) + + def _build_graph_for_future( future: futures.Future, *, @@ -166,12 +187,10 @@ def capture_call_graph( try: while f is not None: # gh-156988: sync gen should not clear the call chain - is_async = isinstance( - f.f_generator, (types.CoroutineType, types.AsyncGeneratorType)) call_stack.append(FrameCallGraphEntry(f)) - if is_async: - if f.f_back is not None and f.f_back.f_generator is None: + if _is_async_frame(f): + if f.f_back is not None and _is_sync_frame(f.f_back): # We've reached the bottom of the coroutine stack, which # must be the Task that runs it. break @@ -228,36 +247,12 @@ def add_line(line: str) -> None: ) for ste in st.call_stack: f = ste.frame - - if f.f_generator is None: - f = ste.frame - add_line( - f' | File {f.f_code.co_filename!r},' - f' line {f.f_lineno}, in' - f' {f.f_code.co_qualname}()' - ) - else: - c = f.f_generator - - try: - f = c.cr_frame - code = c.cr_code - tag = 'async' - except AttributeError: - try: - f = c.ag_frame - code = c.ag_code - tag = 'async generator' - except AttributeError: - f = c.gi_frame - code = c.gi_code - tag = 'generator' - - add_line( - f' | File {f.f_code.co_filename!r},' - f' line {f.f_lineno}, in' - f' {tag} {code.co_qualname}()' - ) + tag = _FRAME_TAGS.get(f.f_code.co_flags & _CO_GEN_FLAGS, '') + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag}{f.f_code.co_qualname}()' + ) if st.awaited_by: add_line( diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 1f1c76e4aef69f..71fd41d27ca812 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -1,4 +1,5 @@ import asyncio +import inspect import io import sys import types @@ -20,19 +21,23 @@ def walk(s): if isinstance(s.future, asyncio.Task) else 'F' ] + def tag(frame): + # A coroutine awaited right after it is called may run + # without a coroutine object until it first suspends, so + # classify frames by their code rather than f_generator. + flags = frame.f_code.co_flags + if flags & inspect.CO_COROUTINE: + return "a" + if flags & inspect.CO_ASYNC_GENERATOR: + return "ag" + if flags & inspect.CO_GENERATOR: + return "g" + return "s" + ret.append( [ - ( - f"s {entry.frame.f_code.co_name}" - if entry.frame.f_generator is None else - ( - f"a {entry.frame.f_generator.cr_code.co_name}" - if hasattr(entry.frame.f_generator, 'cr_code') else - f"ag {entry.frame.f_generator.ag_code.co_name}" - if hasattr(entry.frame.f_generator, 'ag_code') else - f"g {entry.frame.f_generator.gi_code.co_name}" - ) - ) for entry in s.call_stack + f"{tag(entry.frame)} {entry.frame.f_code.co_name}" + for entry in s.call_stack # Skip the pure Python Future.__await__ generator frame if entry.frame.f_code is not future_await_code ] @@ -986,6 +991,26 @@ async def child(): self.assertEqual(stack[0][2], [['T', ['a test_stack_eager_task'], []]]) + async def test_capture_call_graph_through_generator_based_coroutine(self): + # A native coroutine driven by a generator-based coroutine + # (yield from) keeps the frames of its callers. + stack = None + + async def native(): + nonlocal stack + graph = asyncio.capture_call_graph() + stack = [entry.frame.f_code.co_name for entry in graph.call_stack] + + @types.coroutine + def legacy(): + return (yield from native()) + + async def main(): + await legacy() + + await main() + self.assertEqual(stack[:3], ['native', 'legacy', 'main']) + @unittest.skipIf( not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index c5900da09501b3..03d62ea10d412c 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1,6 +1,8 @@ import ast import contextlib import copy +import builtins +import gc import inspect import pickle import sys @@ -2594,6 +2596,595 @@ def test_unawaited_warning_during_shutdown(self): assert_python_ok("-c", code) +@support.cpython_only +class LazyCoroutineTest(unittest.TestCase): + """A coroutine that is awaited right away (``await f()``) runs in + the calling thread's frame and only gets a coroutine object when + it first suspends. This must not be observable, except that no + coroutine object exists before the first suspension.""" + + @staticmethod + def coroutines_for(code): + return [o for o in gc.get_objects() + if type(o) is types.CoroutineType and o.cr_code is code] + + def test_no_object_before_first_suspension(self): + seen = [] + + async def inner(): + seen.append(len(self.coroutines_for(inner.__code__))) + await AsyncYield(1) + seen.append(len(self.coroutines_for(inner.__code__))) + return 42 + + async def outer(): + return await inner() + + self.assertEqual(run_async(outer()), ([1], 42)) + self.assertEqual(seen, [0, 1]) + + def test_return_without_suspending(self): + async def inner(x): + return x + 1 + + async def outer(): + total = 0 + for i in range(100): + total += await inner(i) + return total + + self.assertEqual(run_async(outer()), ([], sum(range(1, 101)))) + + def test_nested_chain(self): + async def chain(n): + if n == 0: + await AsyncYield("bottom") + return 0 + return 1 + await chain(n - 1) + + self.assertEqual(run_async(chain(20)), (["bottom"], 20)) + + def test_exception_before_first_suspension(self): + async def inner(): + raise ValueError("boom") + + async def outer(): + try: + await inner() + except ValueError as exc: + tb = exc.__traceback__ + names = [] + while tb is not None: + names.append(tb.tb_frame.f_code.co_name) + tb = tb.tb_next + return names + + self.assertEqual(run_async(outer()), ([], ["outer", "inner"])) + + def test_stopiteration_before_first_suspension(self): + async def inner(): + raise StopIteration + + async def outer(): + await inner() + + with self.assertRaisesRegex(RuntimeError, + "coroutine raised StopIteration"): + run_async(outer()) + + def test_frame_object(self): + async def inner(): + x = 1 + frame = sys._getframe() + self.assertIsNone(frame.f_generator) + self.assertEqual(frame.f_locals["x"], 1) + self.assertEqual(frame.f_back.f_code.co_name, "outer") + await AsyncYield(1) + coro = frame.f_generator + self.assertIsInstance(coro, types.CoroutineType) + self.assertIs(coro.cr_frame, frame) + self.assertIs(coro.cr_code, inner.__code__) + self.assertEqual(frame.f_locals["x"], 1) + self.assertEqual(frame.f_back.f_code.co_name, "outer") + x = 2 + return frame + + async def outer(): + return await inner() + + _, frame = run_async(outer()) + self.assertIsNone(frame.f_generator) # finished + self.assertEqual(frame.f_locals["x"], 2) + + def test_exc_info_suspend_inside_handler(self): + async def inner(): + try: + raise KeyError + except KeyError: + await AsyncYield(1) + self.assertIsInstance(sys.exc_info()[1], KeyError) + await AsyncYield(2) + self.assertIsInstance(sys.exc_info()[1], KeyError) + # The caller's context is visible after the handler. + self.assertIsInstance(sys.exc_info()[1], TypeError) + await AsyncYield(3) + self.assertIsInstance(sys.exc_info()[1], TypeError) + return "ok" + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exc_info()[1], TypeError) + self.assertIsNone(sys.exc_info()[1]) + return result + + self.assertEqual(run_async(outer()), ([1, 2, 3], "ok")) + + def test_exc_info_handler_without_suspending(self): + async def inner(): + try: + raise KeyError + except KeyError: + self.assertIsInstance(sys.exc_info()[1], KeyError) + self.assertIsInstance(sys.exc_info()[1], TypeError) + return "ok" + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exc_info()[1], TypeError) + self.assertIsNone(sys.exc_info()[1]) + return result + + self.assertEqual(run_async(outer()), ([], "ok")) + + def test_throw_into_materialized_coroutine(self): + async def inner(): + try: + await AsyncYield(1) + except ZeroDivisionError: + return "caught" + return "not caught" + + async def outer(): + return await inner() + + coro = outer() + self.assertEqual(coro.send(None), 1) + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertEqual(cm.exception.value, "caught") + + def test_close_materialized_coroutine(self): + closed = [] + + async def inner(): + try: + await AsyncYield(1) + finally: + closed.append(True) + + async def outer(): + await inner() + + coro = outer() + self.assertEqual(coro.send(None), 1) + coro.close() + self.assertEqual(closed, [True]) + + def test_async_with(self): + log = [] + + class CM: + async def __aenter__(self): + log.append("enter") + return self + + async def __aexit__(self, *args): + log.append("exit") + return False + + async def outer(): + async with CM() as cm: + log.append(type(cm).__name__) + await AsyncYield(1) + return "done" + + self.assertEqual(run_async(outer()), ([1], "done")) + self.assertEqual(log, ["enter", "CM", "exit"]) + + def test_async_comprehension(self): + async def inner(x): + return x * 2 + + async def outer(): + return [await inner(x) for x in range(5)] + + self.assertEqual(run_async(outer()), ([], [0, 2, 4, 6, 8])) + + def test_origin_tracking(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + + async def inner(): + await AsyncYield(1) + return sys._getframe().f_generator.cr_origin + + async def outer(): + coro = inner() + eager = (await coro)[0] + lazy = (await inner())[0] + return eager, lazy + + sys.set_coroutine_origin_tracking_depth(2) + try: + _, (eager, lazy) = run_async(outer()) + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + # Origin tracking disables the lazy path: the origin is the + # call, not the await, and is captured at creation. + self.assertEqual(eager[0], __file__) + self.assertEqual(eager[2], "outer") + self.assertEqual(lazy[0], __file__) + self.assertEqual(lazy[2], "outer") + self.assertEqual(lazy[1], eager[1] + 2) + + def test_origin_tracking_multiline_await(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + + async def inner(): + await AsyncYield(1) + return sys._getframe().f_generator.cr_origin + + async def outer(): + origin = await ( + inner() + ) + return origin[0][1] - outer.__code__.co_firstlineno + + sys.set_coroutine_origin_tracking_depth(1) + try: + _, line = run_async(outer()) + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + # The line of the call, not of the await. + self.assertEqual(line, 2) + + def test_settrace_events(self): + async def inner(): + await AsyncYield(1) + return 1 + + async def eager(): + coro = inner() + return await coro + + async def lazy(): + return await inner() + + def trace(coro_fn): + events = [] + + def tracer(frame, event, arg): + if frame.f_code is inner.__code__: + events.append((event, frame.f_lineno - frame.f_code.co_firstlineno)) + return tracer + + sys.settrace(tracer) + try: + result = run_async(coro_fn()) + finally: + sys.settrace(None) + return events, result + + self.assertEqual(trace(lazy), trace(eager)) + + def test_monitoring_events(self): + async def inner(): + await AsyncYield(1) + return 1 + + async def eager(): + coro = inner() + return await coro + + async def lazy(): + return await inner() + + E = sys.monitoring.events + # No LINE events: they instrument the caller's SEND, which + # makes the await take the regular path. + names = {E.PY_START: "start", E.PY_RESUME: "resume", + E.PY_RETURN: "return", E.PY_YIELD: "yield", + E.STOP_ITERATION: "stop_iteration"} + codes = {inner.__code__: "inner", eager.__code__: "outer", + lazy.__code__: "outer"} + + def monitor(coro_fn): + events = [] + tool = sys.monitoring.DEBUGGER_ID + sys.monitoring.use_tool_id(tool, "test") + try: + for event, name in names.items(): + def callback(code, offset, *args, name=name): + if code in codes: + events.append( + (codes[code], name, type(args[-1]) if args else None)) + sys.monitoring.register_callback(tool, event, callback) + sys.monitoring.set_events(tool, sum(names)) + result = run_async(coro_fn()) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + return events, result + + self.assertEqual(monitor(lazy), monitor(eager)) + self.assertIn(("outer", "stop_iteration", StopIteration), + monitor(lazy)[0]) + + def test_stop_iteration_event_without_suspending(self): + async def inner(): + return 42 + + async def outer(): + return await inner() + + E = sys.monitoring.events + events = [] + tool = sys.monitoring.DEBUGGER_ID + sys.monitoring.use_tool_id(tool, "test") + try: + def callback(code, offset, exc): + if code is outer.__code__: + events.append(exc.value) + sys.monitoring.register_callback(tool, E.STOP_ITERATION, callback) + sys.monitoring.set_events(tool, E.STOP_ITERATION) + self.assertEqual(run_async(outer()), ([], 42)) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + self.assertEqual(events, [42]) + + def test_exc_state_after_suspending_inside_handler(self): + # The coroutine's own exception state must not keep the + # caller's exception once its handler is done. + async def inner(): + try: + raise KeyError + except KeyError: + await AsyncYield(1) + await AsyncYield(2) + return sys.exc_info()[1] + + async def outer(): + try: + raise TypeError + except TypeError: + return await inner() + + coro = outer() + coro.send(None) + inner_coro = coro.cr_await + inner_coro.send(None) + self.assertNotIn(TypeError, + map(type, gc.get_referents(inner_coro))) + # Resumed directly, not through outer: no handled exception. + with self.assertRaises(StopIteration) as cm: + inner_coro.send(None) + self.assertIsNone(cm.exception.value) + coro.close() + + def test_throw_into_coroutine_suspended_inside_handler(self): + async def inner(): + try: + raise KeyError + except KeyError: + try: + await AsyncYield(1) + except ZeroDivisionError: + pass + return sys.exception() + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exception(), TypeError) + return result + + coro = outer() + self.assertEqual(coro.send(None), 1) + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertIsNone(cm.exception.value) + + def test_import_hook_locals(self): + seen = [] + orig_import = builtins.__import__ + + def hook(name, globals=None, locals=None, fromlist=(), level=0): + seen.append(locals) + return orig_import(name, globals, locals, fromlist, level) + + async def inner(): + import sys as _sys + return _sys + + async def outer(): + try: + raise TypeError + except TypeError: + return await inner() + + with support.swap_attr(builtins, "__import__", hook): + self.assertEqual(run_async(outer()), ([], sys)) + self.assertEqual(seen, [None]) + + def test_traceback_frame_keeps_no_caller_exception(self): + async def inner(): + raise ValueError + + async def outer(): + try: + raise TypeError + except TypeError: + try: + await inner() + except ValueError as exc: + return exc.__traceback__.tb_next.tb_frame + + _, frame = run_async(outer()) + self.assertIs(frame.f_code, inner.__code__) + self.assertNotIn(TypeError, map(type, gc.get_referents(frame))) + + def test_code_replaced_before_first_suspension(self): + # The coroutine object is created at the first suspension, so it + # must be sized for the code the frame runs, not func.__code__. + async def small(): + big, small # same free variables as big() + await AsyncYield(1) + + async def big(): + a = b = c = d = e = f = g = h = i = j = k = m = n = o = p = 1 + big.__code__ = small.__code__ + await AsyncYield(1) + return (a, b, c, d, e, f, g, h, i, j, k, m, n, o, p) + + async def outer(): + return await big() + + self.assertEqual(run_async(outer()), ([1], (1,) * 15)) + + def test_finished_frame_has_no_f_back(self): + async def inner(): + return sys._getframe() + + async def outer(): + return await inner() + + _, frame = run_async(outer()) + self.assertIs(frame.f_code, inner.__code__) + self.assertIsNone(frame.f_back) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_memory_error_at_first_suspension(self): + @types.coroutine + def suspend(): + yield 1 + + awaitable = suspend() + + async def inner(): + _testcapi.set_nomemory(0, 1) + try: + await awaitable + finally: + _testcapi.remove_mem_hooks() + return "resumed" + + async def outer(): + return await inner() + + coro = outer() + try: + with self.assertRaises(MemoryError): + coro.send(None) + finally: + _testcapi.remove_mem_hooks() + awaitable.close() + + def test_call_forms(self): + class C: + async def meth(self, x, *, y=0): + if y: + await AsyncYield(y) + return ("meth", x, y) + + async def func(*args, **kwargs): + if kwargs.get("suspend"): + await AsyncYield("f") + return (args, kwargs) + + obj = C() + bound = obj.meth + + async def outer(): + return [ + await obj.meth(1), + await obj.meth(2, y=3), + await bound(4), + await bound(5, y=6), + await func(7), + await func(*[8, 9]), + await func(**{"a": 10}), + await func(11, suspend=True), + ] + + for _ in range(3): # also with specialized calls + self.assertEqual(run_async(outer()), ( + [3, 6, "f"], + [("meth", 1, 0), ("meth", 2, 3), ("meth", 4, 0), + ("meth", 5, 6), ((7,), {}), ((8, 9), {}), + ((), {"a": 10}), ((11,), {"suspend": True})], + )) + + def test_send_specialized_for_other_awaitables(self): + # The SEND of an await site may have been specialized for another + # kind of awaitable before a coroutine function is awaited there. + async def agen(): + yield 1 + + class Iterator: + async def __anext__(self): + return 2 + + class SuspendingIterator: + async def __anext__(self): + await AsyncYield("s") + return 3 + + async def outer(it): + return await type(it).__anext__(it) + + def check(it, expected): + self.assertEqual(run_async(outer(it)), expected) + + for _ in range(3): + for _ in range(20): + check(agen(), ([], 1)) + check(Iterator(), ([], 2)) + check(SuspendingIterator(), (["s"], 3)) + + def test_traceback_like_eager_coroutine(self): + async def inner(): + raise ValueError + + async def outer(): + await inner() + + def format_tb(): + try: + run_async(outer()) + except ValueError: + return traceback.format_exc() + + lazy = format_tb() + # Origin tracking makes awaited calls create the coroutine eagerly. + old_depth = sys.get_coroutine_origin_tracking_depth() + sys.set_coroutine_origin_tracking_depth(1) + try: + eager = format_tb() + finally: + sys.set_coroutine_origin_tracking_depth(old_depth) + # The whole `await inner()` line, not just `inner()` with carets. + self.assertIn(" await inner()\n File", lazy) + self.assertEqual(lazy, eager) + + @support.cpython_only @unittest.skipIf(_testcapi is None, "requires _testcapi") class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 0419542f6fa2e6..bc836b01daf7bc 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -597,6 +597,39 @@ def _with(c): _with.__code__.co_firstlineno + 3, ) +async def _await_call(f): + return await f(1) + +dis_await_call = """\ +%4d RETURN_GENERATOR + POP_TOP + L1: RESUME 0 + +%4d PUSH_NULL + LOAD_FAST_BORROW 0 (f) + PUSH_NULL + LOAD_SMALL_INT 1 + CALL 3 (1 + awaited) + GET_AWAITABLE 4 (after call) + L2: SEND 4 (to L5) + L3: YIELD_VALUE 1 + L4: RESUME 3 + JUMP_BACKWARD_NO_INTERRUPT 6 (to L2) + L5: END_SEND + RETURN_VALUE + L6: CLEANUP_THROW + L7: JUMP_BACKWARD_NO_INTERRUPT 4 (to L5) + + -- L8: CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) + RERAISE 1 +ExceptionTable: + L1 to L3 -> L8 [0] lasti + L3 to L4 -> L6 [3] + L4 to L7 -> L8 [0] lasti +""" % (_await_call.__code__.co_firstlineno, + _await_call.__code__.co_firstlineno + 1, + ) + async def _asyncwith(c): async with c: x = 1 @@ -1241,6 +1274,9 @@ def test_disassemble_with(self): def test_disassemble_asyncwith(self): self.do_disassembly_test(_asyncwith, dis_asyncwith) + def test_disassemble_await_call(self): + self.do_disassembly_test(_await_call, dis_await_call) + def test_disassemble_try_finally(self): self.do_disassembly_test(_tryfinally, dis_tryfinally) self.do_disassembly_test(_tryfinallyconst, dis_tryfinallyconst) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5f7c15f93a8332..f2570f4e220ae1 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1630,8 +1630,15 @@ def run_async(coro): except StopIteration: break + class Suspend: + def __await__(self): + yield + class CM: async def __aenter__(self): + # A coroutine that is awaited right away and never + # suspends doesn't execute the caller's SEND at all. + await Suspend() return self async def __aexit__(self, *exc): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst new file mode 100644 index 00000000000000..e7ba18578dfe11 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-27-12-00-00.gh-issue-161170.lazyCo.rst @@ -0,0 +1,3 @@ +Speed up ``await`` of a coroutine that completes without suspending by about +40% by running it without creating a coroutine object. Until such a coroutine +first suspends, :attr:`frame.f_generator` is ``None``. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index c8f508f1ecf8d1..421d253f24c2d2 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -1896,6 +1896,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } @@ -2200,6 +2203,12 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2338,6 +2347,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2972,6 +2987,13 @@ } ex_frame = PyStackRef_Wrap(new_frame); } + // _MAYBE_LAZY_CORO + { + new_frame = ex_frame; + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2983,7 +3005,6 @@ } // _PUSH_FRAME { - new_frame = ex_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3150,6 +3171,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -3466,6 +3490,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -3605,6 +3632,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -3802,6 +3835,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -4694,6 +4733,12 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -4803,6 +4848,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -5197,9 +5248,6 @@ null_in = stack_pointer[-3]; sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6159,7 +6207,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -6826,32 +6874,47 @@ _PyStackRef top; _PyStackRef iter; _PyStackRef null; - _PyStackRef none; + _PyStackRef v; top = stack_pointer[-1]; bottom = stack_pointer[-2]; - _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; - assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), - oparg & GET_AWAITABLE_WHERE_MASK); - _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(awaitable); - _PyFrame_StackPointerInvalidate(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); + if ((oparg & GET_AWAITABLE_AFTER_CALL) && !PyStackRef_IsNull(bottom)) { + if (PyStackRef_IsTaggedInt(bottom)) { + iter = PyStackRef_NULL; + null = PyStackRef_NULL; + } + else { + assert(PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(bottom))); + iter = bottom; + null = PyStackRef_None; + } + v = top; } - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - none = PyStackRef_None; - stack_pointer[0] = iter; - stack_pointer[1] = null; - stack_pointer[2] = none; - stack_pointer += 3; + else { + _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; + assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), + oparg & GET_AWAITABLE_WHERE_MASK); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(awaitable); + _PyFrame_StackPointerInvalidate(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + v = PyStackRef_None; + stack_pointer += 2; + } + stack_pointer[-2] = iter; + stack_pointer[-1] = null; + stack_pointer[0] = v; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7199,6 +7262,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } @@ -7366,6 +7432,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -7518,6 +7587,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -7660,8 +7732,10 @@ value = stack_pointer[-1]; index_or_null = stack_pointer[-2]; receiver = stack_pointer[-3]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (PyStackRef_IsNull(receiver) || + PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver)) || + PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(receiver))) + { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -7677,7 +7751,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -8294,6 +8368,9 @@ _PyStackRef val; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _YIELD_VALUE_EVENT { val = stack_pointer[-1]; @@ -8326,8 +8403,31 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + stack_pointer[-1] = retval; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -8342,6 +8442,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -8351,12 +8454,21 @@ Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -11453,32 +11565,57 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + if (exc_info != NULL) { + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11540,6 +11677,7 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; + _PyStackRef null; _PyStackRef null_or_index; _PyStackRef v; _PyStackRef retval; @@ -11561,10 +11699,23 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _SEND { v = stack_pointer[-1]; - null_or_index = stack_pointer[-2]; + null_or_index = null; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && @@ -11637,6 +11788,8 @@ next_instr += 2; INSTRUCTION_STATS(SEND_ASYNC_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef null; _PyStackRef iter; _PyStackRef null_in; _PyStackRef v; @@ -11644,9 +11797,23 @@ _PyStackRef null_out; _PyStackRef retval; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _GUARD_3OS_ASYNC_GEN_ASEND { - iter = stack_pointer[-3]; + iter = receiver; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (!PyAsyncGenASend_CheckExact(iter_o)) { UPDATE_MISS_STATS(SEND); @@ -11657,7 +11824,7 @@ // _SEND_ASYNC_GEN { v = stack_pointer[-1]; - null_in = stack_pointer[-2]; + null_in = null; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); assert(PyAsyncGenASend_CheckExact(iter_o)); PyObject *val = PyStackRef_AsPyObjectBorrow(v); @@ -11702,10 +11869,25 @@ INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; + _PyStackRef null; _PyStackRef v; _PyStackRef gen_frame; _PyStackRef new_frame; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { @@ -11717,7 +11899,6 @@ // _SEND_GEN_FRAME { v = stack_pointer[-1]; - receiver = stack_pointer[-3]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); @@ -11775,6 +11956,8 @@ next_instr += 2; INSTRUCTION_STATS(SEND_VIRTUAL); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef null; _PyStackRef val; _PyStackRef nos; _PyStackRef iter; @@ -11782,6 +11965,20 @@ _PyStackRef none; _PyStackRef next; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _GUARD_TOS_IS_NONE { val = stack_pointer[-1]; @@ -11793,7 +11990,7 @@ } // _GUARD_NOS_NOT_NULL { - nos = stack_pointer[-2]; + nos = null; if (PyStackRef_IsNull(nos)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); @@ -11804,7 +12001,7 @@ { none = val; null_or_index = nos; - iter = stack_pointer[-3]; + iter = receiver; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -13585,6 +13782,9 @@ opcode = YIELD_VALUE; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _MAKE_HEAP_SAFE { value = stack_pointer[-1]; @@ -13594,8 +13794,31 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + stack_pointer[-1] = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -13610,6 +13833,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -13619,12 +13845,21 @@ Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -13777,6 +14012,9 @@ JUMP_TO_LABEL(error); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; + if (dying->lazy_coro) { + frame->instr_ptr += frame->return_offset + 1; + } _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4..e250145e169360 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -4,7 +4,6 @@ static void *opcode_targets_table[256] = { &&TARGET_BINARY_SLICE, &&TARGET_BUILD_TEMPLATE, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_CHECK_EG_MATCH, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, @@ -17,8 +16,8 @@ static void *opcode_targets_table[256] = { &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_GET_LEN, - &&TARGET_RESERVED, &&TARGET_INTERPRETER_EXIT, + &&TARGET_RESERVED, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, &&TARGET_MAKE_FUNCTION, @@ -51,6 +50,7 @@ static void *opcode_targets_table[256] = { &&TARGET_BUILD_STRING, &&TARGET_BUILD_TUPLE, &&TARGET_CALL, + &&TARGET_CALL_FUNCTION_EX, &&TARGET_CALL_INTRINSIC_1, &&TARGET_CALL_INTRINSIC_2, &&TARGET_CALL_KW, diff --git a/Objects/genobject.c b/Objects/genobject.c index 82a1bb1ff42068..f5982cbdb3bb19 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1104,9 +1104,8 @@ PyTypeObject PyGen_Type = { }; static PyObject * -make_gen(PyTypeObject *type, PyFunctionObject *func) +make_gen(PyTypeObject *type, PyFunctionObject *func, PyCodeObject *code) { - PyCodeObject *code = (PyCodeObject *)func->func_code; int slots = _PyFrame_NumSlotsForCodeObject(code); PyGenObject *gen = PyObject_GC_NewVar(PyGenObject, type, slots); if (gen == NULL) { @@ -1127,18 +1126,44 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) return (PyObject *)gen; } +static PyObject * +make_coro(PyFunctionObject *func, PyCodeObject *code, + _PyInterpreterFrame *caller) +{ + PyObject *coro = make_gen(&PyCoro_Type, func, code); + if (!coro) { + return NULL; + } + PyThreadState *tstate = _PyThreadState_GET(); + int origin_depth = tstate->coroutine_origin_tracking_depth; + + if (origin_depth == 0) { + ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; + } else { + _PyInterpreterFrame *frame = _PyFrame_GetFirstComplete(caller); + PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); + ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; + if (!cr_origin) { + Py_DECREF(coro); + return NULL; + } + } + return coro; +} + PyObject * _Py_MakeCoro(PyFunctionObject *func) { - int coro_flags = ((PyCodeObject *)func->func_code)->co_flags & + PyCodeObject *code = (PyCodeObject *)func->func_code; + int coro_flags = code->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); assert(coro_flags); if (coro_flags == CO_GENERATOR) { - return make_gen(&PyGen_Type, func); + return make_gen(&PyGen_Type, func, code); } if (coro_flags == CO_ASYNC_GENERATOR) { PyAsyncGenObject *ag; - ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); + ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func, code); if (ag == NULL) { return NULL; } @@ -1150,28 +1175,32 @@ _Py_MakeCoro(PyFunctionObject *func) } assert (coro_flags == CO_COROUTINE); - PyObject *coro = make_gen(&PyCoro_Type, func); - if (!coro) { - return NULL; - } PyThreadState *tstate = _PyThreadState_GET(); - int origin_depth = tstate->coroutine_origin_tracking_depth; - - if (origin_depth == 0) { - ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; - } else { - _PyInterpreterFrame *frame = tstate->current_frame; - assert(frame); - assert(_PyFrame_IsIncomplete(frame)); - frame = _PyFrame_GetFirstComplete(frame->previous); - PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); - ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; - if (!cr_origin) { - Py_DECREF(coro); - return NULL; - } - } - return coro; + _PyInterpreterFrame *frame = tstate->current_frame; + assert(frame); + assert(_PyFrame_IsIncomplete(frame)); + return make_coro(func, code, frame->previous); +} + +/* Create the coroutine object for a coroutine whose body has been + * executing directly in `frame`, a frame owned by the thread, because + * its caller awaited the result immediately (see RETURN_GENERATOR in + * bytecodes.c). Called when the body suspends for the first time. + * The frame itself is moved into the coroutine by the caller. + * The body may have replaced func.__code__ in the meantime, so the + * coroutine is sized for the code the frame is actually running. */ +PyObject * +_PyCoro_MaterializeLazy(_PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(frame->lazy_coro); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = + (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyCodeObject *code = _PyFrame_GetCode(frame); + assert((code->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) == CO_COROUTINE); + return make_coro(func, code, frame->previous); } static PyObject * diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index ebcb38b18dec90..d8adaf26ec18b3 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -3,17 +3,17 @@ unsigned char M_test_frozenmain[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, 0,0,0,0,0,115,188,0,0,0,128,0,0,0,90,0, 77,7,69,0,112,0,90,0,77,7,69,4,112,1,89,2, - 31,0,78,1,50,1,0,0,0,0,0,0,29,0,89,2, - 31,0,78,2,89,0,76,6,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,50,2,0,0,0,0, - 0,0,29,0,89,1,76,8,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,31,0,50,0,0,0, - 0,0,0,0,78,3,42,26,0,0,0,0,0,0,0,0, + 30,0,78,1,49,2,0,0,0,0,0,0,28,0,89,2, + 30,0,78,2,89,0,76,6,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,49,4,0,0,0,0, + 0,0,28,0,89,1,76,8,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,30,0,49,0,0,0, + 0,0,0,0,78,3,41,26,0,0,0,0,0,0,0,0, 0,0,112,5,78,6,67,0,0,0,65,24,0,0,112,6, - 89,2,31,0,78,4,89,6,12,0,78,5,89,5,89,6, - 42,26,0,0,0,0,0,0,0,0,0,0,12,0,48,4, - 50,1,0,0,0,0,0,0,29,0,71,26,0,0,9,0, - 28,0,77,7,33,0,41,7,105,0,0,0,0,122,18,70, + 89,2,30,0,78,4,89,6,11,0,78,5,89,5,89,6, + 41,26,0,0,0,0,0,0,0,0,0,0,11,0,47,4, + 49,2,0,0,0,0,0,0,28,0,71,26,0,0,8,0, + 27,0,77,7,32,0,41,7,105,0,0,0,0,122,18,70, 114,111,122,101,110,32,72,101,108,108,111,32,87,111,114,108, 100,122,8,115,121,115,46,97,114,103,118,218,6,99,111,110, 102,105,103,122,7,99,111,110,102,105,103,32,122,2,58,32, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ab03bd08b51c0e..428027d306127f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -432,12 +432,17 @@ dummy_func( (void)index_or_null; DEAD(value); DEAD(index_or_null); - PyStackRef_CLOSE(receiver); + // NULL if an awaited call returned without creating a coroutine. + PyStackRef_XCLOSE(receiver); } tier1 inst(INSTRUMENTED_END_SEND, (receiver, index_or_null, value -- val)) { - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + /* A NULL receiver is a coroutine that returned before it was + * created (see GET_AWAITABLE). */ + if (PyStackRef_IsNull(receiver) || + PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver)) || + PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(receiver))) + { int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); if (err) { ERROR_NO_POP(); @@ -447,7 +452,7 @@ dummy_func( (void)index_or_null; DEAD(index_or_null); DEAD(value); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); } macro(UNARY_NEGATIVE) = _UNARY_NEGATIVE + POP_TOP; @@ -1649,22 +1654,44 @@ dummy_func( awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); } - // Also pushes the NULL and None that the following SEND loop starts - // with. The compiler pushes a NULL above the awaitable, or, with - // GET_AWAITABLE_AFTER_CALL, below the callable of an awaited call, - // so `top` is its result. - inst(GET_AWAITABLE, (bottom, top -- iter, null, none)) { - _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; - assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), - oparg & GET_AWAITABLE_WHERE_MASK); - DEAD(top); - DEAD(bottom); - PyStackRef_CLOSE(awaitable); - ERROR_IF(iter_o == NULL); - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - none = PyStackRef_None; + // Also pushes the receiver, NULL and value that the following SEND + // loop starts with. The compiler pushes a NULL above the awaitable, + // or, with GET_AWAITABLE_AFTER_CALL, below the callable of an awaited + // call, so `top` is its result. A coroutine body that the call ran + // without creating the coroutine object (see RETURN_GENERATOR) left + // in that slot: + // - a tagged int if the body returned `top`: SEND gets a NULL + // receiver and jumps to END_SEND; + // - the coroutine if the body suspended, yielding `top`: SEND gets + // None instead of NULL and goes on to YIELD_VALUE. + inst(GET_AWAITABLE, (bottom, top -- iter, null, v)) { + if ((oparg & GET_AWAITABLE_AFTER_CALL) && !PyStackRef_IsNull(bottom)) { + if (PyStackRef_IsTaggedInt(bottom)) { + iter = PyStackRef_NULL; + null = PyStackRef_NULL; + } + else { + assert(PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(bottom))); + iter = bottom; + null = PyStackRef_None; + } + v = top; + DEAD(top); + DEAD(bottom); + } + else { + _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; + assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), + oparg & GET_AWAITABLE_WHERE_MASK); + DEAD(top); + DEAD(bottom); + PyStackRef_CLOSE(awaitable); + ERROR_IF(iter_o == NULL); + iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + v = PyStackRef_None; + } } family(SEND, INLINE_CACHE_ENTRIES_SEND) = { @@ -1731,7 +1758,28 @@ dummy_func( } } - macro(SEND) = _SPECIALIZE_SEND + _SEND; + // See GET_AWAITABLE: a NULL receiver means that an awaited call + // returned `v` without creating a coroutine object, and None in + // place of NULL that the coroutine it created has already yielded `v`. + // Every form of SEND starts with this, since an await site may have + // been specialized for whatever its awaitable was before. + replaced op(_SEND_LAZY_CORO, (receiver, null, v -- receiver, null, v)) { + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + null = PyStackRef_NULL; + SYNC_SP(); + DISPATCH(); + } + } + + tier2 op(_GUARD_SEND_NOT_LAZY_CORO, (receiver, null, v -- receiver, null, v)) { + EXIT_IF(PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)); + } + + macro(SEND) = _SPECIALIZE_SEND + _SEND_LAZY_CORO + _SEND; op(_SEND_GEN_FRAME, (receiver, null, v -- receiver, null, gen_frame)) { PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); @@ -1752,6 +1800,7 @@ dummy_func( macro(SEND_GEN) = unused/1 + _RECORD_3OS_GEN_FUNC + + _SEND_LAZY_CORO + _CHECK_PEP_523 + _SEND_GEN_FRAME + _PUSH_FRAME; @@ -1806,6 +1855,7 @@ dummy_func( macro(SEND_VIRTUAL) = unused/1 + + _SEND_LAZY_CORO + _GUARD_TOS_IS_NONE + _GUARD_NOS_NOT_NULL + _SEND_VIRTUAL; @@ -1838,6 +1888,7 @@ dummy_func( macro(SEND_ASYNC_GEN) = unused/1 + + _SEND_LAZY_CORO + _GUARD_3OS_ASYNC_GEN_ASEND + _SEND_ASYNC_GEN; @@ -1860,13 +1911,44 @@ dummy_func( EXIT_IF(what == PYGEN_RETURN); } + // A lazily started coroutine (see RETURN_GENERATOR) is materialized + // by tier one (see _YIELD_VALUE), which also resumes the caller at a + // different place. Tier two replaces this with _GUARD_NOT_LAZY_CORO. + replaced op(_CHECK_LAZY_CORO, (--)) { + } + + tier2 op(_GUARD_NOT_LAZY_CORO, (--)) { + EXIT_IF(frame->lazy_coro); + } + op(_YIELD_VALUE, (retval -- value)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. + // The one exception is a lazily started coroutine (see + // RETURN_GENERATOR) that cannot be allocated: the await then + // raises MemoryError through CLEANUP_THROW. assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + if (gen == NULL) { + ERROR_NO_POP(); + } + /* Move the body's exception state (see RETURN_GENERATOR) + * into the coroutine, where SEND_GEN would have put it. */ + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -1879,6 +1961,9 @@ dummy_func( _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -1889,16 +1974,29 @@ dummy_func( Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif RELOAD_STACK(); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + /* The caller is right after the awaited call: hand the + * coroutine to its GET_AWAITABLE through the slot below + * the call. */ + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } macro(YIELD_VALUE) = + _CHECK_LAZY_CORO + _MAKE_HEAP_SAFE + _YIELD_VALUE; @@ -1918,6 +2016,7 @@ dummy_func( } macro(INSTRUMENTED_YIELD_VALUE) = + _CHECK_LAZY_CORO + _YIELD_VALUE_EVENT + _MAKE_HEAP_SAFE + _YIELD_VALUE; @@ -1973,9 +2072,9 @@ dummy_func( tier1 inst(CLEANUP_THROW, (sub_iter, null_in, last_sent_val, exc_value_st -- none, null_out, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif + /* Reached on throw()/close(), or when a lazily started + * coroutine could not be created at its first suspension + * (see _YIELD_VALUE). */ assert(exc_value && PyExceptionInstance_Check(exc_value)); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); if (matches) { @@ -4383,6 +4482,9 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = INSTRUCTION_SIZE; DISPATCH_INLINED(new_frame); } @@ -4471,6 +4573,7 @@ dummy_func( _CHECK_FUNCTION_VERSION + _CHECK_RECURSION_REMAINING + _PY_FRAME_GENERAL + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -4504,6 +4607,7 @@ dummy_func( flush + // so that self is in the argument array _CHECK_RECURSION_REMAINING + _PY_FRAME_GENERAL + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -4624,6 +4728,7 @@ dummy_func( _CHECK_STACK_SPACE + _CHECK_RECURSION_REMAINING + _INIT_CALL_PY_EXACT_ARGS + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -4636,6 +4741,7 @@ dummy_func( _CHECK_STACK_SPACE + _CHECK_RECURSION_REMAINING + _INIT_CALL_PY_EXACT_ARGS + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -5428,6 +5534,9 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert(INSTRUCTION_SIZE == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = INSTRUCTION_SIZE; DISPATCH_INLINED(new_frame); @@ -5493,6 +5602,7 @@ dummy_func( _CHECK_FUNCTION_VERSION_KW + _CHECK_RECURSION_REMAINING + _PY_FRAME_KW + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -5525,6 +5635,7 @@ dummy_func( _EXPAND_METHOD_KW + flush + // so that self is in the argument array _PY_FRAME_KW + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -5673,6 +5784,9 @@ dummy_func( if (new_frame == NULL) { ERROR_NO_POP(); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert(INSTRUCTION_SIZE == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = INSTRUCTION_SIZE; DISPATCH_INLINED(new_frame); @@ -5745,6 +5859,7 @@ dummy_func( _MAKE_CALLARGS_A_TUPLE + _CHECK_IS_PY_CALLABLE_EX + _PY_FRAME_EX + + _MAYBE_LAZY_CORO + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -5817,25 +5932,53 @@ dummy_func( inst(RETURN_GENERATOR, (-- res)) { assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - ERROR_IF(gen == NULL); - assert(STACK_LEVEL() <= 2); - SAVE_STACK(); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - RELOAD_STACK(); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + /* The body gets its own exception state, as SEND_GEN + * would have given it. It lives on the data stack right + * after this frame; clear_thread_frame() and _YIELD_VALUE + * pop it. */ + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + if (exc_info == NULL) { + /* Out of memory: take the regular path. */ + _PyErr_Clear(tstate); + } + } + #endif + if (exc_info != NULL) { + /* An awaited call (see _MAYBE_LAZY_CORO): keep running the + * body in this frame, and only create the coroutine object + * if the body suspends (see _YIELD_VALUE). */ + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + /* The following POP_TOP discards this. */ + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + ERROR_IF(gen == NULL); + assert(STACK_LEVEL() <= 2); + SAVE_STACK(); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + RELOAD_STACK(); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + } } inst(BUILD_SLICE, (args[oparg] -- slice)) { @@ -6112,6 +6255,12 @@ dummy_func( DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, framesize)); } + replaced op(_MAYBE_LAZY_CORO, (new_frame -- new_frame)) { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } + op(_SAVE_RETURN_OFFSET, (--)) { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); @@ -6390,7 +6539,8 @@ dummy_func( tier2 op(_RECORD_3OS_GEN_FUNC, (gen, nos, tos -- gen, nos, tos)) { PyObject *obj = PyStackRef_AsPyObjectBorrow(gen); - if (PyGen_Check(obj)) { + // `gen` is NULL after an awaited call (see GET_AWAITABLE). + if (obj != NULL && PyGen_Check(obj)) { PyGenObject *gen_obj = (PyGenObject *)obj; _PyStackRef func = gen_obj->gi_iframe.f_funcobj; if (!PyStackRef_IsNull(func)) { @@ -6531,6 +6681,11 @@ dummy_func( // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; + if (dying->lazy_coro) { + /* Raise from the caller's SEND, as if the coroutine had been + * created (see GET_AWAITABLE). */ + frame->instr_ptr += frame->return_offset + 1; + } _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { diff --git a/Python/ceval.c b/Python/ceval.c index ccc0edca2fc794..bbd73b1fff96db 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2005,10 +2005,38 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, return -1; } +/* Pop the exception state that RETURN_GENERATOR gave a lazily started + * coroutine body, restoring the caller's. Returns the body's exception, + * if any (a new reference). */ +static PyObject * +pop_lazy_coro_exc_state(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + assert(frame->lazy_coro); + _PyErr_StackItem *exc_info = tstate->exc_info; + tstate->exc_info = exc_info->previous_item; + PyObject *exc_value = exc_info->exc_value; + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + frame->lazy_coro = 0; + return exc_value; +} + static void clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { assert(frame->owner == FRAME_OWNED_BY_THREAD); + PyObject *lazy_exc = NULL; + if (frame->lazy_coro) { + lazy_exc = pop_lazy_coro_exc_state(tstate, frame); + /* The body finished without creating its coroutine object: tell + * the caller's GET_AWAITABLE through the slot that the compiler + * pushed below the awaited call. */ + _PyInterpreterFrame *caller = frame->previous; + assert(PyStackRef_IsNull(caller->stackpointer[-1])); + caller->stackpointer[-1] = PyStackRef_TagInt(0); + // Like clear_gen_frame(): a frame object that outlives the + // coroutine must not keep the caller's frame alive via f_back. + frame->previous = NULL; + } // Make sure that this is, indeed, the top frame. We can't check this in // _PyThreadState_PopFrame, since f_code is already cleared at that point: assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == @@ -2017,6 +2045,7 @@ clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) _PyFrame_ClearExceptCode(frame); PyStackRef_CLEAR(frame->f_executable); _PyThreadState_PopFrame(tstate, frame); + Py_XDECREF(lazy_exc); } static void @@ -3686,6 +3715,29 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name } } +/* YIELD_VALUE support: move the frame of a lazily started coroutine + * from the thread's data stack into `gen`. Returns the new frame. */ +_PyInterpreterFrame * +_PyEval_MoveLazyCoroFrame(PyThreadState *tstate, PyGenObject *gen, + _PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(frame->lazy_coro); + assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == + tstate->datastack_top); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_Copy(frame, gen_frame); + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + PyFrameObject *f = gen_frame->frame_obj; + if (f != NULL) { + Py_BEGIN_CRITICAL_SECTION(f); + f->f_frame = gen_frame; + Py_END_CRITICAL_SECTION(); + } + _PyThreadState_PopFrame(tstate, frame); + return gen_frame; +} + PyObject * _PyEval_GetAwaitable(PyObject *iterable, int oparg) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 36921e35533433..56d546dadd1680 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -156,6 +156,27 @@ # define LEAVE_TRACING() tracing_mode = 0 #endif +/* An awaited call (the low bit of oparg is set) of a coroutine function lets + * the body run in the pushed frame without creating the coroutine object + * until it first suspends (see RETURN_GENERATOR). Not while the JIT records + * a trace, which expects the regular path, nor while coroutine origin + * tracking is on, since the origin is captured when the object is created. */ +#if _Py_TIER2 +# define LAZY_CORO_ALLOWED() (!IS_JIT_TRACING()) +#else +# define LAZY_CORO_ALLOWED() 1 +#endif +#define MAYBE_START_LAZY_CORO(FRAME) \ + do { \ + _PyInterpreterFrame *lazy_frame_ = (FRAME); \ + if ((_PyFrame_GetCode(lazy_frame_)->co_flags & CO_COROUTINE) && \ + tstate->coroutine_origin_tracking_depth == 0 && \ + LAZY_CORO_ALLOWED()) \ + { \ + lazy_frame_->lazy_coro = 1; \ + } \ + } while (0) + #if _Py_TIER2 #define STOP_TRACING() \ do { \ diff --git a/Python/codegen.c b/Python/codegen.c index 38a3e2d4336234..1df2f3c5ec5e23 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -281,7 +281,6 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) * with the low bit set when the result is awaited right away. */ #define CALL_OPARG(NARGS, AWAITED) (((NARGS) << 1) | (AWAITED)) - #define ADDOP_I(C, LOC, OP, O) \ RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) @@ -4584,7 +4583,7 @@ codegen_call_helper_impl(compiler *c, location loc, if (nkwelts == 0) { ADDOP(c, loc, PUSH_NULL); } - ADDOP(c, loc, CALL_FUNCTION_EX); + ADDOP_I(c, loc, CALL_FUNCTION_EX, awaited); return SUCCESS; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 4314dd408600ac..28d71021276d45 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2622,7 +2622,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = val; _tos_cache1 = PyStackRef_ZERO_BITS; @@ -8913,39 +8913,153 @@ _PyStackRef bottom; _PyStackRef iter; _PyStackRef null; - _PyStackRef none; + _PyStackRef v; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); top = _stack_item_1; bottom = _stack_item_0; - _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; - assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); - stack_pointer[0] = bottom; - stack_pointer[1] = top; - stack_pointer += 2; + if ((oparg & GET_AWAITABLE_AFTER_CALL) && !PyStackRef_IsNull(bottom)) { + if (PyStackRef_IsTaggedInt(bottom)) { + iter = PyStackRef_NULL; + null = PyStackRef_NULL; + } + else { + assert(PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(bottom))); + iter = bottom; + null = PyStackRef_None; + } + v = top; + } + else { + _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; + assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); + stack_pointer[0] = bottom; + stack_pointer[1] = top; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), + oparg & GET_AWAITABLE_WHERE_MASK); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(awaitable); + _PyFrame_StackPointerInvalidate(frame); + if (iter_o == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + v = PyStackRef_None; + } + _tos_cache2 = v; + _tos_cache1 = null; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + /* _SEND_LAZY_CORO is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_SEND_NOT_LAZY_CORO_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null; + _PyStackRef receiver; + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = null; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), - oparg & GET_AWAITABLE_WHERE_MASK); - _PyFrame_StackPointerInvalidate(frame); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_SEND_NOT_LAZY_CORO_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null; + _PyStackRef receiver; + _PyStackRef _stack_item_0 = _tos_cache0; + null = stack_pointer[-1]; + receiver = stack_pointer[-2]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = null; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(awaitable); - _PyFrame_StackPointerInvalidate(frame); - if (iter_o == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_SEND_NOT_LAZY_CORO_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null; + _PyStackRef receiver; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + null = _stack_item_0; + receiver = stack_pointer[-1]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = null; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - none = PyStackRef_None; - _tos_cache2 = none; + _tos_cache2 = _stack_item_1; _tos_cache1 = null; - _tos_cache0 = iter; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_SEND_NOT_LAZY_CORO_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null; + _PyStackRef receiver; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + null = _stack_item_1; + receiver = _stack_item_0; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = null; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = null; + _tos_cache0 = receiver; SET_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -9500,6 +9614,78 @@ break; } + /* _CHECK_LAZY_CORO is not a viable micro-op for tier 2 because it is replaced */ + + case _GUARD_NOT_LAZY_CORO_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + SET_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOT_LAZY_CORO_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOT_LAZY_CORO_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOT_LAZY_CORO_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + if (frame->lazy_coro) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _YIELD_VALUE_r11: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -9509,11 +9695,40 @@ oparg = CURRENT_OPARG(); retval = _stack_item_0; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + stack_pointer[0] = retval; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + stack_pointer += 1; + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); @@ -9523,6 +9738,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -9532,12 +9750,21 @@ Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); _tos_cache0 = value; @@ -20876,33 +21103,58 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + if (exc_info != NULL) { + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); _tos_cache0 = res; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; @@ -23007,6 +23259,8 @@ break; } + /* _MAYBE_LAZY_CORO is not a viable micro-op for tier 2 because it is replaced */ + case _SAVE_RETURN_OFFSET_r00: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ddd57d8782f833..e2c77eccb830cc 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1896,6 +1896,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } @@ -2200,6 +2203,12 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2338,6 +2347,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2972,6 +2987,13 @@ } ex_frame = PyStackRef_Wrap(new_frame); } + // _MAYBE_LAZY_CORO + { + new_frame = ex_frame; + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -2983,7 +3005,6 @@ } // _PUSH_FRAME { - new_frame = ex_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3150,6 +3171,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -3466,6 +3490,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -3605,6 +3632,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -3802,6 +3835,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -4694,6 +4733,12 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -4803,6 +4848,12 @@ } new_frame = PyStackRef_Wrap(temp); } + // _MAYBE_LAZY_CORO + { + if (oparg & 1) { + MAYBE_START_LAZY_CORO(PyStackRef_Unwrap(new_frame)); + } + } // _SAVE_RETURN_OFFSET { #if TIER_ONE @@ -5197,9 +5248,6 @@ null_in = stack_pointer[-3]; sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6159,7 +6207,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -6826,32 +6874,47 @@ _PyStackRef top; _PyStackRef iter; _PyStackRef null; - _PyStackRef none; + _PyStackRef v; top = stack_pointer[-1]; bottom = stack_pointer[-2]; - _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; - assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), - oparg & GET_AWAITABLE_WHERE_MASK); - _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(awaitable); - _PyFrame_StackPointerInvalidate(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); + if ((oparg & GET_AWAITABLE_AFTER_CALL) && !PyStackRef_IsNull(bottom)) { + if (PyStackRef_IsTaggedInt(bottom)) { + iter = PyStackRef_NULL; + null = PyStackRef_NULL; + } + else { + assert(PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(bottom))); + iter = bottom; + null = PyStackRef_None; + } + v = top; } - iter = PyStackRef_FromPyObjectSteal(iter_o); - null = PyStackRef_NULL; - none = PyStackRef_None; - stack_pointer[0] = iter; - stack_pointer[1] = null; - stack_pointer[2] = none; - stack_pointer += 3; + else { + _PyStackRef awaitable = (oparg & GET_AWAITABLE_AFTER_CALL) ? top : bottom; + assert(PyStackRef_IsNull((oparg & GET_AWAITABLE_AFTER_CALL) ? bottom : top)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(awaitable), + oparg & GET_AWAITABLE_WHERE_MASK); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(awaitable); + _PyFrame_StackPointerInvalidate(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + v = PyStackRef_None; + stack_pointer += 2; + } + stack_pointer[-2] = iter; + stack_pointer[-1] = null; + stack_pointer[0] = v; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7199,6 +7262,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL) { + MAYBE_START_LAZY_CORO(new_frame); + } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } @@ -7366,6 +7432,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if (oparg & 1) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); @@ -7518,6 +7587,9 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } + if ((oparg & 1) && opcode != INSTRUMENTED_CALL_KW) { + MAYBE_START_LAZY_CORO(new_frame); + } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -7660,8 +7732,10 @@ value = stack_pointer[-1]; index_or_null = stack_pointer[-2]; receiver = stack_pointer[-3]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (PyStackRef_IsNull(receiver) || + PyGen_Check(PyStackRef_AsPyObjectBorrow(receiver)) || + PyCoro_CheckExact(PyStackRef_AsPyObjectBorrow(receiver))) + { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -7677,7 +7751,7 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyStackRef_CLOSE(receiver); + PyStackRef_XCLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -8293,6 +8367,9 @@ _PyStackRef val; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _YIELD_VALUE_EVENT { val = stack_pointer[-1]; @@ -8325,8 +8402,31 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + stack_pointer[-1] = retval; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -8341,6 +8441,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -8350,12 +8453,21 @@ Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -11450,32 +11562,57 @@ _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + _PyErr_StackItem *exc_info = NULL; + #if TIER_ONE + if (frame->lazy_coro) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + _PyFrame_StackPointerInvalidate(frame); + if (exc_info == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + if (exc_info != NULL) { + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + res = PyStackRef_None; + } + else { + frame->lazy_coro = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11537,6 +11674,7 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; + _PyStackRef null; _PyStackRef null_or_index; _PyStackRef v; _PyStackRef retval; @@ -11558,10 +11696,23 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _SEND { v = stack_pointer[-1]; - null_or_index = stack_pointer[-2]; + null_or_index = null; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && @@ -11634,6 +11785,8 @@ next_instr += 2; INSTRUCTION_STATS(SEND_ASYNC_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef null; _PyStackRef iter; _PyStackRef null_in; _PyStackRef v; @@ -11641,9 +11794,23 @@ _PyStackRef null_out; _PyStackRef retval; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _GUARD_3OS_ASYNC_GEN_ASEND { - iter = stack_pointer[-3]; + iter = receiver; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (!PyAsyncGenASend_CheckExact(iter_o)) { UPDATE_MISS_STATS(SEND); @@ -11654,7 +11821,7 @@ // _SEND_ASYNC_GEN { v = stack_pointer[-1]; - null_in = stack_pointer[-2]; + null_in = null; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); assert(PyAsyncGenASend_CheckExact(iter_o)); PyObject *val = PyStackRef_AsPyObjectBorrow(v); @@ -11699,10 +11866,25 @@ INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; + _PyStackRef null; _PyStackRef v; _PyStackRef gen_frame; _PyStackRef new_frame; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { @@ -11714,7 +11896,6 @@ // _SEND_GEN_FRAME { v = stack_pointer[-1]; - receiver = stack_pointer[-3]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); @@ -11772,6 +11953,8 @@ next_instr += 2; INSTRUCTION_STATS(SEND_VIRTUAL); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef null; _PyStackRef val; _PyStackRef nos; _PyStackRef iter; @@ -11779,6 +11962,20 @@ _PyStackRef none; _PyStackRef next; /* Skip 1 cache entry */ + // _SEND_LAZY_CORO + { + null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + if (PyStackRef_IsNull(receiver) | PyStackRef_IsNone(null)) { + if (PyStackRef_IsNull(receiver)) { + JUMPBY(oparg); + DISPATCH(); + } + null = PyStackRef_NULL; + stack_pointer[-2] = null; + DISPATCH(); + } + } // _GUARD_TOS_IS_NONE { val = stack_pointer[-1]; @@ -11790,7 +11987,7 @@ } // _GUARD_NOS_NOT_NULL { - nos = stack_pointer[-2]; + nos = null; if (PyStackRef_IsNull(nos)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); @@ -11801,7 +11998,7 @@ { none = val; null_or_index = nos; - iter = stack_pointer[-3]; + iter = receiver; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -13582,6 +13779,9 @@ opcode = YIELD_VALUE; _PyStackRef value; _PyStackRef retval; + // _CHECK_LAZY_CORO + { + } // _MAKE_HEAP_SAFE { value = stack_pointer[-1]; @@ -13591,8 +13791,31 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->lazy_coro; + if (lazy) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + stack_pointer[-1] = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + tstate->exc_info = &gen->gi_exc_state; + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -13607,6 +13830,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -13616,12 +13842,21 @@ Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; - assert(opcode == SEND || opcode == FOR_ITER); + assert(lazy ? (opcode == CALL || opcode == CALL_KW || + opcode == CALL_FUNCTION_EX) + : (opcode == SEND || opcode == FOR_ITER)); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-1] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LOAD_IP(frame->return_offset); + } + else { + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -13774,6 +14009,9 @@ JUMP_TO_LABEL(error); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; + if (dying->lazy_coro) { + frame->instr_ptr += frame->return_offset + 1; + } _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { diff --git a/Python/intrinsics.c b/Python/intrinsics.c index dfa81baa654e12..e3b0fd406a612a 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -144,7 +144,11 @@ static PyObject * stopiteration_error(PyThreadState* tstate, PyObject *exc) { _PyInterpreterFrame *frame = tstate->current_frame; - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + /* A coroutine that is awaited immediately runs in a thread-owned + * frame until it first suspends (see RETURN_GENERATOR). */ + assert(frame->owner == FRAME_OWNED_BY_GENERATOR || + (frame->owner == FRAME_OWNED_BY_THREAD && + (_PyFrame_GetCode(frame)->co_flags & CO_COROUTINE))); assert(PyExceptionInstance_Check(exc)); const char *msg = NULL; if (PyErr_GivenExceptionMatches(exc, PyExc_StopIteration)) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4..e250145e169360 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -4,7 +4,6 @@ static void *opcode_targets_table[256] = { &&TARGET_BINARY_SLICE, &&TARGET_BUILD_TEMPLATE, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_CALL_FUNCTION_EX, &&TARGET_CHECK_EG_MATCH, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, @@ -17,8 +16,8 @@ static void *opcode_targets_table[256] = { &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_GET_LEN, - &&TARGET_RESERVED, &&TARGET_INTERPRETER_EXIT, + &&TARGET_RESERVED, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, &&TARGET_MAKE_FUNCTION, @@ -51,6 +50,7 @@ static void *opcode_targets_table[256] = { &&TARGET_BUILD_STRING, &&TARGET_BUILD_TUPLE, &&TARGET_CALL, + &&TARGET_CALL_FUNCTION_EX, &&TARGET_CALL_INTRINSIC_1, &&TARGET_CALL_INTRINSIC_2, &&TARGET_CALL_KW, diff --git a/Python/optimizer.c b/Python/optimizer.c index e05adb344c8d06..bc161d7a85200f 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -502,6 +502,9 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_LOAD_BYTECODE] = _NOP, [_SEND_VIRTUAL] = _SEND_VIRTUAL_TIER_TWO, [_SEND_ASYNC_GEN] = _SEND_ASYNC_GEN_TIER_TWO, + [_SEND_LAZY_CORO] = _GUARD_SEND_NOT_LAZY_CORO, + [_MAYBE_LAZY_CORO] = _NOP, + [_CHECK_LAZY_CORO] = _GUARD_NOT_LAZY_CORO, }; static const uint8_t @@ -966,6 +969,9 @@ _PyJit_translate_single_bytecode_to_trace( target = next_inst; } } + else if (uop == _NOP || uop == _GUARD_NOT_LAZY_CORO) { + // _MAYBE_LAZY_CORO and _CHECK_LAZY_CORO don't jump. + } else { int extended_arg = orig_oparg > 255; uint32_t jump_target = next_inst + orig_oparg + extended_arg; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index be4dcec949e916..04f88b0daa9426 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -206,11 +206,11 @@ dummy_func(void) { res = PyJitRef_Borrow(sym_new_null(ctx)); } - op(_GET_AWAITABLE, (bottom, top -- iter, null, none)) { + op(_GET_AWAITABLE, (bottom, top -- iter, null, v)) { // After an awaited call, any of the outputs may be NULL. iter = sym_new_unknown(ctx); null = sym_new_unknown(ctx); - none = sym_new_unknown(ctx); + v = sym_new_unknown(ctx); } op(_GUARD_TOS_INT, (value -- value)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 176e149d6a7ad3..c38618ea761439 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1889,19 +1889,25 @@ case _GET_AWAITABLE: { JitOptRef iter; JitOptRef null; - JitOptRef none; + JitOptRef v; iter = sym_new_unknown(ctx); null = sym_new_unknown(ctx); - none = sym_new_unknown(ctx); + v = sym_new_unknown(ctx); CHECK_STACK_BOUNDS(1); stack_pointer[-2] = iter; stack_pointer[-1] = null; - stack_pointer[0] = none; + stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } + /* _SEND_LAZY_CORO is not a viable micro-op for tier 2 */ + + case _GUARD_SEND_NOT_LAZY_CORO: { + break; + } + case _SEND_GEN_FRAME: { JitOptRef v; JitOptRef receiver; @@ -1972,6 +1978,12 @@ break; } + /* _CHECK_LAZY_CORO is not a viable micro-op for tier 2 */ + + case _GUARD_NOT_LAZY_CORO: { + break; + } + case _YIELD_VALUE: { JitOptRef retval; JitOptRef value; @@ -5590,6 +5602,8 @@ break; } + /* _MAYBE_LAZY_CORO is not a viable micro-op for tier 2 */ + case _SAVE_RETURN_OFFSET: { break; } diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 75b5ba65f972f1..c0a33c6e1b201f 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -52,7 +52,7 @@ void _PyOpcode_RecordFunction_3OS_GEN_FUNC(_PyInterpreterFrame *frame, _PyStackR _PyStackRef gen; gen = stack_pointer[-3]; PyObject *obj = PyStackRef_AsPyObjectBorrow(gen); - if (PyGen_Check(obj)) { + if (obj != NULL && PyGen_Check(obj)) { PyGenObject *gen_obj = (PyGenObject *)obj; _PyStackRef func = gen_obj->gi_iframe.f_funcobj; if (!PyStackRef_IsNull(func)) { diff --git a/Python/specialize.c b/Python/specialize.c index cb84fb4b81866d..ce8cf19ff11f73 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2738,11 +2738,12 @@ _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT Py_NO_INLINE void _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) { - PyObject *receiver = PyStackRef_AsPyObjectBorrow(receiver_st); - assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND); - PyTypeObject *tp = Py_TYPE(receiver); + /* NULL is a coroutine that an awaited call ran to completion without + * creating it (see GET_AWAITABLE); SEND_GEN handles that too. */ + PyTypeObject *tp = PyStackRef_IsNull(receiver_st) ? &PyCoro_Type + : PyStackRef_TYPE(receiver_st); if (tp == &PyGen_Type || tp == &PyCoro_Type) { /* Don't specialize if PEP 523 is active */ if (!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) { @@ -2761,7 +2762,8 @@ _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) return; } SPECIALIZATION_FAIL(SEND, - _PySpecialization_ClassifyIterator(receiver)); + _PySpecialization_ClassifyIterator( + PyStackRef_AsPyObjectBorrow(receiver_st))); failure: unspecialize(instr); }