From 6afae50557f8ae8742b4c7cca3c01085366c72a1 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Tue, 3 Nov 2026 09:49:35 +0100 Subject: [PATCH 1/2] Add KIND_OP: one generic specialized instruction, driven by the DSL Specializations marked `kind_op` in bytecodes.c get no opcode of their own. The cases generator emits them as cases of a single KIND_OP instruction, numbers them, and generates the tables that map a kind back to its family's generic instruction, its name and its tier-2 expansion. Each kind keeps the cache layout of its family, so the instruction length and the cache entries after the first are what the generic instruction uses. A `kind_candidate` annotation marks a specialization that is folded only when the generator runs with KIND_OPS=2, so the number of real opcodes is a one-word choice per specialization. The kind lives in the first cache entry, in the high bits of the adaptive counter's value field, next to a six-bit miss counter. A failed guard re-executes the generic instruction, whose ADVANCE_ADAPTIVE_COUNTER counts the miss down; when the misses run out the instruction returns to its adaptive form with a triggering counter and re-specializes on the operands that missed, the same life cycle as an ordinary specialized instruction. Pseudo-opcodes `NAME = 512 + KIND_NAME` keep `specialize(instr, NAME)` call sites unchanged. Instrumentation, _Py_GetBaseCodeUnit, unspecialize, the tier-2 trace translator and dis map KIND_OP through its kind, and test_opcache recognizes kinds by the name dis prints. Two kinds come with the mechanism: - BINARY_OP_INPLACE: in-place float and compact-int arithmetic, including float true division, when one operand is a uniquely referenced temporary - TO_BOOL_SIZED: truthiness of dict, tuple and set read from the size field Co-Authored-By: Claude Fable 5.1 --- Include/internal/pycore_code.h | 63 ++++ Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 57 ++- Include/internal/pycore_uop_ids.h | 13 +- Include/internal/pycore_uop_metadata.h | 51 +++ Include/opcode_ids.h | 87 ++--- Lib/_opcode_metadata.py | 92 ++--- Lib/dis.py | 9 + Lib/opcode.py | 2 +- Lib/test/test_opcache.py | 19 +- Modules/_imp* | 0 Modules/_testinternalcapi/test_cases.c.h | 118 +++++- Modules/_testinternalcapi/test_targets.h | 9 +- Python/bytecodes.c | 143 ++++++-- Python/ceval_macros.h | 114 ++++++ Python/executor_cases.c.h | 336 ++++++++++++++++++ Python/generated_cases.c.h | 118 +++++- Python/instrumentation.c | 27 +- Python/opcode_targets.h | 9 +- Python/optimizer.c | 22 +- Python/optimizer_cases.c.h | 21 ++ Python/specialize.c | 130 ++++++- Tools/cases_generator/analyzer.py | 74 +++- Tools/cases_generator/lexer.py | 2 + .../opcode_metadata_generator.py | 68 +++- Tools/cases_generator/parsing.py | 8 +- .../cases_generator/py_metadata_generator.py | 8 + Tools/cases_generator/tier1_generator.py | 98 +++++ 28 files changed, 1521 insertions(+), 180 deletions(-) create mode 100644 Modules/_imp* diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index aeb2ad06ad07cb7..7a71e18bdcfdf6d 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -160,6 +160,25 @@ typedef struct { #define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache) +/* TO_BOOL_SIZED: types whose truthiness is a size field at a fixed offset. */ +typedef struct { + PyTypeObject *type; + uint16_t offset; +} _PyToBoolSizedDescr; + +enum _PyToBoolSizedKind { + TO_BOOL_SIZED_DICT = 0, + TO_BOOL_SIZED_FROZENDICT, + TO_BOOL_SIZED_TUPLE, + TO_BOOL_SIZED_BYTES, + TO_BOOL_SIZED_BYTEARRAY, + TO_BOOL_SIZED_SET, + TO_BOOL_SIZED_FROZENSET, + TO_BOOL_SIZED_NKINDS, +}; + +PyAPI_DATA(const _PyToBoolSizedDescr) _PyToBoolSized_Table[TO_BOOL_SIZED_NKINDS]; + typedef struct { _Py_BackoffCounter counter; } _PyContainsOpCache; @@ -517,6 +536,50 @@ typedef struct { PyTypeObject *rhs_type; } _PyBinaryOpSpecializationDescr; +/* Kinds for BINARY_OP_INPLACE. Bit 0 selects the mutated operand + (0 = left, 1 = right); the kinds for one operator are consecutive. */ +enum _PyBinaryOpInplaceKind { + INPLACE_FLOAT_ADD_L = 0, + INPLACE_FLOAT_ADD_R, + INPLACE_FLOAT_SUB_L, + INPLACE_FLOAT_SUB_R, + INPLACE_FLOAT_MUL_L, + INPLACE_FLOAT_MUL_R, + INPLACE_FLOAT_TRUEDIV_L, + INPLACE_FLOAT_TRUEDIV_R, + INPLACE_INT_ADD_L, + INPLACE_INT_ADD_R, + INPLACE_INT_SUB_L, + INPLACE_INT_SUB_R, + INPLACE_INT_MUL_L, + INPLACE_INT_MUL_R, + INPLACE_NKINDS, +}; +#define INPLACE_FIRST_INT_KIND INPLACE_INT_ADD_L + +/* KIND_OP: one generic specialized instruction shared by several + instruction families. Specializations marked `kind_op` in bytecodes.c have + no opcode of their own; the cases generator emits them as cases of KIND_OP + and numbers them (KIND_ in pycore_opcode_metadata.h). The kind takes + the place of the adaptive counter in the first cache entry, so the + instruction keeps the cache layout of its family. _PyKindOp_Deopt[kind] is + the generic instruction KIND_OP re-dispatches to when a guard fails. */ +/* KIND_OP keeps its kind in the high bits of the adaptive counter's value + field and a miss counter in the low bits, so that a specialization folded + into KIND_OP survives sporadic misses like an ordinary specialized + instruction does: a miss runs the generic instruction once and counts + down through ADVANCE_ADAPTIVE_COUNTER; when the misses run out, the + instruction goes back to its adaptive form and re-specializes on the + operands it has then. */ +#define KIND_OP_MISS_BITS 6 +#define KIND_OP_MISS_MASK ((1 << KIND_OP_MISS_BITS) - 1) +#define KIND_OP_MISS_COOLDOWN ADAPTIVE_COOLDOWN_VALUE +#define KIND_OP_KIND_SHIFT (BACKOFF_BITS + KIND_OP_MISS_BITS) +#define KIND_OP_MAX_KINDS (1 << (16 - KIND_OP_KIND_SHIFT)) +#define _PyKindOp_Kind(instr) ((instr)[1].cache >> KIND_OP_KIND_SHIFT) +#define _PyKindOp_Counter(kind, backoff) \ + MAKE_VALUE_AND_BACKOFF(((kind) << KIND_OP_MISS_BITS) | KIND_OP_MISS_COOLDOWN, (backoff)) + /* Comparison bit masks. */ /* Note this evaluates its arguments twice each */ diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 15d3ad6f3a336b7..0409258c2eabc8c 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -304,6 +304,7 @@ Known values: Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) Python 3.16a1 3706 (Add INTRINSIC_MATCH_CLASS_ISINSTANCE) + Python 3.16a1 3707 (Add KIND_OP) Python 3.17 will start with 3750 @@ -313,7 +314,7 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3706 +#define PYC_MAGIC_NUMBER 3707 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d3546119bc9ee0e..f211e702fe871d0 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -298,6 +298,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case JUMP_NO_INTERRUPT: return 0; + case KIND_OP: + return 0; case LIST_APPEND: return 2 + (oparg-1); case LIST_EXTEND: @@ -793,6 +795,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case JUMP_NO_INTERRUPT: return 0; + case KIND_OP: + return 0; case LIST_APPEND: return 1 + (oparg-1); case LIST_EXTEND: @@ -1025,15 +1029,16 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { enum InstructionFormat { INSTR_FMT_IB = 1, INSTR_FMT_IBC = 2, - INSTR_FMT_IBC00 = 3, - INSTR_FMT_IBC000 = 4, - INSTR_FMT_IBC0000 = 5, - INSTR_FMT_IBC00000000 = 6, - INSTR_FMT_IX = 7, - INSTR_FMT_IXC = 8, - INSTR_FMT_IXC00 = 9, - INSTR_FMT_IXC000 = 10, - INSTR_FMT_IXC0000 = 11, + INSTR_FMT_IBC0 = 3, + INSTR_FMT_IBC00 = 4, + INSTR_FMT_IBC000 = 5, + INSTR_FMT_IBC0000 = 6, + INSTR_FMT_IBC00000000 = 7, + INSTR_FMT_IX = 8, + INSTR_FMT_IXC = 9, + INSTR_FMT_IXC00 = 10, + INSTR_FMT_IXC000 = 11, + INSTR_FMT_IXC0000 = 12, }; #define IS_VALID_OPCODE(OP) \ @@ -1226,6 +1231,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [JUMP_BACKWARD_NO_INTERRUPT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [JUMP_BACKWARD_NO_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, + [KIND_OP] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG }, [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1549,6 +1555,34 @@ _PyOpcode_macro_expansion[256] = { }; #endif // NEED_OPCODE_METADATA +/* Specializations dispatched by KIND_OP. The kind replaces the adaptive + counter in the first cache entry. The pseudo-opcodes let specialize.c + name a kind like an opcode: specialize() maps them to KIND_OP. */ +#define KIND_OP_NKINDS 2 +#define KIND_OP_PSEUDO_BASE 512 +#define IS_KIND_OP_PSEUDO_OPCODE(OP) ((OP) >= KIND_OP_PSEUDO_BASE) +#define KIND_BINARY_OP_INPLACE 0 +#define KIND_TO_BOOL_SIZED 1 +#define BINARY_OP_INPLACE (KIND_OP_PSEUDO_BASE + KIND_BINARY_OP_INPLACE) +#define TO_BOOL_SIZED (KIND_OP_PSEUDO_BASE + KIND_TO_BOOL_SIZED) +PyAPI_DATA(const uint8_t) _PyKindOp_Deopt[2]; +PyAPI_DATA(const char *) _PyKindOp_Name[2]; +extern const struct opcode_macro_expansion _PyKindOp_macro_expansion[2]; +#ifdef NEED_OPCODE_METADATA +const uint8_t _PyKindOp_Deopt[2] = { + [KIND_BINARY_OP_INPLACE] = BINARY_OP, + [KIND_TO_BOOL_SIZED] = TO_BOOL, +}; +const char *_PyKindOp_Name[2] = { + [KIND_BINARY_OP_INPLACE] = "BINARY_OP_INPLACE", + [KIND_TO_BOOL_SIZED] = "TO_BOOL_SIZED", +}; +const struct opcode_macro_expansion _PyKindOp_macro_expansion[2] = { + [KIND_BINARY_OP_INPLACE] = { .nuops = 2, .uops = { { _GUARD_BINARY_OP_INPLACE, 1, 1 }, { _BINARY_OP_INPLACE, 1, 1 } } }, + [KIND_TO_BOOL_SIZED] = { .nuops = 1, .uops = { { _TO_BOOL_SIZED, 1, 1 } } }, +}; +#endif // NEED_OPCODE_METADATA + PyAPI_DATA(const char) *_PyOpcode_OpName[267]; #ifdef NEED_OPCODE_METADATA const char *_PyOpcode_OpName[267] = { @@ -1683,6 +1717,7 @@ const char *_PyOpcode_OpName[267] = { [JUMP_IF_FALSE] = "JUMP_IF_FALSE", [JUMP_IF_TRUE] = "JUMP_IF_TRUE", [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", + [KIND_OP] = "KIND_OP", [LIST_APPEND] = "LIST_APPEND", [LIST_EXTEND] = "LIST_EXTEND", [LOAD_ATTR] = "LOAD_ATTR", @@ -1800,6 +1835,7 @@ const char *_PyOpcode_OpName[267] = { PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { + [KIND_OP] = 2, [RESUME] = 1, [TO_BOOL] = 3, [STORE_SUBSCR] = 1, @@ -1828,7 +1864,6 @@ const uint8_t _PyOpcode_Caches[256] = { PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [117] = 117, [118] = 118, [119] = 119, [120] = 120, @@ -1979,6 +2014,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT, [JUMP_BACKWARD_NO_JIT] = JUMP_BACKWARD, [JUMP_FORWARD] = JUMP_FORWARD, + [KIND_OP] = KIND_OP, [LIST_APPEND] = LIST_APPEND, [LIST_EXTEND] = LIST_EXTEND, [LOAD_ATTR] = LOAD_ATTR, @@ -2089,7 +2125,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 117: \ case 118: \ case 119: \ case 120: \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ba6e0252024a6df..c24510b566d3606 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -22,6 +22,7 @@ enum { _BINARY_OP_ADD_INT_INPLACE_RIGHT, _BINARY_OP_ADD_UNICODE, _BINARY_OP_EXTEND, + _BINARY_OP_INPLACE, _BINARY_OP_INPLACE_ADD_UNICODE, _BINARY_OP_MULTIPLY_FLOAT, _BINARY_OP_MULTIPLY_FLOAT_INPLACE, @@ -131,6 +132,7 @@ enum { _GUARD_BINARY_OP_EXTEND, _GUARD_BINARY_OP_EXTEND_LHS, _GUARD_BINARY_OP_EXTEND_RHS, + _GUARD_BINARY_OP_INPLACE, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, _GUARD_BIT_IS_SET_POP, _GUARD_BIT_IS_SET_POP_4, @@ -353,6 +355,7 @@ enum { _TO_BOOL, _TO_BOOL_INT, _TO_BOOL_LIST, + _TO_BOOL_SIZED, _TO_BOOL_STR, _UNARY_INVERT, _UNARY_NEGATIVE, @@ -366,7 +369,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 654 +#define MAX_UOP_ID 657 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -466,6 +469,7 @@ enum { _BINARY_OP_ADD_UNICODE_r13, _BINARY_OP_ADD_UNICODE_r23, _BINARY_OP_EXTEND_r23, + _BINARY_OP_INPLACE_r21, _BINARY_OP_INPLACE_ADD_UNICODE_r21, _BINARY_OP_MULTIPLY_FLOAT_r03, _BINARY_OP_MULTIPLY_FLOAT_r13, @@ -707,6 +711,10 @@ enum { _GUARD_BINARY_OP_EXTEND_RHS_r12, _GUARD_BINARY_OP_EXTEND_RHS_r22, _GUARD_BINARY_OP_EXTEND_RHS_r33, + _GUARD_BINARY_OP_INPLACE_r02, + _GUARD_BINARY_OP_INPLACE_r12, + _GUARD_BINARY_OP_INPLACE_r22, + _GUARD_BINARY_OP_INPLACE_r33, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22, @@ -1417,6 +1425,7 @@ enum { _TO_BOOL_NONE_r11, _TO_BOOL_NONE_r22, _TO_BOOL_NONE_r33, + _TO_BOOL_SIZED_r11, _TO_BOOL_STR_r02, _TO_BOOL_STR_r12, _TO_BOOL_STR_r23, @@ -1444,7 +1453,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1655 +#define MAX_UOP_REGS_ID 1664 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 86b449158f00d0d..1b84d5e8a8e64a2 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -99,6 +99,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_NOS_UNICODE] = HAS_EXIT_FLAG, [_GUARD_TOS_UNICODE] = HAS_EXIT_FLAG, [_TO_BOOL_STR] = 0, + [_TO_BOOL_SIZED] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_REPLACE_WITH_TRUE] = 0, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, @@ -120,6 +121,8 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_GUARD_BINARY_OP_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_INPLACE] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_ADD_FLOAT_INPLACE] = 0, [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = 0, [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = 0, @@ -1023,6 +1026,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_TO_BOOL_SIZED] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _TO_BOOL_SIZED_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_REPLACE_WITH_TRUE] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -1212,6 +1224,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_BINARY_OP_INPLACE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_INPLACE_r02 }, + { 2, 1, _GUARD_BINARY_OP_INPLACE_r12 }, + { 2, 2, _GUARD_BINARY_OP_INPLACE_r22 }, + { 3, 3, _GUARD_BINARY_OP_INPLACE_r33 }, + }, + }, + [_BINARY_OP_INPLACE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_INPLACE_r21 }, + { -1, -1, -1 }, + }, + }, [_BINARY_OP_ADD_FLOAT_INPLACE] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -4113,6 +4143,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_TO_BOOL_STR_r02] = _TO_BOOL_STR, [_TO_BOOL_STR_r12] = _TO_BOOL_STR, [_TO_BOOL_STR_r23] = _TO_BOOL_STR, + [_TO_BOOL_SIZED_r11] = _TO_BOOL_SIZED, [_REPLACE_WITH_TRUE_r02] = _REPLACE_WITH_TRUE, [_REPLACE_WITH_TRUE_r12] = _REPLACE_WITH_TRUE, [_REPLACE_WITH_TRUE_r23] = _REPLACE_WITH_TRUE, @@ -4181,6 +4212,11 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT_r03] = _BINARY_OP_SUBTRACT_FLOAT, [_BINARY_OP_SUBTRACT_FLOAT_r13] = _BINARY_OP_SUBTRACT_FLOAT, [_BINARY_OP_SUBTRACT_FLOAT_r23] = _BINARY_OP_SUBTRACT_FLOAT, + [_GUARD_BINARY_OP_INPLACE_r02] = _GUARD_BINARY_OP_INPLACE, + [_GUARD_BINARY_OP_INPLACE_r12] = _GUARD_BINARY_OP_INPLACE, + [_GUARD_BINARY_OP_INPLACE_r22] = _GUARD_BINARY_OP_INPLACE, + [_GUARD_BINARY_OP_INPLACE_r33] = _GUARD_BINARY_OP_INPLACE, + [_BINARY_OP_INPLACE_r21] = _BINARY_OP_INPLACE, [_BINARY_OP_ADD_FLOAT_INPLACE_r03] = _BINARY_OP_ADD_FLOAT_INPLACE, [_BINARY_OP_ADD_FLOAT_INPLACE_r13] = _BINARY_OP_ADD_FLOAT_INPLACE, [_BINARY_OP_ADD_FLOAT_INPLACE_r23] = _BINARY_OP_ADD_FLOAT_INPLACE, @@ -4899,6 +4935,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_ADD_UNICODE_r23] = "_BINARY_OP_ADD_UNICODE_r23", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", [_BINARY_OP_EXTEND_r23] = "_BINARY_OP_EXTEND_r23", + [_BINARY_OP_INPLACE] = "_BINARY_OP_INPLACE", + [_BINARY_OP_INPLACE_r21] = "_BINARY_OP_INPLACE_r21", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_INPLACE_ADD_UNICODE_r21] = "_BINARY_OP_INPLACE_ADD_UNICODE_r21", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", @@ -5262,6 +5300,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_BINARY_OP_EXTEND_RHS_r12] = "_GUARD_BINARY_OP_EXTEND_RHS_r12", [_GUARD_BINARY_OP_EXTEND_RHS_r22] = "_GUARD_BINARY_OP_EXTEND_RHS_r22", [_GUARD_BINARY_OP_EXTEND_RHS_r33] = "_GUARD_BINARY_OP_EXTEND_RHS_r33", + [_GUARD_BINARY_OP_INPLACE] = "_GUARD_BINARY_OP_INPLACE", + [_GUARD_BINARY_OP_INPLACE_r02] = "_GUARD_BINARY_OP_INPLACE_r02", + [_GUARD_BINARY_OP_INPLACE_r12] = "_GUARD_BINARY_OP_INPLACE_r12", + [_GUARD_BINARY_OP_INPLACE_r22] = "_GUARD_BINARY_OP_INPLACE_r22", + [_GUARD_BINARY_OP_INPLACE_r33] = "_GUARD_BINARY_OP_INPLACE_r33", [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS", [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02", [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12", @@ -6154,6 +6197,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_TO_BOOL_NONE_r11] = "_TO_BOOL_NONE_r11", [_TO_BOOL_NONE_r22] = "_TO_BOOL_NONE_r22", [_TO_BOOL_NONE_r33] = "_TO_BOOL_NONE_r33", + [_TO_BOOL_SIZED] = "_TO_BOOL_SIZED", + [_TO_BOOL_SIZED_r11] = "_TO_BOOL_SIZED_r11", [_TO_BOOL_STR] = "_TO_BOOL_STR", [_TO_BOOL_STR_r02] = "_TO_BOOL_STR_r02", [_TO_BOOL_STR_r12] = "_TO_BOOL_STR_r12", @@ -6326,6 +6371,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _TO_BOOL_STR: return 1; + case _TO_BOOL_SIZED: + return 1; case _REPLACE_WITH_TRUE: return 1; case _UNARY_INVERT: @@ -6368,6 +6415,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; + case _GUARD_BINARY_OP_INPLACE: + return 0; + case _BINARY_OP_INPLACE: + return 2; case _BINARY_OP_ADD_FLOAT_INPLACE: return 2; case _BINARY_OP_SUBTRACT_FLOAT_INPLACE: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 11342ae451b9f6c..4c4d8fc4c4fe3f6 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -84,49 +84,50 @@ extern "C" { #define JUMP_BACKWARD 71 #define JUMP_BACKWARD_NO_INTERRUPT 72 #define JUMP_FORWARD 73 -#define LIST_APPEND 74 -#define LIST_EXTEND 75 -#define LOAD_ATTR 76 -#define LOAD_COMMON_CONSTANT 77 -#define LOAD_CONST 78 -#define LOAD_DEREF 79 -#define LOAD_FAST 80 -#define LOAD_FAST_AND_CLEAR 81 -#define LOAD_FAST_BORROW 82 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 83 -#define LOAD_FAST_CHECK 84 -#define LOAD_FAST_LOAD_FAST 85 -#define LOAD_FROM_DICT_OR_DEREF 86 -#define LOAD_FROM_DICT_OR_GLOBALS 87 -#define LOAD_GLOBAL 88 -#define LOAD_NAME 89 -#define LOAD_SMALL_INT 90 -#define LOAD_SPECIAL 91 -#define LOAD_SUPER_ATTR 92 -#define MAKE_CELL 93 -#define MAP_ADD 94 -#define MATCH_CLASS 95 -#define POP_JUMP_IF_FALSE 96 -#define POP_JUMP_IF_NONE 97 -#define POP_JUMP_IF_NOT_NONE 98 -#define POP_JUMP_IF_TRUE 99 -#define RAISE_VARARGS 100 -#define RERAISE 101 -#define SEND 102 -#define SET_ADD 103 -#define SET_FUNCTION_ATTRIBUTE 104 -#define SET_UPDATE 105 -#define STORE_ATTR 106 -#define STORE_DEREF 107 -#define STORE_FAST 108 -#define STORE_FAST_LOAD_FAST 109 -#define STORE_FAST_STORE_FAST 110 -#define STORE_GLOBAL 111 -#define STORE_NAME 112 -#define SWAP 113 -#define UNPACK_EX 114 -#define UNPACK_SEQUENCE 115 -#define YIELD_VALUE 116 +#define KIND_OP 74 +#define LIST_APPEND 75 +#define LIST_EXTEND 76 +#define LOAD_ATTR 77 +#define LOAD_COMMON_CONSTANT 78 +#define LOAD_CONST 79 +#define LOAD_DEREF 80 +#define LOAD_FAST 81 +#define LOAD_FAST_AND_CLEAR 82 +#define LOAD_FAST_BORROW 83 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 84 +#define LOAD_FAST_CHECK 85 +#define LOAD_FAST_LOAD_FAST 86 +#define LOAD_FROM_DICT_OR_DEREF 87 +#define LOAD_FROM_DICT_OR_GLOBALS 88 +#define LOAD_GLOBAL 89 +#define LOAD_NAME 90 +#define LOAD_SMALL_INT 91 +#define LOAD_SPECIAL 92 +#define LOAD_SUPER_ATTR 93 +#define MAKE_CELL 94 +#define MAP_ADD 95 +#define MATCH_CLASS 96 +#define POP_JUMP_IF_FALSE 97 +#define POP_JUMP_IF_NONE 98 +#define POP_JUMP_IF_NOT_NONE 99 +#define POP_JUMP_IF_TRUE 100 +#define RAISE_VARARGS 101 +#define RERAISE 102 +#define SEND 103 +#define SET_ADD 104 +#define SET_FUNCTION_ATTRIBUTE 105 +#define SET_UPDATE 106 +#define STORE_ATTR 107 +#define STORE_DEREF 108 +#define STORE_FAST 109 +#define STORE_FAST_LOAD_FAST 110 +#define STORE_FAST_STORE_FAST 111 +#define STORE_GLOBAL 112 +#define STORE_NAME 113 +#define SWAP 114 +#define UNPACK_EX 115 +#define UNPACK_SEQUENCE 116 +#define YIELD_VALUE 117 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index df92eae151d2488..9c4c0a5b453bc99 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -226,6 +226,11 @@ UNPACK_SEQUENCE_TWO_TUPLE=218, ) +_kind_ops = ( + "BINARY_OP_INPLACE", + "TO_BOOL_SIZED", +) + opmap = frozendict( CACHE=0, RESERVED=17, @@ -304,49 +309,50 @@ JUMP_BACKWARD=71, JUMP_BACKWARD_NO_INTERRUPT=72, JUMP_FORWARD=73, - LIST_APPEND=74, - LIST_EXTEND=75, - LOAD_ATTR=76, - LOAD_COMMON_CONSTANT=77, - LOAD_CONST=78, - LOAD_DEREF=79, - LOAD_FAST=80, - LOAD_FAST_AND_CLEAR=81, - LOAD_FAST_BORROW=82, - LOAD_FAST_BORROW_LOAD_FAST_BORROW=83, - LOAD_FAST_CHECK=84, - LOAD_FAST_LOAD_FAST=85, - LOAD_FROM_DICT_OR_DEREF=86, - LOAD_FROM_DICT_OR_GLOBALS=87, - LOAD_GLOBAL=88, - LOAD_NAME=89, - LOAD_SMALL_INT=90, - LOAD_SPECIAL=91, - LOAD_SUPER_ATTR=92, - MAKE_CELL=93, - MAP_ADD=94, - MATCH_CLASS=95, - POP_JUMP_IF_FALSE=96, - POP_JUMP_IF_NONE=97, - POP_JUMP_IF_NOT_NONE=98, - POP_JUMP_IF_TRUE=99, - RAISE_VARARGS=100, - RERAISE=101, - SEND=102, - SET_ADD=103, - SET_FUNCTION_ATTRIBUTE=104, - SET_UPDATE=105, - STORE_ATTR=106, - STORE_DEREF=107, - STORE_FAST=108, - STORE_FAST_LOAD_FAST=109, - STORE_FAST_STORE_FAST=110, - STORE_GLOBAL=111, - STORE_NAME=112, - SWAP=113, - UNPACK_EX=114, - UNPACK_SEQUENCE=115, - YIELD_VALUE=116, + KIND_OP=74, + LIST_APPEND=75, + LIST_EXTEND=76, + LOAD_ATTR=77, + LOAD_COMMON_CONSTANT=78, + LOAD_CONST=79, + LOAD_DEREF=80, + LOAD_FAST=81, + LOAD_FAST_AND_CLEAR=82, + LOAD_FAST_BORROW=83, + LOAD_FAST_BORROW_LOAD_FAST_BORROW=84, + LOAD_FAST_CHECK=85, + LOAD_FAST_LOAD_FAST=86, + LOAD_FROM_DICT_OR_DEREF=87, + LOAD_FROM_DICT_OR_GLOBALS=88, + LOAD_GLOBAL=89, + LOAD_NAME=90, + LOAD_SMALL_INT=91, + LOAD_SPECIAL=92, + LOAD_SUPER_ATTR=93, + MAKE_CELL=94, + MAP_ADD=95, + MATCH_CLASS=96, + POP_JUMP_IF_FALSE=97, + POP_JUMP_IF_NONE=98, + POP_JUMP_IF_NOT_NONE=99, + POP_JUMP_IF_TRUE=100, + RAISE_VARARGS=101, + RERAISE=102, + SEND=103, + SET_ADD=104, + SET_FUNCTION_ATTRIBUTE=105, + SET_UPDATE=106, + STORE_ATTR=107, + STORE_DEREF=108, + STORE_FAST=109, + STORE_FAST_LOAD_FAST=110, + STORE_FAST_STORE_FAST=111, + STORE_GLOBAL=112, + STORE_NAME=113, + SWAP=114, + UNPACK_EX=115, + UNPACK_SEQUENCE=116, + YIELD_VALUE=117, INSTRUMENTED_END_FOR=233, INSTRUMENTED_POP_ITER=234, INSTRUMENTED_END_SEND=235, diff --git a/Lib/dis.py b/Lib/dis.py index 318729091fa098c..823571e9641a25c 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -17,6 +17,7 @@ _special_method_names, _specializations, _specialized_opmap, + _kind_ops, ) from _opcode import get_executor @@ -35,6 +36,8 @@ FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate') ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] +KIND_OP = opmap['KIND_OP'] +_KIND_OP_KIND_SHIFT = 9 # BACKOFF_BITS + KIND_OP_MISS_BITS IMPORT_NAME = opmap['IMPORT_NAME'] LOAD_GLOBAL = opmap['LOAD_GLOBAL'] LOAD_SMALL_INT = opmap['LOAD_SMALL_INT'] @@ -804,6 +807,12 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N argval, argrepr = arg_resolver.get_argval_argrepr(op, arg, offset) else: argval, argrepr = arg, repr(arg) + if op == KIND_OP and offset + 3 < len(code): + # The kind sits in the high bits of the first cache entry, above + # the backoff bits and the miss counter (KIND_OP_KIND_SHIFT). + kind = (code[offset + 2] | (code[offset + 3] << 8)) >> _KIND_OP_KIND_SHIFT + if kind < len(_kind_ops): + argrepr = f"{argrepr} [{_kind_ops[kind]}]" if argrepr else _kind_ops[kind] caches = _get_cache_size(_all_opname[deop]) # Advance the co_positions iterator: diff --git a/Lib/opcode.py b/Lib/opcode.py index 750d83b2c87af58..a56c62913e4383c 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -13,7 +13,7 @@ import _opcode from _opcode import stack_effect -from _opcode_metadata import (_specializations, _specialized_opmap, opmap, # noqa: F401 +from _opcode_metadata import (_specializations, _specialized_opmap, _kind_ops, opmap, # noqa: F401 HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) # noqa: F401 EXTENDED_ARG = opmap['EXTENDED_ARG'] diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 60879e2774e7077..361ba98b8db8ae2 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -28,15 +28,22 @@ def have_dict_key_versions(): class TestBase(unittest.TestCase): + @staticmethod + def _specialized_names(f): + # A specialization folded into KIND_OP shows up as KIND_OP with the + # kind's name in the argrepr, e.g. "KIND_OP 0 [BINARY_OP_ADD_INT]". + names = set() + for instruction in dis.get_instructions(f, adaptive=True): + names.add(instruction.opname) + if instruction.opname == "KIND_OP": + names.add(instruction.argrepr.rsplit("[", 1)[-1].rstrip("]")) + return names + def assert_specialized(self, f, opname): - instructions = dis.get_instructions(f, adaptive=True) - opnames = {instruction.opname for instruction in instructions} - self.assertIn(opname, opnames) + self.assertIn(opname, self._specialized_names(f)) def assert_no_opcode(self, f, opname): - instructions = dis.get_instructions(f, adaptive=True) - opnames = {instruction.opname for instruction in instructions} - self.assertNotIn(opname, opnames) + self.assertNotIn(opname, self._specialized_names(f)) class TestLoadSuperAttrCache(unittest.TestCase): diff --git a/Modules/_imp* b/Modules/_imp* new file mode 100644 index 000000000000000..e69de29bb2d1d64 diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 3bdc16437e2bc61..39b1e1488d7163a 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -6167,7 +6167,7 @@ int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { - if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]] && og_opcode != KIND_OP) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } opcode = og_opcode; @@ -6185,7 +6185,7 @@ opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] && opcode != KIND_OP) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } DISPATCH_GOTO(); @@ -7712,7 +7712,7 @@ JUMP_TO_LABEL(error); } next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { + if (_PyOpcode_Caches[next_opcode] && next_opcode != KIND_OP) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } assert(next_opcode > 0 && next_opcode < 256); @@ -7798,7 +7798,7 @@ DISPATCH(); } } - if (_PyOpcode_Caches[original_opcode]) { + if (_PyOpcode_Caches[original_opcode] && original_opcode != KIND_OP) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); PAUSE_ADAPTIVE_COUNTER(cache->counter); } @@ -8573,6 +8573,116 @@ DISPATCH(); } + TARGET(KIND_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = KIND_OP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 3; + INSTRUCTION_STATS(KIND_OP); + uint16_t kind = read_u16(&this_instr[1].cache) >> KIND_OP_KIND_SHIFT; + assert(kind < KIND_OP_NKINDS); + switch (kind) { + case KIND_BINARY_OP_INPLACE: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_INPLACE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t kind = read_u16(&this_instr[2].cache); + _PyStackRef target = (kind & 1) ? right : left; + if (!PyStackRef_RefcountOnObject(target)) { + KIND_OP_DEOPT(BINARY_OP); + } + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (kind < INPLACE_FIRST_INT_KIND) { + if (!PyFloat_CheckExact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + else { + if (!_PyLong_CheckExactAndCompact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + if (!_PyLong_CheckExactAndCompact(right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(PyStackRef_AsPyObjectBorrow(target))) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip -1 cache entry */ + // _BINARY_OP_INPLACE + { + uint16_t kind = read_u16(&this_instr[2].cache); + _PyStackRef target; + int inplace_ok = 1; + int zero_division = 0; + BINARY_OP_INPLACE_SWITCH(kind, left, right, target, inplace_ok, zero_division); + if (!inplace_ok) { + KIND_OP_DEOPT(BINARY_OP); + } + if (zero_division) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + STAT_INC(BINARY_OP, hit); + res = target; + } + /* Skip 3 cache entries */ + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_TO_BOOL_SIZED: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_TO_BOOL; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + uint16_t kind = read_u16(&this_instr[2].cache); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + const _PyToBoolSizedDescr *d = &_PyToBoolSized_Table[kind]; + if (Py_TYPE(value_o) != d->type) { + KIND_OP_DEOPT(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + Py_ssize_t size = FT_ATOMIC_LOAD_SSIZE_RELAXED( + *(Py_ssize_t *)((char *)value_o + d->offset)); + res = size ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + /* Skip 1 cache entry */ + DISPATCH(); + } + default: + Py_UNREACHABLE(); + } + } + TARGET(LIST_APPEND) { #if _Py_TAIL_CALL_INTERP int opcode = LIST_APPEND; diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4b..6ad781b1c71033a 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -74,6 +74,7 @@ static void *opcode_targets_table[256] = { &&TARGET_JUMP_BACKWARD, &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, &&TARGET_JUMP_FORWARD, + &&TARGET_KIND_OP, &&TARGET_LIST_APPEND, &&TARGET_LIST_EXTEND, &&TARGET_LOAD_ATTR, @@ -127,7 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -376,7 +376,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -657,6 +657,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARA static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_KIND_OP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); @@ -900,6 +901,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, + [KIND_OP] = _TAIL_CALL_KIND_OP, [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, [LOAD_ATTR] = _TAIL_CALL_LOAD_ATTR, @@ -1005,7 +1007,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1158,6 +1159,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACE_RECORD, [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACE_RECORD, [JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [KIND_OP] = _TAIL_CALL_TRACE_RECORD, [LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, [LIST_EXTEND] = _TAIL_CALL_TRACE_RECORD, [LOAD_ATTR] = _TAIL_CALL_TRACE_RECORD, @@ -1263,7 +1265,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 31eaeab0d67841f..870b50607a99d84 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -489,6 +489,7 @@ dummy_func( TO_BOOL_INT, TO_BOOL_LIST, TO_BOOL_NONE, + TO_BOOL_SIZED, TO_BOOL_STR, }; @@ -513,7 +514,7 @@ dummy_func( macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL; - inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { + kind_candidate inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { EXIT_IF(!PyStackRef_BoolCheck(value)); STAT_INC(TO_BOOL, hit); } @@ -526,7 +527,7 @@ dummy_func( DEAD(value); } - macro(TO_BOOL_INT) = + kind_candidate macro(TO_BOOL_INT) = _GUARD_TOS_EXACT_INT + unused/1 + unused/2 + _TO_BOOL_INT + _POP_TOP_INT; op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) { @@ -544,7 +545,7 @@ dummy_func( EXIT_IF(!PySlice_Check(o)); } - macro(TO_BOOL_LIST) = _GUARD_TOS_LIST + unused/1 + unused/2 + _TO_BOOL_LIST + POP_TOP; + kind_candidate macro(TO_BOOL_LIST) = _GUARD_TOS_LIST + unused/1 + unused/2 + _TO_BOOL_LIST + POP_TOP; op(_TO_BOOL_LIST, (value -- res, v)) { PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); @@ -555,7 +556,7 @@ dummy_func( DEAD(value); } - inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { + kind_candidate inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: EXIT_IF(!PyStackRef_IsNone(value)); DEAD(value); @@ -587,16 +588,34 @@ dummy_func( DEAD(value); } - macro(TO_BOOL_STR) = + kind_candidate macro(TO_BOOL_STR) = _GUARD_TOS_UNICODE + unused/1 + unused/2 + _TO_BOOL_STR + _POP_TOP_UNICODE; + // One opcode for every type whose truthiness is a Py_ssize_t size + // field at a fixed offset: dict and frozendict (ma_used), tuple, + // bytes and bytearray (ob_size), set and frozenset (used). `kind` + // indexes _PyToBoolSized_Table, which holds the type to guard on + // and the offset of the size field. + op(_TO_BOOL_SIZED, (kind/1, value -- res)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + const _PyToBoolSizedDescr *d = &_PyToBoolSized_Table[kind]; + EXIT_IF(Py_TYPE(value_o) != d->type); + STAT_INC(TO_BOOL, hit); + Py_ssize_t size = FT_ATOMIC_LOAD_SSIZE_RELAXED( + *(Py_ssize_t *)((char *)value_o + d->offset)); + res = size ? PyStackRef_True : PyStackRef_False; + PyStackRef_CLOSE(value); + } + + kind_op macro(TO_BOOL_SIZED) = unused/1 + _TO_BOOL_SIZED + unused/1; + op(_REPLACE_WITH_TRUE, (value -- res, v)) { res = PyStackRef_True; v = value; DEAD(value); } - macro(TO_BOOL_ALWAYS_TRUE) = + kind_candidate macro(TO_BOOL_ALWAYS_TRUE) = unused/1 + _RECORD_TOS_TYPE + _GUARD_TYPE_VERSION + @@ -632,6 +651,7 @@ dummy_func( BINARY_OP_SUBSCR_GETITEM, BINARY_OP_INPLACE_ADD_UNICODE, BINARY_OP_EXTEND, + BINARY_OP_INPLACE, }; op(_GUARD_NOS_INT, (left, unused -- left, unused)) { @@ -706,13 +726,13 @@ dummy_func( INPUTS_DEAD(); } - macro(BINARY_OP_MULTIPLY_INT) = + kind_candidate macro(BINARY_OP_MULTIPLY_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_MULTIPLY_INT + _POP_TOP_INT + _POP_TOP_INT; - macro(BINARY_OP_ADD_INT) = + kind_candidate macro(BINARY_OP_ADD_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT + _POP_TOP_INT + _POP_TOP_INT; - macro(BINARY_OP_SUBTRACT_INT) = + kind_candidate macro(BINARY_OP_SUBTRACT_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT; // Inplace compact int ops: mutate the uniquely-referenced operand @@ -842,13 +862,58 @@ dummy_func( INPUTS_DEAD(); } - macro(BINARY_OP_MULTIPLY_FLOAT) = + kind_candidate macro(BINARY_OP_MULTIPLY_FLOAT) = _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT + _POP_TOP_FLOAT + _POP_TOP_FLOAT; - macro(BINARY_OP_ADD_FLOAT) = + kind_candidate macro(BINARY_OP_ADD_FLOAT) = _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_ADD_FLOAT + _POP_TOP_FLOAT + _POP_TOP_FLOAT; - macro(BINARY_OP_SUBTRACT_FLOAT) = + kind_candidate macro(BINARY_OP_SUBTRACT_FLOAT) = _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_SUBTRACT_FLOAT + _POP_TOP_FLOAT + _POP_TOP_FLOAT; + // In-place binary ops for tier 1. A single opcode: `kind` (a uint16 + // in the cache, see _PyBinaryOpInplaceKind) selects the operator and + // the operand that is mutated. The mutated operand must be an owned + // stack reference (not borrowed from a local) to a uniquely + // referenced object, which the specializer checks when it picks the + // kind and the guard re-checks at runtime. + op(_GUARD_BINARY_OP_INPLACE, (kind/1, left, right -- left, right)) { + _PyStackRef target = (kind & 1) ? right : left; + EXIT_IF(!PyStackRef_RefcountOnObject(target)); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (kind < INPLACE_FIRST_INT_KIND) { + EXIT_IF(!PyFloat_CheckExact(left_o)); + EXIT_IF(!PyFloat_CheckExact(right_o)); + } + else { + EXIT_IF(!_PyLong_CheckExactAndCompact(left_o)); + EXIT_IF(!_PyLong_CheckExactAndCompact(right_o)); + } + EXIT_IF(!BINARY_OP_INPLACE_IS_UNIQUE(PyStackRef_AsPyObjectBorrow(target))); + } + + op(_BINARY_OP_INPLACE, (kind/1, left, right -- res)) { + _PyStackRef target; + int inplace_ok = 1; + int zero_division = 0; + // The switch lives in ceval_macros.h: the DSL does not accept + // switch statements. It mutates the target in place and closes + // the other operand. + BINARY_OP_INPLACE_SWITCH(kind, left, right, target, inplace_ok, zero_division); + // A compact int result that is a small int or needs more than + // one digit: nothing was mutated, use the generic instruction. + DEOPT_IF(!inplace_ok); + if (zero_division) { + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); + ERROR_NO_POP(); + } + STAT_INC(BINARY_OP, hit); + res = target; + INPUTS_DEAD(); + } + + kind_op macro(BINARY_OP_INPLACE) = + unused/1 + _GUARD_BINARY_OP_INPLACE + rewind/-1 + _BINARY_OP_INPLACE + unused/3; + // Inplace float ops: mutate the uniquely-referenced left operand // instead of allocating a new float. Tier 2 only. // The optimizer sets l to a borrowed value so the following _POP_TOP_FLOAT @@ -966,7 +1031,7 @@ dummy_func( INPUTS_DEAD(); } - macro(BINARY_OP_ADD_UNICODE) = + kind_candidate macro(BINARY_OP_ADD_UNICODE) = _GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_ADD_UNICODE + _POP_TOP_UNICODE + _POP_TOP_UNICODE; // This is a subtle one. We write NULL to the local @@ -1139,7 +1204,7 @@ dummy_func( macro(STORE_SLICE) = _SPECIALIZE_STORE_SLICE + _STORE_SLICE; - macro(BINARY_OP_SUBSCR_LIST_INT) = + kind_candidate macro(BINARY_OP_SUBSCR_LIST_INT) = _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/5 + _BINARY_OP_SUBSCR_LIST_INT + _POP_TOP_INT + POP_TOP; op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res, ls, ss)) { @@ -1169,7 +1234,7 @@ dummy_func( INPUTS_DEAD(); } - macro(BINARY_OP_SUBSCR_LIST_SLICE) = + kind_candidate macro(BINARY_OP_SUBSCR_LIST_SLICE) = _GUARD_TOS_SLICE + _GUARD_NOS_LIST + unused/5 + _BINARY_OP_SUBSCR_LIST_SLICE + POP_TOP + POP_TOP; op(_BINARY_OP_SUBSCR_LIST_SLICE, (list_st, sub_st -- res, ls, ss)) { @@ -1190,7 +1255,7 @@ dummy_func( INPUTS_DEAD(); } - macro(BINARY_OP_SUBSCR_STR_INT) = + kind_candidate macro(BINARY_OP_SUBSCR_STR_INT) = _GUARD_TOS_INT + _GUARD_NOS_COMPACT_ASCII + unused/5 + _BINARY_OP_SUBSCR_STR_INT + _POP_TOP_INT + _POP_TOP_UNICODE; op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res, s, i)) { @@ -1212,7 +1277,7 @@ dummy_func( res = PyStackRef_FromPyObjectBorrow(res_o); } - macro(BINARY_OP_SUBSCR_USTR_INT) = + kind_candidate macro(BINARY_OP_SUBSCR_USTR_INT) = _GUARD_TOS_INT + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_SUBSCR_USTR_INT + _POP_TOP_INT + _POP_TOP_UNICODE; op(_BINARY_OP_SUBSCR_USTR_INT, (str_st, sub_st -- res, s, i)) { @@ -1245,7 +1310,7 @@ dummy_func( EXIT_IF(!PyTuple_CheckExact(o)); } - macro(BINARY_OP_SUBSCR_TUPLE_INT) = + kind_candidate macro(BINARY_OP_SUBSCR_TUPLE_INT) = _GUARD_TOS_INT + _GUARD_NOS_TUPLE + _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS + @@ -1312,7 +1377,7 @@ dummy_func( EXIT_IF(!PyFrozenDict_CheckExact(o)); } - macro(BINARY_OP_SUBSCR_DICT) = + kind_candidate macro(BINARY_OP_SUBSCR_DICT) = _RECORD_NOS_TYPE + _GUARD_NOS_DICT_SUBSCRIPT + unused/5 + _BINARY_OP_SUBSCR_DICT + POP_TOP + POP_TOP; @@ -1417,7 +1482,7 @@ dummy_func( macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; - macro(STORE_SUBSCR_LIST_INT) = + kind_candidate macro(STORE_SUBSCR_LIST_INT) = _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/1 + _STORE_SUBSCR_LIST_INT + _POP_TOP_INT + POP_TOP; op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- ls, ss)) { @@ -1451,7 +1516,7 @@ dummy_func( Py_DECREF(old_value); } - macro(STORE_SUBSCR_DICT) = + kind_candidate macro(STORE_SUBSCR_DICT) = _RECORD_NOS_TYPE + _GUARD_NOS_DICT_STORE_SUBSCRIPT + unused/1 + _STORE_SUBSCR_DICT + POP_TOP; @@ -2040,7 +2105,7 @@ dummy_func( macro(UNPACK_SEQUENCE) = _SPECIALIZE_UNPACK_SEQUENCE + _UNPACK_SEQUENCE; - macro(UNPACK_SEQUENCE_TWO_TUPLE) = + kind_candidate macro(UNPACK_SEQUENCE_TWO_TUPLE) = _GUARD_TOS_TUPLE + unused/1 + _UNPACK_SEQUENCE_TWO_TUPLE; op(_UNPACK_SEQUENCE_TWO_TUPLE, (seq -- val1, val0)) { @@ -2073,7 +2138,7 @@ dummy_func( _PyStolenTuple_Free(seq_o); } - macro(UNPACK_SEQUENCE_TUPLE) = + kind_candidate macro(UNPACK_SEQUENCE_TUPLE) = _GUARD_TOS_TUPLE + unused/1 + _UNPACK_SEQUENCE_TUPLE; op(_UNPACK_SEQUENCE_TUPLE, (seq -- values[oparg])) { @@ -2102,7 +2167,7 @@ dummy_func( _PyStolenTuple_Free(seq_o); } - macro(UNPACK_SEQUENCE_LIST) = + kind_candidate macro(UNPACK_SEQUENCE_LIST) = _GUARD_TOS_LIST + unused/1 + _UNPACK_SEQUENCE_LIST; op(_UNPACK_SEQUENCE_LIST, (seq -- values[oparg])) { @@ -3221,13 +3286,13 @@ dummy_func( macro(COMPARE_OP) = _SPECIALIZE_COMPARE_OP + _COMPARE_OP; - macro(COMPARE_OP_FLOAT) = + kind_candidate macro(COMPARE_OP_FLOAT) = _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/1 + _COMPARE_OP_FLOAT + _POP_TOP_FLOAT + _POP_TOP_FLOAT; - macro(COMPARE_OP_INT) = + kind_candidate macro(COMPARE_OP_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/1 + _COMPARE_OP_INT + _POP_TOP_INT + _POP_TOP_INT; - macro(COMPARE_OP_STR) = + kind_candidate macro(COMPARE_OP_STR) = _GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/1 + _COMPARE_OP_STR + _POP_TOP_UNICODE + _POP_TOP_UNICODE; op(_COMPARE_OP_FLOAT, (left, right -- res, l, r)) { @@ -3346,7 +3411,7 @@ dummy_func( EXIT_IF(!PyFrozenSet_CheckExact(o)); } - macro(CONTAINS_OP_SET) = _GUARD_TOS_ANY_SET + unused/1 + _CONTAINS_OP_SET + POP_TOP + POP_TOP; + kind_candidate macro(CONTAINS_OP_SET) = _GUARD_TOS_ANY_SET + unused/1 + _CONTAINS_OP_SET + POP_TOP + POP_TOP; op(_CONTAINS_OP_SET, (left, right -- b, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -3365,7 +3430,7 @@ dummy_func( INPUTS_DEAD(); } - macro(CONTAINS_OP_DICT) = _GUARD_TOS_ANY_DICT + unused/1 + _CONTAINS_OP_DICT + POP_TOP + POP_TOP; + kind_candidate macro(CONTAINS_OP_DICT) = _GUARD_TOS_ANY_DICT + unused/1 + _CONTAINS_OP_DICT + POP_TOP + POP_TOP; op(_CONTAINS_OP_DICT, (left, right -- b, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -3547,6 +3612,18 @@ dummy_func( COPY, TO_BOOL, POP_JUMP_IF_TRUE, ]; + // KIND_OP: one generic specialized instruction shared by several + // instruction families. Specializations annotated `kind_op` (and, + // with KIND_OPS=2 at regen time, `kind_candidate`) have no opcode of + // their own: the cases generator emits them as cases of this + // instruction, dispatched on the kind stored in the first cache + // entry in place of the adaptive counter. The body below is only + // used when no specialization is folded. + tier1 inst(KIND_OP, (unused/1, unused/1 -- )) { + (void)oparg; + Py_UNREACHABLE(); + } + tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); @@ -3556,7 +3633,7 @@ dummy_func( int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { - if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]] && og_opcode != KIND_OP) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } opcode = og_opcode; @@ -3576,7 +3653,7 @@ dummy_func( opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] && opcode != KIND_OP) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } DISPATCH_GOTO(); @@ -5930,7 +6007,7 @@ dummy_func( DISPATCH(); } } - if (_PyOpcode_Caches[original_opcode]) { + if (_PyOpcode_Caches[original_opcode] && original_opcode != KIND_OP) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); /* Prevent the underlying instruction from specializing * and overwriting the instrumentation. */ @@ -5946,7 +6023,7 @@ dummy_func( tstate, frame, this_instr); ERROR_IF(next_opcode < 0); next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { + if (_PyOpcode_Caches[next_opcode] && next_opcode != KIND_OP) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } assert(next_opcode > 0 && next_opcode < 256); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 36921e35533433d..42c28f49eccfa94 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -594,6 +594,120 @@ gen_try_set_executing(PyGenObject *gen) return false; } +// BINARY_OP_INPLACE helpers (tier 1). +// MSVC does not inline Py_REFCNT() inside the eval loop, so read the field +// directly in the default build. +#ifdef Py_GIL_DISABLED +# define BINARY_OP_INPLACE_IS_UNIQUE(o) _PyObject_IsUniquelyReferenced(o) +#else +# define BINARY_OP_INPLACE_IS_UNIQUE(o) ((o)->ob_refcnt == 1) +#endif + +// One case of the kind switch: mutate TGT_O in place, hand TGT back as the +// result and release the other operand. +#define BINARY_OP_INPLACE_FLOAT_CASE(KIND, TGT_O, TGT, OTH, OP, TARGET) \ + case KIND: \ + ((PyFloatObject *)(TGT_O))->ob_fval = \ + ((PyFloatObject *)_left_o)->ob_fval OP \ + ((PyFloatObject *)_right_o)->ob_fval; \ + (TARGET) = (TGT); \ + PyStackRef_CLOSE_SPECIALIZED((OTH), _PyFloat_ExactDealloc); \ + break; + +// Float true division: a zero divisor raises, nothing is mutated. +#define BINARY_OP_INPLACE_TRUEDIV_CASE(KIND, TGT_O, TGT, OTH, TARGET, ZDIV) \ + case KIND: { \ + double _divisor = ((PyFloatObject *)_right_o)->ob_fval; \ + if (_divisor == 0.0) { \ + (ZDIV) = 1; \ + break; \ + } \ + ((PyFloatObject *)(TGT_O))->ob_fval = \ + ((PyFloatObject *)_left_o)->ob_fval / _divisor; \ + (TARGET) = (TGT); \ + PyStackRef_CLOSE_SPECIALIZED((OTH), _PyFloat_ExactDealloc); \ + break; \ + } + +// Compact int value without the helper call MSVC would emit for +// _PyLong_CompactValue inside the eval loop. +#define BINARY_OP_INPLACE_COMPACT_VALUE(o) \ + ((Py_ssize_t)(1 - (Py_ssize_t)(((PyLongObject *)(o))->long_value.lv_tag & SIGN_MASK)) \ + * (Py_ssize_t)((PyLongObject *)(o))->long_value.ob_digit[0]) + +// Compact int op: the result must stay a single digit and must not be a +// small int (those are canonical immortal objects). Otherwise decline and +// let the generic instruction allocate. +#define BINARY_OP_INPLACE_INT_CASE(KIND, TGT_O, TGT, OTH, OP, TARGET, OK) \ + case KIND: { \ + Py_ssize_t _ires = BINARY_OP_INPLACE_COMPACT_VALUE(_left_o) OP \ + BINARY_OP_INPLACE_COMPACT_VALUE(_right_o); \ + if (_PY_IS_SMALL_INT(_ires) \ + || (twodigits)((stwodigits)_ires) + PyLong_MASK \ + >= (twodigits)PyLong_MASK + PyLong_BASE) { \ + (OK) = 0; \ + break; \ + } \ + ((PyLongObject *)(TGT_O))->long_value.lv_tag = \ + TAG_FROM_SIGN_AND_SIZE(_ires < 0 ? -1 : 1, 1); \ + ((PyLongObject *)(TGT_O))->long_value.ob_digit[0] = \ + (digit)(_ires < 0 ? -_ires : _ires); \ + (TARGET) = (TGT); \ + PyStackRef_CLOSE_SPECIALIZED((OTH), _PyLong_ExactDealloc); \ + break; \ + } + +#define BINARY_OP_INPLACE_SWITCH(kind, left, right, TARGET, OK, ZDIV) \ + do { \ + PyObject *_left_o = PyStackRef_AsPyObjectBorrow(left); \ + PyObject *_right_o = PyStackRef_AsPyObjectBorrow(right); \ + switch (kind) { \ + BINARY_OP_INPLACE_FLOAT_CASE(INPLACE_FLOAT_ADD_L, _left_o, left, right, +, TARGET) \ + BINARY_OP_INPLACE_FLOAT_CASE(INPLACE_FLOAT_ADD_R, _right_o, right, left, +, TARGET) \ + BINARY_OP_INPLACE_FLOAT_CASE(INPLACE_FLOAT_SUB_L, _left_o, left, right, -, TARGET) \ + BINARY_OP_INPLACE_FLOAT_CASE(INPLACE_FLOAT_SUB_R, _right_o, right, left, -, TARGET) \ + BINARY_OP_INPLACE_FLOAT_CASE(INPLACE_FLOAT_MUL_L, _left_o, left, right, *, TARGET) \ + BINARY_OP_INPLACE_FLOAT_CASE(INPLACE_FLOAT_MUL_R, _right_o, right, left, *, TARGET) \ + BINARY_OP_INPLACE_TRUEDIV_CASE(INPLACE_FLOAT_TRUEDIV_L, _left_o, left, right, TARGET, ZDIV) \ + BINARY_OP_INPLACE_TRUEDIV_CASE(INPLACE_FLOAT_TRUEDIV_R, _right_o, right, left, TARGET, ZDIV) \ + BINARY_OP_INPLACE_INT_CASE(INPLACE_INT_ADD_L, _left_o, left, right, +, TARGET, OK) \ + BINARY_OP_INPLACE_INT_CASE(INPLACE_INT_ADD_R, _right_o, right, left, +, TARGET, OK) \ + BINARY_OP_INPLACE_INT_CASE(INPLACE_INT_SUB_L, _left_o, left, right, -, TARGET, OK) \ + BINARY_OP_INPLACE_INT_CASE(INPLACE_INT_SUB_R, _right_o, right, left, -, TARGET, OK) \ + BINARY_OP_INPLACE_INT_CASE(INPLACE_INT_MUL_L, _left_o, left, right, *, TARGET, OK) \ + BINARY_OP_INPLACE_INT_CASE(INPLACE_INT_MUL_R, _right_o, right, left, *, TARGET, OK) \ + default: \ + Py_UNREACHABLE(); \ + } \ + } while (0) + +// KIND_OP: a failed guard re-executes the instruction as its family's +// generic instruction. The kind shares the counter slot with a miss counter +// (see KIND_OP_MISS_BITS): the generic instruction's ADVANCE_ADAPTIVE_COUNTER +// counts each miss down and, once the misses run out, the instruction goes +// back to its adaptive form with a triggering counter so that it +// re-specializes on the operands that missed -- the same life cycle as an +// ordinary specialized instruction. +#define KIND_OP_DEOPT(BASE) \ + do { \ + uint16_t _kind_vb = read_u16(&this_instr[1].cache); \ + if (((_kind_vb >> BACKOFF_BITS) & KIND_OP_MISS_MASK) == 0) { \ + /* Out of misses: back to the adaptive instruction, with its \ + counter triggering so that it re-specializes right away \ + on the operands that just missed. */ \ + FT_ATOMIC_STORE_UINT16_RELAXED( \ + this_instr[1].counter.value_and_backoff, \ + MAKE_VALUE_AND_BACKOFF(0, _kind_vb & BACKOFF_MASK)); \ + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, (BASE)); \ + } \ + /* Otherwise the generic instruction runs once and its \ + ADVANCE_ADAPTIVE_COUNTER counts the miss down. */ \ + opcode = (BASE); \ + next_instr = this_instr; \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ + } while (0) + // Macro for inplace float binary ops (tier 2 only). // Mutates the uniquely-referenced TARGET operand in place. // TARGET must be either left or right. diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c5b2dfcf5f618f3..dc257445614ed1b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3750,6 +3750,43 @@ break; } + case _TO_BOOL_SIZED_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + const _PyToBoolSizedDescr *d = &_PyToBoolSized_Table[kind]; + if (Py_TYPE(value_o) != d->type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(TO_BOOL, hit); + Py_ssize_t size = FT_ATOMIC_LOAD_SSIZE_RELAXED( + *(Py_ssize_t *)((char *)value_o + d->offset)); + res = size ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _REPLACE_WITH_TRUE_r02: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -5662,6 +5699,305 @@ break; } + case _GUARD_BINARY_OP_INPLACE_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + _PyStackRef target = (kind & 1) ? right : left; + if (!PyStackRef_RefcountOnObject(target)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (kind < INPLACE_FIRST_INT_KIND) { + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + else { + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_CheckExactAndCompact(right_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(PyStackRef_AsPyObjectBorrow(target))) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_BINARY_OP_INPLACE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + right = _stack_item_0; + left = stack_pointer[-1]; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + _PyStackRef target = (kind & 1) ? right : left; + if (!PyStackRef_RefcountOnObject(target)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (kind < INPLACE_FIRST_INT_KIND) { + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + else { + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_CheckExactAndCompact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(PyStackRef_AsPyObjectBorrow(target))) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = right; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_BINARY_OP_INPLACE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + _PyStackRef target = (kind & 1) ? right : left; + if (!PyStackRef_RefcountOnObject(target)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (kind < INPLACE_FIRST_INT_KIND) { + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + else { + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_CheckExactAndCompact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(PyStackRef_AsPyObjectBorrow(target))) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_BINARY_OP_INPLACE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + right = _stack_item_2; + left = _stack_item_1; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + _PyStackRef target = (kind & 1) ? right : left; + if (!PyStackRef_RefcountOnObject(target)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (kind < INPLACE_FIRST_INT_KIND) { + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (!PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + else { + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_CheckExactAndCompact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(PyStackRef_AsPyObjectBorrow(target))) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_INPLACE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + _PyStackRef target; + int inplace_ok = 1; + int zero_division = 0; + BINARY_OP_INPLACE_SWITCH(kind, left, right, target, inplace_ok, zero_division); + if (!inplace_ok) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (zero_division) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); + _PyFrame_StackPointerInvalidate(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + STAT_INC(BINARY_OP, hit); + res = target; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _BINARY_OP_ADD_FLOAT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dd0ce41e4b06b4a..7e88f8f84c8da20 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6167,7 +6167,7 @@ int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { - if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]] && og_opcode != KIND_OP) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } opcode = og_opcode; @@ -6185,7 +6185,7 @@ opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] && opcode != KIND_OP) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } DISPATCH_GOTO(); @@ -7712,7 +7712,7 @@ JUMP_TO_LABEL(error); } next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { + if (_PyOpcode_Caches[next_opcode] && next_opcode != KIND_OP) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } assert(next_opcode > 0 && next_opcode < 256); @@ -7798,7 +7798,7 @@ DISPATCH(); } } - if (_PyOpcode_Caches[original_opcode]) { + if (_PyOpcode_Caches[original_opcode] && original_opcode != KIND_OP) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); PAUSE_ADAPTIVE_COUNTER(cache->counter); } @@ -8572,6 +8572,116 @@ DISPATCH(); } + TARGET(KIND_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = KIND_OP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 3; + INSTRUCTION_STATS(KIND_OP); + uint16_t kind = read_u16(&this_instr[1].cache) >> KIND_OP_KIND_SHIFT; + assert(kind < KIND_OP_NKINDS); + switch (kind) { + case KIND_BINARY_OP_INPLACE: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_INPLACE + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t kind = read_u16(&this_instr[2].cache); + _PyStackRef target = (kind & 1) ? right : left; + if (!PyStackRef_RefcountOnObject(target)) { + KIND_OP_DEOPT(BINARY_OP); + } + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (kind < INPLACE_FIRST_INT_KIND) { + if (!PyFloat_CheckExact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + if (!PyFloat_CheckExact(right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + else { + if (!_PyLong_CheckExactAndCompact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + if (!_PyLong_CheckExactAndCompact(right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(PyStackRef_AsPyObjectBorrow(target))) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip -1 cache entry */ + // _BINARY_OP_INPLACE + { + uint16_t kind = read_u16(&this_instr[2].cache); + _PyStackRef target; + int inplace_ok = 1; + int zero_division = 0; + BINARY_OP_INPLACE_SWITCH(kind, left, right, target, inplace_ok, zero_division); + if (!inplace_ok) { + KIND_OP_DEOPT(BINARY_OP); + } + if (zero_division) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + STAT_INC(BINARY_OP, hit); + res = target; + } + /* Skip 3 cache entries */ + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_TO_BOOL_SIZED: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_TO_BOOL; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + uint16_t kind = read_u16(&this_instr[2].cache); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + const _PyToBoolSizedDescr *d = &_PyToBoolSized_Table[kind]; + if (Py_TYPE(value_o) != d->type) { + KIND_OP_DEOPT(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + Py_ssize_t size = FT_ATOMIC_LOAD_SSIZE_RELAXED( + *(Py_ssize_t *)((char *)value_o + d->offset)); + res = size ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + /* Skip 1 cache entry */ + DISPATCH(); + } + default: + Py_UNREACHABLE(); + } + } + TARGET(LIST_APPEND) { #if _Py_TAIL_CALL_INTERP int opcode = LIST_APPEND; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 25663ce5430d2ee..76ca81318aa95a5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -633,6 +633,18 @@ sanity_check_instrumentation(PyCodeObject *code) #endif +/* KIND_OP is a generic specialized instruction: its base opcode depends on + the kind stored in its cache, which _PyOpcode_Deopt cannot express. */ +static inline int +kind_op_deopt(_Py_CODEUNIT *src_instr, int opcode) +{ + if (opcode == KIND_OP) { + uint16_t kind = FT_ATOMIC_LOAD_UINT16_RELAXED(_PyKindOp_Kind(src_instr)); + return _PyKindOp_Deopt[kind]; + } + return opcode; +} + /* Get the underlying code unit, stripping instrumentation and ENTER_EXECUTOR */ _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int i) @@ -642,13 +654,13 @@ _Py_GetBaseCodeUnit(PyCodeObject *code, int i) .cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t *)src_instr)}; int opcode = inst.op.code; if (opcode < MIN_INSTRUMENTED_OPCODE) { - inst.op.code = _PyOpcode_Deopt[opcode]; + inst.op.code = kind_op_deopt(src_instr, _PyOpcode_Deopt[opcode]); assert(inst.op.code < MIN_SPECIALIZED_OPCODE); return inst; } if (opcode == ENTER_EXECUTOR) { _PyExecutorObject *exec = code->co_executors->executors[inst.op.arg]; - opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; + opcode = kind_op_deopt(src_instr, _PyOpcode_Deopt[exec->vm_data.opcode]); inst.op.code = opcode; inst.op.arg = exec->vm_data.oparg; assert(inst.op.code < MIN_SPECIALIZED_OPCODE); @@ -667,7 +679,7 @@ _Py_GetBaseCodeUnit(PyCodeObject *code, int i) inst.op.code = deinstrumented; } else { - inst.op.code = _PyOpcode_Deopt[opcode]; + inst.op.code = kind_op_deopt(src_instr, _PyOpcode_Deopt[opcode]); } assert(inst.op.code < MIN_SPECIALIZED_OPCODE); return inst; @@ -772,7 +784,7 @@ instrument(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *moni } CHECK(opcode != 0); if (!is_instrumented(opcode)) { - int deopt = _PyOpcode_Deopt[opcode]; + int deopt = kind_op_deopt(instr, _PyOpcode_Deopt[opcode]); int instrumented = INSTRUMENTED_OPCODES[deopt]; assert(instrumented); FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented); @@ -819,7 +831,7 @@ instrument_per_instruction(PyCodeObject *code, _Py_CODEUNIT *bytecode, assert(opcode != 0); assert(_PyOpcode_Deopt[opcode] != 0); assert(_PyOpcode_Deopt[opcode] != RESUME); - monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; + monitoring->per_instruction_opcodes[i] = kind_op_deopt(instr, _PyOpcode_Deopt[opcode]); } assert(monitoring->per_instruction_opcodes[i] > 0); FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_INSTRUCTION); @@ -1506,7 +1518,7 @@ initialize_tools(PyCodeObject *code) opcode = DE_INSTRUMENT[opcode]; assert(opcode != 0); } - opcode = _PyOpcode_Deopt[opcode]; + opcode = kind_op_deopt(instr, _PyOpcode_Deopt[opcode]); if (opcode_has_event(opcode)) { if (instrumented) { int8_t event; @@ -1789,7 +1801,8 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) // we know what the original opcode was. for (int i = 0; i < code_len; i++) { int opcode = _PyCode_CODE(code)[i].op.code; - code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; + code->_co_monitoring->per_instruction_opcodes[i] = + kind_op_deopt(&_PyCode_CODE(code)[i], _PyOpcode_Deopt[opcode]); } } if (multitools && code->_co_monitoring->per_instruction_tools == NULL) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4b..6ad781b1c71033a 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -74,6 +74,7 @@ static void *opcode_targets_table[256] = { &&TARGET_JUMP_BACKWARD, &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, &&TARGET_JUMP_FORWARD, + &&TARGET_KIND_OP, &&TARGET_LIST_APPEND, &&TARGET_LIST_EXTEND, &&TARGET_LOAD_ATTR, @@ -127,7 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -376,7 +376,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -657,6 +657,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARA static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_KIND_OP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); @@ -900,6 +901,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, + [KIND_OP] = _TAIL_CALL_KIND_OP, [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, [LOAD_ATTR] = _TAIL_CALL_LOAD_ATTR, @@ -1005,7 +1007,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1158,6 +1159,7 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACE_RECORD, [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACE_RECORD, [JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [KIND_OP] = _TAIL_CALL_TRACE_RECORD, [LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, [LIST_EXTEND] = _TAIL_CALL_TRACE_RECORD, [LOAD_ATTR] = _TAIL_CALL_TRACE_RECORD, @@ -1263,7 +1265,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer.c b/Python/optimizer.c index e05adb344c8d06d..1488c1e3d840d4d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -711,7 +711,20 @@ _PyJit_translate_single_bytecode_to_trace( oparg = (oparg & ~255) | executor->vm_data.oparg; } - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] > 0) { + /* A specialization folded into KIND_OP: translate it with the kind's own + uop expansion, but reason about flags, caches and deopts through its + family's generic instruction. */ + const struct opcode_macro_expansion *kind_expansion = NULL; + bool is_kind_op = false; + if (opcode == KIND_OP) { + uint16_t kind = _PyKindOp_Kind(this_instr); + assert(kind < KIND_OP_NKINDS); + kind_expansion = &_PyKindOp_macro_expansion[kind]; + opcode = _PyKindOp_Deopt[kind]; + is_kind_op = true; + } + + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] > 0 && !is_kind_op) { uint16_t backoff = (this_instr + 1)->counter.value_and_backoff; // adaptive_counter_cooldown is a fresh specialization. // trigger_backoff_counter is what we set during tracing. @@ -737,6 +750,7 @@ _PyJit_translate_single_bytecode_to_trace( /* Special case the first instruction, * so that we can guarantee forward progress */ if (progress_needed && uop_buffer_length(&tracer->code_buffer) < CODE_SIZE_NO_PROGRESS) { + kind_expansion = NULL; if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } @@ -859,7 +873,8 @@ _PyJit_translate_single_bytecode_to_trace( trace->end -= needs_guard_ip; #if Py_DEBUG - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + const struct opcode_macro_expansion *expansion = (kind_expansion != NULL && kind_expansion->nuops > 0) + ? kind_expansion : &_PyOpcode_macro_expansion[opcode]; int space_needed = expansion->nuops + needs_guard_ip + 2 + (!OPCODE_HAS_NO_SAVE_IP(opcode)); assert(uop_buffer_remaining_space(trace) > space_needed); #endif @@ -910,7 +925,8 @@ _PyJit_translate_single_bytecode_to_trace( break; default: { - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + const struct opcode_macro_expansion *expansion = (kind_expansion != NULL && kind_expansion->nuops > 0) + ? kind_expansion : &_PyOpcode_macro_expansion[opcode]; // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) int nuops = expansion->nuops; if (nuops == 0) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9cd20c495db4418..9932fe4a221623d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -480,6 +480,13 @@ break; } + case _TO_BOOL_SIZED: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + case _REPLACE_WITH_TRUE: { JitOptRef value; JitOptRef res; @@ -1024,6 +1031,20 @@ break; } + case _GUARD_BINARY_OP_INPLACE: { + break; + } + + case _BINARY_OP_INPLACE: { + JitOptRef res; + res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _BINARY_OP_ADD_FLOAT_INPLACE: { JitOptRef res; JitOptRef l; diff --git a/Python/specialize.c b/Python/specialize.c index f658df3bc5d2228..fa7b75f8c89bdab 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -361,10 +361,38 @@ load_counter(_Py_BackoffCounter *counter) } static inline void -specialize(_Py_CODEUNIT *instr, uint8_t specialized_opcode) +specialize(_Py_CODEUNIT *instr, int specialized_opcode) { assert(!PyErr_Occurred()); - if (!set_opcode(instr, specialized_opcode)) { + if (IS_KIND_OP_PSEUDO_OPCODE(specialized_opcode)) { + /* A specialization folded into KIND_OP: the kind takes the place of + the adaptive counter. Write it before the opcode, so that KIND_OP + never runs with a stale counter as its kind. */ + uint16_t kind = (uint16_t)(specialized_opcode - KIND_OP_PSEUDO_BASE); + assert(kind < KIND_OP_NKINDS); + static_assert(KIND_OP_NKINDS <= KIND_OP_MAX_KINDS, + "too many kinds for the KIND_OP counter slot"); + int generic = _PyKindOp_Deopt[kind]; + assert(_PyOpcode_Deopt[instr->op.code] == generic + || (instr->op.code == KIND_OP + && _PyKindOp_Deopt[_PyKindOp_Kind(instr)] == generic)); + /* Keep the backoff bits of the counter: when KIND_OP runs out of + misses it hands the instruction back to its adaptive form, whose + failed specialization attempts back off from them. */ + uint16_t backoff = load_counter((_Py_BackoffCounter *)instr + 1).value_and_backoff & BACKOFF_MASK; + if (backoff > MAX_BACKOFF) { + backoff = MAX_BACKOFF; + } + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].cache, _PyKindOp_Counter(kind, backoff)); + if (!set_opcode(instr, KIND_OP)) { + STAT_INC(generic, failure); + SPECIALIZATION_FAIL(generic, SPEC_FAIL_OTHER); + return; + } + STAT_INC(generic, success); + return; + } + if (!set_opcode(instr, (uint8_t)specialized_opcode)) { STAT_INC(_PyOpcode_Deopt[specialized_opcode], failure); SPECIALIZATION_FAIL(_PyOpcode_Deopt[specialized_opcode], SPEC_FAIL_OTHER); @@ -379,7 +407,9 @@ unspecialize(_Py_CODEUNIT *instr) { assert(!PyErr_Occurred()); uint8_t opcode = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code); - uint8_t generic_opcode = _PyOpcode_Deopt[opcode]; + uint8_t generic_opcode = (opcode == KIND_OP) + ? _PyKindOp_Deopt[_PyKindOp_Kind(instr)] + : _PyOpcode_Deopt[opcode]; STAT_INC(generic_opcode, failure); if (!set_opcode(instr, generic_opcode)) { SPECIALIZATION_FAIL(generic_opcode, SPEC_FAIL_OTHER); @@ -2246,6 +2276,49 @@ LONG_FLOAT_ACTION(compactlong_float_multiply, *) LONG_FLOAT_ACTION(compactlong_float_true_div, /) #undef LONG_FLOAT_ACTION +/* Kind-indexed specializations can be switched off at runtime with + sys._inplace_mode = 0, so that A/B measurements can use one binary. */ +static int +kind_specialization_enabled(void) +{ + PyObject *v = PySys_GetObject("_inplace_mode"); + return v == NULL || !PyLong_CheckExact(v) || PyLong_AsLong(v) != 0; +} + +static int +binary_op_inplace_target(_PyStackRef lhs_st, _PyStackRef rhs_st) +{ + /* The mutated operand must be an owned stack reference to a uniquely + referenced object. A borrowed reference (LOAD_FAST_BORROW) can have a + refcount of 1 while a local still names the object. */ + if (PyStackRef_RefcountOnObject(lhs_st) && + _PyObject_IsUniquelyReferenced(PyStackRef_AsPyObjectBorrow(lhs_st))) { + return 0; + } + if (PyStackRef_RefcountOnObject(rhs_st) && + _PyObject_IsUniquelyReferenced(PyStackRef_AsPyObjectBorrow(rhs_st))) { + return 1; + } + return -1; +} + +static int +specialize_binary_op_inplace(_Py_CODEUNIT *instr, _PyStackRef lhs_st, + _PyStackRef rhs_st, int kind_left) +{ + if (!kind_specialization_enabled()) { + return 0; + } + int side = binary_op_inplace_target(lhs_st, rhs_st); + if (side < 0) { + return 0; + } + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1); + cache->external_cache[0] = (uint16_t)(kind_left + side); + specialize(instr, BINARY_OP_INPLACE); + return 1; +} + static _PyBinaryOpSpecializationDescr binaryop_extend_descrs[] = { /* long-long arithmetic: guards also check _PyLong_IsCompact, so type alone is not sufficient to eliminate the guard. */ @@ -2366,10 +2439,16 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in return; } if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { + if (specialize_binary_op_inplace(instr, lhs_st, rhs_st, INPLACE_INT_ADD_L)) { + return; + } specialize(instr, BINARY_OP_ADD_INT); return; } if (PyFloat_CheckExact(lhs)) { + if (specialize_binary_op_inplace(instr, lhs_st, rhs_st, INPLACE_FLOAT_ADD_L)) { + return; + } specialize(instr, BINARY_OP_ADD_FLOAT); return; } @@ -2380,10 +2459,16 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in break; } if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { + if (specialize_binary_op_inplace(instr, lhs_st, rhs_st, INPLACE_INT_MUL_L)) { + return; + } specialize(instr, BINARY_OP_MULTIPLY_INT); return; } if (PyFloat_CheckExact(lhs)) { + if (specialize_binary_op_inplace(instr, lhs_st, rhs_st, INPLACE_FLOAT_MUL_L)) { + return; + } specialize(instr, BINARY_OP_MULTIPLY_FLOAT); return; } @@ -2394,14 +2479,28 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in break; } if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { + if (specialize_binary_op_inplace(instr, lhs_st, rhs_st, INPLACE_INT_SUB_L)) { + return; + } specialize(instr, BINARY_OP_SUBTRACT_INT); return; } if (PyFloat_CheckExact(lhs)) { + if (specialize_binary_op_inplace(instr, lhs_st, rhs_st, INPLACE_FLOAT_SUB_L)) { + return; + } specialize(instr, BINARY_OP_SUBTRACT_FLOAT); return; } break; + case NB_TRUE_DIVIDE: + case NB_INPLACE_TRUE_DIVIDE: + if (PyFloat_CheckExact(lhs) && PyFloat_CheckExact(rhs)) { + if (specialize_binary_op_inplace(instr, lhs_st, rhs_st, INPLACE_FLOAT_TRUEDIV_L)) { + return; + } + } + break; case NB_SUBSCR: if (PyLong_CheckExact(rhs) && _PyLong_IsNonNegativeCompact((PyLongObject *)rhs)) { if (PyList_CheckExact(lhs)) { @@ -2510,7 +2609,7 @@ _Py_Specialize_CompareOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *i { PyObject *lhs = PyStackRef_AsPyObjectBorrow(lhs_st); PyObject *rhs = PyStackRef_AsPyObjectBorrow(rhs_st); - uint8_t specialized_op; + int specialized_op; assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); @@ -2835,6 +2934,16 @@ check_type_always_true(PyTypeObject *ty) return 0; } +const _PyToBoolSizedDescr _PyToBoolSized_Table[TO_BOOL_SIZED_NKINDS] = { + [TO_BOOL_SIZED_DICT] = {&PyDict_Type, offsetof(PyDictObject, ma_used)}, + [TO_BOOL_SIZED_FROZENDICT] = {&PyFrozenDict_Type, offsetof(PyDictObject, ma_used)}, + [TO_BOOL_SIZED_TUPLE] = {&PyTuple_Type, offsetof(PyVarObject, ob_size)}, + [TO_BOOL_SIZED_BYTES] = {&PyBytes_Type, offsetof(PyVarObject, ob_size)}, + [TO_BOOL_SIZED_BYTEARRAY] = {&PyByteArray_Type, offsetof(PyVarObject, ob_size)}, + [TO_BOOL_SIZED_SET] = {&PySet_Type, offsetof(PySetObject, used)}, + [TO_BOOL_SIZED_FROZENSET] = {&PyFrozenSet_Type, offsetof(PySetObject, used)}, +}; + Py_NO_INLINE void _Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr) { @@ -2842,7 +2951,7 @@ _Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr) assert(_PyOpcode_Caches[TO_BOOL] == INLINE_CACHE_ENTRIES_TO_BOOL); _PyToBoolCache *cache = (_PyToBoolCache *)(instr + 1); PyObject *value = PyStackRef_AsPyObjectBorrow(value_o); - uint8_t specialized_op; + int specialized_op; if (PyBool_Check(value)) { specialized_op = TO_BOOL_BOOL; goto success; @@ -2863,6 +2972,15 @@ _Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr) specialized_op = TO_BOOL_STR; goto success; } + if (kind_specialization_enabled()) { + for (int kind = 0; kind < TO_BOOL_SIZED_NKINDS; kind++) { + if (Py_TYPE(value) == _PyToBoolSized_Table[kind].type) { + cache->version[0] = (uint16_t)kind; + specialized_op = TO_BOOL_SIZED; + goto success; + } + } + } if (PyType_HasFeature(Py_TYPE(value), Py_TPFLAGS_HEAPTYPE)) { unsigned int version = 0; int err = _PyType_Validate(Py_TYPE(value), check_type_always_true, &version); @@ -2915,7 +3033,7 @@ _Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr) PyObject *value = PyStackRef_AsPyObjectBorrow(value_st); assert(ENABLE_SPECIALIZATION); - assert(_PyOpcode_Caches[CONTAINS_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); + assert(_PyOpcode_Caches[CONTAINS_OP] == INLINE_CACHE_ENTRIES_CONTAINS_OP); if (PyAnyDict_CheckExact(value)) { specialize(instr, CONTAINS_OP_DICT); return; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 78dfdc5d6a89d57..7104bfd30e628fc 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1,4 +1,5 @@ -from dataclasses import dataclass +import os +from dataclasses import dataclass, field import itertools import lexer import parser @@ -271,6 +272,9 @@ class Instruction: is_target: bool = False family: Optional["Family"] = None opcode: int = -1 + # >= 0 for a kind instruction: a specialization that has no opcode of + # its own and is dispatched by KIND_OP on this kind number. + kind: int = -1 @property def properties(self) -> Properties: @@ -346,6 +350,61 @@ class Analysis: opmap: dict[str, int] have_arg: int min_instrumented: int + # Specializations folded into KIND_OP, indexed by kind number. + kinds: list[Instruction] = field(default_factory=list) + + +KIND_ANNOTATIONS = ("kind_op", "kind_candidate") + + +def kind_fold_level() -> int: + """How aggressively annotated specializations are folded into KIND_OP. + + KIND_OPS=0 (or none): every annotated specialization gets its own opcode. + KIND_OPS=1 (default): `kind_op` specializations become kinds. + KIND_OPS=2 (or all): `kind_candidate` specializations become kinds too. + """ + value = os.environ.get("KIND_OPS", "1").strip().lower() + if value in ("0", "none", "off"): + return 0 + if value in ("2", "all"): + return 2 + return 1 + + +def fold_kind_instructions( + instructions: dict[str, Instruction], + annotations: dict[str, list[str]], +) -> list[Instruction]: + """Remove the specializations that are folded into KIND_OP from the + instruction table (so that they get no opcode, target or metadata entry) + and return them in kind order. They keep their family, which is the + instruction KIND_OP re-dispatches to when a guard fails.""" + level = kind_fold_level() + kinds: list[Instruction] = [] + for name in sorted(instructions): + annos = annotations.get(name, []) + fold = ("kind_op" in annos and level >= 1) or ( + "kind_candidate" in annos and level >= 2 + ) + if not fold: + continue + inst = instructions[name] + if inst.family is None or inst.family.name == name: + raise analysis_error( + f"kind instruction {name} must be a member of a family", inst.where + ) + kinds.append(inst) + if kinds and "KIND_OP" not in instructions: + raise analysis_error( + "kind instructions need an instruction named KIND_OP", kinds[0].where + ) + for number, inst in enumerate(kinds): + del instructions[inst.name] + assert inst.family is not None + inst.family.members.remove(inst) + inst.kind = number + return kinds def analysis_error(message: str, tkn: lexer.Token) -> SyntaxError: @@ -1370,6 +1429,16 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: pass for uop in uops.values(): uop.instruction_size = get_instruction_size_for_uop(instructions, uop) + kind_annotations: dict[str, list[str]] = {} + for node in forest: + match node: + case parser.Macro(): + kind_annotations[node.name] = node.annotations + case parser.InstDef(name): + if node.kind == "inst": + kind_annotations[node.name] = node.annotations + case _: + pass # Special case BINARY_OP_INPLACE_ADD_UNICODE # BINARY_OP_INPLACE_ADD_UNICODE is not a normal family member, # as it is the wrong size, but we need it to maintain an @@ -1378,9 +1447,10 @@ def analyze_forest(forest: list[parser.AstNode]) -> Analysis: inst = instructions["BINARY_OP_INPLACE_ADD_UNICODE"] inst.family = families["BINARY_OP"] families["BINARY_OP"].members.append(inst) + kinds = fold_kind_instructions(instructions, kind_annotations) opmap, first_arg, min_instrumented = assign_opcodes(instructions, families, pseudos) return Analysis( - instructions, uops, families, pseudos, labels, opmap, first_arg, min_instrumented + instructions, uops, families, pseudos, labels, opmap, first_arg, min_instrumented, kinds ) diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 79bb9c27a556010..78b619b1274fbb3 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -234,6 +234,8 @@ def choice(*opts: str) -> str: "tier1", "tier2", "no_save_ip", + "kind_op", + "kind_candidate", } __all__ = [] diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 00f6804f1724b59..830f25f1fdbe21f 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -229,9 +229,11 @@ def generate_metadata_table(analysis: Analysis, out: CWriter) -> None: out.emit("#endif\n\n") -def generate_expansion_table(analysis: Analysis, out: CWriter) -> None: - expansions_table: dict[str, list[tuple[str, str, int]]] = {} - for inst in sorted(analysis.instructions.values(), key=lambda t: t.name): +def instruction_expansion( + analysis: Analysis, inst: Instruction +) -> list[tuple[str, str, int]] | None: + """The tier 2 expansion of an instruction, or None if it has none.""" + if True: offset: int = 0 # Cache effect offset expansions: list[tuple[str, str, int]] = [] # [(name, size, offset), ...] if inst.is_super(): @@ -249,7 +251,7 @@ def generate_expansion_table(analysis: Analysis, out: CWriter) -> None: for part in instr2.parts: expansions.append((part.name, "OPARG_BOTTOM", 0)) elif not is_viable_expansion(inst): - continue + return None else: for part in inst.parts: size = part.size @@ -273,8 +275,20 @@ def generate_expansion_table(analysis: Analysis, out: CWriter) -> None: internal_offset += cache.size expansions.append((part.name, f"OPERAND1_{part.caches[-1].size}", offset+internal_offset)) offset += part.size - expansions_table[inst.name] = expansions + return expansions + + +def generate_expansion_table(analysis: Analysis, out: CWriter) -> None: + expansions_table: dict[str, list[tuple[str, str, int]]] = {} + for inst in sorted(analysis.instructions.values(), key=lambda t: t.name): + expansions = instruction_expansion(analysis, inst) + if expansions is not None: + expansions_table[inst.name] = expansions max_uops = max(len(ex) for ex in expansions_table.values()) + for inst in analysis.kinds: + expansions = instruction_expansion(analysis, inst) + if expansions is not None: + max_uops = max(max_uops, len(expansions)) out.emit(f"#define MAX_UOP_PER_EXPANSION {max_uops}\n") out.emit("struct opcode_macro_expansion {\n") out.emit("int nuops;\n") @@ -299,6 +313,49 @@ def generate_expansion_table(analysis: Analysis, out: CWriter) -> None: out.emit("#endif // NEED_OPCODE_METADATA\n\n") +def generate_kind_tables(analysis: Analysis, out: CWriter) -> None: + """Specializations folded into KIND_OP: kind numbers, pseudo-opcodes for + the specializer, the family to re-dispatch to and the tier 2 expansion.""" + kinds = analysis.kinds + out.emit("/* Specializations dispatched by KIND_OP. The kind replaces the adaptive\n") + out.emit(" counter in the first cache entry. The pseudo-opcodes let specialize.c\n") + out.emit(" name a kind like an opcode: specialize() maps them to KIND_OP. */\n") + out.emit(f"#define KIND_OP_NKINDS {len(kinds)}\n") + out.emit("#define KIND_OP_PSEUDO_BASE 512\n") + out.emit("#define IS_KIND_OP_PSEUDO_OPCODE(OP) ((OP) >= KIND_OP_PSEUDO_BASE)\n") + for inst in kinds: + out.emit(f"#define KIND_{inst.name} {inst.kind}\n") + for inst in kinds: + out.emit(f"#define {inst.name} (KIND_OP_PSEUDO_BASE + KIND_{inst.name})\n") + size = max(len(kinds), 1) + out.emit(f"PyAPI_DATA(const uint8_t) _PyKindOp_Deopt[{size}];\n") + out.emit(f"PyAPI_DATA(const char *) _PyKindOp_Name[{size}];\n") + out.emit(f"extern const struct opcode_macro_expansion _PyKindOp_macro_expansion[{size}];\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit(f"const uint8_t _PyKindOp_Deopt[{size}] = {{\n") + for inst in kinds: + assert inst.family is not None + out.emit(f"[KIND_{inst.name}] = {inst.family.name},\n") + out.emit("};\n") + out.emit(f"const char *_PyKindOp_Name[{size}] = {{\n") + for inst in kinds: + out.emit(f'[KIND_{inst.name}] = "{inst.name}",\n') + out.emit("};\n") + out.emit(f"const struct opcode_macro_expansion _PyKindOp_macro_expansion[{size}] = {{\n") + for inst in kinds: + expansions = instruction_expansion(analysis, inst) + if expansions is None: + continue + uops = [ + f"{{ {name}, {size}, {offset} }}" for (name, size, offset) in expansions + ] + out.emit( + f'[KIND_{inst.name}] = {{ .nuops = {len(expansions)}, .uops = {{ {", ".join(uops)} }} }},\n' + ) + out.emit("};\n") + out.emit("#endif // NEED_OPCODE_METADATA\n\n") + + def is_viable_expansion(inst: Instruction) -> bool: "An instruction can be expanded if all its parts are viable for tier 2" for part in inst.parts: @@ -383,6 +440,7 @@ def generate_opcode_metadata( generate_oparg_macros(out) generate_metadata_table(analysis, out) generate_expansion_table(analysis, out) + generate_kind_tables(analysis, out) generate_name_table(analysis, out) generate_cache_table(analysis, out) generate_deopt_table(analysis, out) diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index c7fe0d162ac6e41..a4a63d7b9fc25ac 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -301,6 +301,7 @@ class InstDef(Node): class Macro(Node): name: str uops: list[UOp] + annotations: list[str] = field(default_factory=list) @dataclass @@ -508,6 +509,11 @@ def op(self) -> OpName | None: @contextual def macro_def(self) -> Macro | None: + # annotation* macro(NAME) = uop + ... ; + # Only the kind annotations are meaningful on a macro. + annotations = [] + while anno := self.expect(lx.ANNOTATION): + annotations.append(anno.text) if tkn := self.expect(lx.MACRO): if self.expect(lx.LPAREN): if tkn := self.expect(lx.IDENTIFIER): @@ -515,7 +521,7 @@ def macro_def(self) -> Macro | None: if self.expect(lx.EQUALS): if uops := self.uops(): self.require(lx.SEMI) - res = Macro(tkn.text, uops) + res = Macro(tkn.text, uops, annotations) return res return None diff --git a/Tools/cases_generator/py_metadata_generator.py b/Tools/cases_generator/py_metadata_generator.py index 6df72de44e78cb9..1066c9878e589a6 100644 --- a/Tools/cases_generator/py_metadata_generator.py +++ b/Tools/cases_generator/py_metadata_generator.py @@ -60,6 +60,13 @@ def generate_specialized_opmap(analysis: Analysis, out: CWriter) -> None: out.emit(")\n\n") +def generate_kind_ops(analysis: Analysis, out: CWriter) -> None: + out.emit("_kind_ops = (\n") + for inst in analysis.kinds: + out.emit(f' "{inst.name}",\n') + out.emit(")\n\n") + + def generate_opmap(analysis: Analysis, out: CWriter) -> None: specialized = get_specialized(analysis) out.emit("opmap = frozendict(\n") @@ -76,6 +83,7 @@ def generate_py_metadata( out = CWriter(outfile, 0, False) generate_specializations(analysis, out) generate_specialized_opmap(analysis, out) + generate_kind_ops(analysis, out) generate_opmap(analysis, out) out.emit(f"HAVE_ARGUMENT = {analysis.have_arg}\n") out.emit(f"MIN_INSTRUMENTED_OPCODE = {analysis.min_instrumented}\n") diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 0d477cb83c5d436..607c41de11a9a62 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -22,7 +22,11 @@ write_header, type_and_null, Emitter, + TokenIterator, + emit_to, + always_true, ) +from lexer import Token from cwriter import CWriter from typing import TextIO from stack import Local, Stack, StackError, get_stack_effect, Storage @@ -209,6 +213,97 @@ def generate_tier1_labels( emitter.emit_tokens(label, storage, None) emitter.emit("\n\n") +class KindEmitter(Emitter): + """Emitter for specializations folded into KIND_OP: a failed guard + re-dispatches to the family's generic instruction instead of jumping to + its PREDICTED label.""" + + def deopt_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: Uop, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.start_line() + self.out.emit("if (") + lparen = next(tkn_iter) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() + emit_to(self.out, tkn_iter, "RPAREN") + self.emit(") {\n") + next(tkn_iter) # Semi colon + assert inst is not None + assert inst.family is not None + self.emit(f"KIND_OP_DEOPT({inst.family.name});\n") + self.emit("}\n") + return not always_true(first_tkn) + + exit_if = deopt_if + + +def write_instruction_body( + inst: Instruction, out: CWriter, emitter: Emitter +) -> None: + """The uops of an instruction, from the cache size check to DISPATCH().""" + if inst.family is not None: + out.emit( + f"static_assert({inst.family.size} == {inst.size-1}" + ', "incorrect cache size");\n' + ) + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + reachable = True + for part in inst.parts: + if part.properties.records_value: + continue + # Only emit braces if more than one uop + insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 + reachable, offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) + out.start_line() + if reachable: + stack.flush(out) + out.emit("DISPATCH();\n") + + +def write_kind_op(analysis: Analysis, out: CWriter) -> None: + """KIND_OP: one instruction whose cases are the folded specializations. + The kind replaces the adaptive counter in the first cache entry; the + instruction otherwise keeps the cache layout of its family.""" + emitter = KindEmitter(out, analysis.labels) + kind_op = analysis.instructions["KIND_OP"] + out.emit("\n") + out.emit("TARGET(KIND_OP) {\n") + out.emit("#if _Py_TAIL_CALL_INTERP\n") + out.emit("int opcode = KIND_OP;\n") + out.emit("(void)(opcode);\n") + out.emit("#endif\n") + out.emit("_Py_CODEUNIT* const this_instr = next_instr;\n") + out.emit("(void)this_instr;\n") + out.emit("frame->instr_ptr = next_instr;\n") + out.emit(f"next_instr += {kind_op.size};\n") + out.emit("INSTRUCTION_STATS(KIND_OP);\n") + out.emit("uint16_t kind = read_u16(&this_instr[1].cache) >> KIND_OP_KIND_SHIFT;\n") + out.emit("assert(kind < KIND_OP_NKINDS);\n") + out.emit("switch (kind) {\n") + for inst in analysis.kinds: + assert inst.family is not None + out.emit(f"case KIND_{inst.name}: {{\n") + out.emit("// The instruction is as long as its family's generic instruction.\n") + out.emit(f"next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_{inst.family.name};\n") + write_instruction_body(inst, out, emitter) + out.start_line() + out.emit("}\n") + out.emit("default:\n") + out.emit("Py_UNREACHABLE();\n") + out.emit("}\n") + out.start_line() + out.emit("}") + out.emit("\n") + + def get_popped(inst: Instruction, analysis: Analysis) -> str: stack = get_stack_effect(inst) return (-stack.base_offset).to_c() @@ -220,6 +315,9 @@ def generate_tier1_cases( emitter = Emitter(out, analysis.labels) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): + if name == "KIND_OP" and analysis.kinds: + write_kind_op(analysis, out) + continue out.emit("\n") out.emit(f"TARGET({name}) {{\n") popped = get_popped(inst, analysis) From 97d2ec958ca969f1609805d11799b886be44b5b9 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Tue, 3 Nov 2026 09:49:36 +0100 Subject: [PATCH 2/2] Add more kinds to KIND_OP Eleven kinds, all tier-1 counterparts of existing tier-2 uops or of open specialization proposals: - UNARY_NEGATIVE_INPLACE: negate a uniquely referenced float in place (UNARY_NEGATIVE gains an inline cache) - BINARY_OP_TRUEDIV_FLOAT: float / float when the operands are shared - BINARY_OP_FLOOR_DIVIDE_INT, BINARY_OP_REMAINDER_INT: compact ints, nonzero divisor, floor semantics - BINARY_OP_LSHIFT_INT, BINARY_OP_RSHIFT_INT: compact ints, shift count within range - BINARY_OP_SUBSCR_BYTES_INT, BINARY_OP_SUBSCR_BYTEARRAY_INT, STORE_SUBSCR_BYTEARRAY_INT: byte access with a compact in-range index - UNPACK_SEQUENCE_STEAL_TWO_TUPLE, UNPACK_SEQUENCE_STEAL_TUPLE: a tuple referenced only from the stack gives up its items without refcount traffic; a shared tuple takes the increfing path Co-Authored-By: Claude Fable 5.1 --- Include/internal/pycore_code.h | 12 + Include/internal/pycore_magic_number.h | 2 +- Include/internal/pycore_opcode_metadata.h | 85 +- Include/internal/pycore_uop_ids.h | 37 +- Include/internal/pycore_uop_metadata.h | 198 ++++ Lib/_opcode_metadata.py | 13 + Lib/opcode.py | 4 + Modules/_testinternalcapi/test_cases.c.h | 734 ++++++++++++++- Python/bytecodes.c | 291 +++++- Python/executor_cases.c.h | 1017 +++++++++++++++++++-- Python/generated_cases.c.h | 734 ++++++++++++++- Python/optimizer_cases.c.h | 143 +++ Python/specialize.c | 76 ++ Tools/cases_generator/analyzer.py | 6 + 14 files changed, 3241 insertions(+), 111 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 7a71e18bdcfdf6d..3fc79072bd3ecd5 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -160,6 +160,17 @@ typedef struct { #define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache) +typedef struct { + _Py_BackoffCounter counter; + uint16_t kind; +} _PyUnaryNegativeCache; + +#define INLINE_CACHE_ENTRIES_UNARY_NEGATIVE CACHE_ENTRIES(_PyUnaryNegativeCache) + +enum _PyUnaryNegativeInplaceKind { + UNARY_NEGATIVE_INPLACE_FLOAT = 0, +}; + /* TO_BOOL_SIZED: types whose truthiness is a size field at a fixed offset. */ typedef struct { PyTypeObject *type; @@ -345,6 +356,7 @@ PyAPI_FUNC(void) _Py_Specialize_UnpackSequence(_PyStackRef seq, _Py_CODEUNIT *in PyAPI_FUNC(void) _Py_Specialize_ForIter(_PyStackRef iter, _PyStackRef null_or_index, _Py_CODEUNIT *instr, int oparg); PyAPI_FUNC(void) _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); PyAPI_FUNC(void) _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_Specialize_UnaryNegative(_PyStackRef value, _Py_CODEUNIT *instr); PyAPI_FUNC(void) _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); PyAPI_FUNC(void) _Py_GatherStats_GetIter(_PyStackRef iterable); PyAPI_FUNC(void) _Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr); diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 0409258c2eabc8c..73ce784c4e10dda 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -304,7 +304,7 @@ Known values: Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) Python 3.16a1 3706 (Add INTRINSIC_MATCH_CLASS_ISINSTANCE) - Python 3.16a1 3707 (Add KIND_OP) + Python 3.16a1 3707 (Add KIND_OP, inline cache for UNARY_NEGATIVE) Python 3.17 will start with 3750 diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index f211e702fe871d0..59d361ba481691d 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1036,9 +1036,10 @@ enum InstructionFormat { INSTR_FMT_IBC00000000 = 7, INSTR_FMT_IX = 8, INSTR_FMT_IXC = 9, - INSTR_FMT_IXC00 = 10, - INSTR_FMT_IXC000 = 11, - INSTR_FMT_IXC0000 = 12, + INSTR_FMT_IXC0 = 10, + INSTR_FMT_IXC00 = 11, + INSTR_FMT_IXC000 = 12, + INSTR_FMT_IXC0000 = 13, }; #define IS_VALID_OPCODE(OP) \ @@ -1328,7 +1329,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TRACE_RECORD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IXC0, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1543,7 +1544,7 @@ _PyOpcode_macro_expansion[256] = { [TO_BOOL_NONE] = { .nuops = 1, .uops = { { _TO_BOOL_NONE, OPARG_SIMPLE, 3 } } }, [TO_BOOL_STR] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _TO_BOOL_STR, OPARG_SIMPLE, 3 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 3 } } }, [UNARY_INVERT] = { .nuops = 2, .uops = { { _UNARY_INVERT, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, - [UNARY_NEGATIVE] = { .nuops = 2, .uops = { { _UNARY_NEGATIVE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [UNARY_NEGATIVE] = { .nuops = 2, .uops = { { _UNARY_NEGATIVE, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, [UNARY_NOT] = { .nuops = 1, .uops = { { _UNARY_NOT, OPARG_SIMPLE, 0 } } }, [UNPACK_EX] = { .nuops = 1, .uops = { { _UNPACK_EX, OPARG_SIMPLE, 0 } } }, [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, OPARG_SIMPLE, 0 } } }, @@ -1558,28 +1559,83 @@ _PyOpcode_macro_expansion[256] = { /* Specializations dispatched by KIND_OP. The kind replaces the adaptive counter in the first cache entry. The pseudo-opcodes let specialize.c name a kind like an opcode: specialize() maps them to KIND_OP. */ -#define KIND_OP_NKINDS 2 +#define KIND_OP_NKINDS 13 #define KIND_OP_PSEUDO_BASE 512 #define IS_KIND_OP_PSEUDO_OPCODE(OP) ((OP) >= KIND_OP_PSEUDO_BASE) -#define KIND_BINARY_OP_INPLACE 0 -#define KIND_TO_BOOL_SIZED 1 +#define KIND_BINARY_OP_FLOOR_DIVIDE_INT 0 +#define KIND_BINARY_OP_INPLACE 1 +#define KIND_BINARY_OP_LSHIFT_INT 2 +#define KIND_BINARY_OP_REMAINDER_INT 3 +#define KIND_BINARY_OP_RSHIFT_INT 4 +#define KIND_BINARY_OP_SUBSCR_BYTEARRAY_INT 5 +#define KIND_BINARY_OP_SUBSCR_BYTES_INT 6 +#define KIND_BINARY_OP_TRUEDIV_FLOAT 7 +#define KIND_STORE_SUBSCR_BYTEARRAY_INT 8 +#define KIND_TO_BOOL_SIZED 9 +#define KIND_UNARY_NEGATIVE_INPLACE 10 +#define KIND_UNPACK_SEQUENCE_STEAL_TUPLE 11 +#define KIND_UNPACK_SEQUENCE_STEAL_TWO_TUPLE 12 +#define BINARY_OP_FLOOR_DIVIDE_INT (KIND_OP_PSEUDO_BASE + KIND_BINARY_OP_FLOOR_DIVIDE_INT) #define BINARY_OP_INPLACE (KIND_OP_PSEUDO_BASE + KIND_BINARY_OP_INPLACE) +#define BINARY_OP_LSHIFT_INT (KIND_OP_PSEUDO_BASE + KIND_BINARY_OP_LSHIFT_INT) +#define BINARY_OP_REMAINDER_INT (KIND_OP_PSEUDO_BASE + KIND_BINARY_OP_REMAINDER_INT) +#define BINARY_OP_RSHIFT_INT (KIND_OP_PSEUDO_BASE + KIND_BINARY_OP_RSHIFT_INT) +#define BINARY_OP_SUBSCR_BYTEARRAY_INT (KIND_OP_PSEUDO_BASE + KIND_BINARY_OP_SUBSCR_BYTEARRAY_INT) +#define BINARY_OP_SUBSCR_BYTES_INT (KIND_OP_PSEUDO_BASE + KIND_BINARY_OP_SUBSCR_BYTES_INT) +#define BINARY_OP_TRUEDIV_FLOAT (KIND_OP_PSEUDO_BASE + KIND_BINARY_OP_TRUEDIV_FLOAT) +#define STORE_SUBSCR_BYTEARRAY_INT (KIND_OP_PSEUDO_BASE + KIND_STORE_SUBSCR_BYTEARRAY_INT) #define TO_BOOL_SIZED (KIND_OP_PSEUDO_BASE + KIND_TO_BOOL_SIZED) -PyAPI_DATA(const uint8_t) _PyKindOp_Deopt[2]; -PyAPI_DATA(const char *) _PyKindOp_Name[2]; -extern const struct opcode_macro_expansion _PyKindOp_macro_expansion[2]; +#define UNARY_NEGATIVE_INPLACE (KIND_OP_PSEUDO_BASE + KIND_UNARY_NEGATIVE_INPLACE) +#define UNPACK_SEQUENCE_STEAL_TUPLE (KIND_OP_PSEUDO_BASE + KIND_UNPACK_SEQUENCE_STEAL_TUPLE) +#define UNPACK_SEQUENCE_STEAL_TWO_TUPLE (KIND_OP_PSEUDO_BASE + KIND_UNPACK_SEQUENCE_STEAL_TWO_TUPLE) +PyAPI_DATA(const uint8_t) _PyKindOp_Deopt[13]; +PyAPI_DATA(const char *) _PyKindOp_Name[13]; +extern const struct opcode_macro_expansion _PyKindOp_macro_expansion[13]; #ifdef NEED_OPCODE_METADATA -const uint8_t _PyKindOp_Deopt[2] = { +const uint8_t _PyKindOp_Deopt[13] = { + [KIND_BINARY_OP_FLOOR_DIVIDE_INT] = BINARY_OP, [KIND_BINARY_OP_INPLACE] = BINARY_OP, + [KIND_BINARY_OP_LSHIFT_INT] = BINARY_OP, + [KIND_BINARY_OP_REMAINDER_INT] = BINARY_OP, + [KIND_BINARY_OP_RSHIFT_INT] = BINARY_OP, + [KIND_BINARY_OP_SUBSCR_BYTEARRAY_INT] = BINARY_OP, + [KIND_BINARY_OP_SUBSCR_BYTES_INT] = BINARY_OP, + [KIND_BINARY_OP_TRUEDIV_FLOAT] = BINARY_OP, + [KIND_STORE_SUBSCR_BYTEARRAY_INT] = STORE_SUBSCR, [KIND_TO_BOOL_SIZED] = TO_BOOL, + [KIND_UNARY_NEGATIVE_INPLACE] = UNARY_NEGATIVE, + [KIND_UNPACK_SEQUENCE_STEAL_TUPLE] = UNPACK_SEQUENCE, + [KIND_UNPACK_SEQUENCE_STEAL_TWO_TUPLE] = UNPACK_SEQUENCE, }; -const char *_PyKindOp_Name[2] = { +const char *_PyKindOp_Name[13] = { + [KIND_BINARY_OP_FLOOR_DIVIDE_INT] = "BINARY_OP_FLOOR_DIVIDE_INT", [KIND_BINARY_OP_INPLACE] = "BINARY_OP_INPLACE", + [KIND_BINARY_OP_LSHIFT_INT] = "BINARY_OP_LSHIFT_INT", + [KIND_BINARY_OP_REMAINDER_INT] = "BINARY_OP_REMAINDER_INT", + [KIND_BINARY_OP_RSHIFT_INT] = "BINARY_OP_RSHIFT_INT", + [KIND_BINARY_OP_SUBSCR_BYTEARRAY_INT] = "BINARY_OP_SUBSCR_BYTEARRAY_INT", + [KIND_BINARY_OP_SUBSCR_BYTES_INT] = "BINARY_OP_SUBSCR_BYTES_INT", + [KIND_BINARY_OP_TRUEDIV_FLOAT] = "BINARY_OP_TRUEDIV_FLOAT", + [KIND_STORE_SUBSCR_BYTEARRAY_INT] = "STORE_SUBSCR_BYTEARRAY_INT", [KIND_TO_BOOL_SIZED] = "TO_BOOL_SIZED", + [KIND_UNARY_NEGATIVE_INPLACE] = "UNARY_NEGATIVE_INPLACE", + [KIND_UNPACK_SEQUENCE_STEAL_TUPLE] = "UNPACK_SEQUENCE_STEAL_TUPLE", + [KIND_UNPACK_SEQUENCE_STEAL_TWO_TUPLE] = "UNPACK_SEQUENCE_STEAL_TWO_TUPLE", }; -const struct opcode_macro_expansion _PyKindOp_macro_expansion[2] = { +const struct opcode_macro_expansion _PyKindOp_macro_expansion[13] = { + [KIND_BINARY_OP_FLOOR_DIVIDE_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_FLOOR_DIVIDE_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, [KIND_BINARY_OP_INPLACE] = { .nuops = 2, .uops = { { _GUARD_BINARY_OP_INPLACE, 1, 1 }, { _BINARY_OP_INPLACE, 1, 1 } } }, + [KIND_BINARY_OP_LSHIFT_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_LSHIFT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [KIND_BINARY_OP_REMAINDER_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_REMAINDER_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [KIND_BINARY_OP_RSHIFT_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_RSHIFT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [KIND_BINARY_OP_SUBSCR_BYTEARRAY_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_BYTEARRAY, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_BYTEARRAY_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [KIND_BINARY_OP_SUBSCR_BYTES_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_BYTES, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_BYTES_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [KIND_BINARY_OP_TRUEDIV_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_TRUEDIV_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, + [KIND_STORE_SUBSCR_BYTEARRAY_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_BYTEARRAY, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_BYTEARRAY_INT, OPARG_SIMPLE, 1 } } }, [KIND_TO_BOOL_SIZED] = { .nuops = 1, .uops = { { _TO_BOOL_SIZED, 1, 1 } } }, + [KIND_UNARY_NEGATIVE_INPLACE] = { .nuops = 1, .uops = { { _UNARY_NEGATIVE_INPLACE, 1, 1 } } }, + [KIND_UNPACK_SEQUENCE_STEAL_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_STEAL_TUPLE, OPARG_SIMPLE, 1 } } }, + [KIND_UNPACK_SEQUENCE_STEAL_TWO_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_STEAL_TWO_TUPLE, OPARG_SIMPLE, 1 } } }, }; #endif // NEED_OPCODE_METADATA @@ -1837,6 +1893,7 @@ PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256]; const uint8_t _PyOpcode_Caches[256] = { [KIND_OP] = 2, [RESUME] = 1, + [UNARY_NEGATIVE] = 2, [TO_BOOL] = 3, [STORE_SUBSCR] = 1, [SEND] = 1, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index c24510b566d3606..fe826b053319038 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -22,14 +22,20 @@ enum { _BINARY_OP_ADD_INT_INPLACE_RIGHT, _BINARY_OP_ADD_UNICODE, _BINARY_OP_EXTEND, + _BINARY_OP_FLOOR_DIVIDE_INT, _BINARY_OP_INPLACE, _BINARY_OP_INPLACE_ADD_UNICODE, + _BINARY_OP_LSHIFT_INT, _BINARY_OP_MULTIPLY_FLOAT, _BINARY_OP_MULTIPLY_FLOAT_INPLACE, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, _BINARY_OP_MULTIPLY_INT, _BINARY_OP_MULTIPLY_INT_INPLACE, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + _BINARY_OP_REMAINDER_INT, + _BINARY_OP_RSHIFT_INT, + _BINARY_OP_SUBSCR_BYTEARRAY_INT, + _BINARY_OP_SUBSCR_BYTES_INT, _BINARY_OP_SUBSCR_CHECK_FUNC, _BINARY_OP_SUBSCR_DICT, _BINARY_OP_SUBSCR_DICT_KNOWN_HASH, @@ -176,6 +182,8 @@ enum { _GUARD_ITERATOR, _GUARD_ITER_VIRTUAL, _GUARD_LOAD_SUPER_ATTR_METHOD, + _GUARD_NOS_BYTEARRAY, + _GUARD_NOS_BYTES, _GUARD_NOS_COMPACT_ASCII, _GUARD_NOS_DICT_STORE_SUBSCRIPT, _GUARD_NOS_DICT_SUBSCRIPT, @@ -336,6 +344,7 @@ enum { _STORE_ATTR_WITH_HINT, _STORE_SLICE, _STORE_SUBSCR, + _STORE_SUBSCR_BYTEARRAY_INT, _STORE_SUBSCR_DICT, _STORE_SUBSCR_DICT_KNOWN_HASH, _STORE_SUBSCR_LIST_INT, @@ -360,8 +369,11 @@ enum { _UNARY_INVERT, _UNARY_NEGATIVE, _UNARY_NEGATIVE_FLOAT_INPLACE, + _UNARY_NEGATIVE_INPLACE, _UNPACK_SEQUENCE, _UNPACK_SEQUENCE_LIST, + _UNPACK_SEQUENCE_STEAL_TUPLE, + _UNPACK_SEQUENCE_STEAL_TWO_TUPLE, _UNPACK_SEQUENCE_TUPLE, _UNPACK_SEQUENCE_TWO_TUPLE, _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, @@ -369,7 +381,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 657 +#define MAX_UOP_ID 669 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -469,8 +481,10 @@ enum { _BINARY_OP_ADD_UNICODE_r13, _BINARY_OP_ADD_UNICODE_r23, _BINARY_OP_EXTEND_r23, + _BINARY_OP_FLOOR_DIVIDE_INT_r23, _BINARY_OP_INPLACE_r21, _BINARY_OP_INPLACE_ADD_UNICODE_r21, + _BINARY_OP_LSHIFT_INT_r23, _BINARY_OP_MULTIPLY_FLOAT_r03, _BINARY_OP_MULTIPLY_FLOAT_r13, _BINARY_OP_MULTIPLY_FLOAT_r23, @@ -489,6 +503,10 @@ enum { _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23, + _BINARY_OP_REMAINDER_INT_r23, + _BINARY_OP_RSHIFT_INT_r23, + _BINARY_OP_SUBSCR_BYTEARRAY_INT_r23, + _BINARY_OP_SUBSCR_BYTES_INT_r23, _BINARY_OP_SUBSCR_CHECK_FUNC_r23, _BINARY_OP_SUBSCR_DICT_r23, _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23, @@ -860,6 +878,14 @@ enum { _GUARD_LOAD_SUPER_ATTR_METHOD_r13, _GUARD_LOAD_SUPER_ATTR_METHOD_r23, _GUARD_LOAD_SUPER_ATTR_METHOD_r33, + _GUARD_NOS_BYTEARRAY_r02, + _GUARD_NOS_BYTEARRAY_r12, + _GUARD_NOS_BYTEARRAY_r22, + _GUARD_NOS_BYTEARRAY_r33, + _GUARD_NOS_BYTES_r02, + _GUARD_NOS_BYTES_r12, + _GUARD_NOS_BYTES_r22, + _GUARD_NOS_BYTES_r33, _GUARD_NOS_COMPACT_ASCII_r02, _GUARD_NOS_COMPACT_ASCII_r12, _GUARD_NOS_COMPACT_ASCII_r22, @@ -1358,6 +1384,7 @@ enum { _STORE_NAME_r10, _STORE_SLICE_r30, _STORE_SUBSCR_r30, + _STORE_SUBSCR_BYTEARRAY_INT_r30, _STORE_SUBSCR_DICT_r31, _STORE_SUBSCR_DICT_KNOWN_HASH_r31, _STORE_SUBSCR_LIST_INT_r32, @@ -1435,6 +1462,10 @@ enum { _UNARY_NEGATIVE_FLOAT_INPLACE_r02, _UNARY_NEGATIVE_FLOAT_INPLACE_r12, _UNARY_NEGATIVE_FLOAT_INPLACE_r23, + _UNARY_NEGATIVE_INPLACE_r01, + _UNARY_NEGATIVE_INPLACE_r11, + _UNARY_NEGATIVE_INPLACE_r22, + _UNARY_NEGATIVE_INPLACE_r33, _UNARY_NOT_r01, _UNARY_NOT_r11, _UNARY_NOT_r22, @@ -1442,6 +1473,8 @@ enum { _UNPACK_EX_r10, _UNPACK_SEQUENCE_r10, _UNPACK_SEQUENCE_LIST_r10, + _UNPACK_SEQUENCE_STEAL_TUPLE_r10, + _UNPACK_SEQUENCE_STEAL_TWO_TUPLE_r12, _UNPACK_SEQUENCE_TUPLE_r10, _UNPACK_SEQUENCE_TWO_TUPLE_r12, _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03, @@ -1453,7 +1486,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1664 +#define MAX_UOP_REGS_ID 1697 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1b84d5e8a8e64a2..54b737d7912c17e 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -84,6 +84,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, [_POP_ITER] = HAS_ESCAPES_FLAG, [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_UNARY_NEGATIVE_INPLACE] = HAS_EXIT_FLAG, [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNARY_NEGATIVE_FLOAT_INPLACE] = 0, [_UNARY_NOT] = 0, @@ -123,6 +124,14 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, [_GUARD_BINARY_OP_INPLACE] = HAS_EXIT_FLAG, [_BINARY_OP_INPLACE] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_FLOOR_DIVIDE_INT] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BINARY_OP_REMAINDER_INT] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BINARY_OP_LSHIFT_INT] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BINARY_OP_RSHIFT_INT] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOS_BYTES] = HAS_EXIT_FLAG, + [_GUARD_NOS_BYTEARRAY] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_BYTES_INT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_BYTEARRAY_INT] = HAS_EXIT_FLAG, [_BINARY_OP_ADD_FLOAT_INPLACE] = 0, [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = 0, [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = 0, @@ -160,6 +169,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_BYTEARRAY_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -188,6 +198,8 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = 0, [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = HAS_ARG_FLAG, + [_UNPACK_SEQUENCE_STEAL_TWO_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_STEAL_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -891,6 +903,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 1, 3, _END_SEND_r31 }, }, }, + [_UNARY_NEGATIVE_INPLACE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _UNARY_NEGATIVE_INPLACE_r01 }, + { 1, 1, _UNARY_NEGATIVE_INPLACE_r11 }, + { 2, 2, _UNARY_NEGATIVE_INPLACE_r22 }, + { 3, 3, _UNARY_NEGATIVE_INPLACE_r33 }, + }, + }, [_UNARY_NEGATIVE] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -1242,6 +1263,78 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_BINARY_OP_FLOOR_DIVIDE_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_FLOOR_DIVIDE_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_REMAINDER_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_REMAINDER_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_LSHIFT_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_LSHIFT_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_RSHIFT_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_RSHIFT_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_BYTES] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_BYTES_r02 }, + { 2, 1, _GUARD_NOS_BYTES_r12 }, + { 2, 2, _GUARD_NOS_BYTES_r22 }, + { 3, 3, _GUARD_NOS_BYTES_r33 }, + }, + }, + [_GUARD_NOS_BYTEARRAY] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_BYTEARRAY_r02 }, + { 2, 1, _GUARD_NOS_BYTEARRAY_r12 }, + { 2, 2, _GUARD_NOS_BYTEARRAY_r22 }, + { 3, 3, _GUARD_NOS_BYTEARRAY_r33 }, + }, + }, + [_BINARY_OP_SUBSCR_BYTES_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_BYTES_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_BYTEARRAY_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_BYTEARRAY_INT_r23 }, + { -1, -1, -1 }, + }, + }, [_BINARY_OP_ADD_FLOAT_INPLACE] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -1575,6 +1668,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 0, 3, _STORE_SUBSCR_r30 }, }, }, + [_STORE_SUBSCR_BYTEARRAY_INT] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _STORE_SUBSCR_BYTEARRAY_INT_r30 }, + }, + }, [_STORE_SUBSCR_LIST_INT] = { .best = { 3, 3, 3, 3 }, .entries = { @@ -1827,6 +1929,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_UNPACK_SEQUENCE_STEAL_TWO_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNPACK_SEQUENCE_STEAL_TWO_TUPLE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_STEAL_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_STEAL_TUPLE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_UNPACK_SEQUENCE_LIST] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -4093,6 +4213,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_END_FOR_r10] = _END_FOR, [_POP_ITER_r20] = _POP_ITER, [_END_SEND_r31] = _END_SEND, + [_UNARY_NEGATIVE_INPLACE_r01] = _UNARY_NEGATIVE_INPLACE, + [_UNARY_NEGATIVE_INPLACE_r11] = _UNARY_NEGATIVE_INPLACE, + [_UNARY_NEGATIVE_INPLACE_r22] = _UNARY_NEGATIVE_INPLACE, + [_UNARY_NEGATIVE_INPLACE_r33] = _UNARY_NEGATIVE_INPLACE, [_UNARY_NEGATIVE_r12] = _UNARY_NEGATIVE, [_UNARY_NEGATIVE_FLOAT_INPLACE_r02] = _UNARY_NEGATIVE_FLOAT_INPLACE, [_UNARY_NEGATIVE_FLOAT_INPLACE_r12] = _UNARY_NEGATIVE_FLOAT_INPLACE, @@ -4217,6 +4341,20 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_BINARY_OP_INPLACE_r22] = _GUARD_BINARY_OP_INPLACE, [_GUARD_BINARY_OP_INPLACE_r33] = _GUARD_BINARY_OP_INPLACE, [_BINARY_OP_INPLACE_r21] = _BINARY_OP_INPLACE, + [_BINARY_OP_FLOOR_DIVIDE_INT_r23] = _BINARY_OP_FLOOR_DIVIDE_INT, + [_BINARY_OP_REMAINDER_INT_r23] = _BINARY_OP_REMAINDER_INT, + [_BINARY_OP_LSHIFT_INT_r23] = _BINARY_OP_LSHIFT_INT, + [_BINARY_OP_RSHIFT_INT_r23] = _BINARY_OP_RSHIFT_INT, + [_GUARD_NOS_BYTES_r02] = _GUARD_NOS_BYTES, + [_GUARD_NOS_BYTES_r12] = _GUARD_NOS_BYTES, + [_GUARD_NOS_BYTES_r22] = _GUARD_NOS_BYTES, + [_GUARD_NOS_BYTES_r33] = _GUARD_NOS_BYTES, + [_GUARD_NOS_BYTEARRAY_r02] = _GUARD_NOS_BYTEARRAY, + [_GUARD_NOS_BYTEARRAY_r12] = _GUARD_NOS_BYTEARRAY, + [_GUARD_NOS_BYTEARRAY_r22] = _GUARD_NOS_BYTEARRAY, + [_GUARD_NOS_BYTEARRAY_r33] = _GUARD_NOS_BYTEARRAY, + [_BINARY_OP_SUBSCR_BYTES_INT_r23] = _BINARY_OP_SUBSCR_BYTES_INT, + [_BINARY_OP_SUBSCR_BYTEARRAY_INT_r23] = _BINARY_OP_SUBSCR_BYTEARRAY_INT, [_BINARY_OP_ADD_FLOAT_INPLACE_r03] = _BINARY_OP_ADD_FLOAT_INPLACE, [_BINARY_OP_ADD_FLOAT_INPLACE_r13] = _BINARY_OP_ADD_FLOAT_INPLACE, [_BINARY_OP_ADD_FLOAT_INPLACE_r23] = _BINARY_OP_ADD_FLOAT_INPLACE, @@ -4307,6 +4445,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LIST_APPEND_r10] = _LIST_APPEND, [_SET_ADD_r10] = _SET_ADD, [_STORE_SUBSCR_r30] = _STORE_SUBSCR, + [_STORE_SUBSCR_BYTEARRAY_INT_r30] = _STORE_SUBSCR_BYTEARRAY_INT, [_STORE_SUBSCR_LIST_INT_r32] = _STORE_SUBSCR_LIST_INT, [_STORE_SUBSCR_DICT_r31] = _STORE_SUBSCR_DICT, [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = _STORE_SUBSCR_DICT_KNOWN_HASH, @@ -4355,6 +4494,8 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13] = _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, [_UNPACK_SEQUENCE_TUPLE_r10] = _UNPACK_SEQUENCE_TUPLE, [_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10] = _UNPACK_SEQUENCE_UNIQUE_TUPLE, + [_UNPACK_SEQUENCE_STEAL_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_STEAL_TWO_TUPLE, + [_UNPACK_SEQUENCE_STEAL_TUPLE_r10] = _UNPACK_SEQUENCE_STEAL_TUPLE, [_UNPACK_SEQUENCE_LIST_r10] = _UNPACK_SEQUENCE_LIST, [_UNPACK_EX_r10] = _UNPACK_EX, [_STORE_ATTR_r20] = _STORE_ATTR, @@ -4935,10 +5076,14 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_ADD_UNICODE_r23] = "_BINARY_OP_ADD_UNICODE_r23", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", [_BINARY_OP_EXTEND_r23] = "_BINARY_OP_EXTEND_r23", + [_BINARY_OP_FLOOR_DIVIDE_INT] = "_BINARY_OP_FLOOR_DIVIDE_INT", + [_BINARY_OP_FLOOR_DIVIDE_INT_r23] = "_BINARY_OP_FLOOR_DIVIDE_INT_r23", [_BINARY_OP_INPLACE] = "_BINARY_OP_INPLACE", [_BINARY_OP_INPLACE_r21] = "_BINARY_OP_INPLACE_r21", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", [_BINARY_OP_INPLACE_ADD_UNICODE_r21] = "_BINARY_OP_INPLACE_ADD_UNICODE_r21", + [_BINARY_OP_LSHIFT_INT] = "_BINARY_OP_LSHIFT_INT", + [_BINARY_OP_LSHIFT_INT_r23] = "_BINARY_OP_LSHIFT_INT_r23", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", [_BINARY_OP_MULTIPLY_FLOAT_r03] = "_BINARY_OP_MULTIPLY_FLOAT_r03", [_BINARY_OP_MULTIPLY_FLOAT_r13] = "_BINARY_OP_MULTIPLY_FLOAT_r13", @@ -4963,6 +5108,14 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03", [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13", [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23", + [_BINARY_OP_REMAINDER_INT] = "_BINARY_OP_REMAINDER_INT", + [_BINARY_OP_REMAINDER_INT_r23] = "_BINARY_OP_REMAINDER_INT_r23", + [_BINARY_OP_RSHIFT_INT] = "_BINARY_OP_RSHIFT_INT", + [_BINARY_OP_RSHIFT_INT_r23] = "_BINARY_OP_RSHIFT_INT_r23", + [_BINARY_OP_SUBSCR_BYTEARRAY_INT] = "_BINARY_OP_SUBSCR_BYTEARRAY_INT", + [_BINARY_OP_SUBSCR_BYTEARRAY_INT_r23] = "_BINARY_OP_SUBSCR_BYTEARRAY_INT_r23", + [_BINARY_OP_SUBSCR_BYTES_INT] = "_BINARY_OP_SUBSCR_BYTES_INT", + [_BINARY_OP_SUBSCR_BYTES_INT_r23] = "_BINARY_OP_SUBSCR_BYTES_INT_r23", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = "_BINARY_OP_SUBSCR_CHECK_FUNC_r23", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", @@ -5493,6 +5646,16 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_LOAD_SUPER_ATTR_METHOD_r13] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r13", [_GUARD_LOAD_SUPER_ATTR_METHOD_r23] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r23", [_GUARD_LOAD_SUPER_ATTR_METHOD_r33] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r33", + [_GUARD_NOS_BYTEARRAY] = "_GUARD_NOS_BYTEARRAY", + [_GUARD_NOS_BYTEARRAY_r02] = "_GUARD_NOS_BYTEARRAY_r02", + [_GUARD_NOS_BYTEARRAY_r12] = "_GUARD_NOS_BYTEARRAY_r12", + [_GUARD_NOS_BYTEARRAY_r22] = "_GUARD_NOS_BYTEARRAY_r22", + [_GUARD_NOS_BYTEARRAY_r33] = "_GUARD_NOS_BYTEARRAY_r33", + [_GUARD_NOS_BYTES] = "_GUARD_NOS_BYTES", + [_GUARD_NOS_BYTES_r02] = "_GUARD_NOS_BYTES_r02", + [_GUARD_NOS_BYTES_r12] = "_GUARD_NOS_BYTES_r12", + [_GUARD_NOS_BYTES_r22] = "_GUARD_NOS_BYTES_r22", + [_GUARD_NOS_BYTES_r33] = "_GUARD_NOS_BYTES_r33", [_GUARD_NOS_COMPACT_ASCII] = "_GUARD_NOS_COMPACT_ASCII", [_GUARD_NOS_COMPACT_ASCII_r02] = "_GUARD_NOS_COMPACT_ASCII_r02", [_GUARD_NOS_COMPACT_ASCII_r12] = "_GUARD_NOS_COMPACT_ASCII_r12", @@ -6109,6 +6272,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_STORE_SLICE_r30] = "_STORE_SLICE_r30", [_STORE_SUBSCR] = "_STORE_SUBSCR", [_STORE_SUBSCR_r30] = "_STORE_SUBSCR_r30", + [_STORE_SUBSCR_BYTEARRAY_INT] = "_STORE_SUBSCR_BYTEARRAY_INT", + [_STORE_SUBSCR_BYTEARRAY_INT_r30] = "_STORE_SUBSCR_BYTEARRAY_INT_r30", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", [_STORE_SUBSCR_DICT_r31] = "_STORE_SUBSCR_DICT_r31", [_STORE_SUBSCR_DICT_KNOWN_HASH] = "_STORE_SUBSCR_DICT_KNOWN_HASH", @@ -6211,6 +6376,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_UNARY_NEGATIVE_FLOAT_INPLACE_r02] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r02", [_UNARY_NEGATIVE_FLOAT_INPLACE_r12] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r12", [_UNARY_NEGATIVE_FLOAT_INPLACE_r23] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r23", + [_UNARY_NEGATIVE_INPLACE] = "_UNARY_NEGATIVE_INPLACE", + [_UNARY_NEGATIVE_INPLACE_r01] = "_UNARY_NEGATIVE_INPLACE_r01", + [_UNARY_NEGATIVE_INPLACE_r11] = "_UNARY_NEGATIVE_INPLACE_r11", + [_UNARY_NEGATIVE_INPLACE_r22] = "_UNARY_NEGATIVE_INPLACE_r22", + [_UNARY_NEGATIVE_INPLACE_r33] = "_UNARY_NEGATIVE_INPLACE_r33", [_UNARY_NOT] = "_UNARY_NOT", [_UNARY_NOT_r01] = "_UNARY_NOT_r01", [_UNARY_NOT_r11] = "_UNARY_NOT_r11", @@ -6222,6 +6392,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_UNPACK_SEQUENCE_r10] = "_UNPACK_SEQUENCE_r10", [_UNPACK_SEQUENCE_LIST] = "_UNPACK_SEQUENCE_LIST", [_UNPACK_SEQUENCE_LIST_r10] = "_UNPACK_SEQUENCE_LIST_r10", + [_UNPACK_SEQUENCE_STEAL_TUPLE] = "_UNPACK_SEQUENCE_STEAL_TUPLE", + [_UNPACK_SEQUENCE_STEAL_TUPLE_r10] = "_UNPACK_SEQUENCE_STEAL_TUPLE_r10", + [_UNPACK_SEQUENCE_STEAL_TWO_TUPLE] = "_UNPACK_SEQUENCE_STEAL_TWO_TUPLE", + [_UNPACK_SEQUENCE_STEAL_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_STEAL_TWO_TUPLE_r12", [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", [_UNPACK_SEQUENCE_TUPLE_r10] = "_UNPACK_SEQUENCE_TUPLE_r10", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", @@ -6341,6 +6515,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _END_SEND: return 3; + case _UNARY_NEGATIVE_INPLACE: + return 1; case _UNARY_NEGATIVE: return 1; case _UNARY_NEGATIVE_FLOAT_INPLACE: @@ -6419,6 +6595,22 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _BINARY_OP_INPLACE: return 2; + case _BINARY_OP_FLOOR_DIVIDE_INT: + return 2; + case _BINARY_OP_REMAINDER_INT: + return 2; + case _BINARY_OP_LSHIFT_INT: + return 2; + case _BINARY_OP_RSHIFT_INT: + return 2; + case _GUARD_NOS_BYTES: + return 0; + case _GUARD_NOS_BYTEARRAY: + return 0; + case _BINARY_OP_SUBSCR_BYTES_INT: + return 2; + case _BINARY_OP_SUBSCR_BYTEARRAY_INT: + return 2; case _BINARY_OP_ADD_FLOAT_INPLACE: return 2; case _BINARY_OP_SUBTRACT_FLOAT_INPLACE: @@ -6493,6 +6685,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _STORE_SUBSCR: return 3; + case _STORE_SUBSCR_BYTEARRAY_INT: + return 3; case _STORE_SUBSCR_LIST_INT: return 3; case _STORE_SUBSCR_DICT: @@ -6549,6 +6743,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _UNPACK_SEQUENCE_UNIQUE_TUPLE: return 1; + case _UNPACK_SEQUENCE_STEAL_TWO_TUPLE: + return 1; + case _UNPACK_SEQUENCE_STEAL_TUPLE: + return 1; case _UNPACK_SEQUENCE_LIST: return 1; case _UNPACK_EX: diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 9c4c0a5b453bc99..0d9b59b1969e5bf 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -7,6 +7,8 @@ "RESUME_CHECK", "RESUME_CHECK_JIT", ), + UNARY_NEGATIVE=( + ), TO_BOOL=( "TO_BOOL_ALWAYS_TRUE", "TO_BOOL_BOOL", @@ -227,8 +229,19 @@ ) _kind_ops = ( + "BINARY_OP_FLOOR_DIVIDE_INT", "BINARY_OP_INPLACE", + "BINARY_OP_LSHIFT_INT", + "BINARY_OP_REMAINDER_INT", + "BINARY_OP_RSHIFT_INT", + "BINARY_OP_SUBSCR_BYTEARRAY_INT", + "BINARY_OP_SUBSCR_BYTES_INT", + "BINARY_OP_TRUEDIV_FLOAT", + "STORE_SUBSCR_BYTEARRAY_INT", "TO_BOOL_SIZED", + "UNARY_NEGATIVE_INPLACE", + "UNPACK_SEQUENCE_STEAL_TUPLE", + "UNPACK_SEQUENCE_STEAL_TWO_TUPLE", ) opmap = frozendict( diff --git a/Lib/opcode.py b/Lib/opcode.py index a56c62913e4383c..c79b89e64a7d267 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -56,6 +56,10 @@ module_keys_version=1, builtin_keys_version=1, ), + UNARY_NEGATIVE=frozendict( + counter=1, + kind=1, + ), BINARY_OP=frozendict( counter=1, descr=4, diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 39b1e1488d7163a..5045b1747d1c140 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -8586,6 +8586,78 @@ uint16_t kind = read_u16(&this_instr[1].cache) >> KIND_OP_KIND_SHIFT; assert(kind < KIND_OP_NKINDS); switch (kind) { + case KIND_BINARY_OP_FLOOR_DIVIDE_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_FLOOR_DIVIDE_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if (b == 0) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t q = a / b; + if ((a % b != 0) && ((a < 0) != (b < 0))) { + q--; + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(q); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } case KIND_BINARY_OP_INPLACE: { // The instruction is as long as its family's generic instruction. next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; @@ -8652,6 +8724,504 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } + case KIND_BINARY_OP_LSHIFT_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_LSHIFT_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if ((size_t)b > 32 || sizeof(Py_ssize_t) < 8) { + KIND_OP_DEOPT(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(a * ((Py_ssize_t)1 << b)); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_BINARY_OP_REMAINDER_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_REMAINDER_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if (b == 0) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t m = a % b; + if (m != 0 && ((m < 0) != (b < 0))) { + m += b; + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(m); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_BINARY_OP_RSHIFT_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_RSHIFT_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if (b < 0) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t shifted; + if (b >= (Py_ssize_t)(8 * sizeof(Py_ssize_t) - 1)) { + shifted = a < 0 ? -1 : 0; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + shifted = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, a, b); + _PyFrame_StackPointerInvalidate(frame); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(shifted); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_BINARY_OP_SUBSCR_BYTEARRAY_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef ba_st; + _PyStackRef sub_st; + _PyStackRef res; + _PyStackRef b; + _PyStackRef i; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_BYTEARRAY + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyByteArray_CheckExact(o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_BYTEARRAY_INT + { + sub_st = value; + ba_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *ba = PyStackRef_AsPyObjectBorrow(ba_st); + assert(PyLong_CheckExact(sub)); + assert(PyByteArray_CheckExact(ba)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyByteArray_GET_SIZE(ba) <= index) { + KIND_OP_DEOPT(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_FromUnsignedChar((unsigned char)PyByteArray_AS_STRING(ba)[index]); + b = ba_st; + i = sub_st; + res = PyStackRef_FromPyObjectBorrow(res_o); + } + // _POP_TOP_INT + { + value = i; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP + { + value = b; + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + case KIND_BINARY_OP_SUBSCR_BYTES_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef bytes_st; + _PyStackRef sub_st; + _PyStackRef res; + _PyStackRef b; + _PyStackRef i; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_BYTES + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyBytes_CheckExact(o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_BYTES_INT + { + sub_st = value; + bytes_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *bytes = PyStackRef_AsPyObjectBorrow(bytes_st); + assert(PyLong_CheckExact(sub)); + assert(PyBytes_CheckExact(bytes)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyBytes_GET_SIZE(bytes) <= index) { + KIND_OP_DEOPT(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_FromUnsignedChar((unsigned char)PyBytes_AS_STRING(bytes)[index]); + b = bytes_st; + i = sub_st; + res = PyStackRef_FromPyObjectBorrow(res_o); + } + // _POP_TOP_INT + { + value = i; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP + { + value = b; + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + case KIND_BINARY_OP_TRUEDIV_FLOAT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_TRUEDIV_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double divisor = ((PyFloatObject *)right_o)->ob_fval; + if (divisor == 0.0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_SetString(PyExc_ZeroDivisionError, + "float division by zero"); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + double dres = ((PyFloatObject *)left_o)->ob_fval / divisor; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + } + // _POP_TOP_FLOAT + { + value = r; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + // _POP_TOP_FLOAT + { + value = l; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_STORE_SUBSCR_BYTEARRAY_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_STORE_SUBSCR; + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef ba_st; + _PyStackRef sub_st; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(STORE_SUBSCR); + } + } + // _GUARD_NOS_BYTEARRAY + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyByteArray_CheckExact(o)) { + KIND_OP_DEOPT(STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_BYTEARRAY_INT + { + sub_st = value; + ba_st = nos; + value = stack_pointer[-3]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *ba = PyStackRef_AsPyObjectBorrow(ba_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + assert(PyLong_CheckExact(sub)); + assert(PyByteArray_CheckExact(ba)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + KIND_OP_DEOPT(STORE_SUBSCR); + } + if (!PyLong_CheckExact(value_o) || !_PyLong_IsNonNegativeCompact((PyLongObject*)value_o)) { + KIND_OP_DEOPT(STORE_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t byte = ((PyLongObject*)value_o)->long_value.ob_digit[0]; + if (PyByteArray_GET_SIZE(ba) <= index || byte > 255) { + KIND_OP_DEOPT(STORE_SUBSCR); + } + STAT_INC(STORE_SUBSCR, hit); + PyByteArray_AS_STRING(ba)[index] = (char)byte; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = ba_st; + ba_st = PyStackRef_NULL; + stack_pointer[-2] = ba_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-3] = value; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + } + DISPATCH(); + } case KIND_TO_BOOL_SIZED: { // The instruction is as long as its family's generic instruction. next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_TO_BOOL; @@ -8678,6 +9248,145 @@ /* Skip 1 cache entry */ DISPATCH(); } + case KIND_UNARY_NEGATIVE_INPLACE: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_UNARY_NEGATIVE; + static_assert(INLINE_CACHE_ENTRIES_UNARY_NEGATIVE == 2, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + uint16_t kind = read_u16(&this_instr[2].cache); + (void)kind; + assert(kind == UNARY_NEGATIVE_INPLACE_FLOAT); + if (!PyStackRef_RefcountOnObject(value)) { + KIND_OP_DEOPT(UNARY_NEGATIVE); + } + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(val_o)) { + KIND_OP_DEOPT(UNARY_NEGATIVE); + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(val_o)) { + KIND_OP_DEOPT(UNARY_NEGATIVE); + } + STAT_INC(UNARY_NEGATIVE, hit); + ((PyFloatObject *)val_o)->ob_fval = -((PyFloatObject *)val_o)->ob_fval; + res = value; + stack_pointer[-1] = res; + DISPATCH(); + } + case KIND_UNPACK_SEQUENCE_STEAL_TUPLE: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + KIND_OP_DEOPT(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_STEAL_TUPLE + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + KIND_OP_DEOPT(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + int owned = PyStackRef_RefcountOnObject(seq); + seq_o = PyStackRef_AsPyObjectSteal(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); + if (owned && BINARY_OP_INPLACE_IS_UNIQUE(seq_o)) { + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectSteal(items[i]); + } + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + else { + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + if (owned) { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(seq_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += 1; + } + } + } + stack_pointer += -1 + oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_UNPACK_SEQUENCE_STEAL_TWO_TUPLE: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + KIND_OP_DEOPT(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_STEAL_TWO_TUPLE + { + seq = tos; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + KIND_OP_DEOPT(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + int owned = PyStackRef_RefcountOnObject(seq); + seq_o = PyStackRef_AsPyObjectSteal(seq); + if (owned && BINARY_OP_INPLACE_IS_UNIQUE(seq_o)) { + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + else { + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + if (owned) { + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(seq_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + } + } + } + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } default: Py_UNREACHABLE(); } @@ -13307,14 +14016,35 @@ (void)(opcode); #endif frame->instr_ptr = next_instr; - next_instr += 1; + next_instr += 3; INSTRUCTION_STATS(UNARY_NEGATIVE); + PREDICTED_UNARY_NEGATIVE:; + _Py_CODEUNIT* const this_instr = next_instr - 3; + (void)this_instr; _PyStackRef value; _PyStackRef res; _PyStackRef v; - // _UNARY_NEGATIVE + // _SPECIALIZE_UNARY_NEGATIVE { value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_UnaryNegative(value, next_instr); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNARY_NEGATIVE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 1 cache entry */ + // _UNARY_NEGATIVE + { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 870b50607a99d84..a536c5c8bb8c1f7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -450,7 +450,40 @@ dummy_func( PyStackRef_CLOSE(receiver); } - macro(UNARY_NEGATIVE) = _UNARY_NEGATIVE + POP_TOP; + family(UNARY_NEGATIVE, INLINE_CACHE_ENTRIES_UNARY_NEGATIVE) = { + UNARY_NEGATIVE_INPLACE, + }; + + specializing op(_SPECIALIZE_UNARY_NEGATIVE, (counter/1, value -- value)) { + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_UnaryNegative(value, next_instr); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNARY_NEGATIVE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + + macro(UNARY_NEGATIVE) = _SPECIALIZE_UNARY_NEGATIVE + unused/1 + _UNARY_NEGATIVE + POP_TOP; + + // Negate a uniquely referenced operand in place. `kind` is reserved + // for more types; only floats for now. + op(_UNARY_NEGATIVE_INPLACE, (kind/1, value -- res)) { + (void)kind; + assert(kind == UNARY_NEGATIVE_INPLACE_FLOAT); + EXIT_IF(!PyStackRef_RefcountOnObject(value)); + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + EXIT_IF(!PyFloat_CheckExact(val_o)); + EXIT_IF(!BINARY_OP_INPLACE_IS_UNIQUE(val_o)); + STAT_INC(UNARY_NEGATIVE, hit); + ((PyFloatObject *)val_o)->ob_fval = -((PyFloatObject *)val_o)->ob_fval; + res = value; + DEAD(value); + } + + kind_op macro(UNARY_NEGATIVE_INPLACE) = unused/1 + _UNARY_NEGATIVE_INPLACE; op(_UNARY_NEGATIVE, (value -- res, v)) { PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); @@ -652,6 +685,13 @@ dummy_func( BINARY_OP_INPLACE_ADD_UNICODE, BINARY_OP_EXTEND, BINARY_OP_INPLACE, + BINARY_OP_TRUEDIV_FLOAT, + BINARY_OP_FLOOR_DIVIDE_INT, + BINARY_OP_REMAINDER_INT, + BINARY_OP_LSHIFT_INT, + BINARY_OP_RSHIFT_INT, + BINARY_OP_SUBSCR_BYTES_INT, + BINARY_OP_SUBSCR_BYTEARRAY_INT, }; op(_GUARD_NOS_INT, (left, unused -- left, unused)) { @@ -914,6 +954,172 @@ dummy_func( kind_op macro(BINARY_OP_INPLACE) = unused/1 + _GUARD_BINARY_OP_INPLACE + rewind/-1 + _BINARY_OP_INPLACE + unused/3; + // Float true division (tier 1 counterpart of the tier 2 uop above, + // for operands that are not uniquely referenced). + kind_op macro(BINARY_OP_TRUEDIV_FLOAT) = + _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_TRUEDIV_FLOAT + _POP_TOP_FLOAT + _POP_TOP_FLOAT; + + // Compact int floor division and remainder (Python semantics: the + // result takes the sign of the divisor). A zero divisor deopts. + op(_BINARY_OP_FLOOR_DIVIDE_INT, (left, right -- res, l, r)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + EXIT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + EXIT_IF(b == 0); + Py_ssize_t q = a / b; + if ((a % b != 0) && ((a < 0) != (b < 0))) { + q--; + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(q); + if (res_o == NULL) { + ERROR_NO_POP(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + INPUTS_DEAD(); + } + + kind_op macro(BINARY_OP_FLOOR_DIVIDE_INT) = + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_FLOOR_DIVIDE_INT + _POP_TOP_INT + _POP_TOP_INT; + + op(_BINARY_OP_REMAINDER_INT, (left, right -- res, l, r)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + EXIT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + EXIT_IF(b == 0); + Py_ssize_t m = a % b; + if (m != 0 && ((m < 0) != (b < 0))) { + m += b; + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(m); + if (res_o == NULL) { + ERROR_NO_POP(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + INPUTS_DEAD(); + } + + kind_op macro(BINARY_OP_REMAINDER_INT) = + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_REMAINDER_INT + _POP_TOP_INT + _POP_TOP_INT; + + // Compact int shifts. A left shift by more than 32 bits, or by a + // negative count, deopts; the product below cannot overflow since + // a compact value has at most 30 significant bits. + op(_BINARY_OP_LSHIFT_INT, (left, right -- res, l, r)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + EXIT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + EXIT_IF((size_t)b > 32 || sizeof(Py_ssize_t) < 8); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(a * ((Py_ssize_t)1 << b)); + if (res_o == NULL) { + ERROR_NO_POP(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + INPUTS_DEAD(); + } + + kind_op macro(BINARY_OP_LSHIFT_INT) = + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_LSHIFT_INT + _POP_TOP_INT + _POP_TOP_INT; + + op(_BINARY_OP_RSHIFT_INT, (left, right -- res, l, r)) { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + EXIT_IF(!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + EXIT_IF(b < 0); + Py_ssize_t shifted; + if (b >= (Py_ssize_t)(8 * sizeof(Py_ssize_t) - 1)) { + shifted = a < 0 ? -1 : 0; + } + else { + shifted = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, a, b); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(shifted); + if (res_o == NULL) { + ERROR_NO_POP(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + INPUTS_DEAD(); + } + + kind_op macro(BINARY_OP_RSHIFT_INT) = + _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_RSHIFT_INT + _POP_TOP_INT + _POP_TOP_INT; + + // bytes[int] and bytearray[int]: the byte as a small int. + op(_GUARD_NOS_BYTES, (nos, unused -- nos, unused)) { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + EXIT_IF(!PyBytes_CheckExact(o)); + } + + op(_GUARD_NOS_BYTEARRAY, (nos, unused -- nos, unused)) { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + EXIT_IF(!PyByteArray_CheckExact(o)); + } + + op(_BINARY_OP_SUBSCR_BYTES_INT, (bytes_st, sub_st -- res, b, i)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *bytes = PyStackRef_AsPyObjectBorrow(bytes_st); + assert(PyLong_CheckExact(sub)); + assert(PyBytes_CheckExact(bytes)); + EXIT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + EXIT_IF(PyBytes_GET_SIZE(bytes) <= index); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_FromUnsignedChar((unsigned char)PyBytes_AS_STRING(bytes)[index]); + b = bytes_st; + i = sub_st; + INPUTS_DEAD(); + res = PyStackRef_FromPyObjectBorrow(res_o); + } + + kind_op macro(BINARY_OP_SUBSCR_BYTES_INT) = + _GUARD_TOS_INT + _GUARD_NOS_BYTES + unused/5 + _BINARY_OP_SUBSCR_BYTES_INT + _POP_TOP_INT + POP_TOP; + + op(_BINARY_OP_SUBSCR_BYTEARRAY_INT, (ba_st, sub_st -- res, b, i)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *ba = PyStackRef_AsPyObjectBorrow(ba_st); + assert(PyLong_CheckExact(sub)); + assert(PyByteArray_CheckExact(ba)); + EXIT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + EXIT_IF(PyByteArray_GET_SIZE(ba) <= index); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_FromUnsignedChar((unsigned char)PyByteArray_AS_STRING(ba)[index]); + b = ba_st; + i = sub_st; + INPUTS_DEAD(); + res = PyStackRef_FromPyObjectBorrow(res_o); + } + + kind_op macro(BINARY_OP_SUBSCR_BYTEARRAY_INT) = + _GUARD_TOS_INT + _GUARD_NOS_BYTEARRAY + unused/5 + _BINARY_OP_SUBSCR_BYTEARRAY_INT + _POP_TOP_INT + POP_TOP; + // Inplace float ops: mutate the uniquely-referenced left operand // instead of allocating a new float. Tier 2 only. // The optimizer sets l to a borrowed value so the following _POP_TOP_FLOAT @@ -969,7 +1175,7 @@ dummy_func( // Float true division --- not specialized at tier 1, emitted by the // tier 2 optimizer when both operands are known floats. - tier2 op(_BINARY_OP_TRUEDIV_FLOAT, (left, right -- res, l, r)) { + op(_BINARY_OP_TRUEDIV_FLOAT, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); @@ -1459,6 +1665,7 @@ dummy_func( family(STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR) = { STORE_SUBSCR_DICT, STORE_SUBSCR_LIST_INT, + STORE_SUBSCR_BYTEARRAY_INT, }; specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { @@ -1485,6 +1692,26 @@ dummy_func( kind_candidate macro(STORE_SUBSCR_LIST_INT) = _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/1 + _STORE_SUBSCR_LIST_INT + _POP_TOP_INT + POP_TOP; + // bytearray[int] = int, both index and value in range. + op(_STORE_SUBSCR_BYTEARRAY_INT, (value, ba_st, sub_st -- )) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *ba = PyStackRef_AsPyObjectBorrow(ba_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + assert(PyLong_CheckExact(sub)); + assert(PyByteArray_CheckExact(ba)); + EXIT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)); + EXIT_IF(!PyLong_CheckExact(value_o) || !_PyLong_IsNonNegativeCompact((PyLongObject*)value_o)); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t byte = ((PyLongObject*)value_o)->long_value.ob_digit[0]; + EXIT_IF(PyByteArray_GET_SIZE(ba) <= index || byte > 255); + STAT_INC(STORE_SUBSCR, hit); + PyByteArray_AS_STRING(ba)[index] = (char)byte; + DECREF_INPUTS(); + } + + kind_op macro(STORE_SUBSCR_BYTEARRAY_INT) = + _GUARD_TOS_INT + _GUARD_NOS_BYTEARRAY + unused/1 + _STORE_SUBSCR_BYTEARRAY_INT; + op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- ls, ss)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); @@ -2080,6 +2307,8 @@ dummy_func( UNPACK_SEQUENCE_TWO_TUPLE, UNPACK_SEQUENCE_TUPLE, UNPACK_SEQUENCE_LIST, + UNPACK_SEQUENCE_STEAL_TWO_TUPLE, + UNPACK_SEQUENCE_STEAL_TUPLE, }; specializing op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) { @@ -2170,6 +2399,64 @@ dummy_func( kind_candidate macro(UNPACK_SEQUENCE_LIST) = _GUARD_TOS_LIST + unused/1 + _UNPACK_SEQUENCE_LIST; + // Unpack a tuple that is only referenced from the stack (a fresh + // return value, typically): steal its items and free the shell + // (tier 1 counterpart of _UNPACK_SEQUENCE_UNIQUE_*). A shared tuple + // takes the increfing path, so only a type or length mismatch deopts. + op(_UNPACK_SEQUENCE_STEAL_TWO_TUPLE, (seq -- val1, val0)) { + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + EXIT_IF(PyTuple_GET_SIZE(seq_o) != 2); + STAT_INC(UNPACK_SEQUENCE, hit); + int owned = PyStackRef_RefcountOnObject(seq); + seq_o = PyStackRef_AsPyObjectSteal(seq); + if (owned && BINARY_OP_INPLACE_IS_UNIQUE(seq_o)) { + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + else { + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + if (owned) { + Py_DECREF(seq_o); + } + } + } + + kind_op macro(UNPACK_SEQUENCE_STEAL_TWO_TUPLE) = + _GUARD_TOS_TUPLE + unused/1 + _UNPACK_SEQUENCE_STEAL_TWO_TUPLE; + + op(_UNPACK_SEQUENCE_STEAL_TUPLE, (seq -- values[oparg])) { + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + EXIT_IF(PyTuple_GET_SIZE(seq_o) != oparg); + STAT_INC(UNPACK_SEQUENCE, hit); + int owned = PyStackRef_RefcountOnObject(seq); + seq_o = PyStackRef_AsPyObjectSteal(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); + if (owned && BINARY_OP_INPLACE_IS_UNIQUE(seq_o)) { + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectSteal(items[i]); + } + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + else { + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + if (owned) { + Py_DECREF(seq_o); + } + } + } + + kind_op macro(UNPACK_SEQUENCE_STEAL_TUPLE) = + _GUARD_TOS_TUPLE + unused/1 + _UNPACK_SEQUENCE_STEAL_TUPLE; + op(_UNPACK_SEQUENCE_LIST, (seq -- values[oparg])) { PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); assert(PyList_CheckExact(seq_o)); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index dc257445614ed1b..43d16fee0e0cb4d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2634,6 +2634,171 @@ break; } + case _UNARY_NEGATIVE_INPLACE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + (void)kind; + assert(kind == UNARY_NEGATIVE_INPLACE_FLOAT); + if (!PyStackRef_RefcountOnObject(value)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(val_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(val_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(UNARY_NEGATIVE, hit); + ((PyFloatObject *)val_o)->ob_fval = -((PyFloatObject *)val_o)->ob_fval; + res = value; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _UNARY_NEGATIVE_INPLACE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + (void)kind; + assert(kind == UNARY_NEGATIVE_INPLACE_FLOAT); + if (!PyStackRef_RefcountOnObject(value)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(val_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(val_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(UNARY_NEGATIVE, hit); + ((PyFloatObject *)val_o)->ob_fval = -((PyFloatObject *)val_o)->ob_fval; + res = value; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _UNARY_NEGATIVE_INPLACE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + (void)kind; + assert(kind == UNARY_NEGATIVE_INPLACE_FLOAT); + if (!PyStackRef_RefcountOnObject(value)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(val_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(val_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(UNARY_NEGATIVE, hit); + ((PyFloatObject *)val_o)->ob_fval = -((PyFloatObject *)val_o)->ob_fval; + res = value; + _tos_cache1 = res; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _UNARY_NEGATIVE_INPLACE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + (void)kind; + assert(kind == UNARY_NEGATIVE_INPLACE_FLOAT); + if (!PyStackRef_RefcountOnObject(value)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(val_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(val_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(UNARY_NEGATIVE, hit); + ((PyFloatObject *)val_o)->ob_fval = -((PyFloatObject *)val_o)->ob_fval; + res = value; + _tos_cache2 = res; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _UNARY_NEGATIVE_r12: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -5880,120 +6045,621 @@ break; } - case _GUARD_BINARY_OP_INPLACE_r33: { - CHECK_CURRENT_CACHED_VALUES(3); + case _GUARD_BINARY_OP_INPLACE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + right = _stack_item_2; + left = _stack_item_1; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + _PyStackRef target = (kind & 1) ? right : left; + if (!PyStackRef_RefcountOnObject(target)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (kind < INPLACE_FIRST_INT_KIND) { + if (!PyFloat_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (!PyFloat_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + else { + if (!_PyLong_CheckExactAndCompact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_CheckExactAndCompact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(PyStackRef_AsPyObjectBorrow(target))) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = right; + _tos_cache1 = left; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_INPLACE_r21: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); + _PyStackRef target; + int inplace_ok = 1; + int zero_division = 0; + BINARY_OP_INPLACE_SWITCH(kind, left, right, target, inplace_ok, zero_division); + if (!inplace_ok) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (zero_division) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); + _PyFrame_StackPointerInvalidate(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + STAT_INC(BINARY_OP, hit); + res = target; + _tos_cache0 = res; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_FLOOR_DIVIDE_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if (b == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t q = a / b; + if ((a % b != 0) && ((a < 0) != (b < 0))) { + q--; + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(q); + if (res_o == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_REMAINDER_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if (b == 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t m = a % b; + if (m != 0 && ((m < 0) != (b < 0))) { + m += b; + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(m); + if (res_o == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_LSHIFT_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if ((size_t)b > 32 || sizeof(Py_ssize_t) < 8) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(a * ((Py_ssize_t)1 << b)); + if (res_o == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_RSHIFT_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef right; + _PyStackRef left; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + right = _stack_item_1; + left = _stack_item_0; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if (b < 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t shifted; + if (b >= (Py_ssize_t)(8 * sizeof(Py_ssize_t) - 1)) { + shifted = a < 0 ? -1 : 0; + } + else { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + shifted = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, a, b); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(shifted); + if (res_o == NULL) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_BYTES_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyBytes_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_BYTES_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyBytes_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_BYTES_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyBytes_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_BYTES_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyBytes_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_BYTEARRAY_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyByteArray_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_BYTEARRAY_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + nos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyByteArray_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_BYTEARRAY_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyByteArray_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_BYTEARRAY_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyByteArray_CheckExact(o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_SUBSCR_BYTES_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef right; - _PyStackRef left; + _PyStackRef sub_st; + _PyStackRef bytes_st; + _PyStackRef res; + _PyStackRef b; + _PyStackRef i; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - right = _stack_item_2; - left = _stack_item_1; - uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); - _PyStackRef target = (kind & 1) ? right : left; - if (!PyStackRef_RefcountOnObject(target)) { + sub_st = _stack_item_1; + bytes_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *bytes = PyStackRef_AsPyObjectBorrow(bytes_st); + assert(PyLong_CheckExact(sub)); + assert(PyBytes_CheckExact(bytes)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { UOP_STAT_INC(uopcode, miss); - _tos_cache2 = right; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); + _tos_cache1 = sub_st; + _tos_cache0 = bytes_st; + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - if (kind < INPLACE_FIRST_INT_KIND) { - if (!PyFloat_CheckExact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = right; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - if (!PyFloat_CheckExact(right_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = right; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - else { - if (!_PyLong_CheckExactAndCompact(left_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = right; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - if (!_PyLong_CheckExactAndCompact(right_o)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = right; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - } - if (!BINARY_OP_INPLACE_IS_UNIQUE(PyStackRef_AsPyObjectBorrow(target))) { + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyBytes_GET_SIZE(bytes) <= index) { UOP_STAT_INC(uopcode, miss); - _tos_cache2 = right; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); + _tos_cache1 = sub_st; + _tos_cache0 = bytes_st; + SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - _tos_cache2 = right; - _tos_cache1 = left; - _tos_cache0 = _stack_item_0; + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_FromUnsignedChar((unsigned char)PyBytes_AS_STRING(bytes)[index]); + b = bytes_st; + i = sub_st; + res = PyStackRef_FromPyObjectBorrow(res_o); + _tos_cache2 = i; + _tos_cache1 = b; + _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _BINARY_OP_INPLACE_r21: { + case _BINARY_OP_SUBSCR_BYTEARRAY_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef right; - _PyStackRef left; + _PyStackRef sub_st; + _PyStackRef ba_st; _PyStackRef res; + _PyStackRef b; + _PyStackRef i; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - right = _stack_item_1; - left = _stack_item_0; - uint16_t kind = (uint16_t)CURRENT_OPERAND0_16(); - _PyStackRef target; - int inplace_ok = 1; - int zero_division = 0; - BINARY_OP_INPLACE_SWITCH(kind, left, right, target, inplace_ok, zero_division); - if (!inplace_ok) { + sub_st = _stack_item_1; + ba_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *ba = PyStackRef_AsPyObjectBorrow(ba_st); + assert(PyLong_CheckExact(sub)); + assert(PyByteArray_CheckExact(ba)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { UOP_STAT_INC(uopcode, miss); - _tos_cache1 = right; - _tos_cache0 = left; + _tos_cache1 = sub_st; + _tos_cache0 = ba_st; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - if (zero_division) { - stack_pointer[0] = left; - stack_pointer[1] = right; - stack_pointer += 2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyErr_SetString(PyExc_ZeroDivisionError, "division by zero"); - _PyFrame_StackPointerInvalidate(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyByteArray_GET_SIZE(ba) <= index) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = sub_st; + _tos_cache0 = ba_st; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_OP, hit); - res = target; + PyObject *res_o = _PyLong_FromUnsignedChar((unsigned char)PyByteArray_AS_STRING(ba)[index]); + b = ba_st; + i = sub_st; + res = PyStackRef_FromPyObjectBorrow(res_o); + _tos_cache2 = i; + _tos_cache1 = b; _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -8762,6 +9428,82 @@ break; } + case _STORE_SUBSCR_BYTEARRAY_INT_r30: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef sub_st; + _PyStackRef ba_st; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub_st = _stack_item_2; + ba_st = _stack_item_1; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *ba = PyStackRef_AsPyObjectBorrow(ba_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + assert(PyLong_CheckExact(sub)); + assert(PyByteArray_CheckExact(ba)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = ba_st; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (!PyLong_CheckExact(value_o) || !_PyLong_IsNonNegativeCompact((PyLongObject*)value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = ba_st; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t byte = ((PyLongObject*)value_o)->long_value.ob_digit[0]; + if (PyByteArray_GET_SIZE(ba) <= index || byte > 255) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = ba_st; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(STORE_SUBSCR, hit); + PyByteArray_AS_STRING(ba)[index] = (char)byte; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer[1] = ba_st; + stack_pointer[2] = sub_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = ba_st; + ba_st = PyStackRef_NULL; + stack_pointer[1] = ba_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[0] = value; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _STORE_SUBSCR_LIST_INT_r32: { CHECK_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -10267,6 +11009,105 @@ break; } + case _UNPACK_SEQUENCE_STEAL_TWO_TUPLE_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = seq; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(UNPACK_SEQUENCE, hit); + int owned = PyStackRef_RefcountOnObject(seq); + seq_o = PyStackRef_AsPyObjectSteal(seq); + if (owned && BINARY_OP_INPLACE_IS_UNIQUE(seq_o)) { + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + else { + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + if (owned) { + stack_pointer[0] = val1; + stack_pointer[1] = val0; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(seq_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + } + } + _tos_cache1 = val0; + _tos_cache0 = val1; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _UNPACK_SEQUENCE_STEAL_TUPLE_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef seq; + _PyStackRef *values; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + seq = _stack_item_0; + values = &stack_pointer[0]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = seq; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(UNPACK_SEQUENCE, hit); + int owned = PyStackRef_RefcountOnObject(seq); + seq_o = PyStackRef_AsPyObjectSteal(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); + if (owned && BINARY_OP_INPLACE_IS_UNIQUE(seq_o)) { + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectSteal(items[i]); + } + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + else { + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + if (owned) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(seq_o); + _PyFrame_StackPointerInvalidate(frame); + } + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _UNPACK_SEQUENCE_LIST_r10: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7e88f8f84c8da20..3fe0fa3b1f598c5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8585,6 +8585,78 @@ uint16_t kind = read_u16(&this_instr[1].cache) >> KIND_OP_KIND_SHIFT; assert(kind < KIND_OP_NKINDS); switch (kind) { + case KIND_BINARY_OP_FLOOR_DIVIDE_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_FLOOR_DIVIDE_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if (b == 0) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t q = a / b; + if ((a % b != 0) && ((a < 0) != (b < 0))) { + q--; + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(q); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } case KIND_BINARY_OP_INPLACE: { // The instruction is as long as its family's generic instruction. next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; @@ -8651,6 +8723,504 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } + case KIND_BINARY_OP_LSHIFT_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_LSHIFT_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if ((size_t)b > 32 || sizeof(Py_ssize_t) < 8) { + KIND_OP_DEOPT(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(a * ((Py_ssize_t)1 << b)); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_BINARY_OP_REMAINDER_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_REMAINDER_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if (b == 0) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t m = a % b; + if (m != 0 && ((m < 0) != (b < 0))) { + m += b; + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(m); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_BINARY_OP_RSHIFT_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_RSHIFT_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + if (!_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t a = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t b = _PyLong_CompactValue((PyLongObject *)right_o); + if (b < 0) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t shifted; + if (b >= (Py_ssize_t)(8 * sizeof(Py_ssize_t) - 1)) { + shifted = a < 0 ? -1 : 0; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + shifted = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, a, b); + _PyFrame_StackPointerInvalidate(frame); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyLong_FromSsize_t(shifted); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_BINARY_OP_SUBSCR_BYTEARRAY_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef ba_st; + _PyStackRef sub_st; + _PyStackRef res; + _PyStackRef b; + _PyStackRef i; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_BYTEARRAY + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyByteArray_CheckExact(o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_BYTEARRAY_INT + { + sub_st = value; + ba_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *ba = PyStackRef_AsPyObjectBorrow(ba_st); + assert(PyLong_CheckExact(sub)); + assert(PyByteArray_CheckExact(ba)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyByteArray_GET_SIZE(ba) <= index) { + KIND_OP_DEOPT(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_FromUnsignedChar((unsigned char)PyByteArray_AS_STRING(ba)[index]); + b = ba_st; + i = sub_st; + res = PyStackRef_FromPyObjectBorrow(res_o); + } + // _POP_TOP_INT + { + value = i; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP + { + value = b; + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + case KIND_BINARY_OP_SUBSCR_BYTES_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef bytes_st; + _PyStackRef sub_st; + _PyStackRef res; + _PyStackRef b; + _PyStackRef i; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_BYTES + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyBytes_CheckExact(o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_BYTES_INT + { + sub_st = value; + bytes_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *bytes = PyStackRef_AsPyObjectBorrow(bytes_st); + assert(PyLong_CheckExact(sub)); + assert(PyBytes_CheckExact(bytes)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + KIND_OP_DEOPT(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyBytes_GET_SIZE(bytes) <= index) { + KIND_OP_DEOPT(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = _PyLong_FromUnsignedChar((unsigned char)PyBytes_AS_STRING(bytes)[index]); + b = bytes_st; + i = sub_st; + res = PyStackRef_FromPyObjectBorrow(res_o); + } + // _POP_TOP_INT + { + value = i; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP + { + value = b; + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + case KIND_BINARY_OP_TRUEDIV_FLOAT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_BINARY_OP; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + KIND_OP_DEOPT(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_TRUEDIV_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double divisor = ((PyFloatObject *)right_o)->ob_fval; + if (divisor == 0.0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_SetString(PyExc_ZeroDivisionError, + "float division by zero"); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + double dres = ((PyFloatObject *)left_o)->ob_fval / divisor; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + } + // _POP_TOP_FLOAT + { + value = r; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + // _POP_TOP_FLOAT + { + value = l; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_STORE_SUBSCR_BYTEARRAY_INT: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_STORE_SUBSCR; + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef ba_st; + _PyStackRef sub_st; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + KIND_OP_DEOPT(STORE_SUBSCR); + } + } + // _GUARD_NOS_BYTEARRAY + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyByteArray_CheckExact(o)) { + KIND_OP_DEOPT(STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_BYTEARRAY_INT + { + sub_st = value; + ba_st = nos; + value = stack_pointer[-3]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *ba = PyStackRef_AsPyObjectBorrow(ba_st); + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + assert(PyLong_CheckExact(sub)); + assert(PyByteArray_CheckExact(ba)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + KIND_OP_DEOPT(STORE_SUBSCR); + } + if (!PyLong_CheckExact(value_o) || !_PyLong_IsNonNegativeCompact((PyLongObject*)value_o)) { + KIND_OP_DEOPT(STORE_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t byte = ((PyLongObject*)value_o)->long_value.ob_digit[0]; + if (PyByteArray_GET_SIZE(ba) <= index || byte > 255) { + KIND_OP_DEOPT(STORE_SUBSCR); + } + STAT_INC(STORE_SUBSCR, hit); + PyByteArray_AS_STRING(ba)[index] = (char)byte; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = ba_st; + ba_st = PyStackRef_NULL; + stack_pointer[-2] = ba_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-3] = value; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + } + DISPATCH(); + } case KIND_TO_BOOL_SIZED: { // The instruction is as long as its family's generic instruction. next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_TO_BOOL; @@ -8677,6 +9247,145 @@ /* Skip 1 cache entry */ DISPATCH(); } + case KIND_UNARY_NEGATIVE_INPLACE: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_UNARY_NEGATIVE; + static_assert(INLINE_CACHE_ENTRIES_UNARY_NEGATIVE == 2, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + uint16_t kind = read_u16(&this_instr[2].cache); + (void)kind; + assert(kind == UNARY_NEGATIVE_INPLACE_FLOAT); + if (!PyStackRef_RefcountOnObject(value)) { + KIND_OP_DEOPT(UNARY_NEGATIVE); + } + PyObject *val_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(val_o)) { + KIND_OP_DEOPT(UNARY_NEGATIVE); + } + if (!BINARY_OP_INPLACE_IS_UNIQUE(val_o)) { + KIND_OP_DEOPT(UNARY_NEGATIVE); + } + STAT_INC(UNARY_NEGATIVE, hit); + ((PyFloatObject *)val_o)->ob_fval = -((PyFloatObject *)val_o)->ob_fval; + res = value; + stack_pointer[-1] = res; + DISPATCH(); + } + case KIND_UNPACK_SEQUENCE_STEAL_TUPLE: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + KIND_OP_DEOPT(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_STEAL_TUPLE + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + KIND_OP_DEOPT(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + int owned = PyStackRef_RefcountOnObject(seq); + seq_o = PyStackRef_AsPyObjectSteal(seq); + PyObject **items = _PyTuple_ITEMS(seq_o); + if (owned && BINARY_OP_INPLACE_IS_UNIQUE(seq_o)) { + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectSteal(items[i]); + } + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + else { + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + if (owned) { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(seq_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += 1; + } + } + } + stack_pointer += -1 + oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + case KIND_UNPACK_SEQUENCE_STEAL_TWO_TUPLE: { + // The instruction is as long as its family's generic instruction. + next_instr = this_instr + 1 + INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + KIND_OP_DEOPT(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_STEAL_TWO_TUPLE + { + seq = tos; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + KIND_OP_DEOPT(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + int owned = PyStackRef_RefcountOnObject(seq); + seq_o = PyStackRef_AsPyObjectSteal(seq); + if (owned && BINARY_OP_INPLACE_IS_UNIQUE(seq_o)) { + val0 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectSteal(PyTuple_GET_ITEM(seq_o, 1)); + PyObject_GC_UnTrack(seq_o); + _PyStolenTuple_Free(seq_o); + } + else { + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + if (owned) { + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(seq_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + } + } + } + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } default: Py_UNREACHABLE(); } @@ -13304,14 +14013,35 @@ (void)(opcode); #endif frame->instr_ptr = next_instr; - next_instr += 1; + next_instr += 3; INSTRUCTION_STATS(UNARY_NEGATIVE); + PREDICTED_UNARY_NEGATIVE:; + _Py_CODEUNIT* const this_instr = next_instr - 3; + (void)this_instr; _PyStackRef value; _PyStackRef res; _PyStackRef v; - // _UNARY_NEGATIVE + // _SPECIALIZE_UNARY_NEGATIVE { value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_UnaryNegative(value, next_instr); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNARY_NEGATIVE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 1 cache entry */ + // _UNARY_NEGATIVE + { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9932fe4a221623d..137fa1ac410cb27 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -211,6 +211,13 @@ break; } + case _UNARY_NEGATIVE_INPLACE: { + JitOptRef res; + res = sym_new_not_null(ctx); + stack_pointer[-1] = res; + break; + } + case _UNARY_NEGATIVE: { JitOptRef value; JitOptRef res; @@ -1045,6 +1052,110 @@ break; } + case _BINARY_OP_FLOOR_DIVIDE_INT: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_REMAINDER_INT: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_LSHIFT_INT: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_RSHIFT_INT: { + JitOptRef res; + JitOptRef l; + JitOptRef r; + res = sym_new_not_null(ctx); + l = sym_new_not_null(ctx); + r = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_BYTES: { + break; + } + + case _GUARD_NOS_BYTEARRAY: { + break; + } + + case _BINARY_OP_SUBSCR_BYTES_INT: { + JitOptRef res; + JitOptRef b; + JitOptRef i; + res = sym_new_not_null(ctx); + b = sym_new_not_null(ctx); + i = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = b; + stack_pointer[0] = i; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _BINARY_OP_SUBSCR_BYTEARRAY_INT: { + JitOptRef res; + JitOptRef b; + JitOptRef i; + res = sym_new_not_null(ctx); + b = sym_new_not_null(ctx); + i = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-2] = res; + stack_pointer[-1] = b; + stack_pointer[0] = i; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _BINARY_OP_ADD_FLOAT_INPLACE: { JitOptRef res; JitOptRef l; @@ -1750,6 +1861,13 @@ break; } + case _STORE_SUBSCR_BYTEARRAY_INT: { + CHECK_STACK_BOUNDS(-3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _STORE_SUBSCR_LIST_INT: { JitOptRef sub_st; JitOptRef list_st; @@ -2145,6 +2263,31 @@ break; } + case _UNPACK_SEQUENCE_STEAL_TWO_TUPLE: { + JitOptRef val1; + JitOptRef val0; + val1 = sym_new_not_null(ctx); + val0 = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _UNPACK_SEQUENCE_STEAL_TUPLE: { + JitOptRef *values; + values = &stack_pointer[-1]; + for (int _i = oparg; --_i >= 0;) { + values[_i] = sym_new_not_null(ctx); + } + CHECK_STACK_BOUNDS(-1 + oparg); + stack_pointer += -1 + oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _UNPACK_SEQUENCE_LIST: { JitOptRef *values; values = &stack_pointer[-1]; diff --git a/Python/specialize.c b/Python/specialize.c index fa7b75f8c89bdab..5cde5063159fc65 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1650,6 +1650,12 @@ _Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_COD specialize(instr, STORE_SUBSCR_DICT); return; } + if (container_type == &PyByteArray_Type + && PyLong_CheckExact(sub) && _PyLong_IsNonNegativeCompact((PyLongObject *)sub)) + { + specialize(instr, STORE_SUBSCR_BYTEARRAY_INT); + return; + } SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container, sub)); unspecialize(instr); } @@ -2499,6 +2505,38 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in if (specialize_binary_op_inplace(instr, lhs_st, rhs_st, INPLACE_FLOAT_TRUEDIV_L)) { return; } + specialize(instr, BINARY_OP_TRUEDIV_FLOAT); + return; + } + break; + case NB_FLOOR_DIVIDE: + case NB_INPLACE_FLOOR_DIVIDE: + case NB_REMAINDER: + case NB_INPLACE_REMAINDER: + if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs) + && !_PyLong_IsZero((PyLongObject *)rhs)) + { + int floor_div = (oparg == NB_FLOOR_DIVIDE || oparg == NB_INPLACE_FLOOR_DIVIDE); + specialize(instr, floor_div ? BINARY_OP_FLOOR_DIVIDE_INT : BINARY_OP_REMAINDER_INT); + return; + } + break; + case NB_LSHIFT: + case NB_INPLACE_LSHIFT: + if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs) + && (size_t)_PyLong_CompactValue((PyLongObject *)rhs) <= 32 && sizeof(Py_ssize_t) >= 8) + { + specialize(instr, BINARY_OP_LSHIFT_INT); + return; + } + break; + case NB_RSHIFT: + case NB_INPLACE_RSHIFT: + if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs) + && !_PyLong_IsNegative((PyLongObject *)rhs)) + { + specialize(instr, BINARY_OP_RSHIFT_INT); + return; } break; case NB_SUBSCR: @@ -2511,6 +2549,14 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in specialize(instr, BINARY_OP_SUBSCR_TUPLE_INT); return; } + if (PyBytes_CheckExact(lhs)) { + specialize(instr, BINARY_OP_SUBSCR_BYTES_INT); + return; + } + if (PyByteArray_CheckExact(lhs)) { + specialize(instr, BINARY_OP_SUBSCR_BYTEARRAY_INT); + return; + } if (PyUnicode_CheckExact(lhs) && _PyLong_IsNonNegativeCompact((PyLongObject*)rhs)) { if (PyUnicode_IS_COMPACT_ASCII(lhs)) { specialize(instr, BINARY_OP_SUBSCR_STR_INT); @@ -2680,6 +2726,16 @@ _Py_Specialize_UnpackSequence(_PyStackRef seq_st, _Py_CODEUNIT *instr, int oparg unspecialize(instr); return; } + /* A tuple that is only referenced from the stack (a fresh return + value, typically) can give up its items without refcount traffic. + The handler falls back to increfing when a later tuple is shared. */ + if (PyStackRef_RefcountOnObject(seq_st) && + _PyObject_IsUniquelyReferenced(seq)) + { + specialize(instr, oparg == 2 ? UNPACK_SEQUENCE_STEAL_TWO_TUPLE + : UNPACK_SEQUENCE_STEAL_TUPLE); + return; + } if (PyTuple_GET_SIZE(seq) == 2) { specialize(instr, UNPACK_SEQUENCE_TWO_TUPLE); return; @@ -3008,6 +3064,26 @@ _Py_Specialize_ToBool(_PyStackRef value_o, _Py_CODEUNIT *instr) specialize(instr, specialized_op); } +Py_NO_INLINE void +_Py_Specialize_UnaryNegative(_PyStackRef value_st, _Py_CODEUNIT *instr) +{ + assert(ENABLE_SPECIALIZATION); + assert(_PyOpcode_Caches[UNARY_NEGATIVE] == INLINE_CACHE_ENTRIES_UNARY_NEGATIVE); + _PyUnaryNegativeCache *cache = (_PyUnaryNegativeCache *)(instr + 1); + PyObject *value = PyStackRef_AsPyObjectBorrow(value_st); + if (kind_specialization_enabled() && + PyFloat_CheckExact(value) && + PyStackRef_RefcountOnObject(value_st) && + _PyObject_IsUniquelyReferenced(value)) + { + cache->kind = UNARY_NEGATIVE_INPLACE_FLOAT; + specialize(instr, UNARY_NEGATIVE_INPLACE); + return; + } + SPECIALIZATION_FAIL(UNARY_NEGATIVE, SPEC_FAIL_OTHER); + unspecialize(instr); +} + #ifdef Py_STATS static int containsop_fail_kind(PyObject *value) { diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 7104bfd30e628fc..6900097bc52e0a7 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -687,6 +687,12 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_TYPE", "PyStackRef_True", "PyBytes_GET_SIZE", + "PyBytes_AS_STRING", + "PyBytes_CheckExact", + "PyByteArray_GET_SIZE", + "PyByteArray_AS_STRING", + "PyByteArray_CheckExact", + "_PyLong_FromUnsignedChar", "PyDict_GET_SIZE", "PySet_GET_SIZE", "PyTuple_GET_ITEM",