From a6fd04259e647d410791a505b782661a4c3fc48f Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 18 Mar 2026 13:25:31 +0100 Subject: [PATCH 1/5] Retrieve code units from constants by index --- .../builtins/modules/MarshalModuleBuiltins.java | 5 +++++ .../compiler/bytecode_dsl/RootNodeCompiler.java | 3 ++- .../nodes/bytecode_dsl/PBytecodeDSLRootNode.java | 14 ++++++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java index 8ec4564be5..afd63099cb 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MarshalModuleBuiltins.java @@ -1512,6 +1512,11 @@ private void writeBytecodeCodeUnit(BytecodeCodeUnit code) throws IOException { } private void writeBytecodeDSLCodeUnit(BytecodeDSLCodeUnit code) throws IOException { + /* + * Nested code units referenced by MakeFunction are stored in co_consts; the + * MakeFunction instruction itself carries only the integer index into this constants + * array. + */ byte[] serialized = code.getSerialized(context); writeBytes(serialized); writeString(code.name); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java index f6a72b8127..5932cdd717 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java @@ -4350,8 +4350,9 @@ private void emitMakeFunction(BytecodeDSLCodeUnit codeUnit, Object scopeKey, Str // Register these in the Python constants list. addConstant(codeUnit); + int codeIndex = constants.get(codeUnit); - b.beginMakeFunction(functionName, qualifiedName, codeUnit); + b.beginMakeFunction(functionName, qualifiedName, codeIndex); if (defaultArgsLocal != null) { assert argsForDefaults == null; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java index 36d53bec19..d78ba3fa3d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java @@ -1465,19 +1465,20 @@ public static Object perform(VirtualFrame frame, LocalAccessor attributes, Objec @Operation(storeBytecodeIndex = true, forceCached = true) @ConstantOperand(type = TruffleString.class, name = "name") @ConstantOperand(type = TruffleString.class, name = "qualifiedName") - @ConstantOperand(type = BytecodeDSLCodeUnit.class) + @ConstantOperand(type = int.class) public static final class MakeFunction { @Specialization(guards = "isSingleContext(rootNode)") public static Object functionSingleContext(VirtualFrame frame, TruffleString name, TruffleString qualifiedName, - BytecodeDSLCodeUnit codeUnit, + @SuppressWarnings("unused") int codeIndex, Object[] defaults, Object[] kwDefaultsObject, Object closure, Object annotations, @Bind PBytecodeDSLRootNode rootNode, - @Cached("getCode(frame, codeUnit)") PCode cachedCode, + @Bind("getCodeUnit(rootNode, codeIndex)") BytecodeDSLCodeUnit codeUnit, + @Cached("getCode(frame, codeUnit)") PCode cachedCode, @Shared @Cached("createCodeStableAssumption()") Assumption codeStableAssumption, @Shared @Cached DynamicObject.PutNode putNode) { return createFunction(frame, name, qualifiedName, codeUnit.getDocstring(), @@ -1488,7 +1489,7 @@ public static Object functionSingleContext(VirtualFrame frame, public static Object functionMultiContext(VirtualFrame frame, TruffleString name, TruffleString qualifiedName, - BytecodeDSLCodeUnit codeUnit, + int codeIndex, Object[] defaults, Object[] kwDefaultsObject, Object closure, @@ -1496,6 +1497,7 @@ public static Object functionMultiContext(VirtualFrame frame, @Bind PBytecodeDSLRootNode rootNode, @Shared @Cached("createCodeStableAssumption()") Assumption codeStableAssumption, @Shared @Cached DynamicObject.PutNode putNode) { + BytecodeDSLCodeUnit codeUnit = getCodeUnit(rootNode, codeIndex); PCode code = getCode(frame, codeUnit); return createFunction(frame, name, qualifiedName, codeUnit.getDocstring(), code, defaults, kwDefaultsObject, closure, annotations, codeStableAssumption, rootNode, putNode); @@ -1517,6 +1519,10 @@ protected static PCode getCode(VirtualFrame frame, BytecodeDSLCodeUnit codeUnit) return thisCode.getOrCreateChildCode(codeUnit); } + protected static BytecodeDSLCodeUnit getCodeUnit(PBytecodeDSLRootNode rootNode, int codeIndex) { + return (BytecodeDSLCodeUnit) rootNode.getCodeUnit().constants[codeIndex]; + } + protected static PFunction createFunction(VirtualFrame frame, TruffleString name, TruffleString qualifiedName, TruffleString doc, PCode code, Object[] defaults, From 1b41821f604acfa484e64f1595e9e25d3ac13c3f Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 18 Mar 2026 14:27:14 +0100 Subject: [PATCH 2/5] Keep code units for reparse --- .../bytecode_dsl/BytecodeDSLCompiler.java | 6 +- .../bytecode_dsl/RootNodeCompiler.java | 140 ++++++++++-------- 2 files changed, 80 insertions(+), 66 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java index 4378439ba8..91f9d66d92 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/BytecodeDSLCompiler.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -41,6 +41,8 @@ package com.oracle.graal.python.compiler.bytecode_dsl; import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; import com.oracle.graal.python.PythonLanguage; import com.oracle.graal.python.compiler.Compiler; @@ -94,6 +96,8 @@ public static class BytecodeDSLCompilerContext { public final int futureLineNumber; public final ParserCallbacksImpl errorCallback; public final ScopeEnvironment scopeEnvironment; + // Store code units for possible reparses + public final Map codeUnits = new HashMap<>(); public BytecodeDSLCompilerContext(PythonLanguage language, ModTy mod, Source source, int optimizationLevel, EnumSet futureFeatures, int futureLineNumber, ParserCallbacksImpl errorCallback, ScopeEnvironment scopeEnvironment) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java index 5932cdd717..f56ca0a29d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/bytecode_dsl/RootNodeCompiler.java @@ -408,7 +408,10 @@ private String getNewScopeQualName(String name, CompilationScope scopeType) { return name; } - private BytecodeDSLCompilerResult compileRootNode(String name, ArgumentInfo argumentInfo, SourceRange sourceRange, BytecodeParser parser) { + private record CodeUnitKey(SSTNode node, CompilationScope scope) { + } + + private BytecodeDSLCompilerResult compileRootNode(String name, ArgumentInfo argumentInfo, SSTNode node, BytecodeParser parser) { qualName = getNewScopeQualName(name, scopeType); BytecodeRootNodes nodes = PBytecodeDSLRootNodeGen.create(ctx.language, BytecodeConfig.WITH_SOURCE, parser); @@ -416,56 +419,61 @@ private BytecodeDSLCompilerResult compileRootNode(String name, ArgumentInfo argu assert nodeList.size() == 1; PBytecodeDSLRootNode rootNode = nodeList.get(0); - int flags = PCode.CO_OPTIMIZED | PCode.CO_NEWLOCALS; - flags |= argumentInfo.takesVarArgs ? PCode.CO_VARARGS : 0; - flags |= argumentInfo.takesVarKeywordArgs ? PCode.CO_VARKEYWORDS : 0; - if (scope.isNested()) { - flags |= PCode.CO_NESTED; + CodeUnitKey key = new CodeUnitKey(node, scopeType); + BytecodeDSLCodeUnit codeUnit = ctx.codeUnits.get(key); + if (codeUnit == null) { + int flags = PCode.CO_OPTIMIZED | PCode.CO_NEWLOCALS; + flags |= argumentInfo.takesVarArgs ? PCode.CO_VARARGS : 0; + flags |= argumentInfo.takesVarKeywordArgs ? PCode.CO_VARKEYWORDS : 0; + if (scope.isNested()) { + flags |= PCode.CO_NESTED; + } + if (scope.isModule()) { + flags |= PCode.CO_GRAALPYHON_MODULE; + } + if (scope.isGenerator() && scope.isCoroutine()) { + flags |= PCode.CO_ASYNC_GENERATOR; + } else if (scope.isGenerator()) { + flags |= PCode.CO_GENERATOR; + } else if (scope.isCoroutine()) { + flags |= PCode.CO_COROUTINE; + } + for (FutureFeature flag : futureFeatures) { + flags |= flag.flagValue; + } + + int classcellIndex = -1; + if (freeLocals.containsKey(J___CLASS__)) { + classcellIndex = freeLocals.get(J___CLASS__).getLocalOffset(); + } + + int selfIndex = -1; + if (argumentInfo.nonEmpty()) { + selfIndex = 0; + if (selfCellName != null) { + selfIndex = cellLocals.get(selfCellName).getLocalOffset(); + } + } + SourceRange sourceRange = node.getSourceRange(); + codeUnit = new BytecodeDSLCodeUnit(toInternedTruffleStringUncached(name), toInternedTruffleStringUncached(qualName), + argumentInfo.argCount, argumentInfo.kwOnlyArgCount, argumentInfo.positionalOnlyArgCount, + flags, orderedTruffleStringArray(names), + orderedTruffleStringArray(varnames), + orderedTruffleStringArray(cellvars), + orderedTruffleStringArray(freevars), + cell2arg, + orderedKeys(constants, new Object[0]), + sourceRange.startLine, + sourceRange.startColumn, + sourceRange.endLine, + sourceRange.endColumn, + classcellIndex, + selfIndex, + yieldFromGenerator != null ? yieldFromGenerator.getLocalIndex() : -1, + instrumentationDataLocal.getLocalIndex(), + new BytecodeSupplier(nodes)); + ctx.codeUnits.put(key, codeUnit); } - if (scope.isModule()) { - flags |= PCode.CO_GRAALPYHON_MODULE; - } - if (scope.isGenerator() && scope.isCoroutine()) { - flags |= PCode.CO_ASYNC_GENERATOR; - } else if (scope.isGenerator()) { - flags |= PCode.CO_GENERATOR; - } else if (scope.isCoroutine()) { - flags |= PCode.CO_COROUTINE; - } - for (FutureFeature flag : futureFeatures) { - flags |= flag.flagValue; - } - - int classcellIndex = -1; - if (freeLocals.containsKey(J___CLASS__)) { - classcellIndex = freeLocals.get(J___CLASS__).getLocalOffset(); - } - - int selfIndex = -1; - if (argumentInfo.nonEmpty()) { - selfIndex = 0; - if (selfCellName != null) { - selfIndex = cellLocals.get(selfCellName).getLocalOffset(); - } - } - - BytecodeDSLCodeUnit codeUnit = new BytecodeDSLCodeUnit(toInternedTruffleStringUncached(name), toInternedTruffleStringUncached(qualName), - argumentInfo.argCount, argumentInfo.kwOnlyArgCount, argumentInfo.positionalOnlyArgCount, - flags, orderedTruffleStringArray(names), - orderedTruffleStringArray(varnames), - orderedTruffleStringArray(cellvars), - orderedTruffleStringArray(freevars), - cell2arg, - orderedKeys(constants, new Object[0]), - sourceRange.startLine, - sourceRange.startColumn, - sourceRange.endLine, - sourceRange.endColumn, - classcellIndex, - selfIndex, - yieldFromGenerator != null ? yieldFromGenerator.getLocalIndex() : -1, - instrumentationDataLocal.getLocalIndex(), - new BytecodeSupplier(nodes)); rootNode.setMetadata(codeUnit, ctx.errorCallback); return new BytecodeDSLCompilerResult(rootNode, codeUnit); } @@ -784,7 +792,9 @@ boolean beginSourceSection(SSTNode node, Builder b) { return beginSourceSection(node.getSourceRange(), b); } - /** {@link #beginSourceSection(SSTNode, Builder)} */ + /** + * {@link #beginSourceSection(SSTNode, Builder)} + */ boolean beginSourceSection(SourceRange sourceRange, Builder b) { SourceRange oldSourceRange = this.currentLocation; this.currentLocation = sourceRange; @@ -873,7 +883,7 @@ String maybeMangleAndAddName(String name) { @Override public BytecodeDSLCompilerResult visit(ModTy.Module node) { - return compileRootNode("", ArgumentInfo.NO_ARGS, node.getSourceRange(), b -> { + return compileRootNode("", ArgumentInfo.NO_ARGS, node, b -> { beginRootNode(node, null, b); visitModuleBody(node.body, b, true); endRootNode(b); @@ -882,7 +892,7 @@ public BytecodeDSLCompilerResult visit(ModTy.Module node) { @Override public BytecodeDSLCompilerResult visit(ModTy.Expression node) { - return compileRootNode("", ArgumentInfo.NO_ARGS, node.getSourceRange(), b -> { + return compileRootNode("", ArgumentInfo.NO_ARGS, node, b -> { beginRootNode(node, null, b); beginReturn(b); new StatementCompiler(b).visitNode(node.body); @@ -893,7 +903,7 @@ public BytecodeDSLCompilerResult visit(ModTy.Expression node) { @Override public BytecodeDSLCompilerResult visit(ModTy.Interactive node) { - return compileRootNode("", ArgumentInfo.NO_ARGS, node.getSourceRange(), b -> { + return compileRootNode("", ArgumentInfo.NO_ARGS, node, b -> { beginRootNode(node, null, b); visitModuleBody(node.body, b, false); endRootNode(b); @@ -985,8 +995,8 @@ private static TruffleString getDocstring(StmtTy[] body) { } public BytecodeDSLCompilerResult compileFunctionDef(StmtTy node, String name, ArgumentsTy args, StmtTy[] body) { - return compileRootNode(name, ArgumentInfo.fromArguments(args), node.getSourceRange(), - b -> emitFunctionDefBody(node, args, body, b, getDocstring(body), false)); + return compileRootNode(name, ArgumentInfo.fromArguments(args), + node, b -> emitFunctionDefBody(node, args, body, b, getDocstring(body), false)); } /** @@ -1013,7 +1023,7 @@ private BytecodeDSLCompilerResult compileFunctionTypeParams(BytecodeDSLCodeUnit typeParamsUnitArgs = NO_ARGS; } ArgumentInfo argInfo = ArgumentInfo.fromArguments(typeParamsUnitArgs); - return compileRootNode(name, argInfo, node.getSourceRange(), b -> { + return compileRootNode(name, argInfo, node, b -> { beginRootNode(node, typeParamsUnitArgs, b); StatementCompiler statementCompiler = new StatementCompiler(b); @@ -1057,7 +1067,7 @@ private BytecodeDSLCompilerResult compileFunctionTypeParams(BytecodeDSLCodeUnit private BytecodeDSLCompilerResult compileBoundTypeVar(TypeVar node) { assert node.bound != null; - return compileRootNode(node.name, ArgumentInfo.NO_ARGS, node.getSourceRange(), b -> { + return compileRootNode(node.name, ArgumentInfo.NO_ARGS, node, b -> { beginRootNode(node, null, b); b.beginReturn(); node.bound.accept(new StatementCompiler(b)); @@ -1068,7 +1078,7 @@ private BytecodeDSLCompilerResult compileBoundTypeVar(TypeVar node) { private BytecodeDSLCompilerResult compileTypeAliasBody(TypeAlias node) { String name = ((ExprTy.Name) node.name).id; - return compileRootNode(name, ArgumentInfo.NO_ARGS, node.getSourceRange(), b -> { + return compileRootNode(name, ArgumentInfo.NO_ARGS, node, b -> { // Make None the first constant, so the evaluate function can't have a docstring. addObject(constants, PNone.NONE); beginRootNode(node, null, b); @@ -1082,7 +1092,7 @@ private BytecodeDSLCompilerResult compileTypeAliasBody(TypeAlias node) { private BytecodeDSLCompilerResult compileTypeAliasTypeParameters(String name, BytecodeDSLCodeUnit codeUnit, TypeAlias node) { assert this.scopeType == CompilationScope.TypeParams; String typeParamsName = ""; - return compileRootNode(typeParamsName, ArgumentInfo.NO_ARGS, node.getSourceRange(), b -> { + return compileRootNode(typeParamsName, ArgumentInfo.NO_ARGS, node, b -> { beginRootNode(node, null, b); StatementCompiler statementCompiler = new StatementCompiler(b); statementCompiler.emitBuildTypeAlias(codeUnit, node); @@ -1092,8 +1102,8 @@ private BytecodeDSLCompilerResult compileTypeAliasTypeParameters(String name, By @Override public BytecodeDSLCompilerResult visit(ExprTy.Lambda node) { - return compileRootNode("", ArgumentInfo.fromArguments(node.args), node.getSourceRange(), - b -> emitFunctionDefBody(node, node.args, new SSTNode[]{node.body}, b, null, true)); + return compileRootNode("", ArgumentInfo.fromArguments(node.args), + node, b -> emitFunctionDefBody(node, node.args, new SSTNode[]{node.body}, b, null, true)); } private void emitFunctionDefBody(SSTNode node, ArgumentsTy args, SSTNode[] body, Builder b, Object docstring, boolean isLambda) { @@ -1138,7 +1148,7 @@ private void emitFunctionDefBody(SSTNode node, ArgumentsTy args, SSTNode[] body, } public BytecodeDSLCompilerResult compileClassDefBody(StmtTy.ClassDef node) { - return compileRootNode(node.name, ArgumentInfo.NO_ARGS, node.getSourceRange(), b -> { + return compileRootNode(node.name, ArgumentInfo.NO_ARGS, node, b -> { beginRootNode(node, null, b); beginStoreLocal("__module__", b); @@ -1210,7 +1220,7 @@ public BytecodeDSLCompilerResult compileClassDefBody(StmtTy.ClassDef node) { public BytecodeDSLCompilerResult compileClassTypeParams(StmtTy.ClassDef node, BytecodeDSLCodeUnit classBody) { assert this.scopeType == CompilationScope.TypeParams; - return compileRootNode(node.name, ArgumentInfo.NO_ARGS, node.getSourceRange(), b -> { + return compileRootNode(node.name, ArgumentInfo.NO_ARGS, node, b -> { beginRootNode(node, null, b); StatementCompiler statementCompiler = new StatementCompiler(b); statementCompiler.emitBuildClass(classBody, node); @@ -1317,7 +1327,7 @@ private BytecodeDSLCompilerResult buildComprehensionCodeUnit(SSTNode node, Compr if (scope.isCoroutine() && type != ComprehensionType.GENEXPR && scopeType != CompilationScope.AsyncFunction && scopeType != CompilationScope.Comprehension) { throw ctx.errorCallback.onError(ErrorType.Syntax, currentLocation, "asynchronous comprehension outside of an asynchronous function"); } - return compileRootNode(type.name, new ArgumentInfo(1, 0, 0, false, false), node.getSourceRange(), b -> { + return compileRootNode(type.name, new ArgumentInfo(1, 0, 0, false, false), node, b -> { beginRootNode(node, null, b); assert scope.isGenerator() == (type == ComprehensionType.GENEXPR); From 39aa350e166c1769412f1a03856d0c346bb2dbb2 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 18 Mar 2026 17:49:34 +0100 Subject: [PATCH 3/5] Re-enable disabled test --- .../com.oracle.graal.python.test/src/tests/test_code.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_code.py b/graalpython/com.oracle.graal.python.test/src/tests/test_code.py index 8c944d6377..915ce2ef8f 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_code.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_code.py @@ -301,8 +301,7 @@ def bar(): bar = foo() assert bar.__code__ is foo().__code__ i = foo.__code__.co_consts.index(bar.__code__) - # TODO this is currently broken on the DSL interpreter because the code unit in constants is a separate copy - # assert bar.__code__ is foo.__code__.co_consts[i] + assert bar.__code__ is foo.__code__.co_consts[i] assert bar.__code__ is bar().f_code foo_copy = types.FunctionType(marshal.loads(marshal.dumps(foo.__code__)), globals=foo.__globals__, closure=foo.__closure__) From 51bd0413ace0817287d64cd9236756269ae4ff4b Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Wed, 18 Mar 2026 19:53:34 +0100 Subject: [PATCH 4/5] Use constants array directly instead of childCode map --- .../python/builtins/objects/code/PCode.java | 159 +++++++----------- .../nodes/bytecode/MakeFunctionNode.java | 10 +- .../nodes/bytecode/PBytecodeRootNode.java | 10 +- .../bytecode_dsl/PBytecodeDSLRootNode.java | 8 +- 4 files changed, 74 insertions(+), 113 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java index 397659c088..9700af6e37 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java @@ -41,30 +41,20 @@ package com.oracle.graal.python.builtins.objects.code; import static com.oracle.graal.python.nodes.StringLiterals.J_EMPTY_STRING; -import static com.oracle.graal.python.util.PythonUtils.EMPTY_OBJECT_ARRAY; import static com.oracle.graal.python.util.PythonUtils.EMPTY_TRUFFLESTRING_ARRAY; import static com.oracle.graal.python.util.PythonUtils.toInternedTruffleStringUncached; import java.math.BigInteger; -import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; import com.oracle.graal.python.PythonLanguage; -import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.bytes.PBytes; -import com.oracle.graal.python.builtins.objects.ellipsis.PEllipsis; import com.oracle.graal.python.builtins.objects.function.Signature; import com.oracle.graal.python.builtins.objects.generator.PGenerator; import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject; import com.oracle.graal.python.builtins.objects.tuple.PTuple; import com.oracle.graal.python.compiler.BytecodeCodeUnit; import com.oracle.graal.python.compiler.CodeUnit; -import com.oracle.graal.python.compiler.OpCodes; import com.oracle.graal.python.nodes.PRootNode; import com.oracle.graal.python.nodes.bytecode.PBytecodeGeneratorFunctionRootNode; import com.oracle.graal.python.nodes.bytecode.PBytecodeGeneratorRootNode; @@ -136,8 +126,6 @@ public final class PCode extends PythonBuiltinObject { // qualified name with which this code object was defined private TruffleString qualname; - private Map childCode; - // number of first line in Python source code private int firstlineno = -1; // is a string encoding the mapping from bytecode offsets to line numbers @@ -293,55 +281,23 @@ private static TruffleString[] extractVarnames(RootNode node) { return EMPTY_TRUFFLESTRING_ARRAY; } + private Object[] ensureConstants() { + if (constants == null) { + CodeUnit codeUnit = getCodeUnit(getRootNode()); + constants = codeUnit != null ? new Object[codeUnit.constants.length] : PythonUtils.EMPTY_OBJECT_ARRAY; + } + return constants; + } + @TruffleBoundary - private Object[] extractConstants(RootNode node) { - RootNode rootNode = rootNodeForExtraction(node); - if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) { - if (rootNode instanceof PBytecodeDSLRootNode bytecodeDSLRootNode) { - BytecodeDSLCodeUnit co = bytecodeDSLRootNode.getCodeUnit(); - List constants = new ArrayList<>(); - for (int i = 0; i < co.constants.length; i++) { - Object constant = convertConstantToPythonSpace(co.constants[i]); - constants.add(constant); - } - return constants.toArray(new Object[0]); - } - } else if (rootNode instanceof PBytecodeRootNode bytecodeRootNode) { - BytecodeCodeUnit co = bytecodeRootNode.getCodeUnit(); - Set bytecodeConstants = new HashSet<>(); - for (int bci = 0; bci < co.code.length;) { - OpCodes op = OpCodes.fromOpCode(co.code[bci]); - if (op.quickens != null) { - op = op.quickens; - } - if (op == OpCodes.LOAD_BYTE) { - bytecodeConstants.add(Byte.toUnsignedInt(co.code[bci + 1])); - } else if (op == OpCodes.LOAD_NONE) { - bytecodeConstants.add(PNone.NONE); - } else if (op == OpCodes.LOAD_TRUE) { - bytecodeConstants.add(true); - } else if (op == OpCodes.LOAD_FALSE) { - bytecodeConstants.add(false); - } else if (op == OpCodes.LOAD_ELLIPSIS) { - bytecodeConstants.add(PEllipsis.INSTANCE); - } else if (op == OpCodes.LOAD_INT || op == OpCodes.LOAD_LONG) { - bytecodeConstants.add(co.primitiveConstants[Byte.toUnsignedInt(co.code[bci + 1])]); - } else if (op == OpCodes.LOAD_DOUBLE) { - bytecodeConstants.add(Double.longBitsToDouble(co.primitiveConstants[Byte.toUnsignedInt(co.code[bci + 1])])); - } - bci += op.length(); - } - List constants = new ArrayList<>(); - for (int i = 0; i < co.constants.length; i++) { - Object constant = convertConstantToPythonSpace(co.constants[i]); - if (constant != PNone.NONE || !bytecodeConstants.contains(PNone.NONE)) { - constants.add(constant); - } - } - constants.addAll(bytecodeConstants); - return constants.toArray(new Object[0]); + private Object getOrCreateConstant(int index) { + Object[] cachedConstants = ensureConstants(); + Object constant = cachedConstants[index]; + if (constant == null) { + constant = convertConstantToPythonSpace(index); + cachedConstants[index] = constant; } - return EMPTY_OBJECT_ARRAY; + return constant; } @TruffleBoundary @@ -419,9 +375,11 @@ public void fixCoFilename(TruffleString filename) { * New code objects inherit the filename from parent, so no need to eagerly construct them * here */ - if (childCode != null) { - for (PCode code : childCode.values()) { - code.filename = filename; + if (constants != null) { + for (Object constant : constants) { + if (constant instanceof PCode code) { + code.filename = filename; + } } } } @@ -525,65 +483,66 @@ public CodeUnit getCodeUnit() { } public Object[] getConstants() { - if (constants == null) { - constants = extractConstants(getRootNode()); + Object[] cachedConstants = ensureConstants(); + for (int i = 0; i < cachedConstants.length; i++) { + getOrCreateConstant(i); } - return constants; + return cachedConstants; } - @TruffleBoundary - public PCode getOrCreateChildCode(BytecodeDSLCodeUnit codeUnit) { - PCode code = null; - if (childCode == null) { - childCode = new HashMap<>(); - } else { - code = childCode.get(codeUnit); - } + public PCode getOrCreateChildCode(int index, BytecodeDSLCodeUnit codeUnit) { + Object[] cachedConstants = ensureConstants(); + PCode code = (PCode) cachedConstants[index]; if (code == null) { - PBytecodeDSLRootNode outerRootNode = (PBytecodeDSLRootNode) getRootNode(); - PythonLanguage language = outerRootNode.getLanguage(); - RootCallTarget callTarget = language.createCachedCallTarget(l -> codeUnit.createRootNode(PythonContext.get(null), outerRootNode.getSource()), codeUnit); - PBytecodeDSLRootNode rootNode = (PBytecodeDSLRootNode) callTarget.getRootNode(); - code = PFactory.createCode(language, callTarget, rootNode.getSignature(), codeUnit, getFilename()); - childCode.put(codeUnit, code); + code = createCode(codeUnit); + cachedConstants[index] = code; } return code; } @TruffleBoundary - public PCode getOrCreateChildCode(BytecodeCodeUnit codeUnit) { - PCode code = null; - if (childCode == null) { - childCode = new HashMap<>(); - } else { - code = childCode.get(codeUnit); - } + private PCode createCode(BytecodeDSLCodeUnit codeUnit) { + PBytecodeDSLRootNode outerRootNode = (PBytecodeDSLRootNode) getRootNode(); + PythonLanguage language = outerRootNode.getLanguage(); + RootCallTarget callTarget = language.createCachedCallTarget(l -> codeUnit.createRootNode(PythonContext.get(null), outerRootNode.getSource()), codeUnit); + PBytecodeDSLRootNode rootNode = (PBytecodeDSLRootNode) callTarget.getRootNode(); + return PFactory.createCode(language, callTarget, rootNode.getSignature(), codeUnit, getFilename()); + } + + public PCode getOrCreateChildCode(int index, BytecodeCodeUnit codeUnit) { + Object[] cachedConstants = ensureConstants(); + PCode code = (PCode) cachedConstants[index]; if (code == null) { - PBytecodeRootNode outerRootNode = (PBytecodeRootNode) getRootNodeForExtraction(); - PythonLanguage language = outerRootNode.getLanguage(); - RootCallTarget callTarget = language.createCachedCallTarget( - l -> PBytecodeRootNode.createMaybeGenerator(language, codeUnit, outerRootNode.getLazySource(), outerRootNode.isInternal()), - codeUnit); - RootNode rootNode = callTarget.getRootNode(); - if (rootNode instanceof PBytecodeGeneratorFunctionRootNode generatorRoot) { - rootNode = generatorRoot.getBytecodeRootNode(); - } - code = PFactory.createCode(language, callTarget, ((PBytecodeRootNode) rootNode).getSignature(), codeUnit, getFilename()); - childCode.put(codeUnit, code); + code = createCode(codeUnit); + cachedConstants[index] = code; } return code; } @TruffleBoundary - private Object convertConstantToPythonSpace(Object o) { + private PCode createCode(BytecodeCodeUnit codeUnit) { + PBytecodeRootNode outerRootNode = (PBytecodeRootNode) getRootNodeForExtraction(); + PythonLanguage language = outerRootNode.getLanguage(); + RootCallTarget callTarget = language.createCachedCallTarget( + l -> PBytecodeRootNode.createMaybeGenerator(language, codeUnit, outerRootNode.getLazySource(), outerRootNode.isInternal()), codeUnit); + RootNode rootNode = callTarget.getRootNode(); + if (rootNode instanceof PBytecodeGeneratorFunctionRootNode generatorRoot) { + rootNode = generatorRoot.getBytecodeRootNode(); + } + return PFactory.createCode(language, callTarget, ((PBytecodeRootNode) rootNode).getSignature(), codeUnit, getFilename()); + } + + @TruffleBoundary + private Object convertConstantToPythonSpace(int index) { + Object o = getCodeUnit().constants[index]; PythonLanguage language = PythonLanguage.get(null); if (o instanceof CodeUnit) { if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) { BytecodeDSLCodeUnit code = (BytecodeDSLCodeUnit) o; - return getOrCreateChildCode(code); + return getOrCreateChildCode(index, code); } else { BytecodeCodeUnit code = (BytecodeCodeUnit) o; - return getOrCreateChildCode(code); + return getOrCreateChildCode(index, code); } } else if (o instanceof BigInteger) { return PFactory.createInt(language, (BigInteger) o); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/MakeFunctionNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/MakeFunctionNode.java index 0dee0e1405..6c1ed0cb3f 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/MakeFunctionNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/MakeFunctionNode.java @@ -65,6 +65,7 @@ import com.oracle.truffle.api.frame.VirtualFrame; public abstract class MakeFunctionNode extends PNodeWithContext { + private final int codeIndex; private final BytecodeCodeUnit codeUnit; @CompilationFinal private PCode cachedCode; @@ -72,7 +73,8 @@ public abstract class MakeFunctionNode extends PNodeWithContext { public abstract int execute(VirtualFrame frame, Object globals, int initialStackTop, int flags); - public MakeFunctionNode(BytecodeCodeUnit codeUnit) { + public MakeFunctionNode(int codeIndex, BytecodeCodeUnit codeUnit) { + this.codeIndex = codeIndex; this.codeUnit = codeUnit; } @@ -84,7 +86,7 @@ int makeFunction(VirtualFrame frame, Object globals, int initialStackTop, int fl PCode code = cachedCode; if (code == null) { - code = PArguments.getCodeObject(frame).getOrCreateChildCode(codeUnit); + code = PArguments.getCodeObject(frame).getOrCreateChildCode(codeIndex, codeUnit); if (PythonLanguage.get(this).isSingleContext()) { CompilerDirectives.transferToInterpreterAndInvalidate(); /* @@ -130,7 +132,7 @@ int makeFunction(VirtualFrame frame, Object globals, int initialStackTop, int fl } @NeverDefault - public static MakeFunctionNode create(BytecodeCodeUnit codeUnit) { - return MakeFunctionNodeGen.create(codeUnit); + public static MakeFunctionNode create(int codeIndex, BytecodeCodeUnit codeUnit) { + return MakeFunctionNodeGen.create(codeIndex, codeUnit); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java index 70c3cbb2c9..76108c5313 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java @@ -2183,7 +2183,7 @@ private Object bytecodeLoop(VirtualFrame virtualFrame, Frame localFrame, Bytecod case OpCodesConstants.MAKE_FUNCTION: { oparg |= Byte.toUnsignedInt(localBC[++bci]); int flags = Byte.toUnsignedInt(localBC[++bci]); - stackTop = bytecodeMakeFunction(virtualFrame, globals, stackTop, localNodes, beginBci, flags, localConsts[oparg]); + stackTop = bytecodeMakeFunction(virtualFrame, globals, stackTop, localNodes, beginBci, oparg, flags, localConsts[oparg]); break; } case OpCodesConstants.SETUP_ANNOTATIONS: { @@ -2727,9 +2727,9 @@ private void bytecodeSetupAnnotations(VirtualFrame virtualFrame, boolean useCach } @BytecodeInterpreterSwitch - private int bytecodeMakeFunction(VirtualFrame virtualFrame, Object globals, int stackTop, Node[] localNodes, int beginBci, int flags, Object localConsts) { + private int bytecodeMakeFunction(VirtualFrame virtualFrame, Object globals, int stackTop, Node[] localNodes, int beginBci, int codeIndex, int flags, Object localConsts) { BytecodeCodeUnit codeUnit = (BytecodeCodeUnit) localConsts; - MakeFunctionNode makeFunctionNode = insertMakeFunctionNode(localNodes, beginBci, codeUnit); + MakeFunctionNode makeFunctionNode = insertMakeFunctionNode(localNodes, beginBci, codeIndex, codeUnit); return makeFunctionNode.execute(virtualFrame, globals, stackTop, flags); } @@ -2910,8 +2910,8 @@ private Object notifyEnter(VirtualFrame virtualFrame, InstrumentationSupport ins return null; } - private MakeFunctionNode insertMakeFunctionNode(Node[] localNodes, int beginBci, BytecodeCodeUnit codeUnit) { - return insertChildNode(localNodes, beginBci, MakeFunctionNodeGen.class, () -> MakeFunctionNode.create(codeUnit)); + private MakeFunctionNode insertMakeFunctionNode(Node[] localNodes, int beginBci, int codeIndex, BytecodeCodeUnit codeUnit) { + return insertChildNode(localNodes, beginBci, MakeFunctionNodeGen.class, () -> MakeFunctionNode.create(codeIndex, codeUnit)); } public void materializeContainedFunctionsForInstrumentation(Set> materializedTags) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java index d78ba3fa3d..85f4760536 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode_dsl/PBytecodeDSLRootNode.java @@ -1478,7 +1478,7 @@ public static Object functionSingleContext(VirtualFrame frame, Object annotations, @Bind PBytecodeDSLRootNode rootNode, @Bind("getCodeUnit(rootNode, codeIndex)") BytecodeDSLCodeUnit codeUnit, - @Cached("getCode(frame, codeUnit)") PCode cachedCode, + @Cached("getCode(frame, codeIndex, codeUnit)") PCode cachedCode, @Shared @Cached("createCodeStableAssumption()") Assumption codeStableAssumption, @Shared @Cached DynamicObject.PutNode putNode) { return createFunction(frame, name, qualifiedName, codeUnit.getDocstring(), @@ -1498,7 +1498,7 @@ public static Object functionMultiContext(VirtualFrame frame, @Shared @Cached("createCodeStableAssumption()") Assumption codeStableAssumption, @Shared @Cached DynamicObject.PutNode putNode) { BytecodeDSLCodeUnit codeUnit = getCodeUnit(rootNode, codeIndex); - PCode code = getCode(frame, codeUnit); + PCode code = getCode(frame, codeIndex, codeUnit); return createFunction(frame, name, qualifiedName, codeUnit.getDocstring(), code, defaults, kwDefaultsObject, closure, annotations, codeStableAssumption, rootNode, putNode); } @@ -1514,9 +1514,9 @@ static Assumption createCodeStableAssumption() { } @NeverDefault - protected static PCode getCode(VirtualFrame frame, BytecodeDSLCodeUnit codeUnit) { + protected static PCode getCode(VirtualFrame frame, int codeIndex, BytecodeDSLCodeUnit codeUnit) { PCode thisCode = PArguments.getCodeObject(frame); - return thisCode.getOrCreateChildCode(codeUnit); + return thisCode.getOrCreateChildCode(codeIndex, codeUnit); } protected static BytecodeDSLCodeUnit getCodeUnit(PBytecodeDSLRootNode rootNode, int codeIndex) { From 5600cd0d7d996a9ee864ee82171cd36813559b7c Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 19 Mar 2026 09:49:12 +0100 Subject: [PATCH 5/5] Remve unwanted tests for co_consts --- .../src/tests/test_code.py | 6 -- .../src/tests/test_parser.py | 56 +------------------ 2 files changed, 1 insertion(+), 61 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_code.py b/graalpython/com.oracle.graal.python.test/src/tests/test_code.py index 915ce2ef8f..3e85b1eb51 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_code.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_code.py @@ -180,17 +180,11 @@ def inner(): code = compile(codestr, "", "exec") assert "module doc" in code.co_consts - assert 1 in code.co_consts assert "fn doc" not in code.co_consts for const in code.co_consts: if type(const) == types.CodeType: code = const assert "fn doc" in code.co_consts - assert "this is fun" not in code.co_consts - for const in code.co_consts: - if type(const) == types.CodeType: - code = const - assert "this is fun" in code.co_consts def test_generator_code_consts(): diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_parser.py b/graalpython/com.oracle.graal.python.test/src/tests/test_parser.py index 49edd17402..ddd96cc260 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_parser.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_parser.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -779,7 +779,6 @@ def test_annotations_in_function(): exec(code,test_globals) assert len(test_globals['__annotations__']) == 0 assert len(test_globals['fn'].__annotations__) == 0 - assert 1 not in test_globals['fn'].__code__.co_consts # the annotation is ignored in function source = '''def fn(): a:int =1 @@ -789,7 +788,6 @@ def test_annotations_in_function(): assert len(test_globals['__annotations__']) == 0 assert hasattr(test_globals['fn'], '__annotations__') assert len(test_globals['fn'].__annotations__) == 0 - assert 1 in test_globals['fn'].__code__.co_consts def test_annotations_in_class(): @@ -849,58 +847,6 @@ def test_annotations_in_class(): assert test_globals['Style'].__annotations__['_path'] == str assert '_path' in dir(test_globals['Style']) -def test_negative_float(): - - def check_const(fn, expected): - for const in fn.__code__.co_consts: - if repr(const) == repr(expected): - return True - else: - return False - - def fn1(): - return -0.0 - - assert check_const(fn1, -0.0) - - -def find_count_in(collection, what): - count = 0; - for item in collection: - if item == what: - count +=1 - return count - -def test_same_consts(): - def fn1(): a = 1; b = 1; return a + b - assert find_count_in(fn1.__code__.co_consts, 1) == 1 - - def fn2(): a = 'a'; b = 'a'; return a + b - assert find_count_in(fn2.__code__.co_consts, 'a') == 1 - -def test_tuple_in_const(): - def fn1() : return (0,) - assert (0,) in fn1.__code__.co_consts - assert 0 not in fn1.__code__.co_consts - - def fn2() : return (1, 2, 3, 1, 2, 3) - assert (1, 2, 3, 1, 2, 3) in fn2.__code__.co_consts - assert 1 not in fn2.__code__.co_consts - assert 2 not in fn2.__code__.co_consts - assert 3 not in fn2.__code__.co_consts - assert find_count_in(fn2.__code__.co_consts, (1, 2, 3, 1, 2, 3)) == 1 - - def fn3() : a = 1; return (1, 2, 1) - assert (1, 2, 1) in fn3.__code__.co_consts - assert find_count_in(fn3.__code__.co_consts, 1) == 1 - assert 2 not in fn3.__code__.co_consts - - def fn4() : a = 1; b = (1,2,3); c = 4; return (1, 2, 3, 1, 2, 3) - assert (1, 2, 3) in fn4.__code__.co_consts - assert (1, 2, 3, 1, 2, 3) in fn4.__code__.co_consts - assert 2 not in fn4.__code__.co_consts - assert find_count_in(fn4.__code__.co_consts, 1) == 1 - assert find_count_in(fn4.__code__.co_consts, 4) == 1 def test_ComprehensionGeneratorExpr(): def create_list(gen):