[GR-55107] Enable NFI Panama by default on GraalPy on HotSpot

PullRequest: graalpython/3848

Author: eregon

CHANGELOG.md

@@ -17,6 +17,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 * Enable FTS3, FTS4, FTS5, RTREE, and math function features in the bundled sqlite3 library.
 * Add support patches for Torch 2.7.0, PyGObject 3.52.3, xmlschema 4.0.0, lxml < 5.4.0, SciPy 1.15, jq 1.8.0, NumPy < 2.3, ormsgpack < 1.9.1, pandas 2.2.3, PyArrow 19.0, PyMuPDF 1.25.4.
 * The GraalPy Native standalone on Linux now uses the G1 garbage collector which is much faster.
+* Speedup native extensions by using the Panama NFI backend for faster native calls, available on GraalVM 25, JDK 25 and in the GraalPy JVM standalone.
 
 ## Version 24.2.0
 * Updated developer metadata of Maven artifacts.

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "eed61bcd1e6a728edbbe22e323ca383f179faeb2" }
+{ "overlay": "734294916a9baacff3c31118a6052aa3eee6efd3" }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

@@ -61,6 +61,7 @@
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.Option;
 import com.oracle.truffle.api.TruffleLanguage.Env;
+import com.oracle.truffle.api.TruffleOptions;
 import com.oracle.truffle.api.dsl.Idempotent;
 import com.oracle.truffle.api.exception.AbstractTruffleException;
 import com.oracle.truffle.api.nodes.ExplodeLoop;
@@ -362,8 +363,8 @@ public static void checkBytecodeDSLEnv() {
     @Option(category = OptionCategory.EXPERT, usageSyntax = "<bytes>", help = "Initial native memory heap size that triggers a GC (default: 256 MB).") //
     public static final OptionKey<Long> InitialNativeMemory = new OptionKey<>(1L << 28);
 
-    @Option(category = OptionCategory.EXPERT, usageSyntax = "true|false", help = "Use the experimental panama backend for NFI.", stability = OptionStability.EXPERIMENTAL) //
-    public static final OptionKey<Boolean> UsePanama = new OptionKey<>(false);
+    @Option(category = OptionCategory.EXPERT, usageSyntax = "true|false", help = "Use the panama backend for NFI.", stability = OptionStability.EXPERIMENTAL) //
+    public static final OptionKey<Boolean> UsePanama = new OptionKey<>(!TruffleOptions.AOT && Runtime.version().feature() >= 22);
 
     @Option(category = OptionCategory.EXPERT, usageSyntax = "true|false", help = "Set by the launcher to true (false means that GraalPy is being embedded in an application).") //
     public static final OptionKey<Boolean> RunViaLauncher = new OptionKey<>(false);

mx.graalpython/mx_graalpython.py

@@ -356,7 +356,7 @@ def do_run_python(args, extra_vm_args=None, env=None, jdk=None, extra_dists=None
         cp_prefix = ver_dep if cp_prefix is None else (ver_dep + os.pathsep + cp_prefix)
     else:
         dists = ['GRAALPYTHON']
-    dists += ['TRUFFLE_NFI', 'TRUFFLE_NFI_LIBFFI', 'GRAALPYTHON-LAUNCHER']
+    dists += ['GRAALPYTHON-LAUNCHER']
 
     vm_args, graalpython_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False)
     if minimal:
@@ -378,7 +378,7 @@ def do_run_python(args, extra_vm_args=None, env=None, jdk=None, extra_dists=None
 
     graalpython_args.insert(0, '--experimental-options=true')
 
-    vm_args += mx.get_runtime_jvm_args(dists, jdk=jdk, cp_prefix=cp_prefix, cp_suffix=cp_suffix)
+    vm_args += mx.get_runtime_jvm_args(dists, jdk=jdk, cp_prefix=cp_prefix, cp_suffix=cp_suffix, force_cp=True)
 
     if not jdk:
         jdk = get_jdk()
@@ -396,7 +396,7 @@ def do_run_python(args, extra_vm_args=None, env=None, jdk=None, extra_dists=None
 
 def node_footprint_analyzer(args, **kwargs):
     main_class = 'com.oracle.graal.python.test.advanced.NodeFootprintAnalyzer'
-    vm_args = mx.get_runtime_jvm_args(['GRAALPYTHON_UNIT_TESTS', 'GRAALPYTHON', 'TRUFFLE_NFI', 'TRUFFLE_NFI_LIBFFI'])
+    vm_args = mx.get_runtime_jvm_args(['GRAALPYTHON_UNIT_TESTS', 'GRAALPYTHON'])
     return mx.run_java(vm_args + [main_class] + args, **kwargs)
 
 
@@ -1155,7 +1155,6 @@ def graalpython_gate_runner(args, tasks):
     with Task('GraalPython JUnit', tasks, tags=[GraalPythonTags.junit]) as task:
         if task:
             run_mx(["build"], env={**os.environ, **LATEST_JAVA_HOME})
-            run_mx(["build", "--only", "JLINE3"], env={**os.environ})
             if WIN32:
                 punittest(
                     [
@@ -1401,7 +1400,8 @@ def graalpython_gate_runner(args, tasks):
             run_mx(["build"], env={**os.environ, **LATEST_JAVA_HOME})
             args =['--verbose']
             vm_args = ['-Dpolyglot.engine.WarnInterpreterOnly=false']
-            mx_unittest.unittest(vm_args + ['org.graalvm.python.embedding.vfs.test'] + args + ["--use-graalvm"])
+            has_compiler = bool(mx.suite('compiler', fatalIfMissing=False))
+            mx_unittest.unittest(vm_args + ['org.graalvm.python.embedding.vfs.test'] + args + (["--use-graalvm"] if has_compiler else []))
 
     with Task('GraalPython Python tests', tasks, tags=[GraalPythonTags.tagged]) as task:
         if task:
@@ -2539,7 +2539,7 @@ def run_leak_launcher(input_args):
 
     env = os.environ.copy()
 
-    dists = ['GRAALPYTHON', 'GRAALPYTHON_RESOURCES', 'TRUFFLE_NFI', 'TRUFFLE_NFI_LIBFFI', 'GRAALPYTHON_UNIT_TESTS']
+    dists = ['GRAALPYTHON', 'GRAALPYTHON_RESOURCES', 'GRAALPYTHON_UNIT_TESTS']
 
     vm_args, graalpython_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False)
     vm_args += mx.get_runtime_jvm_args(dists)

mx.graalpython/mx_graalpython_benchmark.py

@@ -73,7 +73,6 @@
 CONFIGURATION_NATIVE_MULTI_TIER = "native-multi-tier"
 CONFIGURATION_SANDBOXED = "sandboxed"
 CONFIGURATION_SANDBOXED_MULTI = "sandboxed-multi"
-CONFIGURATION_PANAMA = "panama"
 
 PYTHON_JAVA_EMBEDDING_VM_REGISTRY_NAME = "PythonJavaDriver"
 CONFIGURATION_JAVA_EMBEDDING_MULTI = "java-driver-multi-default"
@@ -1042,7 +1041,6 @@ def add_graalpy_vm(name, *extra_polyglot_args):
     add_graalpy_vm(CONFIGURATION_NATIVE_MULTI, '--experimental-options', '-multi-context')
     add_graalpy_vm(CONFIGURATION_NATIVE_INTERPRETER_MULTI, '--experimental-options', '-multi-context', '--engine.Compilation=false')
     add_graalpy_vm(CONFIGURATION_NATIVE_MULTI_TIER, '--experimental-options', '--engine.MultiTier=true')
-    add_graalpy_vm(CONFIGURATION_PANAMA, '--experimental-options', '--python.UsePanama=true')
 
     # all of the graalpy vms, but with bc dsl
     for name, extra_polyglot_args in graalpy_vms[:]:

mx.graalpython/suite.py

@@ -1079,7 +1079,8 @@
                 "sdk:NATIVEIMAGE",
                 "sdk:COLLECTIONS",
                 "truffle:TRUFFLE_NFI",
-                "truffle:TRUFFLE_NFI_LIBFFI", # runtime dependency for convenience
+                "truffle:TRUFFLE_NFI_LIBFFI", # runtime dependency
+                "truffle:TRUFFLE_NFI_PANAMA", # runtime dependency
                 "truffle:TRUFFLE_ICU4J",
                 "truffle:TRUFFLE_XZ",
             ],