diff --git a/src/core/IronPython/Compiler/Ast/PythonAst.cs b/src/core/IronPython/Compiler/Ast/PythonAst.cs index e809f4113..3b5979578 100644 --- a/src/core/IronPython/Compiler/Ast/PythonAst.cs +++ b/src/core/IronPython/Compiler/Ast/PythonAst.cs @@ -46,6 +46,7 @@ public sealed class PythonAst : ScopeStatement { private CompilationMode.ConstantInfo _contextInfo; private Dictionary _globalVariables = new Dictionary(); internal readonly Profiler _profiler; // captures timing data if profiling + private bool? _adaptiveCompilationDisabled; // whether -X NoAdaptiveCompilation applies to this source, once known internal const string GlobalContextName = "$globalContext"; internal static MSAst.ParameterExpression _functionCode = Ast.Variable(typeof(FunctionCode), "$functionCode"); @@ -557,6 +558,15 @@ internal CompilationMode CompilationMode { } } + /// + /// Returns whether -X NoAdaptiveCompilation applies to this source. + /// + /// + /// It depends only on the options and the source unit, so it is evaluated once and remains valid in . + /// + internal bool IsAdaptiveCompilationDisabled + => _adaptiveCompilationDisabled ??= PyContext.IsAdaptiveCompilationDisabled(_compilerContext.SourceUnit); + private MSAst.Expression GetGlobalContext() { if (_contextInfo != null) { return _contextInfo.Expression; diff --git a/src/core/IronPython/Compiler/Ast/ScopeStatement.cs b/src/core/IronPython/Compiler/Ast/ScopeStatement.cs index 9aeb9ccfb..334a8bcbd 100644 --- a/src/core/IronPython/Compiler/Ast/ScopeStatement.cs +++ b/src/core/IronPython/Compiler/Ast/ScopeStatement.cs @@ -166,11 +166,11 @@ nonlocal.Kind is VariableKind.Nonlocal && internal bool ShouldInterpret { get { if (GlobalParent.CompilationMode == CompilationMode.Lookup) { - return true; + // code from compile, exec and eval is compiled adaptively, unless -X NoAdaptiveCompilation applies to it + return !GlobalParent.IsAdaptiveCompilationDisabled; } - CompilerContext context = GlobalParent.CompilerContext; - return ((PythonContext)context.SourceUnit.LanguageContext).ShouldInterpret((PythonCompilerOptions)context.Options, context.SourceUnit); + return PyContext.ShouldInterpret(GlobalParent); } } diff --git a/src/core/IronPython/Compiler/PythonScriptCode.cs b/src/core/IronPython/Compiler/PythonScriptCode.cs index fdda7b685..25d2cf360 100644 --- a/src/core/IronPython/Compiler/PythonScriptCode.cs +++ b/src/core/IronPython/Compiler/PythonScriptCode.cs @@ -123,7 +123,7 @@ private LookupCompilationDelegate CompileBody(LightExpression lambda) { diff --git a/src/core/IronPython/Compiler/RuntimeScriptCode.cs b/src/core/IronPython/Compiler/RuntimeScriptCode.cs index 6e0069b72..92188186d 100644 --- a/src/core/IronPython/Compiler/RuntimeScriptCode.cs +++ b/src/core/IronPython/Compiler/RuntimeScriptCode.cs @@ -105,10 +105,9 @@ private void EnsureCompiled() { } private Func/*!*/ Compile() { - var pco = (PythonCompilerOptions)Ast.CompilerContext.Options; var pc = (PythonContext)SourceUnit.LanguageContext; - - if (pc.ShouldInterpret(pco, SourceUnit)) { + + if (pc.ShouldInterpret(Ast)) { return ((Microsoft.Scripting.Ast.LightExpression>)Ast.GetLambda()).Compile(pc.Options.CompilationThreshold); } else { return ((Microsoft.Scripting.Ast.LightExpression>)Ast.GetLambda()).ReduceToLambda().Compile(pc.EmitDebugSymbols(SourceUnit)); diff --git a/src/core/IronPython/Hosting/PythonOptionsParser.cs b/src/core/IronPython/Hosting/PythonOptionsParser.cs index 9aee202e4..8f4cc0c31 100644 --- a/src/core/IronPython/Hosting/PythonOptionsParser.cs +++ b/src/core/IronPython/Hosting/PythonOptionsParser.cs @@ -269,6 +269,18 @@ protected override void HandleImplementationSpecificOption(string arg, string? v LanguageSetup.Options["Debug"] = ScriptingRuntimeHelpers.True; break; + case "NoAdaptiveCompilation": + LanguageSetup.Options["NoAdaptiveCompilation"] = ScriptingRuntimeHelpers.True; + if (val != null) { + if (val.Length == 0) throw new InvalidOptionException($"Argument expected for the -X {arg}= option."); + try { + LanguageSetup.Options["NoAdaptiveCompilationFilter"] = new Regex(val); + } catch { + throw new InvalidOptionException($"The argument for the -X {arg} option must be a valid regex pattern."); + } + } + break; + case "NoDebug": if (string.IsNullOrEmpty(val)) throw new InvalidOptionException($"Argument expected for the -X NoDebug option."); try { @@ -324,7 +336,8 @@ public override void GetHelp(out string commandLine, out string[,] options, out // only keep the implementation-specific options from the DLR List> standardOptionsList = new(); for (var i = 0; i < standardOptions.GetLength(0); i++) { - if (standardOptions[i, 0].StartsWith("-X:", StringComparison.Ordinal)) { + // -X NoAdaptiveCompilation takes an optional argument here, so it is described below + if (standardOptions[i, 0].StartsWith("-X:", StringComparison.Ordinal) && standardOptions[i, 0] != "-X:NoAdaptiveCompilation") { standardOptionsList.Add(new KeyValuePair(standardOptions[i, 0].Replace(' ', '=').Replace(':', ' '), standardOptions[i, 1])); } } @@ -363,6 +376,8 @@ public override void GetHelp(out string commandLine, out string[,] options, out { "-X MaxRecursion=","Set the maximum recursion level" }, { "-X Debug", "Enable application debugging (preferred over -D)" }, { "-X NoDebug=", "Provides a regular expression of files which should not be emitted in debug mode"}, + { "-X NoAdaptiveCompilation[=]", + "Disable adaptive compilation (compiles immediately), in all code or only in files matching the regular expression" }, { "-X MTA", "Run in multithreaded apartment" }, { "-X EnableProfiler", "Enables profiling support in the compiler" }, { "-X LightweightScopes", "Generate optimized scopes that can be garbage collected" }, diff --git a/src/core/IronPython/Runtime/PythonContext.cs b/src/core/IronPython/Runtime/PythonContext.cs index 9c7a026ad..f82e70d6b 100644 --- a/src/core/IronPython/Runtime/PythonContext.cs +++ b/src/core/IronPython/Runtime/PythonContext.cs @@ -774,12 +774,24 @@ private void InitializeSysFlags() { flags.utf8_mode = PythonOptions.Utf8Mode ? 1 : 0; } - internal bool ShouldInterpret(PythonCompilerOptions options, SourceUnit source) { + internal bool ShouldInterpret(PyAst.PythonAst ast) { + if (ast.IsAdaptiveCompilationDisabled) { + return false; + } + + CompilerContext context = ast.CompilerContext; + // We have to turn off adaptive compilation in debug mode to - // support mangaged debuggers. Also turn off in optimized mode. - bool adaptiveCompilation = !PythonOptions.NoAdaptiveCompilation && !EmitDebugSymbols(source); + // support mangaged debuggers. + return ((PythonCompilerOptions)context.Options).Interpreted || !EmitDebugSymbols(context.SourceUnit); + } - return options.Interpreted || adaptiveCompilation; + /// + /// Returns whether -X NoAdaptiveCompilation applies to the source, even if it was compiled by compile, exec or eval. + /// + internal bool IsAdaptiveCompilationDisabled(SourceUnit source) { + var filter = PythonOptions.NoAdaptiveCompilationFilter; + return PythonOptions.NoAdaptiveCompilation && (filter == null || source.HasPath && filter.IsMatch(source.Path)); } private static PyAst.PythonAst ParseAndBindAst(CompilerContext context) { diff --git a/src/core/IronPython/Runtime/PythonOptions.cs b/src/core/IronPython/Runtime/PythonOptions.cs index d5dc14385..c7a623aa1 100644 --- a/src/core/IronPython/Runtime/PythonOptions.cs +++ b/src/core/IronPython/Runtime/PythonOptions.cs @@ -127,6 +127,15 @@ public sealed class PythonOptions : LanguageOptions { /// public Regex? NoDebug { get; } + /// + /// Returns a regular expression of Python files in which adaptive compilation is disabled. + /// + /// + /// Only used when is set. If null, adaptive compilation is disabled in all code, + /// including code compiled by compile, exec and eval, which otherwise is always compiled adaptively. + /// + public Regex? NoAdaptiveCompilationFilter { get; } + public bool Quiet { get; } /// @@ -163,6 +172,7 @@ public PythonOptions(IDictionary? options) GCStress = GetOption(options, "GCStress", null); Tracing = GetOption(options, "Tracing", false); NoDebug = GetOption(options, "NoDebug", (Regex?)null); + NoAdaptiveCompilationFilter = GetOption(options, "NoAdaptiveCompilationFilter", (Regex?)null); Quiet = GetOption(options, "Quiet", false); NoImportLib = GetOption(options, "NoImportLib", false); Isolated = GetOption(options, "Isolated", false); diff --git a/tests/suite/test_noadaptivecompilation.py b/tests/suite/test_noadaptivecompilation.py index 06ec30672..39b2ba425 100644 --- a/tests/suite/test_noadaptivecompilation.py +++ b/tests/suite/test_noadaptivecompilation.py @@ -3,12 +3,15 @@ # See the LICENSE file in the project root for more information. """ -Test that functions are compiled up front with -X:NoAdaptiveCompilation. +Test that functions are compiled up front with -X NoAdaptiveCompilation. """ +import os +import subprocess +import sys import unittest -from iptest import IronPythonTestCase, is_cli, run_test +from iptest import IronPythonTestCase, is_cli, run_test, skipUnlessIronPython if is_cli: import clr @@ -55,4 +58,52 @@ def method(self, n): func(*args) # for a generator, this runs the function that creates the generator object self.assertFalse(is_interpreted(func), func.__name__) + def test_imported_and_exec(self): + # importlib compiles modules with compile(), like exec, which is compiled adaptively unless the option says otherwise + import textwrap + textwrap.indent('text', ' ') + self.assertFalse(is_interpreted(textwrap.indent)) + + ns = {} + exec('def in_exec(n):\n return n\n', ns) + ns['in_exec'](1) + self.assertFalse(is_interpreted(ns['in_exec'])) + +child_module = ''' +def f(n): + return n +''' + +child_script = ''' +import clr +from System.Reflection import BindingFlags +import nac_filtered_module + +def is_interpreted(func): + field = clr.GetClrType(type(func.__code__)).GetField('_normalDelegate', BindingFlags.NonPublic | BindingFlags.Instance) + return field.GetValue(func.__code__).Target.GetType().Name == 'LightLambda' + +def in_main(n): + return n + +ns = {} +exec('def in_exec(n):\\n return n\\n', ns) +for func in (in_main, nac_filtered_module.f, ns['in_exec']): + func(1) +print(is_interpreted(in_main), is_interpreted(nac_filtered_module.f), is_interpreted(ns['in_exec'])) +''' + +@skipUnlessIronPython() +class NoAdaptiveCompilationFilterTest(IronPythonTestCase): + def test_filter(self): + directory = os.path.join(self.temporary_dir, 'nac_filter_%d' % os.getpid()) + os.makedirs(directory, exist_ok=True) + self.write_to_file(os.path.join(directory, 'nac_filtered_module.py'), child_module) + script = os.path.join(directory, 'nac_main.py') + self.write_to_file(script, child_script) + + output = subprocess.check_output([sys.executable, '-S', '-X', 'NoAdaptiveCompilation=nac_filtered_module', script]) + # only the module matching the regex is compiled up front: not the script itself, nor code run by exec + self.assertEqual(output.split(), [b'True', b'False', b'True']) + run_test(__name__)