diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs index 93e5a09c395..c6e28f68b4b 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs @@ -81,6 +81,17 @@ public static HyperlightCodeActProviderOptions CreateForJavaScript() /// public CodeActApprovalMode ApprovalMode { get; set; } = CodeActApprovalMode.NeverRequire; + /// + /// Gets or sets whether host exception details are exposed to generated code and in execution errors. + /// Defaults to . + /// + /// + /// Detailed errors include exception types, stack traces, and inner exceptions. + /// Enable only for a trusted channel: diagnostics can contain sensitive host data. + /// This is independent of the outer chat client's error policy and sensitive-data logging. + /// + public bool IncludeDetailedErrors { get; set; } + /// /// Gets or sets an optional host directory exposed to the sandbox as its /// /input directory. diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs index 4c13b10d60a..27f8e184e60 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs @@ -52,6 +52,7 @@ public RunSnapshot( this.HeapSize = options.HeapSize; this.StackSize = options.StackSize; this.HostInputDirectory = options.HostInputDirectory; + this.IncludeDetailedErrors = options.IncludeDetailedErrors; this.ToolRegistryVersion = toolRegistryVersion; this.ConfigFingerprint = ComputeFingerprint( this.Tools, @@ -62,7 +63,8 @@ public RunSnapshot( this.Backend, this.ModulePath, this.HeapSize, - this.StackSize); + this.StackSize, + this.IncludeDetailedErrors); } public IReadOnlyList Tools { get; } @@ -81,6 +83,8 @@ public RunSnapshot( public string? HostInputDirectory { get; } + public bool IncludeDetailedErrors { get; } + public Guid ToolRegistryVersion { get; } /// @@ -100,7 +104,8 @@ internal static string ComputeFingerprint( SandboxBackend backend = SandboxBackend.JavaScript, string? modulePath = null, string? heapSize = null, - string? stackSize = null) + string? stackSize = null, + bool includeDetailedErrors = false) { var buffer = new ArrayBufferWriter(); using (var writer = new Utf8JsonWriter(buffer)) @@ -111,6 +116,7 @@ internal static string ComputeFingerprint( writer.WriteString("heapSize", heapSize); writer.WriteString("stackSize", stackSize); writer.WriteString("hostInputDirectory", hostInputDirectory); + writer.WriteBoolean("includeDetailedErrors", includeDetailedErrors); writer.WriteString("toolRegistryVersion", toolRegistryVersion); writer.WritePropertyName("tools"); @@ -225,20 +231,24 @@ public async Task ExecuteAsync(RunSnapshot snapshot, string code, Cancel this._sandbox!.Restore(this._warmSnapshot); } - ExecutionResult result; + return BuildResult(this._sandbox!.Run(code)); + } +#pragma warning disable CA1031 // Return ordinary lifecycle failures as policy-controlled execution errors. + catch (Exception ex) when (ex is not OperationCanceledException) + { + Exception failure = ex; try { - result = this._sandbox!.Run(code); + this.ClearSandboxState(); } -#pragma warning disable CA1031 // Surface sandbox execution failures as structured JSON rather than propagating. - catch (Exception ex) -#pragma warning restore CA1031 + catch (Exception cleanupError) when (cleanupError is not OperationCanceledException) { - return BuildErrorResult(ex.Message); + failure = new AggregateException(ex, cleanupError); } - return BuildResult(result); + return BuildErrorResult(snapshot.IncludeDetailedErrors ? failure.ToString() : "Code execution failed."); } +#pragma warning restore CA1031 finally { this._executionLock.Release(); @@ -254,12 +264,25 @@ private void EnsureInitialized(RunSnapshot snapshot) // Configuration changed (or first run) — dispose the previous sandbox // so the new one picks up the current capabilities and runtime options. - this._warmSnapshot?.Dispose(); - this._sandbox?.Dispose(); + this.ClearSandboxState(); + this.BuildAndWarmUp(snapshot); + } + + private void ClearSandboxState() + { + var warmSnapshot = this._warmSnapshot; + var sandbox = this._sandbox; this._warmSnapshot = null; this._sandbox = null; - - this.BuildAndWarmUp(snapshot); + this._lastConfigFingerprint = null; + try + { + warmSnapshot?.Dispose(); + } + finally + { + sandbox?.Dispose(); + } } private void BuildAndWarmUp(RunSnapshot snapshot) @@ -298,9 +321,10 @@ private void BuildAndWarmUp(RunSnapshot snapshot) } var sandbox = builder.Build(); + this._sandbox = sandbox; // Tools must be registered before the first Run() call. - ToolBridge.RegisterAll(sandbox, snapshot.Tools); + ToolBridge.RegisterAll(sandbox, snapshot.Tools, snapshot.IncludeDetailedErrors); foreach (var allowedDomain in snapshot.AllowedDomains) { @@ -314,7 +338,6 @@ private void BuildAndWarmUp(RunSnapshot snapshot) // upstream HyperlightSandbox.Extensions.AI CodeExecutionTool reference. _ = sandbox.Run(snapshot.Backend == SandboxBackend.JavaScript ? "void 0;" : "None"); this._warmSnapshot = sandbox.Snapshot(); - this._sandbox = sandbox; this._lastConfigFingerprint = snapshot.ConfigFingerprint; } @@ -340,8 +363,13 @@ public void Dispose() } this._disposed = true; - this._warmSnapshot?.Dispose(); - this._sandbox?.Dispose(); - this._executionLock.Dispose(); + try + { + this.ClearSandboxState(); + } + finally + { + this._executionLock.Dispose(); + } } } diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs index b2f735474e4..e951b36efb8 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs @@ -21,24 +21,33 @@ internal static class ToolBridge /// Registers every entry against the provided /// as a raw JSON-in / JSON-out async tool. /// - public static void RegisterAll(Sandbox sandbox, IReadOnlyList tools) + public static void RegisterAll(Sandbox sandbox, IReadOnlyList tools, bool includeDetailedErrors = false) { foreach (var tool in tools) { - RegisterOne(sandbox, tool); + RegisterOne(sandbox, tool, includeDetailedErrors); } } - private static void RegisterOne(Sandbox sandbox, AIFunction tool) + private static void RegisterOne(Sandbox sandbox, AIFunction tool, bool includeDetailedErrors) => sandbox.RegisterToolAsync( tool.Name, - async (string argsJson) => await InvokeAsync(tool, argsJson).ConfigureAwait(false)); + async (string argsJson) => await InvokeAsync(tool, argsJson, includeDetailedErrors).ConfigureAwait(false)); - internal static async Task InvokeAsync(AIFunction tool, string argsJson) + internal static async Task InvokeAsync(AIFunction tool, string argsJson, bool includeDetailedErrors = false) { try { - var arguments = ParseArguments(argsJson); + IDictionary arguments; + try + { + arguments = ParseArguments(argsJson); + } + catch (Exception ex) when (ex is JsonException or ArgumentException) + { + return SerializeError(includeDetailedErrors ? ex.ToString() : ex.Message); + } + var result = await tool.InvokeAsync(new AIFunctionArguments(arguments)).ConfigureAwait(false); return SerializeResult(result); } @@ -46,10 +55,13 @@ internal static async Task InvokeAsync(AIFunction tool, string argsJson) catch (Exception ex) #pragma warning restore CA1031 { - return JsonSerializer.Serialize(new HyperlightToolError(ex.Message), HyperlightJsonContext.Default.HyperlightToolError); + return SerializeError(includeDetailedErrors ? ex.ToString() : "Host tool execution failed."); } } + private static string SerializeError(string message) => + JsonSerializer.Serialize(new HyperlightToolError(message), HyperlightJsonContext.Default.HyperlightToolError); + internal static IDictionary ParseArguments(string argsJson) { if (string.IsNullOrWhiteSpace(argsJson)) diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md index 8596db97644..f6a19d4fe51 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md @@ -35,6 +35,25 @@ Both surfaces support: the meantime. * A Hyperlight Python guest module when using `SandboxBackend.Wasm`. +## Error diagnostics + +Host tool failures return a generic JSON error by default. Host execution +exceptions returned by the executor also use a generic message. +`HyperlightCodeActProviderOptions.IncludeDetailedErrors` defaults to `false` +for both the provider and standalone function. Set it to `true` only for a +trusted channel: full exception details, including stack traces and inner +exceptions, can contain sensitive data. Argument-validation errors keep their +safe messages by default and include stack traces only with this opt-in. + +This option is independent of the outer `FunctionInvokingChatClient` error policy +and sensitive-data logging. Outer error handling cannot redact diagnostics that +generated code has already caught and included in normal output. Run snapshots +capture the policy, and sandbox configuration fingerprints include it so cached +callbacks cannot retain a different disclosure policy. +The same boundary covers initialization, warm-up, snapshot restore, and execution +failures. Ordinary failures dispose and clear failed sandbox state before returning +an error; cancellation still propagates. + ## Network access and guest packages `HyperlightCodeActProviderOptions.AllowedDomains` controls outbound requests diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs index 51d6a7136d1..abbf23dcfe2 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs @@ -39,15 +39,18 @@ public CodeExecutor( /// Immutable snapshot of provider state captured at the start of an invocation. public sealed class RunSnapshot { - public RunSnapshot(IReadOnlyList tools, IReadOnlyList fileMounts) + public RunSnapshot(IReadOnlyList tools, IReadOnlyList fileMounts, bool includeDetailedErrors = false) { this.Tools = tools; this.FileMounts = fileMounts; + this.IncludeDetailedErrors = includeDetailedErrors; } public IReadOnlyList Tools { get; } public IReadOnlyList FileMounts { get; } + + public bool IncludeDetailedErrors { get; } } public async Task> ExecuteAsync(RunSnapshot snapshot, string code, CancellationToken cancellationToken) @@ -65,7 +68,8 @@ public async Task> ExecuteAsync(RunSnapshot snapshot, string cod snapshot.Tools, this._limits, this._environment, - this._workingDirectory); + this._workingDirectory, + snapshot.IncludeDetailedErrors); var result = await bridge.RunAsync(code, cancellationToken).ConfigureAwait(false); diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs index dee86154c38..d94e19d0116 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs @@ -31,6 +31,7 @@ internal sealed class ProcessBridge private readonly ProcessExecutionLimits _limits; private readonly IReadOnlyDictionary? _environment; private readonly string? _workingDirectory; + private readonly bool _includeDetailedErrors; public ProcessBridge( string pythonExecutable, @@ -38,7 +39,8 @@ public ProcessBridge( IReadOnlyList tools, ProcessExecutionLimits limits, IReadOnlyDictionary? environment, - string? workingDirectory) + string? workingDirectory, + bool includeDetailedErrors = false) { this._pythonExecutable = pythonExecutable; this._runnerScript = runnerScript; @@ -46,6 +48,7 @@ public ProcessBridge( this._limits = limits; this._environment = environment; this._workingDirectory = workingDirectory; + this._includeDetailedErrors = includeDetailedErrors; } /// Represents the parsed final result returned by the Python runner. @@ -264,7 +267,9 @@ await SendToolResponseAsync(process, callId, ok: false, result: null, #pragma warning restore CA1031 { await SendToolResponseAsync(process, callId, ok: false, result: null, - excType: ex.GetType().Name, excMessage: ex.Message, cancellationToken).ConfigureAwait(false); + excType: this._includeDetailedErrors ? ex.GetType().Name : "RuntimeError", + excMessage: this._includeDetailedErrors ? ex.ToString() : "Host tool execution failed.", + cancellationToken).ConfigureAwait(false); } } diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs index 33f9205f735..4cdb3b38d86 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs @@ -37,6 +37,7 @@ public sealed class LocalCodeActProvider : AIContextProvider, IDisposable private readonly CodeExecutor _executor; private readonly LocalCodeActApprovalMode _approvalMode; + private readonly bool _includeDetailedErrors; private readonly ConcurrentDictionary _tools = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _fileMounts = new(StringComparer.Ordinal); @@ -68,6 +69,7 @@ public LocalCodeActProvider(string pythonExecutablePath, LocalCodeActProviderOpt } this._approvalMode = options.ApprovalMode; + this._includeDetailedErrors = options.IncludeDetailedErrors; this._executor = new CodeExecutor( pythonExecutablePath, @@ -190,7 +192,8 @@ protected override ValueTask ProvideAIContextAsync(InvokingContext co this.ThrowIfDisposed(); snapshot = new CodeExecutor.RunSnapshot( this._tools.Values.ToList(), - this._fileMounts.Values.ToList()); + this._fileMounts.Values.ToList(), + this._includeDetailedErrors); FeatureUsageMarker.MarkUsed(); var approvalRequired = ComputeApprovalRequired(this._approvalMode, snapshot.Tools); diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs index 51a161a20ea..528901f11c6 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs @@ -31,6 +31,17 @@ public sealed class LocalCodeActProviderOptions /// public LocalCodeActApprovalMode ApprovalMode { get; set; } = LocalCodeActApprovalMode.NeverRequire; + /// + /// Gets or sets whether host exception details are exposed to generated code. + /// Defaults to . + /// + /// + /// Detailed errors include exception types, stack traces, and inner exceptions. + /// Enable only for a trusted channel: diagnostics can contain sensitive host data. + /// This is independent of the outer chat client's error policy and sensitive-data logging. + /// + public bool IncludeDetailedErrors { get; set; } + /// /// Gets or sets the initial set of file mounts exposed to generated code. /// diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs index a8b0fd34e22..5bb0804208d 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs @@ -72,7 +72,7 @@ public LocalExecuteCodeFunction(string pythonExecutablePath, LocalCodeActProvide options.Environment, options.WorkingDirectory); - this._snapshot = new CodeExecutor.RunSnapshot(tools, fileMounts); + this._snapshot = new CodeExecutor.RunSnapshot(tools, fileMounts, options.IncludeDetailedErrors); this._inner = AIFunctionFactory.Create( this.ExecuteCodeAsync, new AIFunctionFactoryOptions diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md index e4ae51298bf..4dc7e823c2f 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md @@ -37,6 +37,20 @@ using var provider = new LocalCodeActProvider("/usr/bin/python3", options); // Register provider with your AIAgent's context providers. ``` +## Error diagnostics + +Host tool failures send a generic failed response to generated Python code by +default. `LocalCodeActProviderOptions.IncludeDetailedErrors` defaults to `false` +for both the provider and standalone function. Set it to `true` only for a +trusted channel: full exception details, including stack traces and inner +exceptions, can contain sensitive data. + +The policy is captured at construction and passed through run snapshots. It is +independent of the outer `FunctionInvokingChatClient` error policy and +sensitive-data logging. Outer error handling still applies to uncaught failures, +but cannot redact diagnostics that generated code has already caught and printed +or returned. Successful results and failure signaling are unchanged. + ## What the Package Controls - **AST validation** (default on): Validates generated code against allow-lists diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs index 58b9ebceb6a..93405aea4a4 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs @@ -2,6 +2,7 @@ using System; using System.Linq; +using System.Reflection; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.AI; @@ -56,4 +57,81 @@ public async Task ExecuteCode_PythonPrint_ReturnsStdoutAsync() Assert.Contains("hi", doc.RootElement.GetProperty("stdout").GetString()!); Assert.Equal(0, doc.RootElement.GetProperty("exit_code").GetInt32()); } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_HostErrorDetailsRequireOptInAsync(bool includeDetailedErrors) + { + // Arrange + if (string.IsNullOrWhiteSpace(GuestPath)) + { + Assert.Skip(SkipReason); + return; + } + + const string Detail = "synthetic backend diagnostic"; + var options = HyperlightCodeActProviderOptions.CreateForWasm(GuestPath!); + options.IncludeDetailedErrors = includeDetailedErrors; + options.Tools = [AIFunctionFactory.Create( + (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool")]; + using var function = new HyperlightExecuteCodeFunction(options); + + // Act + var result = await function.InvokeAsync(new AIFunctionArguments + { + ["code"] = """ + try: + print(call_tool('failing_tool')) + except Exception as exc: + print(str(exc)) + """, + }); + + // Assert + var json = Assert.IsType(result); + Assert.Equal(includeDetailedErrors, json.Contains(Detail, StringComparison.Ordinal)); + if (!includeDetailedErrors) + { + Assert.Contains("Host tool execution failed.", json, StringComparison.Ordinal); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_RestoreFailureObeysPolicyAndRebuildsSandboxAsync(bool includeDetailedErrors) + { + // Arrange + if (string.IsNullOrWhiteSpace(GuestPath)) + { + Assert.Skip(SkipReason); + return; + } + + var options = HyperlightCodeActProviderOptions.CreateForWasm(GuestPath!); + options.IncludeDetailedErrors = includeDetailedErrors; + using var function = new HyperlightExecuteCodeFunction(options); + await function.InvokeAsync(new AIFunctionArguments { ["code"] = "print('first')" }); + var executor = typeof(HyperlightExecuteCodeFunction) + .GetField("_executor", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(function)!; + var snapshotField = executor.GetType().GetField("_warmSnapshot", BindingFlags.Instance | BindingFlags.NonPublic)!; + Assert.IsAssignableFrom(snapshotField.GetValue(executor)).Dispose(); + + // Act + var failed = await function.InvokeAsync(new AIFunctionArguments { ["code"] = "print('second')" }); + + // Assert + using var error = JsonDocument.Parse(Assert.IsType(failed)); + Assert.False(error.RootElement.GetProperty("success").GetBoolean()); + var message = error.RootElement.GetProperty("stderr").GetString(); + Assert.Equal(!includeDetailedErrors, string.Equals(message, "Code execution failed.", StringComparison.Ordinal)); + Assert.Null(snapshotField.GetValue(executor)); + Assert.Null(executor.GetType().GetField("_sandbox", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(executor)); + Assert.Null(executor.GetType().GetField("_lastConfigFingerprint", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(executor)); + var recovered = await function.InvokeAsync(new AIFunctionArguments { ["code"] = "print('recovered')" }); + using var success = JsonDocument.Parse(Assert.IsType(recovered)); + Assert.True(success.RootElement.GetProperty("success").GetBoolean()); + Assert.Contains("recovered", success.RootElement.GetProperty("stdout").GetString()!, StringComparison.Ordinal); + } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs index 5a528606b60..7e347669efc 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs @@ -2,6 +2,9 @@ using System; using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; using HyperlightSandbox.Api; using Microsoft.Agents.AI.Hyperlight.Internal; using Microsoft.Extensions.AI; @@ -10,6 +13,57 @@ namespace Microsoft.Agents.AI.Hyperlight.UnitTests; public sealed class SandboxExecutorTests { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteAsync_InitializationFailureObeysPolicyAndReleasesLockAsync(bool includeDetailedErrors) + { + // Arrange + const string Detail = "synthetic-backend-diagnostic"; + var options = new HyperlightCodeActProviderOptions + { + HeapSize = Detail, + IncludeDetailedErrors = includeDetailedErrors, + }; + var snapshot = new SandboxExecutor.RunSnapshot([], [], [], options); + using var executor = new SandboxExecutor(); + using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // Act + var first = await executor.ExecuteAsync(snapshot, "void 0;", cancellation.Token); + var second = await executor.ExecuteAsync(snapshot, "void 0;", cancellation.Token); + + // Assert + foreach (var result in new[] { first, second }) + { + using var json = JsonDocument.Parse(result); + Assert.False(json.RootElement.GetProperty("success").GetBoolean()); + Assert.Equal(-1, json.RootElement.GetProperty("exit_code").GetInt32()); + var error = json.RootElement.GetProperty("stderr").GetString()!; + Assert.Equal(includeDetailedErrors, error.Contains(Detail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, error.Contains(" at ", StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, error.Contains(nameof(SandboxExecutor.ExecuteAsync), StringComparison.Ordinal)); + if (!includeDetailedErrors) + { + Assert.Equal("Code execution failed.", error); + } + } + } + + [Fact] + public async Task ExecuteAsync_PreservesCancellationAsync() + { + // Arrange + var snapshot = new SandboxExecutor.RunSnapshot([], [], [], new HyperlightCodeActProviderOptions()); + using var executor = new SandboxExecutor(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + // Act / Assert + await Assert.ThrowsAnyAsync( + () => executor.ExecuteAsync(snapshot, "void 0;", cancellation.Token)); + } + [Fact] public void Fingerprint_DifferentToolSets_DifferentFingerprints() { @@ -254,12 +308,15 @@ public void Fingerprint_DifferentSandboxOptions_DifferentFingerprints() [], [], [], hostInputDirectory: null, heapSize: "20Mi"); var differentStack = SandboxExecutor.RunSnapshot.ComputeFingerprint( [], [], [], hostInputDirectory: null, stackSize: "10Mi"); + var detailedErrors = SandboxExecutor.RunSnapshot.ComputeFingerprint( + [], [], [], hostInputDirectory: null, includeDetailedErrors: true); // Assert Assert.NotEqual(baseline, differentBackend); Assert.NotEqual(baseline, differentModule); Assert.NotEqual(baseline, differentHeap); Assert.NotEqual(baseline, differentStack); + Assert.NotEqual(baseline, detailedErrors); } [Fact] @@ -285,6 +342,7 @@ public void RunSnapshot_CapturesMutableInputs() HeapSize = "10Mi", StackSize = "5Mi", HostInputDirectory = "/host/input", + IncludeDetailedErrors = true, }; // Act @@ -296,6 +354,7 @@ public void RunSnapshot_CapturesMutableInputs() options.HeapSize = "20Mi"; options.StackSize = "10Mi"; options.HostInputDirectory = "/host/other"; + options.IncludeDetailedErrors = false; // Assert Assert.Single(snapshot.Tools); @@ -305,6 +364,7 @@ public void RunSnapshot_CapturesMutableInputs() Assert.Equal("10Mi", snapshot.HeapSize); Assert.Equal("5Mi", snapshot.StackSize); Assert.Equal("/host/input", snapshot.HostInputDirectory); + Assert.True(snapshot.IncludeDetailedErrors); } [Fact] diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs index 9d94d719302..bbd1d6ac43e 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs @@ -24,20 +24,47 @@ public async Task InvokeAsync_PassesArgumentsAndReturnsSerializedResultAsync() Assert.Equal("\"echo:hello\"", result); } - [Fact] - public async Task InvokeAsync_ReturnsErrorJsonOnExceptionAsync() + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task InvokeAsync_ReturnsErrorJsonOnExceptionAsync(bool includeDetailedErrors, bool isAsync) { // Arrange - static int Boom() => throw new InvalidOperationException("nope"); - var tool = AIFunctionFactory.Create(Boom, name: "boom"); + const string Detail = "synthetic backend diagnostic"; + const string InnerDetail = "synthetic inner diagnostic"; + static int Boom() => throw new InvalidOperationException(Detail, new ArgumentException(InnerDetail)); + static async Task BoomAsync() + { + await Task.Yield(); + throw new InvalidOperationException(Detail, new ArgumentException(InnerDetail)); + } + + var tool = isAsync + ? AIFunctionFactory.Create(BoomAsync, name: "boom") + : AIFunctionFactory.Create(Boom, name: "boom"); // Act - var result = await ToolBridge.InvokeAsync(tool, "{}"); + var result = await ToolBridge.InvokeAsync(tool, "{}", includeDetailedErrors); // Assert using var doc = JsonDocument.Parse(result); Assert.True(doc.RootElement.TryGetProperty("error", out var err)); - Assert.Contains("nope", err.GetString()!); + var message = err.GetString()!; + Assert.Equal(includeDetailedErrors, message.Contains(Detail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, message.Contains(InnerDetail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, message.Contains(" at ", StringComparison.Ordinal)); + if (includeDetailedErrors) + { + Assert.StartsWith($"System.InvalidOperationException: {Detail}", message, StringComparison.Ordinal); + Assert.Contains($"System.ArgumentException: {InnerDetail}", message, StringComparison.Ordinal); + Assert.Contains(isAsync ? nameof(BoomAsync) : nameof(Boom), message, StringComparison.Ordinal); + } + else + { + Assert.Equal("Host tool execution failed.", message); + } } [Fact] @@ -54,18 +81,30 @@ public async Task InvokeAsync_EmptyArguments_InvokesToolWithNoArgsAsync() Assert.Equal("\"hi\"", result); } - [Fact] - public async Task InvokeAsync_NonObjectJson_ReturnsErrorAsync() + [Theory] + [InlineData(false, "[1, 2, 3]")] + [InlineData(true, "[1, 2, 3]")] + [InlineData(false, "{")] + [InlineData(true, "{")] + public async Task InvokeAsync_InvalidArgumentsFollowDetailedPolicyAsync(bool includeDetailedErrors, string argsJson) { // Arrange static string Hi() => "hi"; var tool = AIFunctionFactory.Create(Hi, name: "hi"); // Act - var result = await ToolBridge.InvokeAsync(tool, "[1, 2, 3]"); + var result = await ToolBridge.InvokeAsync(tool, argsJson, includeDetailedErrors); // Assert using var doc = JsonDocument.Parse(result); - Assert.True(doc.RootElement.TryGetProperty("error", out _)); + Assert.True(doc.RootElement.TryGetProperty("error", out var error)); + var message = error.GetString()!; + Assert.NotEmpty(message); + Assert.Equal(includeDetailedErrors, message.Contains(" at ", StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, message.Contains(nameof(ToolBridge.ParseArguments), StringComparison.Ordinal)); + if (argsJson.StartsWith("[", StringComparison.Ordinal)) + { + Assert.Contains("Tool arguments must be a JSON object.", message, StringComparison.Ordinal); + } } } diff --git a/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs index c7118a452ab..1e47497ca5d 100644 --- a/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs @@ -49,6 +49,117 @@ public async Task ExecuteCode_PrintsAndReturnsResultAsync() Assert.Contains("3", combined); } + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ExecuteCode_HostErrorsRespectDetailedPolicyAsync(bool includeDetailedErrors, bool isAsync) + { + // Arrange + SkipIfNoPython(); + const string Detail = "synthetic backend diagnostic"; + const string InnerDetail = "synthetic inner diagnostic"; + static string Fail() => throw new InvalidOperationException(Detail, new ArgumentException(InnerDetail)); + static async Task FailAsync() + { + await Task.Yield(); + throw new InvalidOperationException(Detail, new ArgumentException(InnerDetail)); + } + + var tool = isAsync + ? AIFunctionFactory.Create(FailAsync, name: "failing_tool") + : AIFunctionFactory.Create(Fail, name: "failing_tool"); + var function = new LocalExecuteCodeFunction(s_python!, new LocalCodeActProviderOptions + { + Tools = [tool], + IncludeDetailedErrors = includeDetailedErrors, + }); + var args = new AIFunctionArguments + { + ["code"] = """ + try: + await failing_tool() + except Exception as exc: + print(str(exc)) + try: + await call_tool('failing_tool') + except Exception as exc: + print(str(exc)) + """, + }; + + // Act + var result = await function.InvokeAsync(args, CancellationToken.None); + + // Assert + var text = GetResultText(result); + Assert.Contains(includeDetailedErrors ? Detail : "Host tool execution failed.", text, StringComparison.Ordinal); + Assert.Equal(includeDetailedErrors, text.Contains(Detail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, text.Contains(InnerDetail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, text.Contains(" at ", StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, text.Contains(isAsync ? nameof(FailAsync) : nameof(Fail), StringComparison.Ordinal)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_UncaughtHostErrorsPreserveFailureAsync(bool includeDetailedErrors) + { + // Arrange + SkipIfNoPython(); + const string Detail = "synthetic backend diagnostic"; + const string InnerDetail = "synthetic inner diagnostic"; + var tool = AIFunctionFactory.Create( + (Func)(() => throw new InvalidOperationException(Detail, new ArgumentException(InnerDetail))), name: "failing_tool"); + var function = new LocalExecuteCodeFunction(s_python!, new LocalCodeActProviderOptions + { + Tools = [tool], + IncludeDetailedErrors = includeDetailedErrors, + }); + + // Act + var error = await Assert.ThrowsAsync(async () => + await function.InvokeAsync(new AIFunctionArguments { ["code"] = "await failing_tool()" })); + + // Assert + Assert.Contains(includeDetailedErrors ? Detail : "Host tool execution failed.", error.Message, StringComparison.Ordinal); + Assert.Equal(includeDetailedErrors, error.Message.Contains(Detail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, error.Message.Contains(InnerDetail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, error.Message.Contains(" at ", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Provider_HostErrorPolicyIsCapturedAtConstructionAsync(bool includeDetailedErrors) + { + // Arrange + SkipIfNoPython(); + const string Detail = "synthetic backend diagnostic"; + var tool = AIFunctionFactory.Create( + (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool"); + var options = new LocalCodeActProviderOptions + { + Tools = [tool], + IncludeDetailedErrors = includeDetailedErrors, + }; + using var provider = new LocalCodeActProvider(s_python!, options); + options.IncludeDetailedErrors = !includeDetailedErrors; + var context = await provider.InvokingAsync( + new AIContextProvider.InvokingContext(new Moq.Mock().Object, session: null, new AIContext())); + var function = Assert.IsAssignableFrom(Assert.Single(context.Tools!)); + + // Act + var result = await function.InvokeAsync(new AIFunctionArguments + { + ["code"] = "try:\n await failing_tool()\nexcept Exception as exc:\n print(str(exc))", + }); + + // Assert + Assert.Equal(includeDetailedErrors, GetResultText(result).Contains(Detail, StringComparison.Ordinal)); + } + [Fact] public async Task ExecuteCode_AllowsAstValidatedFormattingAsync() { @@ -883,8 +994,10 @@ await call_tool('definitely_not_registered', x=1) Assert.DoesNotContain("NO_ERROR", combined); } - [Fact] - public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsync() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsync(bool includeDetailedErrors) { SkipIfNoPython(); @@ -896,6 +1009,7 @@ public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsyn var options = new LocalCodeActProviderOptions { Tools = new[] { faultyTool }, + IncludeDetailedErrors = includeDetailedErrors, }; var function = new LocalExecuteCodeFunction(s_python!, options); @@ -912,8 +1026,9 @@ await call_tool('faulty', message='boom') var result = await function.InvokeAsync(args, CancellationToken.None); var combined = GetResultText(result); Assert.Contains("GOT_ERROR", combined); - Assert.Contains("InvalidOperationException", combined); - Assert.Contains("intentional: boom", combined); + Assert.Contains(includeDetailedErrors ? "intentional: boom" : "Host tool execution failed.", combined); + Assert.Equal(includeDetailedErrors, combined.Contains("InvalidOperationException", StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, combined.Contains("intentional: boom", StringComparison.Ordinal)); } [Fact] diff --git a/skills/agent-framework-builder-dotnet/references/integrations.md b/skills/agent-framework-builder-dotnet/references/integrations.md index d9a9aec7eef..444f2c3b88f 100644 --- a/skills/agent-framework-builder-dotnet/references/integrations.md +++ b/skills/agent-framework-builder-dotnet/references/integrations.md @@ -40,6 +40,12 @@ otherwise see [Extensibility](extensibility.md#custom-providers). - Durable agents and workflows run on Azure Functions and Durable Task; see [Hosting and deployment](hosting-and-deployment.md). +LocalCodeAct and Hyperlight redact host tool exception diagnostics by default. Enable +`IncludeDetailedErrors` on their provider options only for a trusted channel; full exceptions include +stack traces and inner exceptions. This is separate from the +outer `FunctionInvokingChatClient` error policy and sensitive-data logging; outer redaction cannot repair +diagnostics already caught and returned by generated code. + For AG-UI, remember that thread and run IDs are correlation identifiers, not authentication boundaries. The application owns authorization, durable snapshots, scaling, and rendering policy.