From 2cc859732ba4da624e0deca0adf6d493a549703e Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Mon, 2 Nov 2026 14:17:17 +0100 Subject: [PATCH 1/3] Redact .NET CodeAct host-tool diagnostics by default --- .../HyperlightCodeActProviderOptions.cs | 10 ++ .../Internal/SandboxExecutor.cs | 14 ++- .../Internal/ToolBridge.cs | 26 ++-- .../Microsoft.Agents.AI.Hyperlight/README.md | 14 +++ .../Internal/CodeExecutor.cs | 8 +- .../Internal/ProcessBridge.cs | 9 +- .../LocalCodeActProvider.cs | 5 +- .../LocalCodeActProviderOptions.cs | 10 ++ .../LocalExecuteCodeFunction.cs | 2 +- .../README.md | 13 ++ .../CodeActEndToEndTests.cs | 39 ++++++ .../SandboxExecutorTests.cs | 6 + .../ToolBridgeTests.cs | 29 +++-- ...ocalExecuteCodeFunctionIntegrationTests.cs | 116 +++++++++++++++++- .../references/integrations.md | 5 + 15 files changed, 278 insertions(+), 28 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs index 93e5a09c395..d394d328259 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs @@ -81,6 +81,16 @@ public static HyperlightCodeActProviderOptions CreateForJavaScript() /// public CodeActApprovalMode ApprovalMode { get; set; } = CodeActApprovalMode.NeverRequire; + /// + /// Gets or sets whether host exception details are exposed to generated code and in execution errors. + /// Defaults to . + /// + /// + /// Enable only for a trusted channel: diagnostics can contain sensitive host data. + /// This is independent of the outer chat client's error policy and sensitive-data logging. + /// + public bool IncludeDetailedErrors { get; set; } + /// /// Gets or sets an optional host directory exposed to the sandbox as its /// /input directory. diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs index 4c13b10d60a..6ae6dfbb1d0 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs @@ -52,6 +52,7 @@ public RunSnapshot( this.HeapSize = options.HeapSize; this.StackSize = options.StackSize; this.HostInputDirectory = options.HostInputDirectory; + this.IncludeDetailedErrors = options.IncludeDetailedErrors; this.ToolRegistryVersion = toolRegistryVersion; this.ConfigFingerprint = ComputeFingerprint( this.Tools, @@ -62,7 +63,8 @@ public RunSnapshot( this.Backend, this.ModulePath, this.HeapSize, - this.StackSize); + this.StackSize, + this.IncludeDetailedErrors); } public IReadOnlyList Tools { get; } @@ -81,6 +83,8 @@ public RunSnapshot( public string? HostInputDirectory { get; } + public bool IncludeDetailedErrors { get; } + public Guid ToolRegistryVersion { get; } /// @@ -100,7 +104,8 @@ internal static string ComputeFingerprint( SandboxBackend backend = SandboxBackend.JavaScript, string? modulePath = null, string? heapSize = null, - string? stackSize = null) + string? stackSize = null, + bool includeDetailedErrors = false) { var buffer = new ArrayBufferWriter(); using (var writer = new Utf8JsonWriter(buffer)) @@ -111,6 +116,7 @@ internal static string ComputeFingerprint( writer.WriteString("heapSize", heapSize); writer.WriteString("stackSize", stackSize); writer.WriteString("hostInputDirectory", hostInputDirectory); + writer.WriteBoolean("includeDetailedErrors", includeDetailedErrors); writer.WriteString("toolRegistryVersion", toolRegistryVersion); writer.WritePropertyName("tools"); @@ -234,7 +240,7 @@ public async Task ExecuteAsync(RunSnapshot snapshot, string code, Cancel catch (Exception ex) #pragma warning restore CA1031 { - return BuildErrorResult(ex.Message); + return BuildErrorResult(snapshot.IncludeDetailedErrors ? ex.Message : "Code execution failed."); } return BuildResult(result); @@ -300,7 +306,7 @@ private void BuildAndWarmUp(RunSnapshot snapshot) var sandbox = builder.Build(); // Tools must be registered before the first Run() call. - ToolBridge.RegisterAll(sandbox, snapshot.Tools); + ToolBridge.RegisterAll(sandbox, snapshot.Tools, snapshot.IncludeDetailedErrors); foreach (var allowedDomain in snapshot.AllowedDomains) { diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs index b2f735474e4..aab8072b6ed 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs @@ -21,24 +21,33 @@ internal static class ToolBridge /// Registers every entry against the provided /// as a raw JSON-in / JSON-out async tool. /// - public static void RegisterAll(Sandbox sandbox, IReadOnlyList tools) + public static void RegisterAll(Sandbox sandbox, IReadOnlyList tools, bool includeDetailedErrors = false) { foreach (var tool in tools) { - RegisterOne(sandbox, tool); + RegisterOne(sandbox, tool, includeDetailedErrors); } } - private static void RegisterOne(Sandbox sandbox, AIFunction tool) + private static void RegisterOne(Sandbox sandbox, AIFunction tool, bool includeDetailedErrors) => sandbox.RegisterToolAsync( tool.Name, - async (string argsJson) => await InvokeAsync(tool, argsJson).ConfigureAwait(false)); + async (string argsJson) => await InvokeAsync(tool, argsJson, includeDetailedErrors).ConfigureAwait(false)); - internal static async Task InvokeAsync(AIFunction tool, string argsJson) + internal static async Task InvokeAsync(AIFunction tool, string argsJson, bool includeDetailedErrors = false) { try { - var arguments = ParseArguments(argsJson); + IDictionary arguments; + try + { + arguments = ParseArguments(argsJson); + } + catch (Exception ex) when (ex is JsonException or ArgumentException) + { + return SerializeError(ex.Message); + } + var result = await tool.InvokeAsync(new AIFunctionArguments(arguments)).ConfigureAwait(false); return SerializeResult(result); } @@ -46,10 +55,13 @@ internal static async Task InvokeAsync(AIFunction tool, string argsJson) catch (Exception ex) #pragma warning restore CA1031 { - return JsonSerializer.Serialize(new HyperlightToolError(ex.Message), HyperlightJsonContext.Default.HyperlightToolError); + return SerializeError(includeDetailedErrors ? ex.Message : "Host tool execution failed."); } } + private static string SerializeError(string message) => + JsonSerializer.Serialize(new HyperlightToolError(message), HyperlightJsonContext.Default.HyperlightToolError); + internal static IDictionary ParseArguments(string argsJson) { if (string.IsNullOrWhiteSpace(argsJson)) diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md index 8596db97644..bdd7f3063bb 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md @@ -35,6 +35,20 @@ Both surfaces support: the meantime. * A Hyperlight Python guest module when using `SandboxBackend.Wasm`. +## Error diagnostics + +Host tool failures return a generic JSON error by default. Host execution +exceptions returned by the executor also use a generic message. +`HyperlightCodeActProviderOptions.IncludeDetailedErrors` defaults to `false` +for both the provider and standalone function. Set it to `true` only for a +trusted channel: host exception messages can contain sensitive data. + +This option is independent of the outer `FunctionInvokingChatClient` error policy +and sensitive-data logging. Outer error handling cannot redact diagnostics that +generated code has already caught and included in normal output. Run snapshots +capture the policy, and sandbox configuration fingerprints include it so cached +callbacks cannot retain a different disclosure policy. + ## Network access and guest packages `HyperlightCodeActProviderOptions.AllowedDomains` controls outbound requests diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs index 51d6a7136d1..abbf23dcfe2 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/CodeExecutor.cs @@ -39,15 +39,18 @@ public CodeExecutor( /// Immutable snapshot of provider state captured at the start of an invocation. public sealed class RunSnapshot { - public RunSnapshot(IReadOnlyList tools, IReadOnlyList fileMounts) + public RunSnapshot(IReadOnlyList tools, IReadOnlyList fileMounts, bool includeDetailedErrors = false) { this.Tools = tools; this.FileMounts = fileMounts; + this.IncludeDetailedErrors = includeDetailedErrors; } public IReadOnlyList Tools { get; } public IReadOnlyList FileMounts { get; } + + public bool IncludeDetailedErrors { get; } } public async Task> ExecuteAsync(RunSnapshot snapshot, string code, CancellationToken cancellationToken) @@ -65,7 +68,8 @@ public async Task> ExecuteAsync(RunSnapshot snapshot, string cod snapshot.Tools, this._limits, this._environment, - this._workingDirectory); + this._workingDirectory, + snapshot.IncludeDetailedErrors); var result = await bridge.RunAsync(code, cancellationToken).ConfigureAwait(false); diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs index dee86154c38..6e234779b38 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs @@ -31,6 +31,7 @@ internal sealed class ProcessBridge private readonly ProcessExecutionLimits _limits; private readonly IReadOnlyDictionary? _environment; private readonly string? _workingDirectory; + private readonly bool _includeDetailedErrors; public ProcessBridge( string pythonExecutable, @@ -38,7 +39,8 @@ public ProcessBridge( IReadOnlyList tools, ProcessExecutionLimits limits, IReadOnlyDictionary? environment, - string? workingDirectory) + string? workingDirectory, + bool includeDetailedErrors = false) { this._pythonExecutable = pythonExecutable; this._runnerScript = runnerScript; @@ -46,6 +48,7 @@ public ProcessBridge( this._limits = limits; this._environment = environment; this._workingDirectory = workingDirectory; + this._includeDetailedErrors = includeDetailedErrors; } /// Represents the parsed final result returned by the Python runner. @@ -264,7 +267,9 @@ await SendToolResponseAsync(process, callId, ok: false, result: null, #pragma warning restore CA1031 { await SendToolResponseAsync(process, callId, ok: false, result: null, - excType: ex.GetType().Name, excMessage: ex.Message, cancellationToken).ConfigureAwait(false); + excType: this._includeDetailedErrors ? ex.GetType().Name : "RuntimeError", + excMessage: this._includeDetailedErrors ? ex.Message : "Host tool execution failed.", + cancellationToken).ConfigureAwait(false); } } diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs index 33f9205f735..4cdb3b38d86 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProvider.cs @@ -37,6 +37,7 @@ public sealed class LocalCodeActProvider : AIContextProvider, IDisposable private readonly CodeExecutor _executor; private readonly LocalCodeActApprovalMode _approvalMode; + private readonly bool _includeDetailedErrors; private readonly ConcurrentDictionary _tools = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _fileMounts = new(StringComparer.Ordinal); @@ -68,6 +69,7 @@ public LocalCodeActProvider(string pythonExecutablePath, LocalCodeActProviderOpt } this._approvalMode = options.ApprovalMode; + this._includeDetailedErrors = options.IncludeDetailedErrors; this._executor = new CodeExecutor( pythonExecutablePath, @@ -190,7 +192,8 @@ protected override ValueTask ProvideAIContextAsync(InvokingContext co this.ThrowIfDisposed(); snapshot = new CodeExecutor.RunSnapshot( this._tools.Values.ToList(), - this._fileMounts.Values.ToList()); + this._fileMounts.Values.ToList(), + this._includeDetailedErrors); FeatureUsageMarker.MarkUsed(); var approvalRequired = ComputeApprovalRequired(this._approvalMode, snapshot.Tools); diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs index 51a161a20ea..65c8a759dcd 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs @@ -31,6 +31,16 @@ public sealed class LocalCodeActProviderOptions /// public LocalCodeActApprovalMode ApprovalMode { get; set; } = LocalCodeActApprovalMode.NeverRequire; + /// + /// Gets or sets whether host exception details are exposed to generated code. + /// Defaults to . + /// + /// + /// Enable only for a trusted channel: diagnostics can contain sensitive host data. + /// This is independent of the outer chat client's error policy and sensitive-data logging. + /// + public bool IncludeDetailedErrors { get; set; } + /// /// Gets or sets the initial set of file mounts exposed to generated code. /// diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs index a8b0fd34e22..5bb0804208d 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalExecuteCodeFunction.cs @@ -72,7 +72,7 @@ public LocalExecuteCodeFunction(string pythonExecutablePath, LocalCodeActProvide options.Environment, options.WorkingDirectory); - this._snapshot = new CodeExecutor.RunSnapshot(tools, fileMounts); + this._snapshot = new CodeExecutor.RunSnapshot(tools, fileMounts, options.IncludeDetailedErrors); this._inner = AIFunctionFactory.Create( this.ExecuteCodeAsync, new AIFunctionFactoryOptions diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md index e4ae51298bf..d8d3b5cd5cf 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md @@ -37,6 +37,19 @@ using var provider = new LocalCodeActProvider("/usr/bin/python3", options); // Register provider with your AIAgent's context providers. ``` +## Error diagnostics + +Host tool failures send a generic failed response to generated Python code by +default. `LocalCodeActProviderOptions.IncludeDetailedErrors` defaults to `false` +for both the provider and standalone function. Set it to `true` only for a +trusted channel: original exception types and messages can contain sensitive data. + +The policy is captured at construction and passed through run snapshots. It is +independent of the outer `FunctionInvokingChatClient` error policy and +sensitive-data logging. Outer error handling still applies to uncaught failures, +but cannot redact diagnostics that generated code has already caught and printed +or returned. Successful results and failure signaling are unchanged. + ## What the Package Controls - **AST validation** (default on): Validates generated code against allow-lists diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs index 58b9ebceb6a..255e946a266 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs @@ -56,4 +56,43 @@ public async Task ExecuteCode_PythonPrint_ReturnsStdoutAsync() Assert.Contains("hi", doc.RootElement.GetProperty("stdout").GetString()!); Assert.Equal(0, doc.RootElement.GetProperty("exit_code").GetInt32()); } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_HostErrorDetailsRequireOptInAsync(bool includeDetailedErrors) + { + // Arrange + if (string.IsNullOrWhiteSpace(GuestPath)) + { + Assert.Skip(SkipReason); + return; + } + + const string Detail = "synthetic backend diagnostic"; + var options = HyperlightCodeActProviderOptions.CreateForWasm(GuestPath!); + options.IncludeDetailedErrors = includeDetailedErrors; + options.Tools = [AIFunctionFactory.Create( + (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool")]; + using var function = new HyperlightExecuteCodeFunction(options); + + // Act + var result = await function.InvokeAsync(new AIFunctionArguments + { + ["code"] = """ + try: + print(call_tool('failing_tool')) + except Exception as exc: + print(str(exc)) + """, + }); + + // Assert + var json = Assert.IsType(result); + Assert.Equal(includeDetailedErrors, json.Contains(Detail, StringComparison.Ordinal)); + if (!includeDetailedErrors) + { + Assert.Contains("Host tool execution failed.", json, StringComparison.Ordinal); + } + } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs index 5a528606b60..386a839e65d 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs @@ -254,12 +254,15 @@ public void Fingerprint_DifferentSandboxOptions_DifferentFingerprints() [], [], [], hostInputDirectory: null, heapSize: "20Mi"); var differentStack = SandboxExecutor.RunSnapshot.ComputeFingerprint( [], [], [], hostInputDirectory: null, stackSize: "10Mi"); + var detailedErrors = SandboxExecutor.RunSnapshot.ComputeFingerprint( + [], [], [], hostInputDirectory: null, includeDetailedErrors: true); // Assert Assert.NotEqual(baseline, differentBackend); Assert.NotEqual(baseline, differentModule); Assert.NotEqual(baseline, differentHeap); Assert.NotEqual(baseline, differentStack); + Assert.NotEqual(baseline, detailedErrors); } [Fact] @@ -285,6 +288,7 @@ public void RunSnapshot_CapturesMutableInputs() HeapSize = "10Mi", StackSize = "5Mi", HostInputDirectory = "/host/input", + IncludeDetailedErrors = true, }; // Act @@ -296,6 +300,7 @@ public void RunSnapshot_CapturesMutableInputs() options.HeapSize = "20Mi"; options.StackSize = "10Mi"; options.HostInputDirectory = "/host/other"; + options.IncludeDetailedErrors = false; // Assert Assert.Single(snapshot.Tools); @@ -305,6 +310,7 @@ public void RunSnapshot_CapturesMutableInputs() Assert.Equal("10Mi", snapshot.HeapSize); Assert.Equal("5Mi", snapshot.StackSize); Assert.Equal("/host/input", snapshot.HostInputDirectory); + Assert.True(snapshot.IncludeDetailedErrors); } [Fact] diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs index 9d94d719302..d4bd084f3f8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs @@ -24,20 +24,34 @@ public async Task InvokeAsync_PassesArgumentsAndReturnsSerializedResultAsync() Assert.Equal("\"echo:hello\"", result); } - [Fact] - public async Task InvokeAsync_ReturnsErrorJsonOnExceptionAsync() + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task InvokeAsync_ReturnsErrorJsonOnExceptionAsync(bool includeDetailedErrors, bool isAsync) { // Arrange - static int Boom() => throw new InvalidOperationException("nope"); - var tool = AIFunctionFactory.Create(Boom, name: "boom"); + const string Detail = "synthetic backend diagnostic"; + static int Boom() => throw new InvalidOperationException(Detail); + static async Task BoomAsync() + { + await Task.Yield(); + throw new InvalidOperationException(Detail); + } + + var tool = isAsync + ? AIFunctionFactory.Create(BoomAsync, name: "boom") + : AIFunctionFactory.Create(Boom, name: "boom"); // Act - var result = await ToolBridge.InvokeAsync(tool, "{}"); + var result = await ToolBridge.InvokeAsync(tool, "{}", includeDetailedErrors); // Assert using var doc = JsonDocument.Parse(result); Assert.True(doc.RootElement.TryGetProperty("error", out var err)); - Assert.Contains("nope", err.GetString()!); + Assert.Equal(includeDetailedErrors ? Detail : "Host tool execution failed.", err.GetString()); + Assert.Equal(includeDetailedErrors, result.Contains(Detail, StringComparison.Ordinal)); } [Fact] @@ -66,6 +80,7 @@ public async Task InvokeAsync_NonObjectJson_ReturnsErrorAsync() // Assert using var doc = JsonDocument.Parse(result); - Assert.True(doc.RootElement.TryGetProperty("error", out _)); + Assert.True(doc.RootElement.TryGetProperty("error", out var error)); + Assert.Contains("Tool arguments must be a JSON object.", error.GetString()!, StringComparison.Ordinal); } } diff --git a/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs index c7118a452ab..9db177a097c 100644 --- a/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs @@ -49,6 +49,110 @@ public async Task ExecuteCode_PrintsAndReturnsResultAsync() Assert.Contains("3", combined); } + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ExecuteCode_HostErrorsRespectDetailedPolicyAsync(bool includeDetailedErrors, bool isAsync) + { + // Arrange + SkipIfNoPython(); + const string Detail = "synthetic backend diagnostic"; + static string Fail() => throw new InvalidOperationException(Detail); + static async Task FailAsync() + { + await Task.Yield(); + throw new InvalidOperationException(Detail); + } + + var tool = isAsync + ? AIFunctionFactory.Create(FailAsync, name: "failing_tool") + : AIFunctionFactory.Create(Fail, name: "failing_tool"); + var function = new LocalExecuteCodeFunction(s_python!, new LocalCodeActProviderOptions + { + Tools = [tool], + IncludeDetailedErrors = includeDetailedErrors, + }); + var args = new AIFunctionArguments + { + ["code"] = """ + try: + await failing_tool() + except Exception as exc: + print(str(exc)) + try: + await call_tool('failing_tool') + except Exception as exc: + print(str(exc)) + """, + }; + + // Act + var result = await function.InvokeAsync(args, CancellationToken.None); + + // Assert + var text = GetResultText(result); + Assert.Contains(includeDetailedErrors ? Detail : "Host tool execution failed.", text, StringComparison.Ordinal); + Assert.Equal(includeDetailedErrors, text.Contains(Detail, StringComparison.Ordinal)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_UncaughtHostErrorsPreserveFailureAsync(bool includeDetailedErrors) + { + // Arrange + SkipIfNoPython(); + const string Detail = "synthetic backend diagnostic"; + var tool = AIFunctionFactory.Create( + (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool"); + var function = new LocalExecuteCodeFunction(s_python!, new LocalCodeActProviderOptions + { + Tools = [tool], + IncludeDetailedErrors = includeDetailedErrors, + }); + + // Act + var error = await Assert.ThrowsAsync(async () => + await function.InvokeAsync(new AIFunctionArguments { ["code"] = "await failing_tool()" })); + + // Assert + Assert.Contains(includeDetailedErrors ? Detail : "Host tool execution failed.", error.Message, StringComparison.Ordinal); + Assert.Equal(includeDetailedErrors, error.Message.Contains(Detail, StringComparison.Ordinal)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Provider_HostErrorPolicyIsCapturedAtConstructionAsync(bool includeDetailedErrors) + { + // Arrange + SkipIfNoPython(); + const string Detail = "synthetic backend diagnostic"; + var tool = AIFunctionFactory.Create( + (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool"); + var options = new LocalCodeActProviderOptions + { + Tools = [tool], + IncludeDetailedErrors = includeDetailedErrors, + }; + using var provider = new LocalCodeActProvider(s_python!, options); + options.IncludeDetailedErrors = !includeDetailedErrors; + var context = await provider.InvokingAsync( + new AIContextProvider.InvokingContext(new Moq.Mock().Object, session: null, new AIContext())); + var function = Assert.IsAssignableFrom(Assert.Single(context.Tools!)); + + // Act + var result = await function.InvokeAsync(new AIFunctionArguments + { + ["code"] = "try:\n await failing_tool()\nexcept Exception as exc:\n print(str(exc))", + }); + + // Assert + Assert.Equal(includeDetailedErrors, GetResultText(result).Contains(Detail, StringComparison.Ordinal)); + } + [Fact] public async Task ExecuteCode_AllowsAstValidatedFormattingAsync() { @@ -883,8 +987,10 @@ await call_tool('definitely_not_registered', x=1) Assert.DoesNotContain("NO_ERROR", combined); } - [Fact] - public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsync() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsync(bool includeDetailedErrors) { SkipIfNoPython(); @@ -896,6 +1002,7 @@ public async Task ExecuteCode_ToolThrowingExceptionPropagatesToGeneratedCodeAsyn var options = new LocalCodeActProviderOptions { Tools = new[] { faultyTool }, + IncludeDetailedErrors = includeDetailedErrors, }; var function = new LocalExecuteCodeFunction(s_python!, options); @@ -912,8 +1019,9 @@ await call_tool('faulty', message='boom') var result = await function.InvokeAsync(args, CancellationToken.None); var combined = GetResultText(result); Assert.Contains("GOT_ERROR", combined); - Assert.Contains("InvalidOperationException", combined); - Assert.Contains("intentional: boom", combined); + Assert.Contains(includeDetailedErrors ? "intentional: boom" : "Host tool execution failed.", combined); + Assert.Equal(includeDetailedErrors, combined.Contains("InvalidOperationException", StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, combined.Contains("intentional: boom", StringComparison.Ordinal)); } [Fact] diff --git a/skills/agent-framework-builder-dotnet/references/integrations.md b/skills/agent-framework-builder-dotnet/references/integrations.md index d9a9aec7eef..515214941e4 100644 --- a/skills/agent-framework-builder-dotnet/references/integrations.md +++ b/skills/agent-framework-builder-dotnet/references/integrations.md @@ -40,6 +40,11 @@ otherwise see [Extensibility](extensibility.md#custom-providers). - Durable agents and workflows run on Azure Functions and Durable Task; see [Hosting and deployment](hosting-and-deployment.md). +LocalCodeAct and Hyperlight redact host tool exception diagnostics by default. Enable +`IncludeDetailedErrors` on their provider options only for a trusted channel. This is separate from the +outer `FunctionInvokingChatClient` error policy and sensitive-data logging; outer redaction cannot repair +diagnostics already caught and returned by generated code. + For AG-UI, remember that thread and run IDs are correlation identifiers, not authentication boundaries. The application owns authorization, durable snapshots, scaling, and rendering policy. From 63ac4ee5a7dabdb8b6ec134801b1b488af3e8cab Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Mon, 2 Nov 2026 14:49:01 +0100 Subject: [PATCH 2/3] Apply .NET Hyperlight error policy to full lifecycle --- .../Internal/SandboxExecutor.cs | 52 +++++++++++++------ .../Microsoft.Agents.AI.Hyperlight/README.md | 3 ++ .../CodeActEndToEndTests.cs | 39 ++++++++++++++ .../SandboxExecutorTests.cs | 52 +++++++++++++++++++ 4 files changed, 131 insertions(+), 15 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs index 6ae6dfbb1d0..eaeb82b3d99 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs @@ -231,20 +231,24 @@ public async Task ExecuteAsync(RunSnapshot snapshot, string code, Cancel this._sandbox!.Restore(this._warmSnapshot); } - ExecutionResult result; + return BuildResult(this._sandbox!.Run(code)); + } +#pragma warning disable CA1031 // Return ordinary lifecycle failures as policy-controlled execution errors. + catch (Exception ex) when (ex is not OperationCanceledException) + { + Exception failure = ex; try { - result = this._sandbox!.Run(code); + this.ClearSandboxState(); } -#pragma warning disable CA1031 // Surface sandbox execution failures as structured JSON rather than propagating. - catch (Exception ex) -#pragma warning restore CA1031 + catch (Exception cleanupError) when (cleanupError is not OperationCanceledException) { - return BuildErrorResult(snapshot.IncludeDetailedErrors ? ex.Message : "Code execution failed."); + failure = new AggregateException(ex, cleanupError); } - return BuildResult(result); + return BuildErrorResult(snapshot.IncludeDetailedErrors ? failure.Message : "Code execution failed."); } +#pragma warning restore CA1031 finally { this._executionLock.Release(); @@ -260,12 +264,25 @@ private void EnsureInitialized(RunSnapshot snapshot) // Configuration changed (or first run) — dispose the previous sandbox // so the new one picks up the current capabilities and runtime options. - this._warmSnapshot?.Dispose(); - this._sandbox?.Dispose(); + this.ClearSandboxState(); + this.BuildAndWarmUp(snapshot); + } + + private void ClearSandboxState() + { + var warmSnapshot = this._warmSnapshot; + var sandbox = this._sandbox; this._warmSnapshot = null; this._sandbox = null; - - this.BuildAndWarmUp(snapshot); + this._lastConfigFingerprint = null; + try + { + warmSnapshot?.Dispose(); + } + finally + { + sandbox?.Dispose(); + } } private void BuildAndWarmUp(RunSnapshot snapshot) @@ -304,6 +321,7 @@ private void BuildAndWarmUp(RunSnapshot snapshot) } var sandbox = builder.Build(); + this._sandbox = sandbox; // Tools must be registered before the first Run() call. ToolBridge.RegisterAll(sandbox, snapshot.Tools, snapshot.IncludeDetailedErrors); @@ -320,7 +338,6 @@ private void BuildAndWarmUp(RunSnapshot snapshot) // upstream HyperlightSandbox.Extensions.AI CodeExecutionTool reference. _ = sandbox.Run(snapshot.Backend == SandboxBackend.JavaScript ? "void 0;" : "None"); this._warmSnapshot = sandbox.Snapshot(); - this._sandbox = sandbox; this._lastConfigFingerprint = snapshot.ConfigFingerprint; } @@ -346,8 +363,13 @@ public void Dispose() } this._disposed = true; - this._warmSnapshot?.Dispose(); - this._sandbox?.Dispose(); - this._executionLock.Dispose(); + try + { + this.ClearSandboxState(); + } + finally + { + this._executionLock.Dispose(); + } } } diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md index bdd7f3063bb..e85058e04d5 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md @@ -48,6 +48,9 @@ and sensitive-data logging. Outer error handling cannot redact diagnostics that generated code has already caught and included in normal output. Run snapshots capture the policy, and sandbox configuration fingerprints include it so cached callbacks cannot retain a different disclosure policy. +The same boundary covers initialization, warm-up, snapshot restore, and execution +failures. Ordinary failures dispose and clear failed sandbox state before returning +an error; cancellation still propagates. ## Network access and guest packages diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs index 255e946a266..93405aea4a4 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.IntegrationTests/CodeActEndToEndTests.cs @@ -2,6 +2,7 @@ using System; using System.Linq; +using System.Reflection; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.AI; @@ -95,4 +96,42 @@ public async Task ExecuteCode_HostErrorDetailsRequireOptInAsync(bool includeDeta Assert.Contains("Host tool execution failed.", json, StringComparison.Ordinal); } } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteCode_RestoreFailureObeysPolicyAndRebuildsSandboxAsync(bool includeDetailedErrors) + { + // Arrange + if (string.IsNullOrWhiteSpace(GuestPath)) + { + Assert.Skip(SkipReason); + return; + } + + var options = HyperlightCodeActProviderOptions.CreateForWasm(GuestPath!); + options.IncludeDetailedErrors = includeDetailedErrors; + using var function = new HyperlightExecuteCodeFunction(options); + await function.InvokeAsync(new AIFunctionArguments { ["code"] = "print('first')" }); + var executor = typeof(HyperlightExecuteCodeFunction) + .GetField("_executor", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(function)!; + var snapshotField = executor.GetType().GetField("_warmSnapshot", BindingFlags.Instance | BindingFlags.NonPublic)!; + Assert.IsAssignableFrom(snapshotField.GetValue(executor)).Dispose(); + + // Act + var failed = await function.InvokeAsync(new AIFunctionArguments { ["code"] = "print('second')" }); + + // Assert + using var error = JsonDocument.Parse(Assert.IsType(failed)); + Assert.False(error.RootElement.GetProperty("success").GetBoolean()); + var message = error.RootElement.GetProperty("stderr").GetString(); + Assert.Equal(!includeDetailedErrors, string.Equals(message, "Code execution failed.", StringComparison.Ordinal)); + Assert.Null(snapshotField.GetValue(executor)); + Assert.Null(executor.GetType().GetField("_sandbox", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(executor)); + Assert.Null(executor.GetType().GetField("_lastConfigFingerprint", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(executor)); + var recovered = await function.InvokeAsync(new AIFunctionArguments { ["code"] = "print('recovered')" }); + using var success = JsonDocument.Parse(Assert.IsType(recovered)); + Assert.True(success.RootElement.GetProperty("success").GetBoolean()); + Assert.Contains("recovered", success.RootElement.GetProperty("stdout").GetString()!, StringComparison.Ordinal); + } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs index 386a839e65d..e87e4119872 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs @@ -2,6 +2,9 @@ using System; using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; using HyperlightSandbox.Api; using Microsoft.Agents.AI.Hyperlight.Internal; using Microsoft.Extensions.AI; @@ -10,6 +13,55 @@ namespace Microsoft.Agents.AI.Hyperlight.UnitTests; public sealed class SandboxExecutorTests { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteAsync_InitializationFailureObeysPolicyAndReleasesLockAsync(bool includeDetailedErrors) + { + // Arrange + const string Detail = "synthetic-backend-diagnostic"; + var options = new HyperlightCodeActProviderOptions + { + HeapSize = Detail, + IncludeDetailedErrors = includeDetailedErrors, + }; + var snapshot = new SandboxExecutor.RunSnapshot([], [], [], options); + using var executor = new SandboxExecutor(); + using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // Act + var first = await executor.ExecuteAsync(snapshot, "void 0;", cancellation.Token); + var second = await executor.ExecuteAsync(snapshot, "void 0;", cancellation.Token); + + // Assert + foreach (var result in new[] { first, second }) + { + using var json = JsonDocument.Parse(result); + Assert.False(json.RootElement.GetProperty("success").GetBoolean()); + Assert.Equal(-1, json.RootElement.GetProperty("exit_code").GetInt32()); + var error = json.RootElement.GetProperty("stderr").GetString()!; + Assert.Equal(includeDetailedErrors, error.Contains(Detail, StringComparison.Ordinal)); + if (!includeDetailedErrors) + { + Assert.Equal("Code execution failed.", error); + } + } + } + + [Fact] + public async Task ExecuteAsync_PreservesCancellationAsync() + { + // Arrange + var snapshot = new SandboxExecutor.RunSnapshot([], [], [], new HyperlightCodeActProviderOptions()); + using var executor = new SandboxExecutor(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + // Act / Assert + await Assert.ThrowsAnyAsync( + () => executor.ExecuteAsync(snapshot, "void 0;", cancellation.Token)); + } + [Fact] public void Fingerprint_DifferentToolSets_DifferentFingerprints() { From ae96c2aa855e7be2eb3e69b4bf4f0359708f0459 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Tue, 3 Nov 2026 15:42:40 +0100 Subject: [PATCH 3/3] Include full CodeAct exception details for trusted opt-in --- .../HyperlightCodeActProviderOptions.cs | 1 + .../Internal/SandboxExecutor.cs | 2 +- .../Internal/ToolBridge.cs | 4 +- .../Microsoft.Agents.AI.Hyperlight/README.md | 4 +- .../Internal/ProcessBridge.cs | 2 +- .../LocalCodeActProviderOptions.cs | 1 + .../README.md | 3 +- .../SandboxExecutorTests.cs | 2 + .../ToolBridgeTests.cs | 40 +++++++++++++++---- ...ocalExecuteCodeFunctionIntegrationTests.cs | 13 ++++-- .../references/integrations.md | 3 +- 11 files changed, 57 insertions(+), 18 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs index d394d328259..c6e28f68b4b 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/HyperlightCodeActProviderOptions.cs @@ -86,6 +86,7 @@ public static HyperlightCodeActProviderOptions CreateForJavaScript() /// Defaults to . /// /// + /// Detailed errors include exception types, stack traces, and inner exceptions. /// Enable only for a trusted channel: diagnostics can contain sensitive host data. /// This is independent of the outer chat client's error policy and sensitive-data logging. /// diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs index eaeb82b3d99..27f8e184e60 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/SandboxExecutor.cs @@ -246,7 +246,7 @@ public async Task ExecuteAsync(RunSnapshot snapshot, string code, Cancel failure = new AggregateException(ex, cleanupError); } - return BuildErrorResult(snapshot.IncludeDetailedErrors ? failure.Message : "Code execution failed."); + return BuildErrorResult(snapshot.IncludeDetailedErrors ? failure.ToString() : "Code execution failed."); } #pragma warning restore CA1031 finally diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs index aab8072b6ed..e951b36efb8 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/Internal/ToolBridge.cs @@ -45,7 +45,7 @@ internal static async Task InvokeAsync(AIFunction tool, string argsJson, } catch (Exception ex) when (ex is JsonException or ArgumentException) { - return SerializeError(ex.Message); + return SerializeError(includeDetailedErrors ? ex.ToString() : ex.Message); } var result = await tool.InvokeAsync(new AIFunctionArguments(arguments)).ConfigureAwait(false); @@ -55,7 +55,7 @@ internal static async Task InvokeAsync(AIFunction tool, string argsJson, catch (Exception ex) #pragma warning restore CA1031 { - return SerializeError(includeDetailedErrors ? ex.Message : "Host tool execution failed."); + return SerializeError(includeDetailedErrors ? ex.ToString() : "Host tool execution failed."); } } diff --git a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md index e85058e04d5..f6a19d4fe51 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md +++ b/dotnet/src/Microsoft.Agents.AI.Hyperlight/README.md @@ -41,7 +41,9 @@ Host tool failures return a generic JSON error by default. Host execution exceptions returned by the executor also use a generic message. `HyperlightCodeActProviderOptions.IncludeDetailedErrors` defaults to `false` for both the provider and standalone function. Set it to `true` only for a -trusted channel: host exception messages can contain sensitive data. +trusted channel: full exception details, including stack traces and inner +exceptions, can contain sensitive data. Argument-validation errors keep their +safe messages by default and include stack traces only with this opt-in. This option is independent of the outer `FunctionInvokingChatClient` error policy and sensitive-data logging. Outer error handling cannot redact diagnostics that diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs index 6e234779b38..d94e19d0116 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/Internal/ProcessBridge.cs @@ -268,7 +268,7 @@ await SendToolResponseAsync(process, callId, ok: false, result: null, { await SendToolResponseAsync(process, callId, ok: false, result: null, excType: this._includeDetailedErrors ? ex.GetType().Name : "RuntimeError", - excMessage: this._includeDetailedErrors ? ex.Message : "Host tool execution failed.", + excMessage: this._includeDetailedErrors ? ex.ToString() : "Host tool execution failed.", cancellationToken).ConfigureAwait(false); } } diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs index 65c8a759dcd..528901f11c6 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/LocalCodeActProviderOptions.cs @@ -36,6 +36,7 @@ public sealed class LocalCodeActProviderOptions /// Defaults to . /// /// + /// Detailed errors include exception types, stack traces, and inner exceptions. /// Enable only for a trusted channel: diagnostics can contain sensitive host data. /// This is independent of the outer chat client's error policy and sensitive-data logging. /// diff --git a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md index d8d3b5cd5cf..4dc7e823c2f 100644 --- a/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md +++ b/dotnet/src/Microsoft.Agents.AI.LocalCodeAct/README.md @@ -42,7 +42,8 @@ using var provider = new LocalCodeActProvider("/usr/bin/python3", options); Host tool failures send a generic failed response to generated Python code by default. `LocalCodeActProviderOptions.IncludeDetailedErrors` defaults to `false` for both the provider and standalone function. Set it to `true` only for a -trusted channel: original exception types and messages can contain sensitive data. +trusted channel: full exception details, including stack traces and inner +exceptions, can contain sensitive data. The policy is captured at construction and passed through run snapshots. It is independent of the outer `FunctionInvokingChatClient` error policy and diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs index e87e4119872..7e347669efc 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/SandboxExecutorTests.cs @@ -41,6 +41,8 @@ public async Task ExecuteAsync_InitializationFailureObeysPolicyAndReleasesLockAs Assert.Equal(-1, json.RootElement.GetProperty("exit_code").GetInt32()); var error = json.RootElement.GetProperty("stderr").GetString()!; Assert.Equal(includeDetailedErrors, error.Contains(Detail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, error.Contains(" at ", StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, error.Contains(nameof(SandboxExecutor.ExecuteAsync), StringComparison.Ordinal)); if (!includeDetailedErrors) { Assert.Equal("Code execution failed.", error); diff --git a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs index d4bd084f3f8..bbd1d6ac43e 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hyperlight.UnitTests/ToolBridgeTests.cs @@ -33,11 +33,12 @@ public async Task InvokeAsync_ReturnsErrorJsonOnExceptionAsync(bool includeDetai { // Arrange const string Detail = "synthetic backend diagnostic"; - static int Boom() => throw new InvalidOperationException(Detail); + const string InnerDetail = "synthetic inner diagnostic"; + static int Boom() => throw new InvalidOperationException(Detail, new ArgumentException(InnerDetail)); static async Task BoomAsync() { await Task.Yield(); - throw new InvalidOperationException(Detail); + throw new InvalidOperationException(Detail, new ArgumentException(InnerDetail)); } var tool = isAsync @@ -50,8 +51,20 @@ static async Task BoomAsync() // Assert using var doc = JsonDocument.Parse(result); Assert.True(doc.RootElement.TryGetProperty("error", out var err)); - Assert.Equal(includeDetailedErrors ? Detail : "Host tool execution failed.", err.GetString()); - Assert.Equal(includeDetailedErrors, result.Contains(Detail, StringComparison.Ordinal)); + var message = err.GetString()!; + Assert.Equal(includeDetailedErrors, message.Contains(Detail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, message.Contains(InnerDetail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, message.Contains(" at ", StringComparison.Ordinal)); + if (includeDetailedErrors) + { + Assert.StartsWith($"System.InvalidOperationException: {Detail}", message, StringComparison.Ordinal); + Assert.Contains($"System.ArgumentException: {InnerDetail}", message, StringComparison.Ordinal); + Assert.Contains(isAsync ? nameof(BoomAsync) : nameof(Boom), message, StringComparison.Ordinal); + } + else + { + Assert.Equal("Host tool execution failed.", message); + } } [Fact] @@ -68,19 +81,30 @@ public async Task InvokeAsync_EmptyArguments_InvokesToolWithNoArgsAsync() Assert.Equal("\"hi\"", result); } - [Fact] - public async Task InvokeAsync_NonObjectJson_ReturnsErrorAsync() + [Theory] + [InlineData(false, "[1, 2, 3]")] + [InlineData(true, "[1, 2, 3]")] + [InlineData(false, "{")] + [InlineData(true, "{")] + public async Task InvokeAsync_InvalidArgumentsFollowDetailedPolicyAsync(bool includeDetailedErrors, string argsJson) { // Arrange static string Hi() => "hi"; var tool = AIFunctionFactory.Create(Hi, name: "hi"); // Act - var result = await ToolBridge.InvokeAsync(tool, "[1, 2, 3]"); + var result = await ToolBridge.InvokeAsync(tool, argsJson, includeDetailedErrors); // Assert using var doc = JsonDocument.Parse(result); Assert.True(doc.RootElement.TryGetProperty("error", out var error)); - Assert.Contains("Tool arguments must be a JSON object.", error.GetString()!, StringComparison.Ordinal); + var message = error.GetString()!; + Assert.NotEmpty(message); + Assert.Equal(includeDetailedErrors, message.Contains(" at ", StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, message.Contains(nameof(ToolBridge.ParseArguments), StringComparison.Ordinal)); + if (argsJson.StartsWith("[", StringComparison.Ordinal)) + { + Assert.Contains("Tool arguments must be a JSON object.", message, StringComparison.Ordinal); + } } } diff --git a/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs index 9db177a097c..1e47497ca5d 100644 --- a/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.LocalCodeAct.UnitTests/LocalExecuteCodeFunctionIntegrationTests.cs @@ -59,11 +59,12 @@ public async Task ExecuteCode_HostErrorsRespectDetailedPolicyAsync(bool includeD // Arrange SkipIfNoPython(); const string Detail = "synthetic backend diagnostic"; - static string Fail() => throw new InvalidOperationException(Detail); + const string InnerDetail = "synthetic inner diagnostic"; + static string Fail() => throw new InvalidOperationException(Detail, new ArgumentException(InnerDetail)); static async Task FailAsync() { await Task.Yield(); - throw new InvalidOperationException(Detail); + throw new InvalidOperationException(Detail, new ArgumentException(InnerDetail)); } var tool = isAsync @@ -95,6 +96,9 @@ await call_tool('failing_tool') var text = GetResultText(result); Assert.Contains(includeDetailedErrors ? Detail : "Host tool execution failed.", text, StringComparison.Ordinal); Assert.Equal(includeDetailedErrors, text.Contains(Detail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, text.Contains(InnerDetail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, text.Contains(" at ", StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, text.Contains(isAsync ? nameof(FailAsync) : nameof(Fail), StringComparison.Ordinal)); } [Theory] @@ -105,8 +109,9 @@ public async Task ExecuteCode_UncaughtHostErrorsPreserveFailureAsync(bool includ // Arrange SkipIfNoPython(); const string Detail = "synthetic backend diagnostic"; + const string InnerDetail = "synthetic inner diagnostic"; var tool = AIFunctionFactory.Create( - (Func)(() => throw new InvalidOperationException(Detail)), name: "failing_tool"); + (Func)(() => throw new InvalidOperationException(Detail, new ArgumentException(InnerDetail))), name: "failing_tool"); var function = new LocalExecuteCodeFunction(s_python!, new LocalCodeActProviderOptions { Tools = [tool], @@ -120,6 +125,8 @@ public async Task ExecuteCode_UncaughtHostErrorsPreserveFailureAsync(bool includ // Assert Assert.Contains(includeDetailedErrors ? Detail : "Host tool execution failed.", error.Message, StringComparison.Ordinal); Assert.Equal(includeDetailedErrors, error.Message.Contains(Detail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, error.Message.Contains(InnerDetail, StringComparison.Ordinal)); + Assert.Equal(includeDetailedErrors, error.Message.Contains(" at ", StringComparison.Ordinal)); } [Theory] diff --git a/skills/agent-framework-builder-dotnet/references/integrations.md b/skills/agent-framework-builder-dotnet/references/integrations.md index 515214941e4..444f2c3b88f 100644 --- a/skills/agent-framework-builder-dotnet/references/integrations.md +++ b/skills/agent-framework-builder-dotnet/references/integrations.md @@ -41,7 +41,8 @@ otherwise see [Extensibility](extensibility.md#custom-providers). [Hosting and deployment](hosting-and-deployment.md). LocalCodeAct and Hyperlight redact host tool exception diagnostics by default. Enable -`IncludeDetailedErrors` on their provider options only for a trusted channel. This is separate from the +`IncludeDetailedErrors` on their provider options only for a trusted channel; full exceptions include +stack traces and inner exceptions. This is separate from the outer `FunctionInvokingChatClient` error policy and sensitive-data logging; outer redaction cannot repair diagnostics already caught and returned by generated code.