diff --git a/.vscode/settings.json b/.vscode/settings.json index 3f312ba1bbe..ae5de9243f8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,5 @@ { - "dotnet.defaultSolution": "dotnet/agent-framework-src-and-tests.slnf" + "dotnet.defaultSolution": "dotnet/agent-framework-src-and-tests.slnf", + "$comment": "Revisit disabling the lightweight test experience when upgrading to C# Dev Kit 11. Remove the override only after verifying test discovery, execution, and debugging with the default experience.", + "dotnet.testWindow.enableNextTest": false } \ No newline at end of file diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/AIAgentHostExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/AIAgentHostExecutor.cs index 72090c72218..0409edaddbb 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/AIAgentHostExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/AIAgentHostExecutor.cs @@ -84,29 +84,12 @@ private ValueTask HandleUserInputResponseAsync( IWorkflowContext context, CancellationToken cancellationToken) { - if (!this._userInputHandler!.MarkRequestAsHandled(response.RequestId, out var request)) + if (!this._userInputHandler!.TryRecordResponse(response.RequestId, response)) { throw new InvalidOperationException($"No pending ToolApprovalRequest found with id '{response.RequestId}'."); } - // Merge the external response with any already-buffered regular messages so mixed-content - // resumes can be processed in one invocation. - return this.ProcessTurnMessagesAsync(async (pendingMessages, ctx, ct) => - { - ChatMessage responseMessage = new(ChatRole.User, [response]) - { - CreatedAt = DateTimeOffset.UtcNow, - MessageId = Guid.NewGuid().ToString("N"), - }; - pendingMessages.Add(responseMessage); - - await this.ContinueTurnAsync( - pendingMessages, ctx, this._currentTurnEmitEvents ?? false, ct, - (responseMessage, response, request.ToolCall.CallId)).ConfigureAwait(false); - - // Clear the buffered turn messages because they were consumed by ContinueTurnAsync. - return null; - }, context, cancellationToken); + return this.ResumeWithResponsesAsync(context, cancellationToken); } private ValueTask HandleFunctionResultAsync( @@ -114,23 +97,50 @@ private ValueTask HandleFunctionResultAsync( IWorkflowContext context, CancellationToken cancellationToken) { - if (!this._functionCallHandler!.MarkRequestAsHandled(result.CallId)) + if (!this._functionCallHandler!.TryRecordResponse(result.CallId, result)) { throw new InvalidOperationException($"No pending FunctionCall found with id '{result.CallId}'."); } - // Merge the external response with any already-buffered regular messages so mixed-content - // resumes can be processed in one invocation. + return this.ResumeWithResponsesAsync(context, cancellationToken); + } + + private ValueTask ResumeWithResponsesAsync(IWorkflowContext context, CancellationToken cancellationToken) + { + if (this.HasOutstandingRequests) + { + return default; + } + return this.ProcessTurnMessagesAsync(async (pendingMessages, ctx, ct) => { - pendingMessages.Add(new ChatMessage(ChatRole.Tool, [result]) + List<(AIContent Request, AIContent Response, int Order)> responses = []; + responses.AddRange(this._userInputHandler!.TakeResponses().Select(item => ((AIContent)item.Request, (AIContent)item.Response, item.Order))); + responses.AddRange(this._functionCallHandler!.TakeResponses().Select(item => ((AIContent)item.Request, (AIContent)item.Response, item.Order))); + if (responses.Count == 0) { - AuthorName = this._agent.Name ?? this._agent.Id, - CreatedAt = DateTimeOffset.UtcNow, - MessageId = Guid.NewGuid().ToString("N"), - }); + return pendingMessages; + } + + List<(ChatMessage Message, ToolApprovalResponseContent Response, string CallId)> answeredApprovals = []; + foreach (var item in responses.OrderBy(item => item.Order)) + { + bool isFunctionResult = item.Response is FunctionResultContent; + ChatMessage message = new(isFunctionResult ? ChatRole.Tool : ChatRole.User, [item.Response]) + { + AuthorName = isFunctionResult ? this._agent.Name ?? this._agent.Id : null, + CreatedAt = DateTimeOffset.UtcNow, + MessageId = Guid.NewGuid().ToString("N"), + }; + pendingMessages.Add(message); + if (item.Response is ToolApprovalResponseContent approval && + item.Request is ToolApprovalRequestContent request) + { + answeredApprovals.Add((message, approval, request.ToolCall.CallId)); + } + } - await this.ContinueTurnAsync(pendingMessages, ctx, this._currentTurnEmitEvents ?? false, ct).ConfigureAwait(false); + await this.ContinueTurnAsync(pendingMessages, ctx, this._currentTurnEmitEvents ?? false, ct, answeredApprovals).ConfigureAwait(false); // Clear the buffered turn messages because they were consumed by ContinueTurnAsync. return null; @@ -184,13 +194,13 @@ private async ValueTask ContinueTurnAsync( IWorkflowContext context, bool emitEvents, CancellationToken cancellationToken, - (ChatMessage Message, ToolApprovalResponseContent Response, string CallId)? answeredApproval = null) + IReadOnlyList<(ChatMessage Message, ToolApprovalResponseContent Response, string CallId)>? answeredApprovals = null) { this._currentTurnEmitEvents = emitEvents; if (this._options.ForwardIncomingMessages) { - List incomingMessages = answeredApproval is { } answer - ? messages.Where(m => !ReferenceEquals(m, answer.Message)).ToList() + List incomingMessages = answeredApprovals is not null + ? messages.Where(m => !answeredApprovals.Any(answer => ReferenceEquals(m, answer.Message))).ToList() : messages; if (incomingMessages.Count > 0) { @@ -209,12 +219,18 @@ private async ValueTask ContinueTurnAsync( // causes invalid request errors when the receiving agent uses the Responses API, // because these item types are not valid as input items. List forwardableMessages = FilterForwardableMessages(response.Messages); - if (answeredApproval is { } approval) + if (answeredApprovals is not null) { // Other agents received the approval request; they also need its answer before the tool result. - ChatMessage completedAnswer = approval.Message.Clone(); - completedAnswer.Contents = [CreateCompletedApprovalResponse(approval.Response, approval.CallId, response.Messages)]; - forwardableMessages.InsertRange(0, FilterForwardableMessages([completedAnswer])); + List completedAnswers = []; + foreach (var approval in answeredApprovals) + { + ChatMessage completedAnswer = approval.Message.Clone(); + completedAnswer.Contents = [CreateCompletedApprovalResponse(approval.Response, approval.CallId, response.Messages)]; + completedAnswers.Add(completedAnswer); + } + + forwardableMessages.InsertRange(0, FilterForwardableMessages(completedAnswers)); } if (forwardableMessages.Count > 0) diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/AIAgentUnservicedRequestsCollector.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/AIAgentUnservicedRequestsCollector.cs index 7e4f8c8c9d4..d465871e120 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/AIAgentUnservicedRequestsCollector.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/AIAgentUnservicedRequestsCollector.cs @@ -14,18 +14,29 @@ internal sealed class AIAgentUnservicedRequestsCollector(AIContentExternalHandle { private readonly Dictionary _userInputRequests = []; private readonly Dictionary _functionCalls = []; + private readonly List _requestsInOrder = []; public Task SubmitAsync(IWorkflowContext context, CancellationToken cancellationToken) { - Task userInputTask = userInputHandler != null && this._userInputRequests.Count > 0 - ? userInputHandler.ProcessRequestContentsAsync(this._userInputRequests, context, cancellationToken) - : Task.CompletedTask; - - Task functionCallTask = functionCallHandler != null && this._functionCalls.Count > 0 - ? functionCallHandler.ProcessRequestContentsAsync(this._functionCalls, context, cancellationToken) - : Task.CompletedTask; + List tasks = []; + for (int order = 0; order < this._requestsInOrder.Count; order++) + { + AIContent content = this._requestsInOrder[order]; + if (content is ToolApprovalRequestContent approval && userInputHandler is not null && + this._userInputRequests.TryGetValue(approval.RequestId, out var pendingApproval) && + ReferenceEquals(approval, pendingApproval)) + { + tasks.Add(userInputHandler.ProcessRequestContentAsync(approval.RequestId, approval, context, order, cancellationToken).AsTask()); + } + else if (content is FunctionCallContent call && functionCallHandler is not null && + this._functionCalls.TryGetValue(call.CallId, out var pendingCall) && + ReferenceEquals(call, pendingCall)) + { + tasks.Add(functionCallHandler.ProcessRequestContentAsync(call.CallId, call, context, order, cancellationToken).AsTask()); + } + } - return Task.WhenAll(userInputTask, functionCallTask); + return Task.WhenAll(tasks); } public void ProcessAgentResponseUpdate(AgentResponseUpdate update, Func? functionCallFilter = null) @@ -47,6 +58,7 @@ public void ProcessAIContents(IEnumerable contents, Func contents, Func _pendingRequests = new(); + private ConcurrentOrderedDictionary _pendingRequests = new(); + + internal sealed record RequestState(TRequestContent Request, int Order, TResponseContent? Response = null); public AIContentExternalHandler(ref ProtocolBuilder protocolBuilder, string portId, bool intercepted, Func handler) { @@ -45,20 +46,11 @@ void ConfigureRoutes(RouteBuilder routeBuilder, out PortBinding? portBinding) } } - public bool HasPendingRequests => !this._pendingRequests.IsEmpty; - - public Task ProcessRequestContentsAsync(Dictionary requests, IWorkflowContext context, CancellationToken cancellationToken = default) - { - IEnumerable requestTasks = from string requestId in requests.Keys - select this.ProcessRequestContentAsync(requestId, requests[requestId], context, cancellationToken) - .AsTask(); - - return Task.WhenAll(requestTasks); - } + public bool HasPendingRequests => this._pendingRequests.ToArray().Any(item => item.Value.Response is null); - public ValueTask ProcessRequestContentAsync(string id, TRequestContent requestContent, IWorkflowContext context, CancellationToken cancellationToken = default) + public ValueTask ProcessRequestContentAsync(string id, TRequestContent requestContent, IWorkflowContext context, int order = 0, CancellationToken cancellationToken = default) { - if (!this._pendingRequests.TryAdd(id, requestContent)) + if (!this._pendingRequests.TryAdd(id, new(requestContent, order))) { // Request is already pending; treat as an idempotent re-emission. // Do not repost to the sink because request IDs must remain unique while pending. @@ -76,7 +68,36 @@ public bool MarkRequestAsHandled(string id) } public bool MarkRequestAsHandled(string id, [NotNullWhen(true)] out TRequestContent? request) - => this._pendingRequests.TryRemove(id, out request); + { + if (this._pendingRequests.TryRemove(id, out RequestState? state)) + { + request = state.Request; + return true; + } + + request = null; + return false; + } + + public bool TryRecordResponse(string id, TResponseContent response) + { + while (this._pendingRequests.TryGetValue(id, out RequestState? state) && state.Response is null) + { + if (this._pendingRequests.TryUpdate(id, state with { Response = response }, state)) + { + return true; + } + } + + return false; + } + + public (TRequestContent Request, TResponseContent Response, int Order)[] TakeResponses() + { + return this._pendingRequests.TryDrain(state => state.Response is not null, out var entries) + ? entries.Select(item => (item.Value.Request, item.Value.Response!, item.Value.Order)).ToArray() + : []; + } [MemberNotNullWhen(false, nameof(_portBinding))] private bool IsIntercepted => this._portBinding == null; @@ -87,20 +108,19 @@ public bool MarkRequestAsHandled(string id, [NotNullWhen(true)] out TRequestCont public async ValueTask OnCheckpointingAsync(string id, IWorkflowContext context, CancellationToken cancellationToken = default) { - Dictionary pendingRequestsCopy = new(this._pendingRequests); - await context.QueueStateUpdateAsync(MakeKey(id), pendingRequestsCopy, cancellationToken: cancellationToken) + await context.QueueStateUpdateAsync(MakeKey(id), this._pendingRequests.ToArray(), cancellationToken: cancellationToken) .ConfigureAwait(false); } public async ValueTask OnCheckpointRestoredAsync(string id, IWorkflowContext context, CancellationToken cancellationToken = default) { - Dictionary? loadedState = - await context.ReadStateAsync>(MakeKey(id), cancellationToken: cancellationToken) + KeyValuePair[]? loadedState = + await context.ReadStateAsync[]>(MakeKey(id), cancellationToken: cancellationToken) .ConfigureAwait(false); if (loadedState != null) { - this._pendingRequests = new ConcurrentDictionary(loadedState); + this._pendingRequests = new ConcurrentOrderedDictionary(loadedState); } } } diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/ConcurrentOrderedDictionary.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/ConcurrentOrderedDictionary.cs new file mode 100644 index 00000000000..742a411f5f3 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/ConcurrentOrderedDictionary.cs @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +namespace Microsoft.Agents.AI.Workflows.Specialized; + +/// +/// Provides synchronized key-based access and insertion-ordered snapshots. +/// +internal sealed class ConcurrentOrderedDictionary + where TKey : notnull +{ + private readonly object _syncLock = new(); +#if NET9_0_OR_GREATER + private readonly OrderedDictionary _items = new(); +#else + private readonly Dictionary _items = new(); + private readonly List _keys = []; +#endif + + /// + /// Initializes an empty dictionary. + /// + public ConcurrentOrderedDictionary() + { + } + + /// + /// Initializes a dictionary with entries in the supplied order. + /// + public ConcurrentOrderedDictionary(IEnumerable> items) + { + foreach (KeyValuePair item in items) + { + if (!this.TryAdd(item.Key, item.Value)) + { + throw new ArgumentException("Duplicate keys are not supported.", nameof(items)); + } + } + } + + /// + /// Gets whether the dictionary contains no entries. + /// + public bool IsEmpty + { + get + { + lock (this._syncLock) + { + return this._items.Count == 0; + } + } + } + + /// + /// Adds an entry at the end if the key is not already present. + /// + public bool TryAdd(TKey key, TValue value) + { + lock (this._syncLock) + { +#if NET9_0_OR_GREATER + return this._items.TryAdd(key, value); +#else + if (this._items.ContainsKey(key)) + { + return false; + } + + this._items.Add(key, value); + this._keys.Add(key); + return true; +#endif + } + } + + /// + /// Removes an entry without changing the relative order of the remaining entries. + /// + public bool TryRemove(TKey key, [MaybeNullWhen(false)] out TValue value) + { + lock (this._syncLock) + { +#if NET9_0_OR_GREATER + return this._items.Remove(key, out value); +#else + if (!this._items.TryGetValue(key, out value)) + { + return false; + } + + this._items.Remove(key); + this._keys.Remove(key); + return true; +#endif + } + } + + /// + /// Gets the value associated with a key. + /// + public bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value) + { + lock (this._syncLock) + { + return this._items.TryGetValue(key, out value); + } + } + + /// + /// Updates a matching entry without changing its position. + /// + public bool TryUpdate(TKey key, TValue value, TValue comparisonValue) + { + lock (this._syncLock) + { + if (!this._items.TryGetValue(key, out TValue? current) || + !EqualityComparer.Default.Equals(current, comparisonValue)) + { + return false; + } + + this._items[key] = value; + return true; + } + } + + /// + /// Atomically removes all entries in insertion order if every value satisfies the predicate. + /// + public bool TryDrain(Func predicate, out KeyValuePair[] entries) + { + lock (this._syncLock) + { + if (this._items.Count == 0 || !this._items.Values.All(predicate)) + { + entries = []; + return false; + } + + entries = this.ToArray(); + this._items.Clear(); +#if !NET9_0_OR_GREATER + this._keys.Clear(); +#endif + return true; + } + } + + /// + /// Creates a snapshot of the entries in insertion order under the collection lock. + /// + public KeyValuePair[] ToArray() + { + lock (this._syncLock) + { +#if NET9_0_OR_GREATER + return this._items.ToArray(); +#else + return this._keys.Select(key => new KeyValuePair(key, this._items[key])).ToArray(); +#endif + } + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/AIAgentHostExecutorTests.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/AIAgentHostExecutorTests.cs index 7f35366a6ac..8b2142362c0 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/AIAgentHostExecutorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/AIAgentHostExecutorTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; @@ -8,8 +9,10 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Agents.AI.Workflows.Execution; +using Microsoft.Agents.AI.Workflows.InProc; using Microsoft.Agents.AI.Workflows.Specialized; using Microsoft.Extensions.AI; +using Moq; namespace Microsoft.Agents.AI.Workflows.UnitTests; @@ -269,12 +272,9 @@ List ExtractAndValidateRequestContents() where TRequest : AICo } // Assert 2 - // Since we are not finished, we expect the agent to not have produced a final response (="Remaining: 1") + // A partial response batch must not invoke the agent again. List agentResponseEvents = testContext.Events.OfType().ToList(); - Assert.NotEmpty(agentResponseEvents); - AgentResponseEvent lastResponseEvent = agentResponseEvents.Last(); - - Assert.Equal("Remaining: 1", lastResponseEvent.Response.Text); + Assert.Single(agentResponseEvents); // Act 3 object finalResponse = responses.Last(); @@ -284,11 +284,301 @@ List ExtractAndValidateRequestContents() where TRequest : AICo // Now that we are finished, we expect the agent to have produced a final response agentResponseEvents = testContext.Events.OfType().ToList(); Assert.NotEmpty(agentResponseEvents); - lastResponseEvent = agentResponseEvents.Last(); + AgentResponseEvent lastResponseEvent = agentResponseEvents.Last(); Assert.Equal("Done", lastResponseEvent.Response.Text); } + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Test_AgentHostExecutor_MultipleApprovals_ResumesOnlyAfterAllResponsesAsync( + bool streaming, bool respondTogether) + { + // Arrange + MultipleApprovalHarness harness = new(); + Workflow workflow = AgentWorkflowBuilder.BuildSequential(harness.Agent); + using CancellationTokenSource cts = new(TimeSpan.FromSeconds(30)); + await using StreamingRun run = await InProcessExecution.RunStreamingAsync( + workflow, new List { new(ChatRole.User, "echo a and b") }, cancellationToken: cts.Token); + Assert.True(await run.TrySendMessageAsync(new TurnToken(emitEvents: streaming))); + + List requests = []; + await foreach (WorkflowEvent evt in run.WatchStreamAsync(blockOnPendingRequest: false, cts.Token)) + { + Assert.IsNotType(evt); + Assert.IsNotType(evt); + if (evt is RequestInfoEvent requestInfo) + { + requests.Add(requestInfo.Request); + } + } + + Assert.Equal(2, requests.Count); + Assert.Empty(harness.ExecutedTexts); + Assert.Equal(1, harness.ChatCallCount); + List resumedEvents = []; + + // Act + for (int i = 0; i < requests.Count; i++) + { + ExternalRequest request = requests[i]; + ToolApprovalRequestContent? approval = request.Data.As(); + Assert.NotNull(approval); + await run.SendResponseAsync(request.CreateResponse(approval.CreateResponse(approved: true))); + + if (!respondTogether && i == 0) + { + await foreach (WorkflowEvent evt in run.WatchStreamAsync(blockOnPendingRequest: false, cts.Token)) + { + resumedEvents.Add(evt); + } + + // Assert + Assert.Empty(harness.ExecutedTexts); + Assert.Equal(1, harness.ChatCallCount); + Assert.Empty(resumedEvents.OfType()); + } + } + + await foreach (WorkflowEvent evt in run.WatchStreamAsync(blockOnPendingRequest: false, cts.Token)) + { + resumedEvents.Add(evt); + } + + // Assert + Assert.Empty(resumedEvents.OfType()); + Assert.Empty(resumedEvents.OfType()); + Assert.Equal(["a", "b"], harness.ExecutedTexts.OrderBy(text => text, StringComparer.Ordinal)); + Assert.Equal(2, harness.ChatCallCount); + WorkflowOutputEvent output = Assert.Single(resumedEvents.OfType(), evt => evt.Data is List); + List? messages = output.As>(); + Assert.NotNull(messages); + Assert.Equal("Both tools completed.", messages.Last().Text); + Assert.Equal( + ["c1", "c2"], + messages.SelectMany(message => message.Contents).OfType().Select(answer => answer.ToolCall.CallId)); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task Test_AgentHostExecutor_PartialApprovalBatch_SurvivesCheckpointAsync(bool streaming, bool approveFirst) + { + // Arrange + MultipleApprovalHarness harness = new(); + Workflow workflow = AgentWorkflowBuilder.BuildSequential(harness.Agent); + InProcessExecutionEnvironment environment = InProcessExecution.OffThread.WithCheckpointing(CheckpointManager.CreateJson(new InMemoryJsonStore())); + using CancellationTokenSource cts = new(TimeSpan.FromSeconds(30)); + List requests = []; + CheckpointInfo? partialCheckpoint = null; + await using (StreamingRun run = await environment.RunStreamingAsync(workflow, new List { new(ChatRole.User, "echo a and b") })) + { + Assert.True(await run.TrySendMessageAsync(new TurnToken(emitEvents: streaming))); + await foreach (WorkflowEvent evt in run.WatchStreamAsync(blockOnPendingRequest: false, cts.Token)) + { + Assert.IsNotType(evt); + Assert.IsNotType(evt); + if (evt is RequestInfoEvent requestInfo) + { + requests.Add(requestInfo.Request); + } + } + + Assert.Equal(2, requests.Count); + ToolApprovalRequestContent? secondApproval = requests[1].Data.As(); + Assert.NotNull(secondApproval); + + // Act + await run.SendResponseAsync(requests[1].CreateResponse(secondApproval.CreateResponse(approved: true))); + await foreach (WorkflowEvent evt in run.WatchStreamAsync(blockOnPendingRequest: false, cts.Token)) + { + Assert.IsNotType(evt); + Assert.IsNotType(evt); + if (evt is SuperStepCompletedEvent step && step.CompletionInfo?.Checkpoint is { } checkpoint) + { + partialCheckpoint = checkpoint; + } + } + + Assert.Empty(harness.ExecutedTexts); + Assert.Equal(1, harness.ChatCallCount); + } + + Assert.NotNull(partialCheckpoint); + List resumedEvents = []; + await using (StreamingRun resumed = await environment.ResumeStreamingAsync(workflow, partialCheckpoint)) + { + ToolApprovalRequestContent? firstApproval = requests[0].Data.As(); + Assert.NotNull(firstApproval); + await resumed.SendResponseAsync(requests[0].CreateResponse(firstApproval.CreateResponse(approved: approveFirst))); + await foreach (WorkflowEvent evt in resumed.WatchStreamAsync(blockOnPendingRequest: false, cts.Token)) + { + resumedEvents.Add(evt); + } + } + + // Assert + Assert.Empty(resumedEvents.OfType()); + Assert.Empty(resumedEvents.OfType()); + Assert.Equal(approveFirst ? ["a", "b"] : ["b"], harness.ExecutedTexts.OrderBy(text => text, StringComparer.Ordinal)); + Assert.Equal(2, harness.ChatCallCount); + WorkflowOutputEvent output = Assert.Single(resumedEvents.OfType(), evt => evt.Data is List); + List? messages = output.As>(); + Assert.NotNull(messages); + Assert.Equal( + ["c1", "c2"], + messages.SelectMany(message => message.Contents).OfType().Select(answer => answer.ToolCall.CallId)); + } + + private sealed class MultipleApprovalHarness + { + private int _chatCallCount; + + public ConcurrentQueue ExecutedTexts { get; } = new(); + public int ChatCallCount => Volatile.Read(ref this._chatCallCount); + public ChatClientAgent Agent { get; } + + public MultipleApprovalHarness() + { + AIFunction echo = new ApprovalRequiredAIFunction(AIFunctionFactory.Create( + (string text) => + { + this.ExecutedTexts.Enqueue(text); + return text; + }, + name: "echo")); + Mock client = new(); + client.Setup(c => c.GetResponseAsync( + It.IsAny>(), It.IsAny(), It.IsAny())) + .ReturnsAsync(this.CreateResponse); + client.Setup(c => c.GetStreamingResponseAsync( + It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(() => this.CreateResponse().ToChatResponseUpdates().ToAsyncEnumerable()); + this.Agent = new(client.Object, name: "clerk", tools: [echo]); + } + + private ChatResponse CreateResponse() + { + return Interlocked.Increment(ref this._chatCallCount) == 1 + ? new(new ChatMessage(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "echo", new Dictionary { ["text"] = "a" }), + new FunctionCallContent("c2", "echo", new Dictionary { ["text"] = "b" }), + ])) + : new(new ChatMessage(ChatRole.Assistant, "Both tools completed.")); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Test_AgentHostExecutor_OutOfOrderApprovals_ResumeInRequestOrderAsync(bool streaming) + { + // Arrange + ToolApprovalRequestContent firstRequest = new("request-1", new FunctionCallContent("c1", "echo")); + ToolApprovalRequestContent secondRequest = new("request-2", new FunctionCallContent("c2", "echo")); + List agentMessages = [new(ChatRole.Assistant, [firstRequest, secondRequest])]; + MixedContentAgent agent = new(agentMessages, TestAgentId, TestAgentName); + TestRunContext testContext = new(); + AIAgentHostExecutor executor = new(agent, new() { EmitAgentUpdateEvents = streaming }); + testContext.ConfigureExecutor(executor); + await executor.TakeTurnAsync(new(), testContext.BindWorkflowContext(executor.Id)); + + ExternalRequest[] pendingRequests = testContext.ExternalRequests.ToArray(); + Assert.Equal(2, pendingRequests.Length); + Assert.Equal( + [firstRequest.RequestId, secondRequest.RequestId], + pendingRequests.Select(request => request.Data.As()?.RequestId)); + agentMessages.Clear(); + agentMessages.Add(new(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "echo") { InformationalOnly = true }, + new FunctionCallContent("c2", "echo") { InformationalOnly = true }, + ])); + agentMessages.Add(new(ChatRole.Tool, + [ + new FunctionResultContent("c1", "a"), + new FunctionResultContent("c2", "b"), + ])); + agentMessages.Add(new(ChatRole.Assistant, "Done")); + + // Act + foreach (ExternalRequest pendingRequest in Enumerable.Reverse(pendingRequests)) + { + ToolApprovalRequestContent? approval = pendingRequest.Data.As(); + Assert.NotNull(approval); + var result = await executor.Router.RouteMessageAsync( + pendingRequest.CreateResponse(approval.CreateResponse(approved: true)), + testContext.BindWorkflowContext(executor.Id)); + Assert.Null(result?.Exception); + } + + // Assert + List> resumedInputs = agent.ReceivedMessages.Skip(1).ToList(); + Assert.Equal( + [firstRequest.RequestId, secondRequest.RequestId], + resumedInputs.SelectMany(messages => messages) + .SelectMany(message => message.Contents) + .OfType() + .Select(response => response.RequestId)); + Assert.Single(resumedInputs); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Test_AgentHostExecutor_MixedResponses_ResumeInRequestOrderAsync(bool streaming) + { + // Arrange + ToolApprovalRequestContent first = new("request-1", new FunctionCallContent("c1", "echo")); + FunctionCallContent second = new("c2", "external"); + ToolApprovalRequestContent third = new("request-3", new FunctionCallContent("c3", "echo")); + List agentMessages = [new(ChatRole.Assistant, [first, second, third])]; + MixedContentAgent agent = new(agentMessages, TestAgentId, TestAgentName); + TestRunContext testContext = new(); + AIAgentHostExecutor executor = new(agent, new() { EmitAgentUpdateEvents = streaming }); + testContext.ConfigureExecutor(executor); + await executor.TakeTurnAsync(new(), testContext.BindWorkflowContext(executor.Id)); + ExternalRequest[] requests = testContext.ExternalRequests.ToArray(); + Assert.Equal(3, requests.Length); + agentMessages.Clear(); + agentMessages.Add(new(ChatRole.Assistant, + [ + new FunctionCallContent("c1", "echo") { InformationalOnly = true }, + new FunctionCallContent("c3", "echo") { InformationalOnly = true }, + ])); + agentMessages.Add(new(ChatRole.Tool, [new FunctionResultContent("c1", "a"), new FunctionResultContent("c3", "c")])); + + // Act + foreach (ExternalRequest request in Enumerable.Reverse(requests)) + { + AIContent? content = request.Data.As(); + Assert.NotNull(content); + AIContent response = content is ToolApprovalRequestContent approval + ? approval.CreateResponse(approved: true) + : new FunctionResultContent(Assert.IsType(content).CallId, "b"); + var result = await executor.Router.RouteMessageAsync(request.CreateResponse(response), testContext.BindWorkflowContext(executor.Id)); + Assert.Null(result?.Exception); + } + + // Assert + List resumed = Assert.Single(agent.ReceivedMessages.Skip(1)); + Assert.Equal([ChatRole.User, ChatRole.Tool, ChatRole.User], resumed.Select(message => message.Role)); + Assert.Equal( + ["c1", "c2", "c3"], + resumed.SelectMany(message => message.Contents).Select(content => content switch + { + ToolApprovalResponseContent approval => approval.ToolCall.CallId, + FunctionResultContent result => result.CallId, + _ => throw new InvalidOperationException("Unexpected response content."), + })); + } + #region FilterForwardableMessages tests [Theory] @@ -475,6 +765,8 @@ private sealed class MixedContentAgent(List responseMessages, strin { public Action? BeforeResponse { get; set; } + public List> ReceivedMessages { get; } = []; + protected override string? IdCore => id; public override string? Name => name; @@ -489,12 +781,14 @@ protected override ValueTask SerializeSessionCoreAsync(AgentSession protected override Task RunCoreAsync(IEnumerable messages, AgentSession? session = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default) { + this.ReceivedMessages.Add(messages.ToList()); this.BeforeResponse?.Invoke(); return Task.FromResult(new AgentResponse(responseMessages.ToList()) { AgentId = this.Id }); } protected override async IAsyncEnumerable RunCoreStreamingAsync(IEnumerable messages, AgentSession? session = null, AgentRunOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + this.ReceivedMessages.Add(messages.ToList()); this.BeforeResponse?.Invoke(); foreach (ChatMessage msg in responseMessages) { diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/ConcurrentOrderedDictionaryTests.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/ConcurrentOrderedDictionaryTests.cs new file mode 100644 index 00000000000..375203ef578 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/ConcurrentOrderedDictionaryTests.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Agents.AI.Workflows.Specialized; + +namespace Microsoft.Agents.AI.Workflows.UnitTests; + +public class ConcurrentOrderedDictionaryTests +{ + [Fact] + public void UpdatesAndReinsertionPreserveOrder() + { + // Arrange + ConcurrentOrderedDictionary dictionary = new(); + Assert.True(dictionary.TryAdd("first", 1)); + Assert.True(dictionary.TryAdd("second", 2)); + KeyValuePair[] original = dictionary.ToArray(); + + // Act + Assert.False(dictionary.TryAdd("first", 3)); + Assert.False(dictionary.TryUpdate("first", 3, 0)); + Assert.True(dictionary.TryUpdate("first", 3, 1)); + Assert.Equal(["first", "second"], dictionary.ToArray().Select(item => item.Key)); + Assert.True(dictionary.TryRemove("first", out int removed)); + Assert.False(dictionary.TryRemove("first", out _)); + Assert.True(dictionary.TryAdd("first", 4)); + + // Assert + Assert.Equal(3, removed); + Assert.Equal(["second", "first"], dictionary.ToArray().Select(item => item.Key)); + Assert.Equal([2, 4], dictionary.ToArray().Select(item => item.Value)); + Assert.Equal([1, 2], original.Select(item => item.Value)); + } + + [Fact] + public void DrainWaitsForEveryEntryAndPreservesOrderAfterSerialization() + { + // Arrange + ConcurrentOrderedDictionary dictionary = new(); + Assert.True(dictionary.TryAdd("first", false)); + Assert.True(dictionary.TryAdd("second", false)); + Assert.True(dictionary.TryUpdate("second", true, false)); + + // Act + Assert.False(dictionary.TryDrain(value => value, out var incomplete)); + string json = JsonSerializer.Serialize(dictionary.ToArray()); + KeyValuePair[]? snapshot = JsonSerializer.Deserialize[]>(json); + Assert.NotNull(snapshot); + ConcurrentOrderedDictionary restored = new(snapshot); + Assert.True(restored.TryGetValue("second", out bool second)); + Assert.True(restored.TryUpdate("first", true, false)); + Assert.True(restored.TryDrain(value => value, out var completed)); + + // Assert + Assert.Empty(incomplete); + Assert.True(second); + Assert.Equal(["first", "second"], completed.Select(item => item.Key)); + Assert.True(restored.IsEmpty); + Assert.False(restored.TryDrain(_ => true, out _)); + Assert.True(restored.TryAdd("third", true)); + Assert.Equal("third", Assert.Single(restored.ToArray()).Key); + } + + [Fact] + public async Task ConcurrentWritesAndSnapshotsKeepKeysAndValuesAlignedAsync() + { + // Arrange + const int WorkerCount = 4; + const int EntriesPerWorker = 50; + ConcurrentOrderedDictionary dictionary = new(); + + // Act + await Task.WhenAll(Enumerable.Range(0, WorkerCount).Select(worker => Task.Run(() => + { + for (int index = 0; index < EntriesPerWorker; index++) + { + int key = (worker * EntriesPerWorker) + index; + Assert.True(dictionary.TryAdd(key, key)); + Assert.True(dictionary.TryUpdate(key, -key, key)); + KeyValuePair[] current = dictionary.ToArray(); + Assert.Equal(current.Length, current.Select(item => item.Key).Distinct().Count()); + } + }))); + KeyValuePair[] snapshot = dictionary.ToArray(); + await Task.WhenAll(snapshot.Select(item => Task.Run(() => Assert.True(dictionary.TryRemove(item.Key, out _))))); + + // Assert + Assert.Equal(WorkerCount * EntriesPerWorker, snapshot.Length); + Assert.All(snapshot, item => Assert.Equal(-item.Key, item.Value)); + for (int worker = 0; worker < WorkerCount; worker++) + { + int first = worker * EntriesPerWorker; + Assert.Equal( + Enumerable.Range(first, EntriesPerWorker), + snapshot.Where(item => item.Key >= first && item.Key < first + EntriesPerWorker).Select(item => item.Key)); + } + + Assert.True(dictionary.IsEmpty); + } +}