Repository navigation
.NET: [BREAKING] Batch workflow tool approval responses in request order #10393
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
6a897b0
8b772b3
4fa48f0
7d326cc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,5 @@ | ||
| { | ||
| "dotnet.defaultSolution": "dotnet/agent-framework-src-and-tests.slnf" | ||
| "dotnet.defaultSolution": "dotnet/agent-framework-src-and-tests.slnf", | ||
| "$comment": "Revisit disabling the lightweight test experience when upgrading to C# Dev Kit 11. Remove the override only after verifying test discovery, execution, and debugging with the default experience.", | ||
| "dotnet.testWindow.enableNextTest": false | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,6 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using System.Collections.Concurrent; | ||
| using System.Collections.Generic; | ||
| using System.Diagnostics.CodeAnalysis; | ||
| using System.Linq; | ||
|
|
@@ -17,7 +16,9 @@ internal sealed class AIContentExternalHandler<TRequestContent, TResponseContent | |
| { | ||
| private readonly PortBinding? _portBinding; | ||
| private readonly string _portId; | ||
| private ConcurrentDictionary<string, TRequestContent> _pendingRequests = new(); | ||
| private ConcurrentOrderedDictionary<string, RequestState> _pendingRequests = new(); | ||
|
|
||
| internal sealed record RequestState(TRequestContent Request, int Order, TResponseContent? Response = null); | ||
|
|
||
| public AIContentExternalHandler(ref ProtocolBuilder protocolBuilder, string portId, bool intercepted, Func<TResponseContent, IWorkflowContext, CancellationToken, ValueTask> handler) | ||
| { | ||
|
|
@@ -45,20 +46,11 @@ void ConfigureRoutes(RouteBuilder routeBuilder, out PortBinding? portBinding) | |
| } | ||
| } | ||
|
|
||
| public bool HasPendingRequests => !this._pendingRequests.IsEmpty; | ||
|
|
||
| public Task ProcessRequestContentsAsync(Dictionary<string, TRequestContent> requests, IWorkflowContext context, CancellationToken cancellationToken = default) | ||
| { | ||
| IEnumerable<Task> requestTasks = from string requestId in requests.Keys | ||
| select this.ProcessRequestContentAsync(requestId, requests[requestId], context, cancellationToken) | ||
| .AsTask(); | ||
|
|
||
| return Task.WhenAll(requestTasks); | ||
| } | ||
| public bool HasPendingRequests => this._pendingRequests.ToArray().Any(item => item.Value.Response is null); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Every delivered response calls this property, which allocates and scans a snapshot of the entire batch. Answering an N-request batch therefore copies roughly N entries N times, causing quadratic allocation/CPU and avoidable GC pressure for large model-generated batches. Please expose a lock-protected |
||
|
|
||
| public ValueTask ProcessRequestContentAsync(string id, TRequestContent requestContent, IWorkflowContext context, CancellationToken cancellationToken = default) | ||
| public ValueTask ProcessRequestContentAsync(string id, TRequestContent requestContent, IWorkflowContext context, int order = 0, CancellationToken cancellationToken = default) | ||
| { | ||
| if (!this._pendingRequests.TryAdd(id, requestContent)) | ||
| if (!this._pendingRequests.TryAdd(id, new(requestContent, order))) | ||
| { | ||
| // Request is already pending; treat as an idempotent re-emission. | ||
| // Do not repost to the sink because request IDs must remain unique while pending. | ||
|
|
@@ -76,7 +68,36 @@ public bool MarkRequestAsHandled(string id) | |
| } | ||
|
|
||
| public bool MarkRequestAsHandled(string id, [NotNullWhen(true)] out TRequestContent? request) | ||
| => this._pendingRequests.TryRemove(id, out request); | ||
| { | ||
| if (this._pendingRequests.TryRemove(id, out RequestState? state)) | ||
| { | ||
| request = state.Request; | ||
| return true; | ||
| } | ||
|
|
||
| request = null; | ||
| return false; | ||
| } | ||
|
|
||
| public bool TryRecordResponse(string id, TResponseContent response) | ||
| { | ||
| while (this._pendingRequests.TryGetValue(id, out RequestState? state) && state.Response is null) | ||
| { | ||
| if (this._pendingRequests.TryUpdate(id, state with { Response = response }, state)) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This retains each full external response until every sibling answers. A caller can submit large results for a large batch, withhold one final response, and keep all payloads resident and serialized into checkpoints indefinitely; there is no size/count limit, expiry, cancellation cleanup, or |
||
| { | ||
| return true; | ||
| } | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| public (TRequestContent Request, TResponseContent Response, int Order)[] TakeResponses() | ||
| { | ||
| return this._pendingRequests.TryDrain(state => state.Response is not null, out var entries) | ||
| ? entries.Select(item => (item.Value.Request, item.Value.Response!, item.Value.Order)).ToArray() | ||
| : []; | ||
| } | ||
|
|
||
| [MemberNotNullWhen(false, nameof(_portBinding))] | ||
| private bool IsIntercepted => this._portBinding == null; | ||
|
|
@@ -87,20 +108,19 @@ public bool MarkRequestAsHandled(string id, [NotNullWhen(true)] out TRequestCont | |
|
|
||
| public async ValueTask OnCheckpointingAsync(string id, IWorkflowContext context, CancellationToken cancellationToken = default) | ||
| { | ||
| Dictionary<string, TRequestContent> pendingRequestsCopy = new(this._pendingRequests); | ||
| await context.QueueStateUpdateAsync(MakeKey(id), pendingRequestsCopy, cancellationToken: cancellationToken) | ||
| await context.QueueStateUpdateAsync(MakeKey(id), this._pendingRequests.ToArray(), cancellationToken: cancellationToken) | ||
| .ConfigureAwait(false); | ||
| } | ||
|
|
||
| public async ValueTask OnCheckpointRestoredAsync(string id, IWorkflowContext context, CancellationToken cancellationToken = default) | ||
| { | ||
| Dictionary<string, TRequestContent>? loadedState = | ||
| await context.ReadStateAsync<Dictionary<string, TRequestContent>>(MakeKey(id), cancellationToken: cancellationToken) | ||
| KeyValuePair<string, RequestState>[]? loadedState = | ||
| await context.ReadStateAsync<KeyValuePair<string, RequestState>[]>(MakeKey(id), cancellationToken: cancellationToken) | ||
| .ConfigureAwait(false); | ||
|
|
||
| if (loadedState != null) | ||
| { | ||
| this._pendingRequests = new ConcurrentDictionary<string, TRequestContent>(loadedState); | ||
| this._pendingRequests = new ConcurrentOrderedDictionary<string, RequestState>(loadedState); | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This restarts ordering at zero for every agent invocation. If another
TurnTokenreaches the host while an earlier request remains pending, the new requests collide with the old order values; the later merge adds approval responses before function results, so a newer approval can be replayed ahead of an older function result. Please either defer/reject turns while requests are outstanding or assign a checkpointed executor-wide sequence shared by both handlers.