From f3db78f43067c94443cee30e9ee669d22437c841 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:11:59 +0100 Subject: [PATCH 01/16] .NET: Add Microsoft.Agents.AI.TypeSafe Jev AIFunction tool (alpha) Adds an alpha package that exposes Jev, the TypeSafe System One classification model, to agents as an AIFunction built with JevAIToolBuilder. The function arguments are a JevRequest (state plus typed choice, score, and noul questions) and the result is the evaluator's JevResponse serialized as JSON. - JevAIToolOptions derives from ClientPipelineOptions; the TypeSafe API is called through a System.ClientModel pipeline authenticated with an ApiKeyCredential, retrying 429/529 and capping Retry-After at one minute. - UseEvaluator and UseClient let any other client answer requests, with input and output mappers; UseClient resolves the client from the agent's services in a new scope per call. - Requests from the model and evaluator responses are validated against the Jev contract before they reach the API or the model. - Adds unit tests and the Agent_Step25_ClassificationWithJev sample. --- dotnet/agent-framework-dotnet.slnx | 3 + dotnet/agent-framework-src-and-tests.slnf | 2 + .../Agent_Step25_ClassificationWithJev.csproj | 20 + .../Program.cs | 63 +++ .../README.md | 100 ++++ dotnet/samples/02-agents/Agents/README.md | 1 + .../Core/JevAIFunction.cs | 89 ++++ .../Core/JevApiRequest.cs | 22 + .../Core/JevContractValidator.cs | 214 ++++++++ .../Core/JevHttpEvaluator.cs | 88 ++++ .../Core/JevJsonContext.cs | 20 + .../Core/JevJsonUtilities.cs | 60 +++ .../Core/JevResponseClassifier.cs | 39 ++ .../Core/JevRetryPolicy.cs | 87 ++++ .../JevAIToolBuilder.cs | 231 +++++++++ .../JevAIToolOptions.cs | 112 +++++ .../Microsoft.Agents.AI.TypeSafe/JevAnswer.cs | 24 + .../JevChoiceAnswer.cs | 35 ++ .../JevChoiceQuestion.cs | 28 ++ .../JevNoulAnswer.cs | 20 + .../JevNoulCriteria.cs | 28 ++ .../JevNoulQuestion.cs | 24 + .../JevQuestion.cs | 32 ++ .../JevRequest.cs | 41 ++ .../JevResponse.cs | 41 ++ .../JevScoreAnswer.cs | 42 ++ .../JevScoreQuestion.cs | 27 + .../Microsoft.Agents.AI.TypeSafe/JevUsage.cs | 26 + .../Microsoft.Agents.AI.TypeSafe.csproj | 31 ++ .../JevAIFunctionTests.cs | 463 ++++++++++++++++++ .../JevAIToolBuilderTests.cs | 172 +++++++ .../JevAgentIntegrationTests.cs | 114 +++++ .../JevHttpEvaluatorTests.cs | 291 +++++++++++ .../JevTestData.cs | 105 ++++ .../JevTestHttpHandler.cs | 55 +++ ...rosoft.Agents.AI.TypeSafe.UnitTests.csproj | 17 + .../ScriptedChatClient.cs | 52 ++ 37 files changed, 2819 insertions(+) create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevApiRequest.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ScriptedChatClient.cs diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 77e772cf214..ec1b47fdcbe 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -69,6 +69,7 @@ + @@ -631,6 +632,7 @@ + @@ -689,6 +691,7 @@ + diff --git a/dotnet/agent-framework-src-and-tests.slnf b/dotnet/agent-framework-src-and-tests.slnf index b6639c7a09b..bf22d4be168 100644 --- a/dotnet/agent-framework-src-and-tests.slnf +++ b/dotnet/agent-framework-src-and-tests.slnf @@ -29,6 +29,7 @@ "src\\Microsoft.Agents.AI.OpenAI\\Microsoft.Agents.AI.OpenAI.csproj", "src\\Microsoft.Agents.AI.Purview\\Microsoft.Agents.AI.Purview.csproj", "src\\Microsoft.Agents.AI.Tools.Shell\\Microsoft.Agents.AI.Tools.Shell.csproj", + "src\\Microsoft.Agents.AI.TypeSafe\\Microsoft.Agents.AI.TypeSafe.csproj", "src\\Microsoft.Agents.AI.Valkey\\Microsoft.Agents.AI.Valkey.csproj", "src\\Microsoft.Agents.AI.Workflows.Declarative.Foundry\\Microsoft.Agents.AI.Workflows.Declarative.Foundry.csproj", "src\\Microsoft.Agents.AI.Workflows.Declarative.Mcp\\Microsoft.Agents.AI.Workflows.Declarative.Mcp.csproj", @@ -64,6 +65,7 @@ "tests\\Microsoft.Agents.AI.OpenAI.UnitTests\\Microsoft.Agents.AI.OpenAI.UnitTests.csproj", "tests\\Microsoft.Agents.AI.Purview.UnitTests\\Microsoft.Agents.AI.Purview.UnitTests.csproj", "tests\\Microsoft.Agents.AI.Tools.Shell.UnitTests\\Microsoft.Agents.AI.Tools.Shell.UnitTests.csproj", + "tests\\Microsoft.Agents.AI.TypeSafe.UnitTests\\Microsoft.Agents.AI.TypeSafe.UnitTests.csproj", "tests\\Microsoft.Agents.AI.UnitTests\\Microsoft.Agents.AI.UnitTests.csproj", "tests\\Microsoft.Agents.AI.Valkey.UnitTests\\Microsoft.Agents.AI.Valkey.UnitTests.csproj", "tests\\Microsoft.Agents.AI.Workflows.Declarative.Mcp.UnitTests\\Microsoft.Agents.AI.Workflows.Declarative.Mcp.UnitTests.csproj", diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj new file mode 100644 index 00000000000..26193187de0 --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs new file mode 100644 index 00000000000..c99b7db4f8f --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +// Classification with Jev — Give an agent a fast, calibrated classifier as a tool +// +// Jev, the TypeSafe System One model, does not write text: it answers typed questions +// (choice, score, noul) about a state and returns calibrated probabilities. This sample gives +// a support agent a Jev tool built with JevAIToolBuilder. The agent's model decides which +// questions to ask, Jev answers them, and the model writes the routing decision, sending +// uncertain tickets to a human. + +using System.ClientModel; +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.TypeSafe; +using Microsoft.Extensions.AI; + +var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); +var deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini"; +var typeSafeApiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); + +// +// The tool's arguments are a Jev request (the state plus typed questions) and its result is the Jev response. +AIFunction jev = new JevAIToolBuilder(new ApiKeyCredential(typeSafeApiKey)).Build(); +// + +const string Instructions = """ + You triage customer support tickets. For every ticket, call evaluate_with_jev once with the ticket text as the + state and these questions: + - "department": a choice between billing (payments, invoices, refunds), technical (bugs, outages, integrations), + and sales (pricing, upgrades, new accounts). + - "urgent": a noul asking whether the customer needs help urgently. + - "frustration": a score with the levels calm, annoyed, and angry. + Then reply in one line with the department, whether it is urgent, and the frustration level. When the department + confidence is below 0.6, say the ticket needs human review instead of naming a department. + """; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()) + .AsAIAgent(model: deploymentName, instructions: Instructions, name: "TicketTriage", tools: [jev]); + +string[] tickets = +[ + "I've been trying to connect my Stripe account for 3 days and the integration keeps failing. I'm losing sales. Please help ASAP.", + "Hi, could you send me a copy of last month's invoice when you get a chance? Thanks!", + "I'd like to talk to someone about my account.", +]; + +foreach (string ticket in tickets) +{ + AgentResponse response = await agent.RunAsync(ticket); + + Console.WriteLine($"Ticket: {ticket}"); + foreach (FunctionResultContent result in response.Messages.SelectMany(static m => m.Contents).OfType()) + { + Console.WriteLine($"Jev answers: {result.Result}"); + } + + Console.WriteLine($"Agent: {response.Text}"); + Console.WriteLine(); +} diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md new file mode 100644 index 00000000000..21a96610475 --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md @@ -0,0 +1,100 @@ +# Classification with Jev + +This sample gives an agent a classifier tool built with `JevAIToolBuilder` from `Microsoft.Agents.AI.TypeSafe`. + +Jev, the TypeSafe System One model, does not write text. It answers typed questions about a state and returns +calibrated probabilities. An agent's model writes the replies; Jev makes the fast, structured decisions: + +1. The agent's model calls `evaluate_with_jev` with a Jev request: the `state` to evaluate and the `questions` to + answer about it. +2. Jev answers each question: + + | Question type | What it answers | Answer | + | --- | --- | --- | + | `choice` | Which of these options? | The chosen option, the probability of each option, and a confidence | + | `score` | Which level of an ordered scale (2 to 10 levels)? | A score that can fall between levels, the probability of each level, and a confidence | + | `noul` | Is this yes or no statement true? | The probability, from 0 to 1, that it is true | + +3. The model reads the answers and writes the reply. Here it routes each support ticket and sends uncertain tickets to + human review. + +## Configuring the client + +`JevAIToolBuilder` hides the HTTP client. The API key is an `ApiKeyCredential`, and `JevAIToolOptions` follows the +`ClientPipelineOptions` pattern of the OpenAI and Azure AI client libraries: + +```csharp +var options = new JevAIToolOptions +{ + ModelId = "jev-latest", // default + Transport = new HttpClientPipelineTransport(httpClient), // for example from IHttpClientFactory + NetworkTimeout = TimeSpan.FromSeconds(30), +}; +options.AddPolicy(myPolicy, PipelinePosition.PerCall); // custom pipeline policies + +AIFunction jev = new JevAIToolBuilder(new ApiKeyCredential(apiKey), options).Build(); +``` + +The default retry policy retries rate-limited (429) and overloaded (529) requests, and fails at once when the API asks +for a wait longer than one minute, so the agent run is never blocked for long. `Name` and `Description` set what the +agent's model sees. + +## Using another client + +Any other client can answer the requests instead of the TypeSafe API. Pass a delegate that takes a `JevRequest` and +returns a `JevResponse`, or map between the Jev types and the client's own types: + +```csharp +AIFunction jev = new JevAIToolBuilder() + .UseClient( + myClient.EvaluateAsync, + inputMapper: request => ToMyClientRequest(request), + outputMapper: response => ToJevResponse(response)) + .Build(); +``` + +The tool checks the mapped response before the model sees it: every question needs an answer of the same type, and a +`choice` answer and its probabilities may only use the options the question offered. + +## Dependency injection + +Every evaluator receives the `IServiceProvider` of the agent that calls the tool. `FunctionInvokingChatClient` passes +it in `AIFunctionArguments.Services`, from the services the agent was created with. `UseClient` resolves +a registered client from a new scope on every call and disposes the scope afterwards, so scoped and disposable clients +work even though hosted agents are singletons that receive the root provider: + +```csharp +builder.Services.AddScoped(); +builder.Services.AddAIAgent("triage", Instructions) + .WithAITool(sp => new JevAIToolBuilder() + .UseClient( + (client, request, cancellationToken) => client.EvaluateAsync(request, cancellationToken), + inputMapper: ToMyClientRequest, + outputMapper: ToJevResponse) + .Build()); +``` + +With the TypeSafe API, build the tool in the same `WithAITool` factory and take the credential and the transport from +the container or configuration. + +## Prerequisites + +- .NET 10 SDK or later +- A Microsoft Foundry project with a model deployment +- Azure CLI installed and authenticated (for Azure credential authentication) +- A TypeSafe API key from the [TypeSafe console](https://console.typesafe.ai/keys) + +Set the following environment variables: + +```powershell +$env:FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Foundry project endpoint +$env:FOUNDRY_MODEL="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini +$env:TYPESAFE_API_KEY="" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev +dotnet run +``` diff --git a/dotnet/samples/02-agents/Agents/README.md b/dotnet/samples/02-agents/Agents/README.md index 487e7f269a2..356cf1116bf 100644 --- a/dotnet/samples/02-agents/Agents/README.md +++ b/dotnet/samples/02-agents/Agents/README.md @@ -51,6 +51,7 @@ Before you begin, ensure you have the following prerequisites: |[Switching agent operating mode](./Agent_Step22_AgentMode/)|This sample demonstrates how to use the AgentModeProvider to track and switch an agent's operating mode at runtime, including the built-in plan/execute modes and custom modes, with a simple input loop that switches mode using a slash command.| |[Tracking work with a todo list](./Agent_Step23_TodoList/)|This sample demonstrates how to use the TodoProvider to let an agent plan and track multi-step work using a todo list that persists across turns, printing the evolving todo list after each turn.| |[Routing turns across multiple models](./Agent_Step24_MultiModelRouting/)|This sample demonstrates how to use the RoutePersistingRoutingChatClient to route each agent turn to one of several named chat clients, switching the active model mid-conversation while preserving the conversation history.| +|[Classification with Jev](./Agent_Step25_ClassificationWithJev/)|This sample demonstrates how to give an agent a Jev tool built with JevAIToolBuilder, so the agent's model asks Jev typed choice, score, and noul questions and uses the calibrated answers to route support tickets.| ## Running the samples from the console diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs new file mode 100644 index 00000000000..8001bbd4f59 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The created by : its arguments are a +/// and its result is the of the configured evaluator. +/// +internal sealed class JevAIFunction : AIFunction +{ + private readonly Func> _evaluateAsync; + + public JevAIFunction(string name, string description, Func> evaluateAsync) + { + this.Name = name; + this.Description = description; + this._evaluateAsync = evaluateAsync; + } + + public override string Name { get; } + + public override string Description { get; } + + public override JsonElement JsonSchema => JevJsonUtilities.RequestSchema; + + public override JsonElement? ReturnJsonSchema => JevJsonUtilities.ResponseSchema; + + public override JsonSerializerOptions JsonSerializerOptions => JevJsonUtilities.Options; + + protected override async ValueTask InvokeCoreAsync(AIFunctionArguments arguments, CancellationToken cancellationToken) + { + JevRequest request = ReadRequest(arguments); + JevContractValidator.ValidateRequest(request); + + // FunctionInvokingChatClient sets Services to the IServiceProvider of the agent (ChatClientAgent passes its + // services through), so evaluators can resolve per-call dependencies such as a registered client. + JevResponse? response = await this._evaluateAsync(request, arguments.Services, cancellationToken).ConfigureAwait(false); + if (response is null) + { + Throw.InvalidOperationException("The Jev evaluator returned no response."); + } + + JevContractValidator.ValidateResponse(request, response); + + // The result is returned as JSON, as AIFunctionFactory does. Chat clients serialize any other result object + // with AIJsonUtilities.DefaultOptions, which has no metadata for these types when reflection is disabled + // (for example under native AOT); the model would then receive an empty tool output. + try + { + return JsonSerializer.SerializeToElement(response, JevJsonContext.Default.JevResponse); + } + catch (Exception ex) when (ex is ArgumentException or JsonException or NotSupportedException) + { + // A response that cannot be written is an evaluator fault. Reported as ArgumentException, it would look + // like a mistake in the model's arguments and invite the model to retry the same call. + throw new InvalidOperationException($"The Jev response could not be serialized: {ex.Message}", ex); + } + } + + private static JevRequest ReadRequest(AIFunctionArguments arguments) + { + // The arguments are the top-level properties of the request, so they are reassembled into one object. + var json = new JsonObject(); + foreach (KeyValuePair argument in arguments) + { + json[argument.Key] = JevJsonUtilities.ToNode(argument.Value); + } + + try + { + return json.Deserialize(JevJsonContext.Default.JevRequest) + ?? throw new JsonException("The arguments are null."); + } + catch (Exception ex) when (ex is JsonException or NotSupportedException) + { + // An unknown or missing question type surfaces as NotSupportedException rather than JsonException. + throw new ArgumentException($"The arguments are not a valid Jev request: {ex.Message}", nameof(arguments), ex); + } + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevApiRequest.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevApiRequest.cs new file mode 100644 index 00000000000..718e446afe3 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevApiRequest.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The body of a TypeSafe System One API request: a plus the model that answers it. +/// +internal sealed class JevApiRequest +{ + [JsonPropertyName("state")] + public required JsonElement State { get; init; } + + [JsonPropertyName("model")] + public required string Model { get; init; } + + [JsonPropertyName("questions")] + public required IReadOnlyDictionary Questions { get; init; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs new file mode 100644 index 00000000000..3188ff4333c --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Checks requests before they are evaluated and responses before they are returned to the caller. +/// +/// +/// Requests come from a model, so their errors are reported as messages that say how to +/// fix the arguments. Responses come from an evaluator, possibly a custom client with its own mappers, so their errors +/// are reported as ; they point at an evaluator or mapper bug, not at the model. +/// +internal static class JevContractValidator +{ + /// The most options that the Jev API accepts in one Choice question. + public const int MaxChoiceOptions = 255; + + /// + /// The fewest levels of a Score question. The API also accepts a single level, but the documentation requires two, + /// and a one-level score always returns that level, so it is rejected as a request mistake. + /// + public const int MinScoreLevels = 2; + + /// The most levels that the Jev API accepts in one Score question. + public const int MaxScoreLevels = 10; + + // Jev computes scores and probabilities in floating point, so values may exceed their bounds by rounding error. + private const double Tolerance = 1e-6; + + public static void ValidateRequest(JevRequest request) + { + if (request.State.ValueKind is not (JsonValueKind.String or JsonValueKind.Object or JsonValueKind.Array)) + { + Throw.ArgumentException(nameof(request), "The state must be a string, a JSON object, or a JSON array."); + } + + if (request.Questions is null || request.Questions.Count == 0) + { + Throw.ArgumentException(nameof(request), "The request must contain at least one question."); + } + + foreach (KeyValuePair entry in request.Questions) + { + string id = entry.Key; + if (string.IsNullOrWhiteSpace(id)) + { + Throw.ArgumentException(nameof(request), "Question IDs must not be empty."); + } + + JevQuestion question = entry.Value; + if (question is null) + { + Throw.ArgumentException(nameof(request), $"Question '{id}' is null."); + } + + if (string.IsNullOrWhiteSpace(question.Instructions)) + { + Throw.ArgumentException(nameof(request), $"Question '{id}' must have instructions."); + } + + switch (question) + { + case JevChoiceQuestion choice: + int options = choice.Criteria?.Count ?? 0; + if (options is < 1 or > MaxChoiceOptions) + { + Throw.ArgumentException(nameof(request), $"Choice question '{id}' has {Count(options, "option")}; it needs from 1 to {MaxChoiceOptions}."); + } + + foreach (string option in choice.Criteria!.Keys) + { + if (string.IsNullOrWhiteSpace(option)) + { + Throw.ArgumentException(nameof(request), $"Choice question '{id}' has an empty option name."); + } + } + + break; + + case JevScoreQuestion score: + int levels = score.Criteria?.Count ?? 0; + if (levels is < MinScoreLevels or > MaxScoreLevels) + { + Throw.ArgumentException(nameof(request), $"Score question '{id}' has {Count(levels, "level")}; it needs from {MinScoreLevels} to {MaxScoreLevels}."); + } + + foreach (string level in score.Criteria!) + { + if (string.IsNullOrWhiteSpace(level)) + { + Throw.ArgumentException(nameof(request), $"Score question '{id}' has an empty level description."); + } + } + + break; + } + } + } + + public static void ValidateResponse(JevRequest request, JevResponse response) + { + if (response.Answers is null) + { + Throw.InvalidOperationException("The Jev response has no answers."); + } + + foreach (KeyValuePair entry in request.Questions) + { + string id = entry.Key; + if (!response.Answers.TryGetValue(id, out JevAnswer? answer) || answer is null) + { + Throw.InvalidOperationException($"The Jev response has no answer for question '{id}'."); + } + + switch (entry.Value, answer) + { + case (JevChoiceQuestion question, JevChoiceAnswer choice): + // A choice outside the options would reach the model as a value it never offered, which is how a + // mapper that converts between option names and a client's own values would fail silently. + if (choice.Choice is null || !question.Criteria.ContainsKey(choice.Choice)) + { + Throw.InvalidOperationException($"The answer to question '{id}' chose '{choice.Choice}', which is not one of its options."); + } + + EnsureInRange(choice.Confidence, 0, 1, id, "confidence"); + EnsureProbabilities(choice.Probabilities, id, key => question.Criteria.ContainsKey(key), "one of its options"); + break; + + case (JevScoreQuestion question, JevScoreAnswer score): + EnsureInRange(score.Score, 0, question.Criteria.Count - 1, id, "score"); + EnsureInRange(score.Confidence, 0, 1, id, "confidence"); + + // Score answers key their probabilities and legend by level number ("0", "1", ...). + int levels = question.Criteria.Count; + bool IsLevel(string key) => int.TryParse(key, NumberStyles.None, CultureInfo.InvariantCulture, out int level) && level < levels; + string levelRange = $"a level number from 0 to {(levels - 1).ToString(CultureInfo.InvariantCulture)}"; + EnsureProbabilities(score.Probabilities, id, IsLevel, levelRange); + EnsureKeys(score.Legend?.Keys, id, "legend", IsLevel, levelRange); + break; + + case (JevNoulQuestion, JevNoulAnswer noul): + EnsureInRange(noul.Noul, 0, 1, id, "noul"); + break; + + default: + Throw.InvalidOperationException($"The answer to question '{id}' is a {KindOf(answer)} answer, but the question is a {KindOf(entry.Value)} question."); + break; + } + } + } + + /// + /// Checks that every probability is in range and is keyed by something the question offered. A mapper that fills + /// probabilities with a client's own labels would otherwise show the model options it never asked about. + /// + private static void EnsureProbabilities(IReadOnlyDictionary? probabilities, string id, Func isValidKey, string expected) + { + if (probabilities is null) + { + Throw.InvalidOperationException($"The answer to question '{id}' has no probabilities."); + } + + EnsureKeys(probabilities.Keys, id, "probabilities", isValidKey, expected); + foreach (KeyValuePair probability in probabilities) + { + EnsureInRange(probability.Value, 0, 1, id, $"probability '{probability.Key}'"); + } + } + + private static void EnsureKeys(IEnumerable? keys, string id, string field, Func isValidKey, string expected) + { + foreach (string key in keys ?? []) + { + if (!isValidKey(key)) + { + Throw.InvalidOperationException($"The answer to question '{id}' has {field} for '{key}', which is not {expected}."); + } + } + } + + private static void EnsureInRange(double value, double min, double max, string id, string field) + { + // NaN and infinity fail these comparisons too; they cannot be written as JSON at all. + if (!(value >= min - Tolerance && value <= max + Tolerance)) + { + Throw.InvalidOperationException(string.Format( + CultureInfo.InvariantCulture, + "The answer to question '{0}' has {1} {2}, which is outside the range {3} to {4}.", + id, + field, + value, + min, + max)); + } + } + + private static string Count(int count, string noun) => + count == 1 ? $"1 {noun}" : $"{count.ToString(CultureInfo.InvariantCulture)} {noun}s"; + + private static string KindOf(object value) => + value switch + { + JevChoiceQuestion or JevChoiceAnswer => "Choice", + JevScoreQuestion or JevScoreAnswer => "Score", + JevNoulQuestion or JevNoulAnswer => "Noul", + _ => value.GetType().Name, + }; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs new file mode 100644 index 00000000000..02104e5fed6 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Globalization; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Evaluates a with the TypeSafe System One API through a System.ClientModel pipeline. +/// +internal sealed class JevHttpEvaluator +{ + private const string DefaultEndpoint = "https://api.typesafe.ai/v1/systemone"; + private const int MaxErrorDetailLength = 500; + + private readonly ClientPipeline _pipeline; + private readonly Uri _endpoint; + private readonly string _modelId; + + public JevHttpEvaluator(ApiKeyCredential credential, JevAIToolOptions options) + { + this._endpoint = options.Endpoint ?? new Uri(DefaultEndpoint); + this._modelId = options.ModelId; + + // The credential is applied per try, like the OpenAI client does, so a retried request is signed again. + // Creating the pipeline freezes the options, as with any other client built from them. + this._pipeline = ClientPipeline.Create( + options, + perCallPolicies: [], + perTryPolicies: [ApiKeyAuthenticationPolicy.CreateBearerAuthorizationPolicy(credential)], + beforeTransportPolicies: []); + } + + public async Task EvaluateAsync(JevRequest request, IServiceProvider? services, CancellationToken cancellationToken) + { + using PipelineMessage message = this._pipeline.CreateMessage(); + message.ResponseClassifier = JevResponseClassifier.Instance; + message.Apply(new RequestOptions { CancellationToken = cancellationToken }); + + PipelineRequest httpRequest = message.Request; + httpRequest.Method = "POST"; + httpRequest.Uri = this._endpoint; + httpRequest.Headers.Set("Accept", "application/json"); + httpRequest.Headers.Set("Content-Type", "application/json"); + httpRequest.Content = BinaryContent.Create(BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes( + new JevApiRequest { State = request.State, Model = this._modelId, Questions = request.Questions }, + JevJsonContext.Default.JevApiRequest))); + + // Transports leave the cancellation check to the HTTP handler, so a canceled call is stopped here before + // anything is sent. + cancellationToken.ThrowIfCancellationRequested(); + await this._pipeline.SendAsync(message).ConfigureAwait(false); + + PipelineResponse response = message.Response!; + if (response.IsError) + { + // The service detail says which field was rejected, which is what the model needs to correct its + // arguments, but the default ClientResultException message has only the status. The exception still + // carries the response for GetRawResponse, and never the credential. + string body = response.Content.ToString(); + string detail = body.Length > MaxErrorDetailLength ? string.Concat(body.AsSpan(0, MaxErrorDetailLength), "...") : body; + throw new ClientResultException( + string.Format(CultureInfo.InvariantCulture, "The Jev request failed with status {0} ({1}): {2}", response.Status, response.ReasonPhrase, detail), + response); + } + + return ReadResponse(response.Content); + } + + private static JevResponse ReadResponse(BinaryData content) + { + try + { + return JsonSerializer.Deserialize(content.ToMemory().Span, JevJsonContext.Default.JevResponse) + ?? throw new JsonException("The response body is null."); + } + catch (Exception ex) when (ex is JsonException or NotSupportedException) + { + // An answer with an unknown or missing type surfaces as NotSupportedException rather than JsonException. + throw new JsonException($"The Jev API response could not be read: {ex.Message}", ex); + } + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs new file mode 100644 index 00000000000..4eb54bccb97 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Source-generated serialization metadata for the Jev contract types. +/// +/// +/// Models often write the type discriminator after other properties of a question, so out-of-order metadata +/// is allowed; without it, a valid question such as {"instructions": "...", "type": "noul"} would be rejected. +/// +[JsonSourceGenerationOptions(AllowOutOfOrderMetadataProperties = true)] +[JsonSerializable(typeof(JevRequest))] +[JsonSerializable(typeof(JevResponse))] +[JsonSerializable(typeof(JevApiRequest))] +[JsonSerializable(typeof(Dictionary))] +internal sealed partial class JevJsonContext : JsonSerializerContext; diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs new file mode 100644 index 00000000000..8e8ff710492 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization.Metadata; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Serializer options and JSON schemas shared by the Jev tool. +/// +internal static class JevJsonUtilities +{ + private static readonly Lazy s_requestSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevRequest), serializerOptions: Options)); + private static readonly Lazy s_responseSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevResponse), serializerOptions: Options)); + + /// + /// Gets the options that resolve the Jev contract types first and every other type through + /// . + /// + public static JsonSerializerOptions Options { get; } = CreateOptions(); + + /// Gets the JSON schema of the tool arguments, which is the schema of . + public static JsonElement RequestSchema => s_requestSchema.Value; + + /// Gets the JSON schema of the tool result, which is the schema of . + public static JsonElement ResponseSchema => s_responseSchema.Value; + + /// + /// Converts one tool argument into a JSON node. Chat clients pass arguments as values; + /// callers that invoke the function directly may pass .NET objects instead. + /// + public static JsonNode? ToNode(object? value) => + value switch + { + null => null, + JsonNode node => node.DeepClone(), + JsonElement element => element.ValueKind switch + { + JsonValueKind.Object => JsonObject.Create(element), + JsonValueKind.Array => JsonArray.Create(element), + JsonValueKind.Null or JsonValueKind.Undefined => null, + _ => JsonValue.Create(element), + }, + _ => JsonSerializer.SerializeToNode(value, Options.GetTypeInfo(value.GetType())), + }; + + private static JsonSerializerOptions CreateOptions() + { + var options = new JsonSerializerOptions(JevJsonContext.Default.Options) + { + TypeInfoResolver = JsonTypeInfoResolver.Combine(JevJsonContext.Default, AIJsonUtilities.DefaultOptions.TypeInfoResolver), + }; + + options.MakeReadOnly(); + return options; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs new file mode 100644 index 00000000000..adf42d6ac73 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel.Primitives; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Classifies System One responses: any 2xx status is a success, and 529 is retriable in addition to the statuses +/// that retries (408, 429, 500, 502, 503, and 504). +/// +/// +/// TypeSafe answers 529 when it is overloaded and documents it as transient, but the default classifier treats it as a +/// permanent error. Every other case, including network failures, is left to the default classifier. +/// +internal sealed class JevResponseClassifier : PipelineMessageClassifier +{ + private const int OverloadedStatus = 529; + + public static JevResponseClassifier Instance { get; } = new(); + + public override bool TryClassify(PipelineMessage message, out bool isError) + { + if (message.Response is not { } response) + { + isError = false; + return false; + } + + isError = response.Status is < 200 or >= 300; + return true; + } + + public override bool TryClassify(PipelineMessage message, Exception? exception, out bool isRetriable) + { + isRetriable = exception is null && message.Response?.Status == OverloadedStatus; + return isRetriable; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs new file mode 100644 index 00000000000..b54b892ce09 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The default retry policy of : the standard , except +/// that it never waits longer than . +/// +/// +/// waits for whatever Retry-After the server sends. Inside a tool call that +/// wait blocks the agent run with no feedback, and a very large value makes the delay itself throw, so a longer +/// request fails at once with the service's status instead. +/// +internal sealed class JevRetryPolicy : ClientRetryPolicy +{ + public const int DefaultMaxRetries = 3; + + public static readonly TimeSpan MaxRetryAfter = TimeSpan.FromMinutes(1); + + public JevRetryPolicy(int maxRetries = DefaultMaxRetries, bool enableLogging = true, ILoggerFactory? loggerFactory = null) + : base(maxRetries, enableLogging, loggerFactory) + { + } + + /// + /// Gets or sets a list that, when set, receives each delay instead of waiting it, so tests can check the delays + /// without being slowed down by them. + /// + internal List? RecordedWaitsForTests { get; set; } + + protected override bool ShouldRetry(PipelineMessage message, Exception? exception) => + base.ShouldRetry(message, exception) && IsRetryAfterWithinLimit(message); + + protected override async ValueTask ShouldRetryAsync(PipelineMessage message, Exception? exception) => + await base.ShouldRetryAsync(message, exception).ConfigureAwait(false) && IsRetryAfterWithinLimit(message); + + protected override Task WaitAsync(TimeSpan time, CancellationToken cancellationToken) + { + if (this.RecordedWaitsForTests is { } waits) + { + waits.Add(time); + return Task.CompletedTask; + } + + return base.WaitAsync(time, cancellationToken); + } + + protected override void Wait(TimeSpan time, CancellationToken cancellationToken) + { + if (this.RecordedWaitsForTests is { } waits) + { + waits.Add(time); + return; + } + + base.Wait(time, cancellationToken); + } + + private static bool IsRetryAfterWithinLimit(PipelineMessage message) + { + if (message.Response?.Headers.TryGetValue("Retry-After", out string? value) is not true || string.IsNullOrWhiteSpace(value)) + { + return true; + } + + // Retry-After is either a number of seconds or an HTTP date. + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double seconds)) + { + return seconds <= MaxRetryAfter.TotalSeconds; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset date)) + { + return date - DateTimeOffset.UtcNow <= MaxRetryAfter; + } + + return true; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs new file mode 100644 index 00000000000..d1bb421f9c7 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Builds an that lets an agent classify, route, score, and check content with Jev, the +/// TypeSafe System One model. +/// +/// +/// +/// Jev does not write text. It answers typed questions about a state and returns calibrated probabilities, which +/// makes it a fast and inexpensive classifier for an agent whose own model writes the replies. The function created by +/// is a thin wrapper around an evaluator: its arguments are a written by +/// the agent's model, and its result is the evaluator's , as JSON. +/// +/// +/// With an , the evaluator is the TypeSafe System One API, called through a +/// System.ClientModel pipeline configured by . Any other client can be used instead with +/// or one of the UseClient methods. +/// +/// +/// Every evaluator receives the of the agent that calls the tool +/// (, which FunctionInvokingChatClient sets from the services +/// the agent was created with), so it can resolve its dependencies per call. In a host, register the tool with +/// services.AddAIAgent(...).WithAITool(sp => new JevAIToolBuilder(credential, options).Build()). +/// +/// +public sealed class JevAIToolBuilder +{ + private readonly ApiKeyCredential? _credential; + private readonly JevAIToolOptions _options; + private Func>? _evaluateAsync; + + /// + /// Initializes a new instance of the class that evaluates requests with the + /// TypeSafe System One API. + /// + /// The TypeSafe API key, sent as a bearer token. + /// The tool and client pipeline options. Defaults are used when . + public JevAIToolBuilder(ApiKeyCredential credential, JevAIToolOptions? options = null) + { + this._credential = Throw.IfNull(credential); + this._options = options ?? new JevAIToolOptions(); + } + + /// + /// Initializes a new instance of the class for a custom evaluator, which must be set + /// with or a UseClient method before . + /// + /// + /// The tool options. Only and apply + /// to a custom evaluator, which uses its own client configuration. Defaults are used when . + /// + public JevAIToolBuilder(JevAIToolOptions? options = null) + { + this._options = options ?? new JevAIToolOptions(); + } + + /// + /// Sets the evaluator that answers each request, replacing the TypeSafe API. + /// + /// + /// The evaluator. It receives the written by the model and the agent's services, which are + /// when the caller provides none, and returns one answer per question. + /// + /// This builder. + /// + /// The agent's services are often the application's root provider, because hosted agents are singletons by + /// default. To use a scoped service, create a scope inside the evaluator, or use + /// , which does so for every call. + /// + public JevAIToolBuilder UseEvaluator(Func> evaluateAsync) + { + this._evaluateAsync = Throw.IfNull(evaluateAsync); + return this; + } + + /// + /// Evaluates requests with another client, converting between the Jev contract types and the client's own types. + /// + /// The request type of the client. + /// The response type of the client. + /// The client operation that evaluates a request. + /// Converts the written by the model into the client's request. + /// Converts the client's response into a . + /// This builder. + /// + /// The mapped response is checked against the request before it reaches the model: every question needs an answer + /// of the same kind, a Choice answer and its probabilities may only use the question's options, and probabilities + /// and scores must be in range. A mapper mistake therefore fails the call instead of showing the model values it + /// never offered. + /// + public JevAIToolBuilder UseClient( + Func> evaluateAsync, + Func inputMapper, + Func outputMapper) + { + _ = Throw.IfNull(evaluateAsync); + _ = Throw.IfNull(inputMapper); + _ = Throw.IfNull(outputMapper); + + return this.UseEvaluator(async (request, _, cancellationToken) => + { + TClientRequest clientRequest = inputMapper(request); + TClientResponse clientResponse = await evaluateAsync(clientRequest, cancellationToken).ConfigureAwait(false); + return outputMapper(clientResponse); + }); + } + + /// + /// Evaluates requests with a client registered in the agent's services, converting between the Jev contract types + /// and the client's own types. + /// + /// The client type, resolved from the agent's services on every call. + /// The request type of the client. + /// The response type of the client. + /// The client operation that evaluates a request. + /// Converts the written by the model into the client's request. + /// Converts the client's response into a . + /// This builder. + /// + /// The client is resolved from a new service scope on every call, and the scope is disposed when the call ends. + /// Scoped clients therefore live for one call, and disposable transient clients are released after it, even when + /// the agent's services are the application's root provider. The call fails when the agent has no services or + /// is not registered. The mapped response is checked as described for + /// . + /// + public JevAIToolBuilder UseClient( + Func> evaluateAsync, + Func inputMapper, + Func outputMapper) + where TClient : notnull + { + _ = Throw.IfNull(evaluateAsync); + _ = Throw.IfNull(inputMapper); + _ = Throw.IfNull(outputMapper); + + return this.UseEvaluator(async (request, services, cancellationToken) => + { + if (services is null) + { + Throw.InvalidOperationException( + $"The Jev tool resolves {typeof(TClient).Name} from the agent's services, but this call has none. " + + "Create the agent with an IServiceProvider, for example with AddAIAgent, or pass the client operation to UseClient directly."); + } + + // Hosted agents are singletons that receive the root provider by default. Resolving from it directly would + // reject scoped clients (or keep one forever when scope validation is off) and hold every disposable + // transient client until shutdown, so each call gets its own scope. + if (services.GetService() is not { } scopeFactory) + { + return await EvaluateWithClientAsync(services).ConfigureAwait(false); + } + + AsyncServiceScope scope = scopeFactory.CreateAsyncScope(); + await using (scope.ConfigureAwait(false)) + { + return await EvaluateWithClientAsync(scope.ServiceProvider).ConfigureAwait(false); + } + + async Task EvaluateWithClientAsync(IServiceProvider provider) + { + if (provider.GetService(typeof(TClient)) is not TClient client) + { + Throw.InvalidOperationException($"{typeof(TClient).Name} is not registered in the agent's services."); + return default!; + } + + TClientRequest clientRequest = inputMapper(request); + TClientResponse clientResponse = await evaluateAsync(client, clientRequest, cancellationToken).ConfigureAwait(false); + return outputMapper(clientResponse); + } + }); + } + + /// + /// Creates the function. + /// + /// + /// An whose JSON schema is the schema of and whose result is the + /// evaluator's serialized as JSON. + /// + /// + /// The builder has neither an nor a custom evaluator, or an option that it uses is + /// empty or invalid. + /// + /// + /// With the TypeSafe API, building creates the client pipeline, which makes the options read-only. Invalid arguments + /// from the model fail with an that says what to fix; a function-invoking chat client + /// includes that message in the tool result only when its detailed errors are enabled. + /// + public AIFunction Build() + { + if (string.IsNullOrWhiteSpace(this._options.Name) || string.IsNullOrWhiteSpace(this._options.Description)) + { + Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.Name)} and {nameof(JevAIToolOptions.Description)} must not be empty."); + } + + Func>? evaluateAsync = this._evaluateAsync; + if (evaluateAsync is null) + { + if (this._credential is null) + { + Throw.InvalidOperationException($"Pass an {nameof(ApiKeyCredential)} to the constructor to use the TypeSafe API, or call {nameof(UseEvaluator)} or UseClient."); + } + + // The model and endpoint only matter to the TypeSafe API; a custom evaluator uses its own configuration. + if (string.IsNullOrWhiteSpace(this._options.ModelId)) + { + Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.ModelId)} must not be empty."); + } + + if (this._options.Endpoint?.IsAbsoluteUri is false) + { + Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.Endpoint)} must be an absolute URI."); + } + + evaluateAsync = new JevHttpEvaluator(this._credential, this._options).EvaluateAsync; + } + + return new JevAIFunction(this._options.Name, this._options.Description, evaluateAsync); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs new file mode 100644 index 00000000000..7f08031b762 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Configures the built by and the +/// client pipeline that sends its requests to the TypeSafe System One API. +/// +/// +/// +/// This type follows the pattern of the OpenAI and Azure AI client libraries: +/// selects the HTTP stack (for example an +/// over an HttpClient from IHttpClientFactory), +/// controls retries, and +/// adds custom policies. The pipeline settings, +/// , and apply only when the builder uses the TypeSafe API, that is, when +/// it receives an ; and apply to every +/// evaluator. +/// +/// +/// When is , the pipeline retries up to three +/// times on the statuses that TypeSafe documents as transient (429 and 529, besides the standard 408 and 5xx ones), and +/// fails at once when the API asks for a wait longer than one minute, because the wait would block the agent run +/// without any feedback. That default follows . +/// +/// +/// Like other client options, an instance becomes read-only once a tool has been built with it. +/// +/// +public sealed class JevAIToolOptions : ClientPipelineOptions +{ + private const string DefaultDescription = + "Evaluates content with Jev, a classification model. Jev does not write text: it answers typed questions about the " + + "given state and returns calibrated probabilities. Use it to classify, route, score, or check facts about text or " + + "JSON data, and ask all related questions in one call. A 'choice' question picks exactly one option from its " + + "criteria. A 'score' question rates the state against 2 to 10 ordered levels and can land between levels. A 'noul' " + + "question returns the probability, from 0 to 1, that its yes or no statement is true. Choice and score answers " + + "include a confidence from 0 to 1; treat a low confidence as uncertain."; + + /// + public override void Freeze() + { + // The default policy is assigned here, not in the constructor, because ClientLoggingOptions may be set after + // construction; ClientPipeline.Create freezes the options before it reads RetryPolicy, so the policy is in place. + this.RetryPolicy ??= new JevRetryPolicy( + JevRetryPolicy.DefaultMaxRetries, + this.ClientLoggingOptions?.EnableLogging ?? true, + this.ClientLoggingOptions?.LoggerFactory); + + base.Freeze(); + } + + /// + /// Gets or sets the System One endpoint. When , https://api.typesafe.ai/v1/systemone is used. + /// + public Uri? Endpoint + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } + + /// + /// Gets or sets the Jev model that answers the questions. Defaults to jev-latest. + /// + public string ModelId + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } = "jev-latest"; + + /// + /// Gets or sets the name of the function that the model sees. Defaults to evaluate_with_jev. + /// + public string Name + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } = "evaluate_with_jev"; + + /// + /// Gets or sets the description of the function that the model sees. + /// + /// + /// The default description explains the Choice, Score, and Noul question kinds. A replacement should explain them + /// too, together with when the agent should call the function. + /// + public string Description + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } = DefaultDescription; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs new file mode 100644 index 00000000000..77a1adf5e55 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents Jev's answer to one question of a . +/// +/// +/// The JSON type property matches the kind of the question: choice (), +/// score (), or noul (). +/// +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(JevChoiceAnswer), "choice")] +[JsonDerivedType(typeof(JevScoreAnswer), "score")] +[JsonDerivedType(typeof(JevNoulAnswer), "noul")] +public abstract class JevAnswer +{ + // Only the three answer kinds that the Jev API returns can exist. + private protected JevAnswer() + { + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs new file mode 100644 index 00000000000..9af3f283071 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a . +/// +[Description("The answer to a Choice question.")] +public sealed class JevChoiceAnswer : JevAnswer +{ + /// + /// Gets or sets the option with the highest probability. + /// + [JsonPropertyName("choice")] + [Description("The option with the highest probability.")] + public required string Choice { get; set; } + + /// + /// Gets or sets how certain the answer is, from 0 to 1, derived from how the probability is spread across the options. + /// + [JsonPropertyName("confidence")] + [Description("How certain the answer is, from 0 to 1. Treat low confidence as uncertain.")] + public required double Confidence { get; set; } + + /// + /// Gets or sets the probability of each option. The probabilities sum to 1. + /// + [JsonPropertyName("probabilities")] + [Description("The probability of each option. The probabilities sum to 1.")] + public required IReadOnlyDictionary Probabilities { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs new file mode 100644 index 00000000000..7377a778036 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents a Choice question, which picks exactly one option from a set that the caller defines. +/// +/// +/// The answer is a . +/// +[Description("A Choice question: picks exactly one option from criteria.")] +public sealed class JevChoiceQuestion : JevQuestion +{ + /// + /// Gets or sets the options, keyed by the option name that the answer returns. + /// + /// + /// Each value describes when the option applies, or is when the name is enough. + /// Jev accepts from 1 to 255 options. + /// + [JsonPropertyName("criteria")] + [Description("The options to choose from, keyed by the option name that the answer returns. Each value describes when the option applies. From 1 to 255 options.")] + public required IReadOnlyDictionary Criteria { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs new file mode 100644 index 00000000000..b737c69c977 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a . +/// +[Description("The answer to a Noul question.")] +public sealed class JevNoulAnswer : JevAnswer +{ + /// + /// Gets or sets the probability, from 0 to 1, that the answer is yes. + /// + [JsonPropertyName("noul")] + [Description("The probability, from 0 to 1, that the answer is yes.")] + public required double Noul { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs new file mode 100644 index 00000000000..4540e047ec0 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Describes what a yes and a no mean for a . +/// +public sealed class JevNoulCriteria +{ + /// + /// Gets or sets what a yes, a probability near 1, means. + /// + [JsonPropertyName("true")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("What a yes, a probability near 1, means.")] + public string? True { get; set; } + + /// + /// Gets or sets what a no, a probability near 0, means. + /// + [JsonPropertyName("false")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("What a no, a probability near 0, means.")] + public string? False { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs new file mode 100644 index 00000000000..bee933254dd --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents a Noul question, which returns the probability that a yes or no statement is true. +/// +/// +/// The answer is a . +/// +[Description("A Noul question: returns the probability, from 0 to 1, that the yes or no statement in instructions is true.")] +public sealed class JevNoulQuestion : JevQuestion +{ + /// + /// Gets or sets optional descriptions of what a yes and a no mean. + /// + [JsonPropertyName("criteria")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("Optional descriptions of what a yes and a no mean.")] + public JevNoulCriteria? Criteria { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs new file mode 100644 index 00000000000..af3fe909da1 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents one typed question that Jev answers about the state of a . +/// +/// +/// The JSON type property selects the question kind: choice (), +/// score (), or noul (). +/// +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(JevChoiceQuestion), "choice")] +[JsonDerivedType(typeof(JevScoreQuestion), "score")] +[JsonDerivedType(typeof(JevNoulQuestion), "noul")] +public abstract class JevQuestion +{ + // Only the three question kinds that the Jev API accepts can exist. + private protected JevQuestion() + { + } + + /// + /// Gets or sets what Jev should decide, rate, or check. + /// + [JsonPropertyName("instructions")] + [Description("What Jev should decide, rate, or check. Describe the idea rather than exact words, because Jev matches on meaning.")] + public required string Instructions { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs new file mode 100644 index 00000000000..2899a89ccf8 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the input of a Jev evaluation: the state to evaluate and the typed questions to answer about it. +/// +/// +/// This type mirrors the body of a TypeSafe System One API request without the model, which the tool configuration +/// selects. Its JSON form is the argument shape of the that +/// creates. +/// +public sealed class JevRequest +{ + /// + /// Gets or sets the content to evaluate. + /// + /// + /// A JSON string for plain text, or a JSON object or array for structured data, such as a conversation or + /// related records. + /// + [JsonPropertyName("state")] + [Description("The content to evaluate. Use a string for plain text, or a JSON object or array for structured data. Questions can refer to object fields by name in backticks, for example `order`.")] + public required JsonElement State { get; set; } + + /// + /// Gets or sets the questions to answer, keyed by an ID that the caller chooses. + /// + /// + /// Each answer in uses the ID of its question. + /// + [JsonPropertyName("questions")] + [Description("The questions to answer about the state, keyed by an ID you choose. Each answer comes back under the same ID. Ask all related questions in one call.")] + public required IReadOnlyDictionary Questions { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs new file mode 100644 index 00000000000..944240124d8 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the result of a Jev evaluation: one typed answer per question of the . +/// +/// +/// This type mirrors the body of a TypeSafe System One API response. Its JSON form is the result of the +/// that creates. +/// +public sealed class JevResponse +{ + /// + /// Gets or sets the model that performed the evaluation, when the evaluator reports it. + /// + [JsonPropertyName("model")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("The model that performed the evaluation.")] + public string? Model { get; set; } + + /// + /// Gets or sets the answers, keyed by the ID of the question they answer. + /// + [JsonPropertyName("answers")] + [Description("One answer per question, keyed by the question ID.")] + public required IReadOnlyDictionary Answers { get; set; } + + /// + /// Gets or sets the token usage of the evaluation, when the evaluator reports it. + /// + [JsonPropertyName("usage")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("The token usage of the evaluation.")] + public JevUsage? Usage { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs new file mode 100644 index 00000000000..a4f4be565b0 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a . +/// +[Description("The answer to a Score question.")] +public sealed class JevScoreAnswer : JevAnswer +{ + /// + /// Gets or sets the score, weighted by the probability of each level. It can fall between levels. + /// + [JsonPropertyName("score")] + [Description("The score, weighted by the probability of each level. It can fall between levels.")] + public required double Score { get; set; } + + /// + /// Gets or sets how certain the answer is, from 0 to 1, derived from how the probability is spread across the levels. + /// + [JsonPropertyName("confidence")] + [Description("How certain the answer is, from 0 to 1. Treat low confidence as uncertain.")] + public required double Confidence { get; set; } + + /// + /// Gets or sets the description of each level, keyed by the level number. + /// + [JsonPropertyName("legend")] + [Description("The description of each level, keyed by the level number.")] + public required IReadOnlyDictionary Legend { get; set; } + + /// + /// Gets or sets the probability of each level, keyed by the level number. The probabilities sum to 1. + /// + [JsonPropertyName("probabilities")] + [Description("The probability of each level, keyed by the level number. The probabilities sum to 1.")] + public required IReadOnlyDictionary Probabilities { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs new file mode 100644 index 00000000000..fa8a7954b1a --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents a Score question, which rates the state against ordered levels that the caller defines. +/// +/// +/// The answer is a , whose score is weighted by probability and can fall between levels. +/// +[Description("A Score question: rates the state against ordered levels in criteria.")] +public sealed class JevScoreQuestion : JevQuestion +{ + /// + /// Gets or sets the level descriptions, ordered from level 0 (the lowest) to the highest level. + /// + /// + /// A Score needs from 2 to 10 levels. + /// + [JsonPropertyName("criteria")] + [Description("The level descriptions, ordered from level 0 (the lowest) to the highest level. From 2 to 10 levels.")] + public required IReadOnlyList Criteria { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs new file mode 100644 index 00000000000..1c004164070 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the token usage of a Jev evaluation. +/// +public sealed class JevUsage +{ + /// + /// Gets or sets the number of input tokens. + /// + [JsonPropertyName("input_tokens")] + [Description("The number of input tokens.")] + public required long InputTokens { get; set; } + + /// + /// Gets or sets the number of output tokens. + /// + [JsonPropertyName("output_tokens")] + [Description("The number of output tokens.")] + public required long OutputTokens { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj new file mode 100644 index 00000000000..203d77b4310 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj @@ -0,0 +1,31 @@ + + + + $(TargetFrameworksCore) + Microsoft.Agents.AI.TypeSafe + alpha + + + + true + + + + + + + Microsoft Agent Framework - TypeSafe (Jev) integration + Provides an AIFunction that lets agents classify, route, score, and check content with Jev, the TypeSafe System One model. + + + + + + + + + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs new file mode 100644 index 00000000000..9135880f01b --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs @@ -0,0 +1,463 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevAIFunctionTests +{ + [Fact] + public async Task InvokeAsync_ModelArguments_ReachTheEvaluatorAsTypedRequestAsync() + { + // Arrange + JevRequest? received = null; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + { + received = request; + return Task.FromResult(JevTestData.CreateResponse()); + }).Build(); + + // Act + await tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.NotNull(received); + Assert.Equal(JevTestData.TicketText, received.State.GetString()); + + JevChoiceQuestion department = Assert.IsType(received.Questions["department"]); + Assert.Equal("Which team should handle this", department.Instructions); + Assert.Equal(["billing", "technical", "sales"], department.Criteria.Keys); + Assert.Null(department.Criteria["sales"]); + + JevScoreQuestion frustration = Assert.IsType(received.Questions["frustration"]); + Assert.Equal(3, frustration.Criteria.Count); + Assert.Equal("Calm, just stating facts", frustration.Criteria[0]); + + JevNoulQuestion urgent = Assert.IsType(received.Questions["is_urgent"]); + Assert.Equal("Explicitly time-sensitive", urgent.Criteria!.True); + Assert.Equal("No urgency expressed", urgent.Criteria.False); + } + + [Fact] + public async Task InvokeAsync_ReturnsTheResponseAsJsonWithWireNamesAsync() + { + // Arrange + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + + // Act + object? result = await tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + JsonElement json = Assert.IsType(result); + Assert.Equal("custom-model", json.GetProperty("model").GetString()); + JsonElement answers = json.GetProperty("answers"); + Assert.Equal("choice", answers.GetProperty("department").GetProperty("type").GetString()); + Assert.Equal("technical", answers.GetProperty("department").GetProperty("choice").GetString()); + Assert.Equal(0.78, answers.GetProperty("department").GetProperty("confidence").GetDouble()); + Assert.Equal("score", answers.GetProperty("frustration").GetProperty("type").GetString()); + Assert.Equal("Frustrated", answers.GetProperty("frustration").GetProperty("legend").GetProperty("1").GetString()); + Assert.Equal(0.97, answers.GetProperty("is_urgent").GetProperty("noul").GetDouble()); + Assert.Equal(10, json.GetProperty("usage").GetProperty("input_tokens").GetInt64()); + Assert.Equal(5, json.GetProperty("usage").GetProperty("output_tokens").GetInt64()); + } + + [Fact] + public async Task InvokeAsync_TypeWrittenLastAndObjectState_AreAcceptedAsync() + { + // Arrange + JevRequest? received = null; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + { + received = request; + return Task.FromResult(new JevResponse { Answers = new Dictionary { ["paid"] = new JevNoulAnswer { Noul = 0.1 } } }); + }).Build(); + + // Act + await tool.InvokeAsync(JevTestData.Arguments(""" + { + "questions": { "paid": { "instructions": "Was `order` paid?", "type": "noul" } }, + "state": { "order": { "id": "A-104", "status": "failed" } } + } + """)); + + // Assert + Assert.Equal(JsonValueKind.Object, received!.State.ValueKind); + Assert.Equal("A-104", received.State.GetProperty("order").GetProperty("id").GetString()); + Assert.IsType(received.Questions["paid"]); + } + + [Fact] + public async Task InvokeAsync_DotNetObjectArguments_AreAcceptedAsync() + { + // Arrange + JevRequest? received = null; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + { + received = request; + return Task.FromResult(new JevResponse { Answers = new Dictionary { ["q"] = new JevNoulAnswer { Noul = 0.5 } } }); + }).Build(); + + // Act + await tool.InvokeAsync(new AIFunctionArguments + { + ["state"] = "The payment failed.", + ["questions"] = new Dictionary { ["q"] = new JevNoulQuestion { Instructions = "Did the payment fail?" } }, + }); + + // Assert + Assert.Equal("The payment failed.", received!.State.GetString()); + Assert.Equal("Did the payment fail?", Assert.IsType(received.Questions["q"]).Instructions); + } + + [Theory] + [InlineData("""{"questions":{"q":{"type":"noul","instructions":"x"}}}""", "state")] + [InlineData("""{"state":42,"questions":{"q":{"type":"noul","instructions":"x"}}}""", "The state must be a string")] + [InlineData("""{"state":"x","questions":{}}""", "at least one question")] + [InlineData("""{"state":"x","questions":{"q":{"type":"rank","instructions":"x"}}}""", "not a valid Jev request")] + [InlineData("""{"state":"x","questions":{"q":{"instructions":"x"}}}""", "not a valid Jev request")] + [InlineData("""{"state":"x","questions":{"q":null}}""", "Question 'q' is null")] + [InlineData("""{"state":"x","questions":{"q":{"type":"noul","instructions":" "}}}""", "Question 'q' must have instructions")] + [InlineData("""{"state":"x","questions":{"q":{"type":"choice","instructions":"x","criteria":{}}}}""", "has 0 options; it needs from 1 to 255")] + [InlineData("""{"state":"x","questions":{"q":{"type":"choice","instructions":"x","criteria":{"":"empty"}}}}""", "empty option name")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["only"]}}}""", "has 1 level; it needs from 2 to 10")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["0","1","2","3","4","5","6","7","8","9","10"]}}}""", "has 11 levels")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["low",""]}}}""", "empty level description")] + public async Task InvokeAsync_InvalidArguments_FailBeforeEvaluatingAsync(string arguments, string expectedMessage) + { + // Arrange + int calls = 0; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => + { + calls++; + return Task.FromResult(JevTestData.CreateResponse()); + }).Build(); + + // Act + ArgumentException exception = await Assert.ThrowsAnyAsync(() => tool.InvokeAsync(JevTestData.Arguments(arguments)).AsTask()); + + // Assert + Assert.Contains(expectedMessage, exception.Message, StringComparison.Ordinal); + Assert.Equal(0, calls); + } + + [Fact] + public async Task InvokeAsync_TooManyChoiceOptions_FailBeforeEvaluatingAsync() + { + // Arrange + string options = string.Join(",", Enumerable.Range(0, 256).Select(static i => $"\"o{i}\":null")); + string arguments = "{\"state\":\"x\",\"questions\":{\"q\":{\"type\":\"choice\",\"instructions\":\"x\",\"criteria\":{" + options + "}}}}"; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + + // Act + ArgumentException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(arguments)).AsTask()); + + // Assert + Assert.Contains("has 256 options", exception.Message, StringComparison.Ordinal); + } + + public static TheoryData, string> InvalidResponses => new() + { + { "no answer", static r => Without(r, "is_urgent"), "no answer for question 'is_urgent'" }, + { "wrong kind", static r => With(r, "department", new JevNoulAnswer { Noul = 0.5 }), "is a Noul answer, but the question is a Choice question" }, + { "unknown option", static r => With(r, "department", Choice("refunds", 0.9)), "chose 'refunds', which is not one of its options" }, + { "confidence out of range", static r => With(r, "department", Choice("technical", 1.2)), "has confidence 1.2, which is outside the range 0 to 1" }, + { "score out of range", static r => With(r, "frustration", Score(3)), "has score 3, which is outside the range 0 to 2" }, + { "noul as percent", static r => With(r, "is_urgent", new JevNoulAnswer { Noul = 97 }), "has noul 97, which is outside the range 0 to 1" }, + { "no answers", static r => new JevResponse { Answers = null! }, "has no answers" }, + { "probability for an option not offered", static r => With(r, "department", ChoiceWith("technical", new() { ["TECHNICAL"] = 1 })), "has probabilities for 'TECHNICAL', which is not one of its options" }, + { "probability above one", static r => With(r, "department", ChoiceWith("technical", new() { ["technical"] = 42 })), "has probability 'technical' 42, which is outside the range 0 to 1" }, + { "probability not a number", static r => With(r, "department", ChoiceWith("technical", new() { ["technical"] = double.NaN })), "has probability 'technical' NaN" }, + { "score probability for a missing level", static r => With(r, "frustration", ScoreWith(new() { ["3"] = 1 }, [])), "has probabilities for '3', which is not a level number from 0 to 2" }, + { "legend for a missing level", static r => With(r, "frustration", ScoreWith([], new() { ["high"] = "Very angry" })), "has legend for 'high', which is not a level number from 0 to 2" }, + }; + + [Theory] + [MemberData(nameof(InvalidResponses))] + public async Task InvokeAsync_InvalidEvaluatorResponse_FailsAsync(string scenario, Func corrupt, string expectedMessage) + { + // Arrange + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(corrupt(JevTestData.CreateResponse()))).Build(); + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.True(exception.Message.Contains(expectedMessage, StringComparison.Ordinal), $"{scenario}: {exception.Message}"); + } + + [Fact] + public async Task InvokeAsync_EvaluatorReturnsNull_FailsAsync() + { + // Arrange + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(null!)).Build(); + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.Contains("returned no response", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task InvokeAsync_PassesTheCancellationTokenToTheEvaluatorAsync() + { + // Arrange + using var cancellation = new CancellationTokenSource(); + CancellationToken received = default; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, cancellationToken) => + { + received = cancellationToken; + return Task.FromResult(JevTestData.CreateResponse()); + }).Build(); + + // Act + await tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson), cancellation.Token); + + // Assert + Assert.Equal(cancellation.Token, received); + } + + [Fact] + public async Task UseClient_MapsBetweenTheContractAndTheClientTypesAsync() + { + // Arrange + var client = new LabelClient(label: "technical"); + AIFunction tool = new JevAIToolBuilder().UseClient( + client.ClassifyAsync, + inputMapper: static request => + { + (string id, JevQuestion question) = request.Questions.Single(); + return new LabelRequest(id, request.State.GetString()!, ((JevChoiceQuestion)question).Criteria.Keys.ToList()); + }, + outputMapper: static response => new JevResponse + { + Answers = new Dictionary + { + [response.QuestionId] = Choice(response.Label, response.Score), + }, + }).Build(); + + // Act + object? result = await tool.InvokeAsync(JevTestData.Arguments(""" + {"state":"My integration keeps failing","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} + """)); + + // Assert + Assert.Equal(new LabelRequest("team", "My integration keeps failing", ["billing", "technical"]), client.Received, LabelRequestComparer.Instance); + JsonElement answer = Assert.IsType(result).GetProperty("answers").GetProperty("team"); + Assert.Equal("technical", answer.GetProperty("choice").GetString()); + } + + [Fact] + public async Task UseClient_MapperReturnsAnOptionThatWasNotOffered_FailsAsync() + { + // Arrange + var client = new LabelClient(label: "TECHNICAL"); + AIFunction tool = new JevAIToolBuilder().UseClient( + client.ClassifyAsync, + static request => new LabelRequest("team", request.State.GetString()!, ["billing", "technical"]), + static response => new JevResponse { Answers = new Dictionary { ["team"] = Choice(response.Label, response.Score) } }).Build(); + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(""" + {"state":"x","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} + """)).AsTask()); + + // Assert + Assert.Contains("chose 'TECHNICAL'", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task UseEvaluator_ReceivesTheServicesOfTheCallAsync() + { + // Arrange + using ServiceProvider services = new ServiceCollection().BuildServiceProvider(); + IServiceProvider? received = null; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, callServices, _) => + { + received = callServices; + return Task.FromResult(JevTestData.CreateResponse()); + }).Build(); + + AIFunctionArguments arguments = JevTestData.Arguments(JevTestData.RequestJson); + arguments.Services = services; + + // Act + await tool.InvokeAsync(arguments); + + // Assert + Assert.Same(services, received); + } + + [Fact] + public async Task UseClient_RegisteredClient_IsResolvedFromTheServicesOnEveryCallAsync() + { + // Arrange + int created = 0; + using ServiceProvider services = new ServiceCollection() + .AddTransient(_ => + { + created++; + return new LabelClient(label: "technical"); + }) + .BuildServiceProvider(); + + AIFunction tool = new JevAIToolBuilder().UseClient( + static (client, request, cancellationToken) => client.ClassifyAsync(request, cancellationToken), + static request => new LabelRequest("team", request.State.GetString()!, ["billing", "technical"]), + static response => new JevResponse { Answers = new Dictionary { ["team"] = Choice(response.Label, response.Score) } }).Build(); + + AIFunctionArguments arguments = JevTestData.Arguments(""" + {"state":"x","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} + """); + arguments.Services = services; + + // Act + await tool.InvokeAsync(arguments); + object? result = await tool.InvokeAsync(arguments); + + // Assert + Assert.Equal(2, created); + Assert.Equal("technical", Assert.IsType(result).GetProperty("answers").GetProperty("team").GetProperty("choice").GetString()); + } + + [Theory] + [InlineData(false, "this call has none")] + [InlineData(true, "LabelClient is not registered")] + public async Task UseClient_ClientNotAvailable_FailsWithWhatIsMissingAsync(bool withServices, string expectedMessage) + { + // Arrange + using ServiceProvider services = new ServiceCollection().BuildServiceProvider(); + AIFunction tool = new JevAIToolBuilder().UseClient( + static (client, request, cancellationToken) => client.ClassifyAsync(request, cancellationToken), + static request => new LabelRequest("q", "x", []), + static response => JevTestData.CreateResponse()).Build(); + + AIFunctionArguments arguments = JevTestData.Arguments(JevTestData.RequestJson); + arguments.Services = withServices ? services : null; + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(arguments).AsTask()); + + // Assert + Assert.Contains(expectedMessage, exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task UseClient_ScopedClientFromTheRootProvider_GetsAScopePerCallAsync() + { + // Arrange + var created = new List(); + using ServiceProvider root = new ServiceCollection() + .AddScoped(_ => + { + var client = new ScopedLabelClient(); + created.Add(client); + return client; + }) + .BuildServiceProvider(new ServiceProviderOptions { ValidateScopes = true }); + + AIFunction tool = new JevAIToolBuilder().UseClient( + static (client, text, _) => Task.FromResult(client.IsDisposed ? "disposed" : "technical"), + static request => request.State.GetString()!, + static label => new JevResponse { Answers = new Dictionary { ["team"] = Choice(label, 0.9) } }).Build(); + + AIFunctionArguments arguments = JevTestData.Arguments(""" + {"state":"x","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} + """); + arguments.Services = root; + + // Act + await tool.InvokeAsync(arguments); + await tool.InvokeAsync(arguments); + + // Assert + Assert.Equal(2, created.Count); + Assert.All(created, static client => Assert.True(client.IsDisposed)); + } + + private static JevChoiceAnswer Choice(string choice, double confidence) => new() + { + Choice = choice, + Confidence = confidence, + Probabilities = new Dictionary { [choice] = confidence }, + }; + + private static JevScoreAnswer Score(double score) => new() + { + Score = score, + Confidence = 1, + Legend = new Dictionary(), + Probabilities = new Dictionary(), + }; + + private static JevChoiceAnswer ChoiceWith(string choice, Dictionary probabilities) => new() + { + Choice = choice, + Confidence = 0.5, + Probabilities = probabilities, + }; + + private static JevScoreAnswer ScoreWith(Dictionary probabilities, Dictionary legend) => new() + { + Score = 1, + Confidence = 1, + Legend = legend, + Probabilities = probabilities, + }; + + private static JevResponse With(JevResponse response, string id, JevAnswer answer) + { + var answers = new Dictionary(response.Answers) { [id] = answer }; + return new JevResponse { Answers = answers }; + } + + private static JevResponse Without(JevResponse response, string id) + { + var answers = new Dictionary(response.Answers); + answers.Remove(id); + return new JevResponse { Answers = answers }; + } + + /// The request type of a hypothetical third-party classification client. + private sealed record LabelRequest(string QuestionId, string Text, List Labels); + + /// The response type of a hypothetical third-party classification client. + private sealed record LabelResponse(string QuestionId, string Label, double Score); + + private sealed class LabelClient(string label) + { + public LabelRequest? Received { get; private set; } + + public Task ClassifyAsync(LabelRequest request, CancellationToken cancellationToken) + { + this.Received = request; + return Task.FromResult(new LabelResponse(request.QuestionId, label, 0.9)); + } + } + + /// A disposable client registered as scoped, as clients holding per-request resources usually are. + private sealed class ScopedLabelClient : IDisposable + { + public bool IsDisposed { get; private set; } + + public void Dispose() => this.IsDisposed = true; + } + + private sealed class LabelRequestComparer : IEqualityComparer + { + public static LabelRequestComparer Instance { get; } = new(); + + public bool Equals(LabelRequest? x, LabelRequest? y) => + x is not null && y is not null && x.QuestionId == y.QuestionId && x.Text == y.Text && x.Labels.SequenceEqual(y.Labels); + + public int GetHashCode(LabelRequest? obj) => obj?.QuestionId.GetHashCode(StringComparison.Ordinal) ?? 0; + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs new file mode 100644 index 00000000000..1fd41c9ec41 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevAIToolBuilderTests +{ + [Fact] + public void Build_Defaults_DescribeTheJevContract() + { + // Act + AIFunction tool = new JevAIToolBuilder(new ApiKeyCredential("test-key")).Build(); + + // Assert + Assert.Equal("evaluate_with_jev", tool.Name); + Assert.Contains("'choice'", tool.Description, StringComparison.Ordinal); + Assert.Contains("'score'", tool.Description, StringComparison.Ordinal); + Assert.Contains("'noul'", tool.Description, StringComparison.Ordinal); + + JsonElement schema = tool.JsonSchema; + Assert.Equal(["state", "questions"], schema.GetProperty("required").EnumerateArray().Select(static r => r.GetString())); + JsonElement question = schema.GetProperty("properties").GetProperty("questions").GetProperty("additionalProperties"); + Assert.Equal( + ["choice", "score", "noul"], + question.GetProperty("anyOf").EnumerateArray().Select(static b => b.GetProperty("properties").GetProperty("type").GetProperty("const").GetString())); + + JsonElement returnSchema = tool.ReturnJsonSchema!.Value; + Assert.Equal(["answers"], returnSchema.GetProperty("required").EnumerateArray().Select(static r => r.GetString())); + Assert.True(returnSchema.GetProperty("properties").GetProperty("usage").GetProperty("properties").TryGetProperty("input_tokens", out _)); + } + + [Fact] + public void Options_Defaults_MatchTheTypeSafeApi() + { + // Act + var options = new JevAIToolOptions(); + + // Assert + Assert.Null(options.Endpoint); + Assert.Equal("jev-latest", options.ModelId); + Assert.Equal("evaluate_with_jev", options.Name); + Assert.Null(options.RetryPolicy); + } + + [Fact] + public void Build_WithTheTypeSafeApi_AssignsTheDefaultRetryPolicyOnlyWhenNoneIsSet() + { + // Arrange + var defaults = new JevAIToolOptions(); + var custom = new JevAIToolOptions { RetryPolicy = new System.ClientModel.Primitives.ClientRetryPolicy(maxRetries: 1) }; + + // Act + _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), defaults).Build(); + _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), custom).Build(); + + // Assert + Assert.IsType(defaults.RetryPolicy); + Assert.IsType(custom.RetryPolicy); + } + + [Fact] + public void Build_WithACustomEvaluator_IgnoresTheApiOnlyOptions() + { + // Arrange + var options = new JevAIToolOptions { ModelId = " ", Endpoint = new Uri("/relative", UriKind.Relative) }; + + // Act + AIFunction tool = new JevAIToolBuilder(options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + + // Assert + Assert.Equal("evaluate_with_jev", tool.Name); + } + + [Fact] + public void Build_WithNameAndDescription_UsesTheOptions() + { + // Arrange + var options = new JevAIToolOptions { Name = "triage_ticket", Description = "Classifies support tickets." }; + + // Act + AIFunction tool = new JevAIToolBuilder(options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + + // Assert + Assert.Equal("triage_ticket", tool.Name); + Assert.Equal("Classifies support tickets.", tool.Description); + } + + [Fact] + public void Build_WithTheTypeSafeApi_MakesTheOptionsReadOnly() + { + // Arrange + var options = new JevAIToolOptions(); + + // Act + _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), options).Build(); + + // Assert + Assert.Throws(() => options.Endpoint = new Uri("https://proxy.contoso.com/jev")); + Assert.Throws(() => options.ModelId = "jev-1.13"); + } + + [Fact] + public void Build_WithACustomEvaluator_LeavesTheOptionsWritable() + { + // Arrange + var options = new JevAIToolOptions(); + + // Act + _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + + // Assert + options.ModelId = "jev-1.13"; + Assert.Equal("jev-1.13", options.ModelId); + } + + [Fact] + public void Build_WithoutCredentialOrEvaluator_Throws() + { + // Act + InvalidOperationException exception = Assert.Throws(() => new JevAIToolBuilder().Build()); + + // Assert + Assert.Contains("ApiKeyCredential", exception.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("name")] + [InlineData("description")] + [InlineData("model")] + [InlineData("endpoint")] + public void Build_InvalidOptions_Throw(string invalid) + { + // Arrange + var options = new JevAIToolOptions(); + switch (invalid) + { + case "name": options.Name = " "; break; + case "description": options.Description = ""; break; + case "model": options.ModelId = " "; break; + case "endpoint": options.Endpoint = new Uri("/v1/systemone", UriKind.Relative); break; + } + + // Act & Assert + Assert.Throws(() => new JevAIToolBuilder(new ApiKeyCredential("test-key"), options).Build()); + } + + [Fact] + public void Methods_NullArguments_Throw() + { + // Arrange + var builder = new JevAIToolBuilder(); + Task evaluateAsync(string request, CancellationToken _) => Task.FromResult(request); + string input(JevRequest _) => "request"; + JevResponse output(string _) => JevTestData.CreateResponse(); + Task evaluateWithClientAsync(object _1, string request, CancellationToken _2) => Task.FromResult(request); + + // Act & Assert + Assert.Throws(() => new JevAIToolBuilder((ApiKeyCredential)null!)); + Assert.Throws(() => builder.UseEvaluator(null!)); + Assert.Throws(() => builder.UseClient(null!, input, (Func)output)); + Assert.Throws(() => builder.UseClient((Func>)evaluateAsync, null!, output)); + Assert.Throws(() => builder.UseClient(evaluateAsync, input, null!)); + Assert.Throws(() => builder.UseClient((Func>)evaluateWithClientAsync, null!, output)); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs new file mode 100644 index 00000000000..4a3582d140f --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevAgentIntegrationTests +{ + [Fact] + public async Task RunAsync_AgentCallsTheTool_ModelReceivesTheAnswersAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + AIFunction jev = new JevAIToolBuilder( + new ApiKeyCredential("test-key"), + new JevAIToolOptions { Transport = new HttpClientPipelineTransport(httpClient) }).Build(); + + using var model = new ScriptedChatClient( + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", jev.Name, JevTestData.CallArguments(JevTestData.RequestJson))])), + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Routed to the technical team as urgent."))); + AIAgent agent = model.AsAIAgent(instructions: "Triage support tickets with the Jev tool.", tools: [jev]); + + // Act + AgentResponse response = await agent.RunAsync(JevTestData.TicketText); + + // Assert + Assert.Equal("Routed to the technical team as urgent.", response.Text); + Assert.Single(handler.Requests); + + FunctionResultContent result = model.Calls[1].SelectMany(static m => m.Contents).OfType().Single(); + Assert.Null(result.Exception); + + // Chat clients serialize tool results with AIJsonUtilities.DefaultOptions before sending them to the model. + string sent = JsonSerializer.Serialize(result.Result, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(object))); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(JevTestData.ResponseJson), JsonNode.Parse(sent)), sent); + } + + [Fact] + public async Task GetResponseAsync_InvalidArgumentsWithDetailedErrors_TellTheModelWhatToFixAsync() + { + // Arrange + AIFunction jev = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + const string InvalidArguments = """{"state":"x","questions":{"q":{"type":"score","instructions":"How bad?","criteria":["only"]}}}"""; + + using var model = new ScriptedChatClient( + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", jev.Name, JevTestData.CallArguments(InvalidArguments))])), + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "done"))); + using var client = new FunctionInvokingChatClient(model) { IncludeDetailedErrors = true }; + + // Act + await client.GetResponseAsync("How bad is it?", new ChatOptions { Tools = [jev] }); + + // Assert + FunctionResultContent result = model.Calls[1].SelectMany(static m => m.Contents).OfType().Single(); + Assert.Contains("has 1 level; it needs from 2 to 10", result.Result?.ToString(), StringComparison.Ordinal); + } + + [Fact] + public async Task RunAsync_AgentWithServices_ToolResolvesTheClientFromThemAsync() + { + // Arrange + var classifier = new UrgencyClassifier(probability: 0.93); + using ServiceProvider services = new ServiceCollection().AddSingleton(classifier).BuildServiceProvider(); + + AIFunction jev = new JevAIToolBuilder() + .UseClient( + static (client, text, cancellationToken) => client.ScoreAsync(text, cancellationToken), + inputMapper: static request => request.State.GetString()!, + outputMapper: static probability => new JevResponse + { + Answers = new Dictionary { ["is_urgent"] = new JevNoulAnswer { Noul = probability } }, + }) + .Build(); + + const string Arguments = """{"state":"Please help ASAP","questions":{"is_urgent":{"type":"noul","instructions":"Is this urgent?"}}}"""; + using var model = new ScriptedChatClient( + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", jev.Name, JevTestData.CallArguments(Arguments))])), + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Urgent."))); + AIAgent agent = model.AsAIAgent(tools: [jev], services: services); + + // Act + AgentResponse response = await agent.RunAsync("Please help ASAP"); + + // Assert + Assert.Equal("Urgent.", response.Text); + Assert.Equal("Please help ASAP", classifier.Received); + FunctionResultContent result = model.Calls[1].SelectMany(static m => m.Contents).OfType().Single(); + Assert.Equal(0.93, Assert.IsType(result.Result).GetProperty("answers").GetProperty("is_urgent").GetProperty("noul").GetDouble()); + } + + /// A hypothetical third-party client registered in the application's services. + private sealed class UrgencyClassifier(double probability) + { + public string? Received { get; private set; } + + public Task ScoreAsync(string text, CancellationToken cancellationToken) + { + this.Received = text; + return Task.FromResult(probability); + } + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs new file mode 100644 index 00000000000..7ec9e0ab56c --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs @@ -0,0 +1,291 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevHttpEvaluatorTests +{ + private const string ApiKey = "test-key-9f2c"; + + [Fact] + public async Task InvokeAsync_PostsTheWireRequestAndReturnsTheResponseAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + AIFunction tool = CreateBuilder(httpClient).Build(); + + // Act + object? result = await tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); + Assert.Equal(HttpMethod.Post, request.Method); + Assert.Equal(new Uri("https://api.typesafe.ai/v1/systemone"), request.Uri); + Assert.Equal("Bearer", request.Authorization!.Scheme); + Assert.Equal(ApiKey, request.Authorization.Parameter); + + JsonObject expected = JsonNode.Parse(JevTestData.RequestJson)!.AsObject(); + expected["model"] = "jev-latest"; + Assert.True(JsonNode.DeepEquals(expected, JsonNode.Parse(request.Body)), request.Body); + + JsonElement json = Assert.IsType(result); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(JevTestData.ResponseJson), JsonNode.Parse(json.GetRawText())), json.GetRawText()); + } + + [Fact] + public async Task InvokeAsync_CustomModelAndEndpoint_AreUsedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + var options = new JevAIToolOptions { ModelId = "jev-1.13", Endpoint = new Uri("https://proxy.contoso.com/jev") }; + + // Act + await CreateBuilder(httpClient, options).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); + Assert.Equal(new Uri("https://proxy.contoso.com/jev"), request.Uri); + Assert.Equal("jev-1.13", (string?)JsonNode.Parse(request.Body)!["model"]); + } + + [Fact] + public async Task InvokeAsync_CustomPolicy_RunsInThePipelineAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + var options = new JevAIToolOptions(); + options.AddPolicy(new HeaderPolicy("x-tenant", "contoso"), PipelinePosition.PerCall); + + // Act + await CreateBuilder(httpClient, options).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal("contoso", Assert.Single(handler.Requests).Headers["x-tenant"]); + } + + [Fact] + public async Task InvokeAsync_RateLimitedOrOverloaded_RetriesAsync() + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.TooManyRequests, "{}") + .Reply((HttpStatusCode)529, "{}") + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + object? result = await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal(3, handler.Requests.Count); + Assert.Equal(2, waits.Count); + Assert.Equal("technical", Assert.IsType(result).GetProperty("answers").GetProperty("department").GetProperty("choice").GetString()); + } + + [Fact] + public async Task InvokeAsync_StillRateLimitedAfterRetries_FailsWithTheStatusAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.TooManyRequests, """{"detail":"Rate limit exceeded"}"""); + using var httpClient = new HttpClient(handler); + + // Act + ClientResultException exception = await Assert.ThrowsAsync(() => + CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.Equal(429, exception.Status); + Assert.Equal(4, handler.Requests.Count); + } + + [Fact] + public async Task InvokeAsync_RetryAfterSeconds_IsWaitedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.TooManyRequests, "{}", new RetryConditionHeaderValue(TimeSpan.FromSeconds(5))) + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal(2, handler.Requests.Count); + Assert.Equal(TimeSpan.FromSeconds(5), Assert.Single(waits)); + } + + [Fact] + public async Task InvokeAsync_RetryAfterDate_IsWaitedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply((HttpStatusCode)529, "{}", new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddSeconds(20))) + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal(2, handler.Requests.Count); + Assert.InRange(Assert.Single(waits), TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(21)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InvokeAsync_RetryAfterBeyondTheLimit_FailsAtOnceAsync(bool asDate) + { + // Arrange + RetryConditionHeaderValue retryAfter = asDate + ? new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddHours(1)) + : new RetryConditionHeaderValue(TimeSpan.FromHours(1)); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.TooManyRequests, """{"detail":"Slow down"}""", retryAfter); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + ClientResultException exception = await Assert.ThrowsAsync(() => + CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.Equal(429, exception.Status); + Assert.Single(handler.Requests); + Assert.Empty(waits); + } + + [Fact] + public async Task InvokeAsync_ReplacedRetryPolicy_IsUsedAndStillRetriesOverloadedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply((HttpStatusCode)529, "{}") + .Reply((HttpStatusCode)529, "{}"); + using var httpClient = new HttpClient(handler); + var options = new JevAIToolOptions + { + Transport = new HttpClientPipelineTransport(httpClient), + RetryPolicy = new NoWaitRetryPolicy(maxRetries: 1), + }; + + // Act + ClientResultException exception = await Assert.ThrowsAsync(() => + new JevAIToolBuilder(new ApiKeyCredential(ApiKey), options).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.Equal(529, exception.Status); + Assert.Equal(2, handler.Requests.Count); + } + + [Theory] + [InlineData(HttpStatusCode.Unauthorized, """{"detail":"Invalid API key"}""", "Invalid API key")] + [InlineData(HttpStatusCode.BadRequest, """{"detail":"Too many score levels. Must have at most 10 levels."}""", "Too many score levels")] + [InlineData(HttpStatusCode.UnprocessableEntity, """{"detail":[{"loc":["body","questions"],"msg":"Dictionary should have at least 1 item"}]}""", "Dictionary should have at least 1 item")] + public async Task InvokeAsync_ApiError_FailsWithTheDetailButNotTheKeyAsync(HttpStatusCode status, string body, string expectedMessage) + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(status, body); + using var httpClient = new HttpClient(handler); + + // Act + ClientResultException exception = await Assert.ThrowsAsync(() => + CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.Equal((int)status, exception.Status); + Assert.Contains(expectedMessage, exception.Message, StringComparison.Ordinal); + Assert.DoesNotContain(ApiKey, exception.Message, StringComparison.Ordinal); + Assert.Single(handler.Requests); + } + + [Theory] + [InlineData("choice", "\"confidence\": 0.78, ", "", "confidence")] + [InlineData("unknown type", "\"type\": \"noul\", \"noul\": 0.97", "\"type\": \"rank\", \"noul\": 0.97", "could not be read")] + [InlineData("not json", "{", "", "could not be read")] + public async Task InvokeAsync_UnreadableResponse_FailsAsync(string scenario, string find, string replace, string expectedMessage) + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, ReplaceFirst(JevTestData.ResponseJson, find, replace)); + using var httpClient = new HttpClient(handler); + + // Act + JsonException exception = await Assert.ThrowsAsync(() => + CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.True(exception.Message.Contains(expectedMessage, StringComparison.Ordinal), $"{scenario}: {exception.Message}"); + } + + [Fact] + public async Task InvokeAsync_Canceled_StopsWithoutSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + // Act & Assert + await Assert.ThrowsAnyAsync(() => + CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson), cancellation.Token).AsTask()); + Assert.Empty(handler.Requests); + } + + private static JevAIToolBuilder CreateBuilder(HttpClient httpClient, JevAIToolOptions? options = null, List? waits = null) + { + options ??= new JevAIToolOptions(); + options.Transport = new HttpClientPipelineTransport(httpClient); + options.RetryPolicy = new JevRetryPolicy { RecordedWaitsForTests = waits ?? [] }; + return new JevAIToolBuilder(new ApiKeyCredential(ApiKey), options); + } + + private static string ReplaceFirst(string text, string find, string replace) + { + int index = text.IndexOf(find, StringComparison.Ordinal); + Assert.True(index >= 0, $"'{find}' is not in the test response."); + return string.Concat(text.AsSpan(0, index), replace, text.AsSpan(index + find.Length)); + } + + private sealed class HeaderPolicy(string name, string value) : PipelinePolicy + { + public override void Process(PipelineMessage message, IReadOnlyList pipeline, int currentIndex) + { + message.Request.Headers.Set(name, value); + ProcessNext(message, pipeline, currentIndex); + } + + public override ValueTask ProcessAsync(PipelineMessage message, IReadOnlyList pipeline, int currentIndex) + { + message.Request.Headers.Set(name, value); + return ProcessNextAsync(message, pipeline, currentIndex); + } + } + + private sealed class NoWaitRetryPolicy(int maxRetries) : ClientRetryPolicy(maxRetries) + { + protected override Task WaitAsync(TimeSpan time, CancellationToken cancellationToken) => Task.CompletedTask; + + protected override void Wait(TimeSpan time, CancellationToken cancellationToken) + { + } + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs new file mode 100644 index 00000000000..3afde9ae642 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// A support-ticket request and response taken from the TypeSafe quick start, used across the tests. +/// +internal static class JevTestData +{ + public const string TicketText = "Hi, I've been trying to connect my Stripe account for 3 days and the integration keeps failing. I'm losing sales. Please help ASAP."; + + /// The tool arguments a model would write for the ticket, with the three question kinds. + public const string RequestJson = """ + { + "state": "Hi, I've been trying to connect my Stripe account for 3 days and the integration keeps failing. I'm losing sales. Please help ASAP.", + "questions": { + "department": { + "type": "choice", + "instructions": "Which team should handle this", + "criteria": { "billing": "Payment or subscription issues", "technical": "Bugs or integration problems", "sales": null } + }, + "frustration": { + "type": "score", + "instructions": "How frustrated the customer appears", + "criteria": ["Calm, just stating facts", "Frustrated but civil", "Very angry, strong language"] + }, + "is_urgent": { + "type": "noul", + "instructions": "The message conveys urgency or time-sensitivity", + "criteria": { "true": "Explicitly time-sensitive", "false": "No urgency expressed" } + } + } + } + """; + + /// The API response for . + public const string ResponseJson = """ + { + "model": "jev-1.13.0", + "answers": { + "department": { "type": "choice", "choice": "technical", "confidence": 0.78, "probabilities": { "technical": 0.85, "sales": 0.0, "billing": 0.15 } }, + "frustration": { + "type": "score", "score": 1.0, "confidence": 1.0, + "legend": { "0": "Calm, just stating facts", "1": "Frustrated but civil", "2": "Very angry, strong language" }, + "probabilities": { "0": 0.0, "1": 1.0, "2": 0.0 } + }, + "is_urgent": { "type": "noul", "noul": 0.97 } + }, + "usage": { "input_tokens": 392, "output_tokens": 65 } + } + """; + + /// Parses tool arguments the way chat clients do: one per top-level property. + public static AIFunctionArguments Arguments(string json) + { + var arguments = new AIFunctionArguments(); + using JsonDocument document = JsonDocument.Parse(json); + foreach (JsonProperty property in document.RootElement.EnumerateObject()) + { + arguments[property.Name] = property.Value.Clone(); + } + + return arguments; + } + + /// Parses tool arguments into the dictionary shape that carries. + public static Dictionary CallArguments(string json) + { + var arguments = new Dictionary(); + foreach (KeyValuePair argument in Arguments(json)) + { + arguments[argument.Key] = argument.Value; + } + + return arguments; + } + + /// A correct response to , built from the contract types. + public static JevResponse CreateResponse() => new() + { + Model = "custom-model", + Answers = new Dictionary + { + ["department"] = new JevChoiceAnswer + { + Choice = "technical", + Confidence = 0.78, + Probabilities = new Dictionary { ["technical"] = 0.85, ["billing"] = 0.15, ["sales"] = 0 }, + }, + ["frustration"] = new JevScoreAnswer + { + Score = 1, + Confidence = 1, + Legend = new Dictionary { ["0"] = "Calm", ["1"] = "Frustrated", ["2"] = "Very angry" }, + Probabilities = new Dictionary { ["0"] = 0, ["1"] = 1, ["2"] = 0 }, + }, + ["is_urgent"] = new JevNoulAnswer { Noul = 0.97 }, + }, + Usage = new JevUsage { InputTokens = 10, OutputTokens = 5 }, + }; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs new file mode 100644 index 00000000000..bc53c152cd1 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// A fake TypeSafe endpoint that records each request and replies with queued responses, repeating the last one. +/// +internal sealed class JevTestHttpHandler : HttpMessageHandler +{ + private readonly Queue<(HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter)> _responses = new(); + private (HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter) _last = (HttpStatusCode.OK, JevTestData.ResponseJson, null); + + public List Requests { get; } = []; + + public JevTestHttpHandler Reply(HttpStatusCode status, string body, RetryConditionHeaderValue? retryAfter = null) + { + this._responses.Enqueue((status, body, retryAfter)); + return this; + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string body = await request.Content!.ReadAsStringAsync(cancellationToken); + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (KeyValuePair> header in request.Headers) + { + headers[header.Key] = string.Join(",", header.Value); + } + + this.Requests.Add(new CapturedRequest(request.Method, request.RequestUri!, request.Headers.Authorization, body, headers)); + + if (this._responses.Count > 0) + { + this._last = this._responses.Dequeue(); + } + + var response = new HttpResponseMessage(this._last.Status) + { + Content = new StringContent(this._last.Body, Encoding.UTF8, "application/json"), + }; + response.Headers.RetryAfter = this._last.RetryAfter; + return response; + } + + public sealed record CapturedRequest(HttpMethod Method, Uri Uri, AuthenticationHeaderValue? Authorization, string Body, IReadOnlyDictionary Headers); +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj new file mode 100644 index 00000000000..76656380a93 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj @@ -0,0 +1,17 @@ + + + + $(TargetFrameworksCore) + + + + + + + + + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ScriptedChatClient.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ScriptedChatClient.cs new file mode 100644 index 00000000000..e2b12db3bf7 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ScriptedChatClient.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// A chat client that stands in for the agent's model: each call returns the next scripted response and records +/// the messages it received. +/// +internal sealed class ScriptedChatClient : IChatClient +{ + private readonly Queue, ChatResponse>> _script; + + public ScriptedChatClient(params Func, ChatResponse>[] script) + { + this._script = new Queue, ChatResponse>>(script); + } + + public List> Calls { get; } = []; + + public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + List received = messages.ToList(); + this.Calls.Add(received); + return Task.FromResult(this._script.Dequeue()(received)); + } + + public async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + ChatResponse response = await this.GetResponseAsync(messages, options, cancellationToken); + foreach (ChatResponseUpdate update in response.ToChatResponseUpdates()) + { + yield return update; + } + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + } +} From c4a10b376555c6147b07e5eb9714ec5006a3eb07 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 29 Sep 2026 18:16:44 +0100 Subject: [PATCH 02/16] .NET: Add JevChatClient to Microsoft.Agents.AI.TypeSafe and align Jev types with the TypeSafe SDK Add an IChatClient for Jev, the TypeSafe System One model, ported from the Python TypeSafe connector (agent-framework-typesafe): - JevChatClient sends the conversation as the Jev state and answers the questions passed with ChatOptions.WithJevQuestions (carried by RawRepresentationFactory) or JevChatClientOptions.DefaultQuestions. GetJevResult reads the typed result from ChatResponse and AgentResponse. - Closed-set function calling: internal route and argument questions choose the tool and each const, enum, Boolean, or enum-array argument, with presence questions for optional arguments, previous-call hints, and the same question IDs and limits as the Python connector. MaximumToolCallsPerTurn (default 1) ends the tool loop. - Streaming, generative options, hosted tools, and non-text content are rejected with NotSupportedException. Handled tool approvals are left out of the state, and text sent with an approval continues the approved call's turn. Align the tool contract with the official TypeScript SDK (@typesafe-ai/sdk): JevEntry (text or JSON entries), JevResult and JevResponse types, SDK retry and timeout defaults, JevHttpClient, and UseMapping instead of UseEvaluator. Samples: Agent_With_TypeSafe (chat client, default questions, closed-set tools, optional arguments, approvals), Agent_TypeSafe_Step01_LoopJudge (Jev as the judge of a LoopAgent), and a local UseClient mapping in Agent_Step25_ClassificationWithJev. The three samples are registered in eng/verify-samples and run when TYPESAFE_API_KEY is set. --- .github/workflows/dotnet-verify-samples.yml | 2 + dotnet/agent-framework-dotnet.slnx | 2 + dotnet/eng/verify-samples/AgentsSamples.cs | 66 +++ .../02-agents/AgentProviders/README.md | 7 + .../Agent_TypeSafe_Step01_LoopJudge.csproj | 20 + .../Program.cs | 101 ++++ .../Agent_TypeSafe_Step01_LoopJudge/README.md | 70 +++ .../Agent_With_TypeSafe.csproj | 16 + .../typesafe/Agent_With_TypeSafe/Program.cs | 159 ++++++ .../typesafe/Agent_With_TypeSafe/README.md | 107 ++++ .../Agent_Step25_ClassificationWithJev.csproj | 1 + .../KeywordClassifier.cs | 60 +++ .../Program.cs | 96 +++- .../README.md | 76 ++- dotnet/samples/AGENTS.md | 3 +- .../Core/JevAIFunction.cs | 26 +- .../Core/JevApiRequest.cs | 6 +- .../Core/JevArgumentKind.cs | 21 + .../Core/JevArgumentPlan.cs | 79 +++ .../Core/JevChatClientLog.cs | 15 + .../Core/JevChatState.cs | 279 +++++++++++ .../Core/JevCompiledTool.cs | 36 ++ .../Core/JevContractValidator.cs | 145 +++--- .../Core/JevEntryJsonConverter.cs | 39 ++ .../{JevHttpEvaluator.cs => JevHttpClient.cs} | 41 +- .../Core/JevJsonContext.cs | 3 +- .../Core/JevJsonUtilities.cs | 130 ++++- .../Core/JevPipelineDefaults.cs | 34 ++ .../Core/JevQuestionBudget.cs | 42 ++ .../Core/JevResponseClassifier.cs | 21 +- .../Core/JevRetryPolicy.cs | 86 +++- .../Core/JevToolCall.cs | 11 + .../Core/JevToolCallCompiler.cs | 424 ++++++++++++++++ .../Core/JevToolCallPlan.cs | 65 +++ .../Core/JevToolSchema.cs | 303 ++++++++++++ .../Core/JevUnsupportedToolSchemaException.cs | 30 ++ .../JevAIToolBuilder.cs | 89 ++-- .../JevAIToolOptions.cs | 39 +- .../Microsoft.Agents.AI.TypeSafe/JevAnswer.cs | 24 - .../JevChatClient.cs | 302 ++++++++++++ .../JevChatClientOptions.cs | 104 ++++ .../JevChatOptionsExtensions.cs | 43 ++ .../JevChatRequestOptions.cs | 26 + .../JevChoiceAnswer.cs | 35 -- .../JevChoiceQuestion.cs | 16 +- .../JevChoiceResponse.cs | 38 ++ .../Microsoft.Agents.AI.TypeSafe/JevEntry.cs | 138 ++++++ .../JevNoulAnswer.cs | 20 - .../JevNoulCriteria.cs | 14 +- .../JevNoulQuestion.cs | 11 +- .../JevNoulResponse.cs | 23 + .../JevQuestion.cs | 15 +- .../JevRequest.cs | 22 +- .../JevResponse.cs | 42 +- .../Microsoft.Agents.AI.TypeSafe/JevResult.cs | 40 ++ .../JevResultExtensions.cs | 62 +++ .../JevScoreAnswer.cs | 42 -- .../JevScoreQuestion.cs | 16 +- .../JevScoreResponse.cs | 45 ++ .../Microsoft.Agents.AI.TypeSafe/JevUsage.cs | 2 +- .../Microsoft.Agents.AI.TypeSafe.csproj | 7 +- .../JevAIFunctionTests.cs | 249 ++++++---- .../JevAIToolBuilderTests.cs | 58 ++- .../JevAgentIntegrationTests.cs | 12 +- .../JevChatClientTests.cs | 386 +++++++++++++++ .../JevChatClientToolTests.cs | 463 ++++++++++++++++++ .../JevChatTestData.cs | 54 ++ .../JevEntryTests.cs | 116 +++++ ...valuatorTests.cs => JevHttpClientTests.cs} | 91 +++- .../JevLoopJudgeTests.cs | 81 +++ .../JevTestData.cs | 14 +- .../JevTestHttpHandler.cs | 19 +- .../JevToolCallCompilerTests.cs | 207 ++++++++ .../ListLoggerFactory.cs | 36 ++ 74 files changed, 5064 insertions(+), 559 deletions(-) create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Agent_TypeSafe_Step01_LoopJudge.csproj create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/KeywordClassifier.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevEntryJsonConverter.cs rename dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/{JevHttpEvaluator.cs => JevHttpClient.cs} (66%) create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevPipelineDefaults.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceResponse.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevEntry.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulResponse.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResult.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreResponse.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevEntryTests.cs rename dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/{JevHttpEvaluatorTests.cs => JevHttpClientTests.cs} (76%) create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs diff --git a/.github/workflows/dotnet-verify-samples.yml b/.github/workflows/dotnet-verify-samples.yml index 3855914f6c9..bd60fa1d34e 100644 --- a/.github/workflows/dotnet-verify-samples.yml +++ b/.github/workflows/dotnet-verify-samples.yml @@ -112,6 +112,8 @@ jobs: # Foundry FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + # TypeSafe (Jev) + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} - name: Write Job Summary if: always() diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index ec1b47fdcbe..c60b7e4dded 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -36,6 +36,8 @@ + + diff --git a/dotnet/eng/verify-samples/AgentsSamples.cs b/dotnet/eng/verify-samples/AgentsSamples.cs index f8cc340e9cb..0c125b56c56 100644 --- a/dotnet/eng/verify-samples/AgentsSamples.cs +++ b/dotnet/eng/verify-samples/AgentsSamples.cs @@ -77,6 +77,49 @@ internal static class AgentsSamples ], }, + new SampleDefinition + { + Name = "Agent_With_TypeSafe", + ProjectPath = "samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe", + RequiredEnvironmentVariables = ["TYPESAFE_API_KEY"], + Inputs = ["Y"], + MustContain = + [ + "Direct JevChatClient result:", + "Agent result:", + "Default questions result:", + "Function calling result:", + "better_city:", + "Tool approval:", + "issue_refund(plan: Pro)", + ], + ExpectedOutputDescription = + [ + "The direct result should route the duplicate subscription charge to billing, and the agent result should route the failing checkout integration to technical with a high urgency probability.", + "The default questions result should route the invoice request to billing with a low urgency probability.", + "The function calling result should contain one weather line for Seattle and one for Amsterdam, with temperatures in Fahrenheit, followed by a better_city line that names one of the two cities.", + "The tool approval step should ask to approve issue_refund for the Pro plan and, after approval, show the refund result followed by a department line.", + "The output should not contain error messages or stack traces.", + ], + }, + + new SampleDefinition + { + Name = "Agent_TypeSafe_Step01_LoopJudge", + ProjectPath = "samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge", + RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT", "TYPESAFE_API_KEY"], + OptionalEnvironmentVariables = ["FOUNDRY_MODEL"], + MustContain = ["Answer 1:", "Judge:", "red_sunsets:", "Final answer:"], + ExpectedOutputDescription = + [ + "The first answer should explain only why the sky is blue, and the judge should mark red_sunsets as missing with a low probability.", + "A later answer should also explain why sunsets are red, and the judge should mark red_sunsets as met for it.", + "Every numbered answer (Answer 1, Answer 2, and so on) should be followed by a Judge block with probabilities. The Final answer section repeats the last judged answer and has no Judge block of its own.", + "The final answer should explain both why the sky is blue and why sunsets are red.", + "The output should not contain error messages or stack traces.", + ], + }, + // ── Agents ────────────────────────────────────────────────────────── new SampleDefinition @@ -403,6 +446,29 @@ internal static class AgentsSamples ], }, + new SampleDefinition + { + Name = "Agent_Step25_ClassificationWithJev", + ProjectPath = "samples/02-agents/Agents/Agent_Step25_ClassificationWithJev", + RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT", "TYPESAFE_API_KEY"], + OptionalEnvironmentVariables = ["FOUNDRY_MODEL"], + MustContain = + [ + "Ticket:", + "Tool result:", + "The same tool, answered by a local keyword classifier:", + "local-keyword-classifier", + ], + ExpectedOutputDescription = + [ + "Each of the three tickets should show a tool result with department, urgent, and frustration answers from a jev model, followed by a one-line agent reply.", + "The failing Stripe integration ticket should be routed to technical and marked urgent, and the invoice request should be routed to billing and not urgent.", + "The vague account ticket may be sent to human review when the department confidence is low.", + "The last part should show a tool result from the local-keyword-classifier model for the invoice ticket, followed by an agent reply.", + "The output should not contain error messages or stack traces.", + ], + }, + // ── AgentSkills ───────────────────────────────────────────────────── new SampleDefinition diff --git a/dotnet/samples/02-agents/AgentProviders/README.md b/dotnet/samples/02-agents/AgentProviders/README.md index 407c4e8e4d9..982e35874cd 100644 --- a/dotnet/samples/02-agents/AgentProviders/README.md +++ b/dotnet/samples/02-agents/AgentProviders/README.md @@ -93,6 +93,13 @@ covering basics, function tools, structured output, middleware, MCP, code interp | [Conversation](./openai/Agent_OpenAI_Step05_Conversation/) | Multi-turn conversations with OpenAI | | [Code Interpreter](./openai/Agent_OpenAI_Step06_CodeInterpreterFileDownload/) | Code interpreter with file downloads | +### [TypeSafe](./typesafe/) + +| Sample | Description | +| --- | --- | +| [Agent with TypeSafe](./typesafe/Agent_With_TypeSafe/) | Use Jev, the TypeSafe System One model, as an IChatClient that answers typed questions and calls closed-set tools | +| [Loop judge](./typesafe/Agent_TypeSafe_Step01_LoopJudge/) | Use Jev as the judge of a LoopAgent, with one yes or no question per criterion | + ## Running the samples Navigate to a sample directory and run: diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Agent_TypeSafe_Step01_LoopJudge.csproj b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Agent_TypeSafe_Step01_LoopJudge.csproj new file mode 100644 index 00000000000..48a53996f27 --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Agent_TypeSafe_Step01_LoopJudge.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs new file mode 100644 index 00000000000..4bfe9cb9c7c --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample uses Jev, the TypeSafe System One model, as the judge of a LoopAgent. A Foundry agent writes the +// answer. After each answer, Jev checks every criterion with its own yes or no question, and the loop sends the agent +// the criteria that are still missing, until the answer meets all of them. + +#pragma warning disable MAAI001 // LoopAgent and DelegateLoopEvaluator are experimental. + +using System.ClientModel; +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.TypeSafe; +using Microsoft.Extensions.AI; + +var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); +var deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini"; +var typeSafeApiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); + +// 1. The criteria. Each one becomes a Noul question, whose answer is the probability that the answer meets it, so the +// judge reports exactly which criteria are missing instead of a single verdict. +Dictionary criteria = new() +{ + ["blue_sky"] = "explains why the daytime sky appears blue", + ["red_sunsets"] = "explains why sunsets appear red or orange", + ["plain_language"] = "uses clear language suitable for a general audience", +}; +Dictionary questions = criteria.ToDictionary( + criterion => criterion.Key, + criterion => (JevQuestion)new JevNoulQuestion { Instructions = $"Does the assistant's latest response {criterion.Value}?" }); + +// 2. The agent that writes the answers. Its instructions hold back part of the first answer, so the judge has a gap +// to find and the loop runs more than once. +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIAgent answerer = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()).AsAIAgent( + model: deploymentName, + name: "Answerer", + instructions: """ + You answer science questions for a general audience in one short paragraph. + In your first answer, explain only why the daytime sky is blue, even if the user asks for more. + When feedback lists missing points, rewrite the whole answer so that it covers them too. + """); + +// 3. The judge. AIJudgeLoopEvaluator asks its chat client for JSON structured output, which Jev cannot write, so a +// DelegateLoopEvaluator asks Jev the criteria questions instead. Jev reads the original request and the latest answer +// as a conversation. +using var judge = new JevChatClient(new ApiKeyCredential(typeSafeApiKey)); +bool accepted = false; +var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => +{ + Console.WriteLine($"\nAnswer {context.Iteration}:\n{context.LastResponse.Text}"); + List missing = await JudgeAsync(context.InitialMessages, context.LastResponse.Text, cancellationToken); + accepted = missing.Count == 0; + + // The feedback names the missing criteria, which is the gap analysis that a generative judge would write. + return accepted + ? LoopEvaluation.Stop() + : LoopEvaluation.Continue($"Your answer does not yet: {string.Join("; ", missing)}. Rewrite the whole answer so that it does."); +}); + +// 4. The loop re-runs the agent until the judge stops it. MaxIterations bounds the cost if the answer never passes. +const int MaxIterations = 3; +const string Request = "Explain why the sky is blue and why sunsets are red."; +AIAgent loop = new LoopAgent(answerer, evaluator, new LoopAgentOptions { MaxIterations = MaxIterations, NonStreamingReturnsLastResponseOnly = true }); +AgentResponse response = await loop.RunAsync(Request); + +// LoopAgent stops at MaxIterations before it calls the evaluator, so an answer from the last allowed run has not been +// judged yet. Judging it here keeps the final answer from being presented as accepted when it may miss a criterion. +if (!accepted) +{ + Console.WriteLine($"\nAnswer {MaxIterations} (the last run the loop allows):\n{response.Text}"); + accepted = (await JudgeAsync([new ChatMessage(ChatRole.User, Request)], response.Text, CancellationToken.None)).Count == 0; +} + +Console.WriteLine(accepted ? "\nFinal answer:" : "\nFinal answer (some criteria are still missing):"); +Console.WriteLine(response.Text); + +// Asks Jev whether the answer meets each criterion, prints the probabilities, and returns the missing criteria. +async Task> JudgeAsync(IEnumerable request, string answer, CancellationToken cancellationToken) +{ + List conversation = [.. request, new ChatMessage(ChatRole.Assistant, answer)]; + ChatResponse verdict = await judge.GetResponseAsync(conversation, new ChatOptions().WithJevQuestions(questions), cancellationToken); + JevResult result = verdict.GetJevResult()!; + + Console.WriteLine("Judge:"); + List missing = []; + foreach ((string id, string description) in criteria) + { + double probability = ((JevNoulResponse)result.Answers[id]).Noul; + bool met = probability > 0.5; + Console.WriteLine($" {id}: {(met ? "met" : "missing")} (probability {probability:F2})"); + if (!met) + { + missing.Add(description); + } + } + + return missing; +} diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md new file mode 100644 index 00000000000..be92f66704a --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md @@ -0,0 +1,70 @@ +# Jev as the judge of a loop + +This sample uses `JevChatClient` from `Microsoft.Agents.AI.TypeSafe` as the judge of a `LoopAgent`. A Foundry agent +writes the answer, and Jev, the TypeSafe System One model, decides whether the answer is complete. + +A generative judge writes a verdict and a gap analysis. Jev writes no text, so the sample asks it one `JevNoulQuestion` +per criterion instead. Each answer is the probability, from 0 to 1, that the answer meets that criterion, so the judge +reports exactly which criteria are missing: + +```text +Judge: + blue_sky: met (probability 0.99) + red_sunsets: missing (probability 0.02) + plain_language: met (probability 0.92) +``` + +The loop turns the missing criteria into the feedback for the next run, and stops when every criterion is met or after +three runs. The agent's instructions hold back part of the first answer on purpose, so the loop visibly runs twice. + +`LoopAgent` checks `MaxIterations` before it calls the evaluator, so when the loop reaches its limit, the answer of the +last run has not been judged. The sample judges that answer after the loop and labels the final answer when some +criteria are still missing, instead of presenting it as accepted. + +## Why a delegate evaluator + +`AIJudgeLoopEvaluator` asks its chat client for a `JudgeVerdict` as JSON structured output, which Jev cannot write; +`JevChatClient` rejects the JSON response format with a `NotSupportedException`. The sample wraps the judge in a +`DelegateLoopEvaluator` instead: + +```csharp +var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => +{ + List missing = await JudgeAsync(context.InitialMessages, context.LastResponse.Text, cancellationToken); + accepted = missing.Count == 0; + return accepted + ? LoopEvaluation.Stop() + : LoopEvaluation.Continue($"Your answer does not yet: {string.Join("; ", missing)}. ..."); +}); +``` + +`JudgeAsync` sends Jev the original request and the answer as a conversation, with +`new ChatOptions().WithJevQuestions(questions)`, and returns the criteria whose Noul is 0.5 or lower. + +Jev reads the original request and the latest answer as a conversation, and each question refers to "the assistant's +latest response". The same pattern works with a `JevScoreQuestion` for a graded rubric, or a `JevChoiceQuestion` to +classify what kind of revision is needed. + +## Prerequisites + +- .NET 10 SDK or later +- A Microsoft Foundry project with a deployed chat model +- Azure CLI installed and authenticated (`az login`) +- A TypeSafe API key + +Set the environment variables: + +```powershell +$env:FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:FOUNDRY_MODEL="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini +$env:TYPESAFE_API_KEY="your-typesafe-api-key" +``` + +## Running the sample + +```powershell +dotnet run +``` + +The sample prints each answer with the judge's probabilities, then the final answer, labeled when some criteria are +still missing. The wording of the answers and the exact probabilities vary between runs. diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj new file mode 100644 index 00000000000..b83c02d799f --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj @@ -0,0 +1,16 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs new file mode 100644 index 00000000000..367db584b33 --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to use Jev, the TypeSafe System One model, as an IChatClient and as an AIAgent. +// Jev does not write text: it answers typed questions about the conversation and returns calibrated probabilities. + +using System.ClientModel; +using System.ComponentModel; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.TypeSafe; +using Microsoft.Extensions.AI; + +var apiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); + +// 1. The questions to answer. Each answer comes back under the ID of its question. +Dictionary questions = new() +{ + ["department"] = new JevChoiceQuestion + { + Instructions = "Which team should handle the support request?", + Criteria = new Dictionary + { + ["billing"] = "Payments, subscriptions, invoices, or refunds", + ["technical"] = "Bugs, outages, integrations, or account access", + ["sales"] = "Pricing, upgrades, or new account questions", + }, + }, + ["frustration"] = new JevScoreQuestion + { + Instructions = "How frustrated does the customer appear?", + Criteria = ["Calm and factual", "Frustrated but civil", "Very angry or threatening to leave"], + }, + ["is_urgent"] = new JevNoulQuestion + { + Instructions = "Does the request require urgent attention?", + Criteria = new JevNoulCriteria + { + True = "The customer describes ongoing harm, lost revenue, or an immediate deadline", + False = "The request can wait for the normal support queue", + }, + }, +}; + +// 2. Call the chat client directly. WithJevQuestions passes the questions through ChatOptions. +using IChatClient chatClient = new JevChatClient(new ApiKeyCredential(apiKey)); +ChatResponse direct = await chatClient.GetResponseAsync( + "I was charged twice for the same subscription. Please refund the duplicate charge.", + new ChatOptions().WithJevQuestions(questions)); +PrintEvaluation("Direct JevChatClient result:", direct.GetJevResult()!); + +// 3. Use the same client through an agent. ChatClientAgentRunOptions carries the questions for one run. +AIAgent agent = chatClient.AsAIAgent(name: "JevTicketEvaluator", instructions: "Evaluate the support request."); +AgentResponse agentResponse = await agent.RunAsync( + "Our checkout integration has failed for three days and we are losing sales. Please help ASAP.", + options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(questions))); +PrintEvaluation("Agent result:", agentResponse.GetJevResult()!); + +// 4. Default questions. A client created with DefaultQuestions answers requests that pass no questions, so code that +// knows nothing about Jev can use it like any other IChatClient; here, an agent run without Jev options. +using IChatClient triageClient = new JevChatClient(new ApiKeyCredential(apiKey), new JevChatClientOptions { DefaultQuestions = questions }); +AgentResponse triaged = await triageClient.AsAIAgent(name: "JevTriage").RunAsync( + "Hi, could you send me a copy of last month's invoice when you get a chance? Thanks!"); +PrintEvaluation("Default questions result:", triaged.GetJevResult()!); + +// 5. Function calling. Jev cannot write arguments, so it chooses them from closed sets: an enum, a Boolean, and an +// optional enum that is passed only when Jev decides the user asked for it. The user asks about two cities, so the +// client allows two tool calls per turn instead of the default one. Jev reads the tool's name and description when +// it chooses a tool, so the local function gets a readable name instead of its compiler-generated one. +using IChatClient weatherClient = new JevChatClient(new ApiKeyCredential(apiKey), new JevChatClientOptions { MaximumToolCallsPerTurn = 2 }); +AIAgent weatherAgent = weatherClient.AsAIAgent(name: "WeatherEvaluator", tools: [AIFunctionFactory.Create(GetWeather, "get_weather")]); +var comparison = new Dictionary +{ + ["better_city"] = new JevChoiceQuestion + { + Instructions = "Which city has better weather based on the tool results?", + Criteria = new Dictionary + { + ["Seattle"] = "Seattle has the better weather.", + ["Amsterdam"] = "Amsterdam has the better weather.", + }, + }, +}; +AgentResponse weather = await weatherAgent.RunAsync( + "Give me a detailed weather report for Seattle and Amsterdam in Fahrenheit and tell me where the weather is better.", + options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(comparison))); + +// After tool calls, the text is the tool results followed by one line per Choice and Score answer. +Console.WriteLine("\nFunction calling result:"); +Console.WriteLine(weather.Text); +Console.WriteLine($"Comparison confidence: {((JevChoiceResponse)weather.GetJevResult()!.Answers["better_city"]).Confidence:F3}"); + +// 6. Tool approval. A tool wrapped in ApprovalRequiredAIFunction runs only after the user approves the call that Jev +// chose. The session keeps the pending call, so the approval is sent in a second run with the same questions. +AIAgent refundAgent = chatClient.AsAIAgent(name: "RefundAgent", tools: [new ApprovalRequiredAIFunction(AIFunctionFactory.Create(IssueRefund, "issue_refund"))]); +AgentSession session = await refundAgent.CreateSessionAsync(); +var refundOptions = new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(questions)); +AgentResponse refund = await refundAgent.RunAsync( + "I was charged twice for my Pro plan this month. Please refund the duplicate charge.", + session, + refundOptions); + +Console.WriteLine("\nTool approval:"); +if (refund.Messages.SelectMany(message => message.Contents).OfType().FirstOrDefault() is { ToolCall: FunctionCallContent call } approval) +{ + string arguments = string.Join(", ", call.Arguments?.Select(argument => $"{argument.Key}: {argument.Value}") ?? []); + Console.Write($"Jev wants to call {call.Name}({arguments}). Approve? (Y/N): "); + bool approved = Console.ReadLine()?.Trim().Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false; + refund = await refundAgent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(approved)]), session, refundOptions); +} + +Console.WriteLine(refund.Text); + +static void PrintEvaluation(string label, JevResult result) +{ + var department = (JevChoiceResponse)result.Answers["department"]; + var frustration = (JevScoreResponse)result.Answers["frustration"]; + var urgent = (JevNoulResponse)result.Answers["is_urgent"]; + + Console.WriteLine($"\n{label}"); + Console.WriteLine($"Department: {department.Choice} (confidence {department.Confidence:F3})"); + Console.WriteLine($"Frustration score: {frustration.Score:F3} of {frustration.Legend.Count - 1}"); + Console.WriteLine($"Urgency probability: {urgent.Noul:F3}"); + Console.WriteLine($"Tokens: {result.Usage.InputTokens} in, {result.Usage.OutputTokens} out"); +} + +[Description("Gets the weather forecast.")] +static string GetWeather( + [Description("The city to get the weather for")] City city, + [Description("Whether to include a detailed hourly forecast; true when the user asks for a detailed report")] bool detailed, + [Description("The temperature unit; leave it out to use Celsius")] TemperatureUnit? unit = null) +{ + int celsius = Random.Shared.Next(15, 31); + string temperature = unit == TemperatureUnit.Fahrenheit ? $"{(celsius * 9 / 5) + 32} F" : $"{celsius} C"; + string suffix = detailed ? " with a detailed hourly forecast" : string.Empty; + return $"{city} is sunny and {temperature}{suffix}."; +} + +[Description("Refunds a duplicate charge for a subscription plan.")] +static string IssueRefund([Description("The plan that was charged twice")] Plan plan) => + $"Refunded the duplicate {plan} plan charge."; + +internal enum City +{ + Seattle, + Amsterdam, + Paris, +} + +internal enum TemperatureUnit +{ + Celsius, + Fahrenheit, +} + +internal enum Plan +{ + Basic, + Pro, + Enterprise, +} diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md new file mode 100644 index 00000000000..77ac532b541 --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md @@ -0,0 +1,107 @@ +# Agent with TypeSafe (Jev) + +This sample uses `JevChatClient` from `Microsoft.Agents.AI.TypeSafe`, an `IChatClient` for Jev, the TypeSafe System +One model. Jev does not write text. It answers typed questions about the conversation and returns calibrated +probabilities: + +| Question type | What it answers | Answer | +| --- | --- | --- | +| `JevChoiceQuestion` | Which of these named alternatives? | The selected label, the probability of each label, and a confidence | +| `JevScoreQuestion` | Which score on an ordered rubric (2 to 10 entries)? | An expected score that can fall between rubric entries, the probability of each score, and a confidence | +| `JevNoulQuestion` | Yes or no? | The probability, from 0 to 1, of yes | + +The client follows the TypeSafe connector for Python (`agent-framework-typesafe`), so both languages behave alike. + +## What the sample shows + +1. **The chat client directly.** `WithJevQuestions` puts the questions in `ChatOptions`, and `GetJevResult` reads the + typed answers from the response: + + ```csharp + IChatClient client = new JevChatClient(new ApiKeyCredential(apiKey)); + ChatResponse response = await client.GetResponseAsync(ticket, new ChatOptions().WithJevQuestions(questions)); + JevResult result = response.GetJevResult()!; + ``` + + The response text is the same result as JSON, so code that reads only text still gets every answer. + +2. **The client behind an agent.** A `ChatClientAgentRunOptions` carries the questions for one run. Questions in the + agent's `ChatClientAgentOptions.ChatOptions` apply to every run, and a run's questions replace them. + +3. **Default questions.** A client created with `JevChatClientOptions.DefaultQuestions` answers requests that pass no + questions, so code that knows nothing about Jev can use it like any other `IChatClient`. The sample runs an agent + without any Jev options. + +4. **Function calling.** Jev cannot write arguments, so the client adds internal questions that choose the tool and + each argument from a closed set: a constant, an enum, a Boolean, or an array of enum values. Optional arguments + are passed only when Jev decides the user asked for them; here, the temperature unit is passed because the user + asks for Fahrenheit. The agent runs the tool, and once the turn has used + `JevChatClientOptions.MaximumToolCallsPerTurn` calls (one by default, two in this sample), the client answers the + caller's questions with the tool results in the state. The response text is then the tool results followed by one + `id: answer` line per Choice and Score answer: + + ```text + Seattle is sunny and 75 F with a detailed hourly forecast. + Amsterdam is sunny and 69 F with a detailed hourly forecast. + better_city: Seattle + ``` + + Jev reads each tool's name and description, and each argument's description, to make these choices, so write them + for Jev as you would for a person. Give local functions an explicit name, because their compiler-generated names + mean nothing. In this sample, saying that `detailed` is true "when the user asks for a detailed report" is what makes + Jev choose it reliably. Tools with free-form arguments, such as text or numbers, are left out of the request with a + warning in the log. + +5. **Tool approval.** A tool wrapped in `ApprovalRequiredAIFunction` runs only after the user approves the call that + Jev chose. The first run returns a `ToolApprovalRequestContent`; the sample asks for approval in the console and + sends the answer in a second run on the same session, with the same questions. + +## Limits + +Jev answers all questions at once, so the client rejects what needs generated text: streaming (`RunStreamingAsync`), +sampling and output options such as `Temperature` or a JSON `ResponseFormat`, server-side conversations, tools other +than functions, and message content other than text, reasoning text, and function calls and results. Each fails with a +`NotSupportedException` that names the problem. Tool approval requests and responses are allowed but left out of what +Jev reads, because the call and result they lead to carry the information. + +Because streaming is not supported, run agents with `RunAsync`. A function-invoking client runs approved tools before +it calls the chat client, so a streaming run that resumes a tool approval would run the tool and then fail. + +## Configuring the client + +`JevChatClientOptions` follows the `ClientPipelineOptions` pattern of the OpenAI and Azure AI client libraries, with the +retry and timeout defaults of the official TypeSafe SDK: + +```csharp +var options = new JevChatClientOptions +{ + ModelId = "jev-latest", // ChatOptions.ModelId overrides it per request + MaximumToolCallsPerTurn = 2, + Transport = new HttpClientPipelineTransport(httpClientFactory.CreateClient("jev")), +}; +IChatClient client = new JevChatClient(new ApiKeyCredential(apiKey), options); +``` + +To use Jev as a tool of an agent whose own model writes the replies, see +[Classification with Jev](../../../Agents/Agent_Step25_ClassificationWithJev/). To use it as the judge of a +`LoopAgent`, see [Jev as the judge of a loop](../Agent_TypeSafe_Step01_LoopJudge/). + +## Prerequisites + +- .NET 10 SDK or later +- A TypeSafe API key + +Set the environment variable: + +```powershell +$env:TYPESAFE_API_KEY="your-typesafe-api-key" +``` + +## Running the sample + +```powershell +dotnet run +``` + +The tool approval step asks for `Y` or `N` in the console. Probabilities, scores, and the temperatures in the tool +results vary between runs. diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj index 26193187de0..1c976a48330 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj @@ -10,6 +10,7 @@ + diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/KeywordClassifier.cs b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/KeywordClassifier.cs new file mode 100644 index 00000000000..338dd1e7dc6 --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/KeywordClassifier.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +/// +/// A local classifier that stands in for another classification client. It has its own request and response types, +/// knows nothing about Jev, and needs no network: it scores each option by the words it shares with the text. +/// +[SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Instantiated by the service provider")] +internal sealed class KeywordClassifier +{ + private static readonly char[] s_separators = [' ', ',', '.', '!', '?', ';', ':', '\'', '"', '(', ')', '-', '\n', '\r', '\t']; + + public Task ClassifyAsync(ClassificationRequest request, CancellationToken cancellationToken) + { + HashSet words = Words(request.Text); + var results = new Dictionary(); + foreach (ClassificationTask task in request.Tasks) + { + // One point per shared word, plus one so that every option keeps some probability. + double[] points = [.. task.Options.Select(option => Words(option.Description).Count(words.Contains) + 1.0)]; + double total = points.Sum(); + results[task.Id] = new ClassificationResult( + task.Kind, + [.. task.Options.Select((option, index) => new ScoredOption(option.Key, option.Description, points[index] / total))]); + } + + return Task.FromResult(new ClassificationResponse(results)); + } + + // Words are compared in upper case and without a plural "S", so "invoice" matches "invoices". + private static HashSet Words(string text) => + [.. text.ToUpperInvariant().Split(s_separators, StringSplitOptions.RemoveEmptyEntries).Where(word => word.Length > 3).Select(word => word.TrimEnd('S'))]; +} + +/// What the classifier decides for one task. +internal enum ClassificationKind +{ + PickOne, + Rate, + YesNo, +} + +/// The classifier's request: the text and the tasks to decide about it. +internal sealed record ClassificationRequest(string Text, IReadOnlyList Tasks); + +/// One task: pick one option, rate on ordered options, or decide yes or no. +internal sealed record ClassificationTask(string Id, ClassificationKind Kind, IReadOnlyList Options); + +/// An option of a task and the words that describe it. +internal sealed record ClassificationOption(string Key, string Description); + +/// The classifier's response: one result per task, by task ID. +internal sealed record ClassificationResponse(IReadOnlyDictionary Results); + +/// The result of one task: every option with its probability. +internal sealed record ClassificationResult(ClassificationKind Kind, IReadOnlyList Options); + +/// An option with the probability that the classifier gives it. +internal sealed record ScoredOption(string Key, string Description, double Probability); diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs index c99b7db4f8f..a228d7573d2 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs @@ -6,14 +6,16 @@ // (choice, score, noul) about a state and returns calibrated probabilities. This sample gives // a support agent a Jev tool built with JevAIToolBuilder. The agent's model decides which // questions to ask, Jev answers them, and the model writes the routing decision, sending -// uncertain tickets to a human. +// uncertain tickets to a human. The second part answers the same tool with another client. using System.ClientModel; +using System.Globalization; using Azure.AI.Projects; using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Agents.AI.TypeSafe; using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); var deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini"; @@ -25,12 +27,12 @@ // const string Instructions = """ - You triage customer support tickets. For every ticket, call evaluate_with_jev once with the ticket text as the + You triage customer support tickets. For every ticket, call ask_system_one once with the ticket text as the state and these questions: - "department": a choice between billing (payments, invoices, refunds), technical (bugs, outages, integrations), and sales (pricing, upgrades, new accounts). - "urgent": a noul asking whether the customer needs help urgently. - - "frustration": a score with the levels calm, annoyed, and angry. + - "frustration": a score with the rubric calm, annoyed, and angry. Then reply in one line with the department, whether it is urgent, and the frustration level. When the department confidence is below 0.6, say the ticket needs human review instead of naming a department. """; @@ -38,8 +40,8 @@ // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()) - .AsAIAgent(model: deploymentName, instructions: Instructions, name: "TicketTriage", tools: [jev]); +var projectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()); +AIAgent agent = projectClient.AsAIAgent(model: deploymentName, instructions: Instructions, name: "TicketTriage", tools: [jev]); string[] tickets = [ @@ -49,15 +51,97 @@ ]; foreach (string ticket in tickets) +{ + await TriageAsync(agent, ticket); +} + +// +// The same tool can be answered by another client. UseClient takes the client's operation and two mappers: one from +// the JevRequest that the model writes to the client's request, and one from the client's response to a JevResult. +// The tool resolves the client from the agent's services, in a new scope on every call, and checks the mapped result +// against the request before the model sees it. +AIFunction localJev = new JevAIToolBuilder() + .UseClient( + (classifier, request, cancellationToken) => classifier.ClassifyAsync(request, cancellationToken), + inputMapper: ToClassificationRequest, + outputMapper: ToJevResult) + .Build(); + +using ServiceProvider services = new ServiceCollection() + .AddSingleton() + .BuildServiceProvider(); +AIAgent localAgent = projectClient.AsAIAgent( + model: deploymentName, + instructions: Instructions, + name: "LocalTicketTriage", + tools: [localJev], + services: services); +// + +Console.WriteLine("The same tool, answered by a local keyword classifier:"); +Console.WriteLine(); +await TriageAsync(localAgent, tickets[1]); + +static async Task TriageAsync(AIAgent agent, string ticket) { AgentResponse response = await agent.RunAsync(ticket); Console.WriteLine($"Ticket: {ticket}"); foreach (FunctionResultContent result in response.Messages.SelectMany(static m => m.Contents).OfType()) { - Console.WriteLine($"Jev answers: {result.Result}"); + Console.WriteLine($"Tool result: {result.Result}"); } Console.WriteLine($"Agent: {response.Text}"); Console.WriteLine(); } + +// +// Converts each Jev question into a classifier task. The options carry the words the classifier matches. +static ClassificationRequest ToClassificationRequest(JevRequest request) => + new( + request.State.ToString(), + [.. request.Questions.Select(question => question.Value switch + { + JevChoiceQuestion choice => new ClassificationTask( + question.Key, + ClassificationKind.PickOne, + [.. choice.Criteria.Select(criterion => new ClassificationOption(criterion.Key, $"{criterion.Key} {criterion.Value}"))]), + JevScoreQuestion score => new ClassificationTask( + question.Key, + ClassificationKind.Rate, + [.. score.Criteria.Select((level, index) => new ClassificationOption(index.ToString(CultureInfo.InvariantCulture), level.ToString()))]), + JevNoulQuestion noul => new ClassificationTask( + question.Key, + ClassificationKind.YesNo, + [new ClassificationOption("yes", $"{noul.Instructions} {noul.Criteria?.True}"), new ClassificationOption("no", $"{noul.Criteria?.False}")]), + _ => throw new NotSupportedException($"Question '{question.Key}' has an unknown type."), + })]); + +// Converts each classifier result back into the Jev answer of the same kind. +static JevResult ToJevResult(ClassificationResponse response) => new() +{ + Model = "local-keyword-classifier", + Answers = response.Results.ToDictionary(result => result.Key, result => ToJevResponse(result.Value)), + Usage = new JevUsage { InputTokens = 0, OutputTokens = 0 }, +}; + +static JevResponse ToJevResponse(ClassificationResult result) +{ + ScoredOption best = result.Options.MaxBy(option => option.Probability)!; + Dictionary probabilities = result.Options.ToDictionary(option => option.Key, option => option.Probability); + return result.Kind switch + { + ClassificationKind.PickOne => new JevChoiceResponse { Choice = best.Key, Confidence = best.Probability, Probabilities = probabilities }, + ClassificationKind.Rate => new JevScoreResponse + { + // Like Jev, the score is the expected value over the rubric, so it can fall between levels. + Score = result.Options.Sum(option => int.Parse(option.Key, CultureInfo.InvariantCulture) * option.Probability), + Confidence = best.Probability, + Legend = result.Options.ToDictionary(option => option.Key, option => (JevEntry)option.Description), + Probabilities = probabilities, + }, + _ => new JevNoulResponse { Noul = probabilities["yes"] }, + }; +} +// diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md index 21a96610475..cd289614e72 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md @@ -5,18 +5,27 @@ This sample gives an agent a classifier tool built with `JevAIToolBuilder` from Jev, the TypeSafe System One model, does not write text. It answers typed questions about a state and returns calibrated probabilities. An agent's model writes the replies; Jev makes the fast, structured decisions: -1. The agent's model calls `evaluate_with_jev` with a Jev request: the `state` to evaluate and the `questions` to +1. The agent's model calls `ask_system_one` with a Jev request: the `state` and the `questions` to answer about it. 2. Jev answers each question: | Question type | What it answers | Answer | | --- | --- | --- | - | `choice` | Which of these options? | The chosen option, the probability of each option, and a confidence | - | `score` | Which level of an ordered scale (2 to 10 levels)? | A score that can fall between levels, the probability of each level, and a confidence | - | `noul` | Is this yes or no statement true? | The probability, from 0 to 1, that it is true | + | `choice` | Which of these named alternatives? | The selected label, the probability of each label, and a confidence | + | `score` | Which score on an ordered rubric (2 to 10 entries)? | An expected score that can fall between rubric entries, the probability of each score, and a confidence | + | `noul` | Yes or no? | The probability, from 0 to 1, of yes | + + The state, the instructions, and every criterion can be text or JSON, like `EntryType` in the official TypeSafe SDK + ([`@typesafe-ai/sdk`](https://github.com/typesafe-ai/typesafe-sdk-js)). The Jev contract types mirror the SDK's type + names: `JevRequest`, `JevChoiceQuestion`, `JevResult`, `JevChoiceResponse`, and so on. 3. The model reads the answers and writes the reply. Here it routes each support ticket and sends uncertain tickets to human review. +4. The same tool is answered by a local keyword classifier instead of the TypeSafe API, through `UseClient` and the + agent's services (see [Using another client](#using-another-client)). + +To use Jev as the agent's chat client instead, answering questions about the whole conversation, see +[Agent with TypeSafe](../../AgentProviders/typesafe/Agent_With_TypeSafe/). ## Configuring the client @@ -35,42 +44,61 @@ options.AddPolicy(myPolicy, PipelinePosition.PerCall); // custom pipe AIFunction jev = new JevAIToolBuilder(new ApiKeyCredential(apiKey), options).Build(); ``` -The default retry policy retries rate-limited (429) and overloaded (529) requests, and fails at once when the API asks -for a wait longer than one minute, so the agent run is never blocked for long. `Name` and `Description` set what the -agent's model sees. +Unset pipeline settings get the defaults of the official SDK: two retries on 408, 429, and 5xx responses (529 means +overloaded), honoring `retry-after-ms` or `Retry-After` up to one minute and otherwise backing off from half a second +to five seconds, and a 10-second timeout per attempt. `Name` and `Description` set what the agent's model sees. ## Using another client -Any other client can answer the requests instead of the TypeSafe API. Pass a delegate that takes a `JevRequest` and -returns a `JevResponse`, or map between the Jev types and the client's own types: +Any other client can answer the requests instead of the TypeSafe API. The second part of the sample shows it with +`KeywordClassifier`, a local classifier in [KeywordClassifier.cs](./KeywordClassifier.cs) that has its own request and +response types, knows nothing about Jev, and needs no network. `UseClient` takes the client's operation and two +mappers between the Jev types and the client's types: ```csharp -AIFunction jev = new JevAIToolBuilder() - .UseClient( - myClient.EvaluateAsync, - inputMapper: request => ToMyClientRequest(request), - outputMapper: response => ToJevResponse(response)) +AIFunction localJev = new JevAIToolBuilder() + .UseClient( + (classifier, request, cancellationToken) => classifier.ClassifyAsync(request, cancellationToken), + inputMapper: ToClassificationRequest, + outputMapper: ToJevResult) .Build(); ``` -The tool checks the mapped response before the model sees it: every question needs an answer of the same type, and a -`choice` answer and its probabilities may only use the options the question offered. +`ToClassificationRequest` turns each Jev question into a classifier task, and `ToJevResult` turns each classifier +result back into the Jev answer of the same kind (see the `mappers` region of [Program.cs](./Program.cs)). The tool +checks the mapped result before the model sees it: every question needs an answer of the same type, and a `choice` +answer and its probabilities may only use the labels the question offered. The output mapper receives only the +client's response, so the client's result says which kind of answer it is. + +The keyword classifier is far less certain than Jev, so for the same invoice ticket the agent's confidence rule +usually sends it to human review. Use `UseMapping` instead of `UseClient` for a mapping that is a single delegate from +a `JevRequest` to a `JevResult`. ## Dependency injection -Every evaluator receives the `IServiceProvider` of the agent that calls the tool. `FunctionInvokingChatClient` passes +Every mapping receives the `IServiceProvider` of the agent that calls the tool. `FunctionInvokingChatClient` passes it in `AIFunctionArguments.Services`, from the services the agent was created with. `UseClient` resolves -a registered client from a new scope on every call and disposes the scope afterwards, so scoped and disposable clients -work even though hosted agents are singletons that receive the root provider: +the client from a new scope on every call and disposes the scope afterwards, so scoped and disposable clients work +even though hosted agents are singletons that receive the root provider. The sample registers the classifier in a +`ServiceCollection` and passes the provider to the agent: + +```csharp +using ServiceProvider services = new ServiceCollection() + .AddSingleton() + .BuildServiceProvider(); +AIAgent localAgent = projectClient.AsAIAgent(model: deploymentName, instructions: Instructions, tools: [localJev], services: services); +``` + +In a host, register the client and build the tool in the agent's tool factory: ```csharp -builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddAIAgent("triage", Instructions) .WithAITool(sp => new JevAIToolBuilder() - .UseClient( - (client, request, cancellationToken) => client.EvaluateAsync(request, cancellationToken), - inputMapper: ToMyClientRequest, - outputMapper: ToJevResponse) + .UseClient( + (classifier, request, cancellationToken) => classifier.ClassifyAsync(request, cancellationToken), + inputMapper: ToClassificationRequest, + outputMapper: ToJevResult) .Build()); ``` diff --git a/dotnet/samples/AGENTS.md b/dotnet/samples/AGENTS.md index 7c56735e4c7..417f3434bcc 100644 --- a/dotnet/samples/AGENTS.md +++ b/dotnet/samples/AGENTS.md @@ -26,7 +26,8 @@ dotnet/samples/ │ │ ├── google-gemini/ # Google Gemini provider sample │ │ ├── ollama/ # Ollama provider sample │ │ ├── onnx/ # ONNX Runtime provider sample -│ │ └── openai/ # OpenAI provider samples +│ │ ├── openai/ # OpenAI provider samples +│ │ └── typesafe/ # TypeSafe (Jev) chat client samples │ ├── AgentOpenTelemetry/ # OpenTelemetry integration │ ├── AgentSkills/ # Agent skills patterns │ ├── AgentWithMemory/ # Memory providers (chat history, Mem0, Valkey, Foundry, AgentMemory) diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs index 8001bbd4f59..45fb819c118 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs @@ -13,17 +13,17 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// The created by : its arguments are a -/// and its result is the of the configured evaluator. +/// and its result is the of the configured mapping or the TypeSafe API. /// internal sealed class JevAIFunction : AIFunction { - private readonly Func> _evaluateAsync; + private readonly Func> _systemOneAsync; - public JevAIFunction(string name, string description, Func> evaluateAsync) + public JevAIFunction(string name, string description, Func> systemOneAsync) { this.Name = name; this.Description = description; - this._evaluateAsync = evaluateAsync; + this._systemOneAsync = systemOneAsync; } public override string Name { get; } @@ -32,7 +32,7 @@ public JevAIFunction(string name, string description, Func JevJsonUtilities.RequestSchema; - public override JsonElement? ReturnJsonSchema => JevJsonUtilities.ResponseSchema; + public override JsonElement? ReturnJsonSchema => JevJsonUtilities.ResultSchema; public override JsonSerializerOptions JsonSerializerOptions => JevJsonUtilities.Options; @@ -42,27 +42,27 @@ public JevAIFunction(string name, string description, Func -/// The body of a TypeSafe System One API request: a plus the model that answers it. +/// The body of POST /v1/systemone: a with the model resolved, like +/// SystemOneRequestPayload of the official TypeSafe SDK. /// internal sealed class JevApiRequest { [JsonPropertyName("state")] - public required JsonElement State { get; init; } + public required JevEntry State { get; init; } [JsonPropertyName("model")] public required string Model { get; init; } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs new file mode 100644 index 00000000000..a99e841d048 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// How the value of one tool argument is read from Jev's answers. +/// +internal enum JevArgumentKind +{ + /// The schema allows one value, so no question is asked about it. + Const, + + /// A Choice question selects one value of an enum. + Choice, + + /// A Noul question decides a Boolean. + Boolean, + + /// One Noul question per enum member decides which members an array includes. + Set, +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs new file mode 100644 index 00000000000..825e535cf36 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The questions asked about one tool argument and how its value is read from the answers. +/// +internal sealed class JevArgumentPlan +{ + public required string Name { get; init; } + + public required JevArgumentKind Kind { get; init; } + + /// + /// Gets the ID of the Noul question that decides whether an optional argument is passed at all, or + /// for a required argument. + /// + public string? PresenceQuestionId { get; init; } + + /// Gets the ID of the Choice or Noul question of a or argument. + public string? ValueQuestionId { get; init; } + + /// Gets the enum values of a argument, keyed by their Choice label. + public IReadOnlyDictionary Values { get; init; } = new Dictionary(); + + /// Gets the Noul question ID and the value of each member of a argument. + public IReadOnlyList> Members { get; init; } = []; + + /// Gets the value of a argument. + public JsonElement ConstValue { get; init; } + + /// + /// Reads the argument value from Jev's answers. + /// + /// + /// when an optional argument should be left out, so that the function uses its default. + /// + public bool TryDecode(JevResult result, out JsonElement value) + { + value = default; + if (this.PresenceQuestionId is not null && !JevToolCallPlan.IsYes(result, this.PresenceQuestionId)) + { + return false; + } + + switch (this.Kind) + { + case JevArgumentKind.Const: + value = this.ConstValue; + return true; + + case JevArgumentKind.Boolean: + value = JevJsonUtilities.CreateBoolean(JevToolCallPlan.IsYes(result, this.ValueQuestionId!)); + return true; + + case JevArgumentKind.Choice: + string label = JevToolCallPlan.GetChoice(result, this.ValueQuestionId!); + if (!this.Values.TryGetValue(label, out value)) + { + Throw.InvalidOperationException($"Jev returned the unknown choice '{label}' for the tool argument '{this.Name}'."); + } + + return true; + + case JevArgumentKind.Set: + value = JevJsonUtilities.CreateArray(this.Members.Where(member => JevToolCallPlan.IsYes(result, member.Key)).Select(member => member.Value)); + return true; + + default: + Throw.InvalidOperationException($"The argument plan of '{this.Name}' has the unknown kind {this.Kind}."); + return false; + } + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs new file mode 100644 index 00000000000..3d093ad6a39 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// Source-generated log messages of . +internal static partial class JevChatClientLog +{ + [LoggerMessage( + EventId = 1, + Level = LogLevel.Warning, + Message = "The tool '{ToolName}' is left out of the Jev request because its parameters cannot be expressed as Jev questions: {Reason}")] + public static partial void LogToolExcluded(this ILogger logger, string toolName, string reason); +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs new file mode 100644 index 00000000000..bbbc5a84582 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs @@ -0,0 +1,279 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Converts a conversation into the state that Jev answers questions about, and reads the current turn of it. +/// +/// +/// The state has the shape of the Python TypeSafe connector: {"messages": [{"role", "contents": [...]}], +/// "instructions"}, where each content is text, text_reasoning, function_call, or +/// function_result. Tool approval requests and responses are left out: they are control messages, and the call +/// and result they lead to carry the information. The current turn is everything after the last user message that +/// starts a request: the tool calls and results that answer it. +/// +internal static class JevChatState +{ + public static JevEntry Build(IReadOnlyList messages, string? instructions) + { + var stateMessages = new JsonArray(); + for (int index = 0; index < messages.Count; index++) + { + ChatMessage message = messages[index]; + List unsupported = [.. message.Contents + .Where(content => content is not (TextContent or TextReasoningContent or FunctionCallContent or FunctionResultContent or ToolApprovalRequestContent or ToolApprovalResponseContent)) + .Select(content => content.GetType().Name) + .Distinct(StringComparer.Ordinal) + .OrderBy(name => name, StringComparer.Ordinal)]; + if (unsupported.Count > 0) + { + throw new NotSupportedException( + $"Jev supports text, reasoning text, and function call and result content only; message {index} contains {string.Join(", ", unsupported)}."); + } + + // Approval content is not written: FunctionInvokingChatClient keeps handled approvals in the history of + // later turns, and rejecting them would make every turn after an approval fail. + var contents = new JsonArray(); + foreach (AIContent content in message.Contents) + { + switch (content) + { + case TextContent { Text: { Length: > 0 } text }: + contents.Add((JsonNode)new JsonObject { ["type"] = "text", ["text"] = text }); + break; + + // Only the readable reasoning text is sent; protected (encrypted) reasoning is meaningful only to + // the model that produced it. + case TextReasoningContent { Text: { Length: > 0 } reasoning }: + contents.Add((JsonNode)new JsonObject { ["type"] = "text_reasoning", ["text"] = reasoning }); + break; + + case FunctionCallContent call: + contents.Add((JsonNode)new JsonObject + { + ["type"] = "function_call", + ["call_id"] = call.CallId, + ["name"] = call.Name, + ["arguments"] = call.Arguments is null ? null : ToObject(call.Arguments), + }); + break; + + case FunctionResultContent result: + contents.Add((JsonNode)SerializeResult(result)); + break; + } + } + + if (contents.Count > 0) + { + stateMessages.Add((JsonNode)new JsonObject { ["role"] = message.Role.Value, ["contents"] = contents }); + } + } + + if (stateMessages.Count == 0) + { + Throw.ArgumentException(nameof(messages), "Jev needs at least one message with text or function call content."); + } + + var state = new JsonObject { ["messages"] = stateMessages }; + if (!string.IsNullOrWhiteSpace(instructions)) + { + state["instructions"] = instructions; + } + + return new JevEntry(JevJsonUtilities.Parse(state.ToJsonString())); + } + + /// + /// Gets the index of the first message of the current turn, which follows the last user message that starts a + /// request. + /// + /// + /// User messages that directly follow a function result continue the turn instead of starting one. + /// FunctionInvokingChatClient moves text that was sent together with an approval response, such as "Yes, go + /// ahead", after the approved call and its result. Starting the turn there would hide the approved call from the + /// tool call limit and offer the tool again with no hint that it already ran. + /// + public static int GetCurrentTurnStart(IReadOnlyList messages) + { + int index = messages.Count - 1; + while (index >= 0) + { + if (messages[index].Role != ChatRole.User) + { + index--; + continue; + } + + // Text that followed several approval responses may span consecutive user messages. + int runStart = index; + while (runStart > 0 && messages[runStart - 1].Role == ChatRole.User) + { + runStart--; + } + + if (runStart > 0 && messages[runStart - 1].Contents.Any(content => content is FunctionResultContent)) + { + index = runStart - 1; + continue; + } + + return index + 1; + } + + return 0; + } + + /// Gets the argument sets of the calls in the current turn, by tool name, oldest first. + public static Dictionary>> GetCurrentTurnCalls(IReadOnlyList messages, int turnStart) + { + var calls = new Dictionary>>(StringComparer.Ordinal); + foreach (FunctionCallContent call in CurrentTurnContents(messages, turnStart)) + { + if (string.IsNullOrEmpty(call.Name)) + { + continue; + } + + var arguments = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair argument in call.Arguments ?? new Dictionary()) + { + arguments[argument.Key] = JevJsonUtilities.ToElement(argument.Value); + } + + if (!calls.TryGetValue(call.Name, out List>? list)) + { + calls[call.Name] = list = []; + } + + list.Add(arguments); + } + + return calls; + } + + /// Counts the function calls in the current turn. + public static int CountCurrentTurnCalls(IReadOnlyList messages, int turnStart) => + CurrentTurnContents(messages, turnStart).Count(); + + /// + /// Gets the text of each function result in the current turn, oldest first, unwrapping results of the form + /// {"result": "text"}. + /// + public static List GetCurrentTurnResultTexts(IReadOnlyList messages, int turnStart) => + [.. CurrentTurnContents(messages, turnStart).Select(result => GetResultText(result.Result)).OfType()]; + + private static IEnumerable CurrentTurnContents(IReadOnlyList messages, int turnStart) + where T : AIContent + { + for (int index = turnStart; index < messages.Count; index++) + { + foreach (T content in messages[index].Contents.OfType()) + { + yield return content; + } + } + } + + private static JsonObject ToObject(IDictionary arguments) + { + var json = new JsonObject(); + foreach (KeyValuePair argument in arguments) + { + json[argument.Key] = JevJsonUtilities.ToNodeOrText(argument.Value); + } + + return json; + } + + private static JsonObject SerializeResult(FunctionResultContent result) + { + var json = new JsonObject { ["type"] = "function_result", ["call_id"] = result.CallId }; + IEnumerable? items = result.Result switch + { + AIContent content => [content], + IEnumerable contents => contents, + _ => null, + }; + + if (items is null) + { + json["result"] = JevJsonUtilities.ToNodeOrText(result.Result); + return json; + } + + // Rich results are sent as text items, like the Python connector; other media cannot be described to Jev. + var texts = new List(); + foreach (AIContent item in items) + { + if (item is not TextContent text) + { + throw new NotSupportedException($"Jev supports text function results only; a result of the call '{result.CallId}' contains {item.GetType().Name}."); + } + + texts.Add(text.Text ?? string.Empty); + } + + json["result"] = string.Concat(texts); + var textItems = new JsonArray(); + foreach (string text in texts) + { + textItems.Add((JsonNode)new JsonObject { ["type"] = "text", ["text"] = text }); + } + + json["items"] = textItems; + return json; + } + + private static string? GetResultText(object? result) => + result switch + { + null => null, + string text => Unwrap(text), + JsonElement { ValueKind: JsonValueKind.String } element => Unwrap(element.GetString()!), + JsonElement { ValueKind: JsonValueKind.Null or JsonValueKind.Undefined } => null, + JsonElement element => Unwrap(element.GetRawText()), + TextContent content => content.Text, + IEnumerable contents => string.Concat(contents.OfType().Select(content => content.Text)), + _ => JevJsonUtilities.ToNodeOrText(result)?.ToJsonString(), + }; + + /// + /// Returns the text of a result written as {"result": "text"}, such as an MCP tool result, or the result + /// unchanged. + /// + private static string Unwrap(string result) + { + if (!result.AsSpan().TrimStart().StartsWith("{".AsSpan(), StringComparison.Ordinal)) + { + return result; + } + + try + { + using JsonDocument document = JsonDocument.Parse(result); + JsonElement root = document.RootElement; + if (root.ValueKind == JsonValueKind.Object && + root.EnumerateObject().Count() == 1 && + root.TryGetProperty("result", out JsonElement inner) && + inner.ValueKind == JsonValueKind.String) + { + return inner.GetString()!; + } + } + catch (JsonException) + { + // Text that only looks like JSON is returned as it is. + } + + return result; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs new file mode 100644 index 00000000000..cd4a568618b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// One tool that can be expressed as Jev questions: its route label, its argument questions, and their decoders. +/// +internal sealed class JevCompiledTool +{ + public required AIFunctionDeclaration Function { get; init; } + + /// Gets the label of the tool in the route question, such as t0. + public required string RouteLabel { get; init; } + + public required IReadOnlyDictionary Questions { get; init; } + + public required IReadOnlyList Arguments { get; init; } + + public Dictionary DecodeArguments(JevResult result) + { + var arguments = new Dictionary(this.Arguments.Count); + foreach (JevArgumentPlan argument in this.Arguments) + { + if (argument.TryDecode(result, out JsonElement value)) + { + arguments[argument.Name] = value; + } + } + + return arguments; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs index 3188ff4333c..c27a2db5f0d 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs @@ -9,142 +9,132 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Checks requests before they are evaluated and responses before they are returned to the caller. +/// Checks requests before they are sent and results before they are returned to the caller. /// /// -/// Requests come from a model, so their errors are reported as messages that say how to -/// fix the arguments. Responses come from an evaluator, possibly a custom client with its own mappers, so their errors -/// are reported as ; they point at an evaluator or mapper bug, not at the model. +/// +/// Request rules follow the official TypeSafe SDK (at least one question; a Score rubric of at least two entries) and +/// the limits the API rejects (at most 255 Choice alternatives and 10 Score entries; a Noul needs instructions or +/// criteria; the state and Score rubric entries may not be null, although the SDK types allow it). Checking them here +/// saves a round trip and gives the model a message that says what to fix. +/// +/// +/// Requests come from a model, so their errors are . Results come from the API or from +/// a custom mapping, so their errors are ; they point at a mapping bug, not at +/// the model. +/// /// internal static class JevContractValidator { - /// The most options that the Jev API accepts in one Choice question. - public const int MaxChoiceOptions = 255; - - /// - /// The fewest levels of a Score question. The API also accepts a single level, but the documentation requires two, - /// and a one-level score always returns that level, so it is rejected as a request mistake. - /// - public const int MinScoreLevels = 2; - - /// The most levels that the Jev API accepts in one Score question. - public const int MaxScoreLevels = 10; + public const int MaxChoiceCriteria = 255; + public const int MinScoreCriteria = 2; + public const int MaxScoreCriteria = 10; // Jev computes scores and probabilities in floating point, so values may exceed their bounds by rounding error. private const double Tolerance = 1e-6; public static void ValidateRequest(JevRequest request) { - if (request.State.ValueKind is not (JsonValueKind.String or JsonValueKind.Object or JsonValueKind.Array)) + // The SDK types allow a null state, but the API answers 422 ("Field required"). + if (request.State.IsNull) { - Throw.ArgumentException(nameof(request), "The state must be a string, a JSON object, or a JSON array."); + Throw.ArgumentException(nameof(request), "The state must be text or a JSON object or array."); } if (request.Questions is null || request.Questions.Count == 0) { - Throw.ArgumentException(nameof(request), "The request must contain at least one question."); + Throw.ArgumentException(nameof(request), "At least one question is required."); } foreach (KeyValuePair entry in request.Questions) { string id = entry.Key; - if (string.IsNullOrWhiteSpace(id)) - { - Throw.ArgumentException(nameof(request), "Question IDs must not be empty."); - } - - JevQuestion question = entry.Value; - if (question is null) - { - Throw.ArgumentException(nameof(request), $"Question '{id}' is null."); - } - - if (string.IsNullOrWhiteSpace(question.Instructions)) + switch (entry.Value) { - Throw.ArgumentException(nameof(request), $"Question '{id}' must have instructions."); - } + case null: + Throw.ArgumentException(nameof(request), $"Question '{id}' is null."); + break; - switch (question) - { case JevChoiceQuestion choice: - int options = choice.Criteria?.Count ?? 0; - if (options is < 1 or > MaxChoiceOptions) - { - Throw.ArgumentException(nameof(request), $"Choice question '{id}' has {Count(options, "option")}; it needs from 1 to {MaxChoiceOptions}."); - } - - foreach (string option in choice.Criteria!.Keys) + int alternatives = choice.Criteria?.Count ?? 0; + if (alternatives is < 1 or > MaxChoiceCriteria) { - if (string.IsNullOrWhiteSpace(option)) - { - Throw.ArgumentException(nameof(request), $"Choice question '{id}' has an empty option name."); - } + Throw.ArgumentException(nameof(request), $"Choice question '{id}' has {Count(alternatives, "alternative")}; it needs from 1 to {MaxChoiceCriteria}."); } break; case JevScoreQuestion score: - int levels = score.Criteria?.Count ?? 0; - if (levels is < MinScoreLevels or > MaxScoreLevels) + int scores = score.Criteria?.Count ?? 0; + if (scores is < MinScoreCriteria or > MaxScoreCriteria) { - Throw.ArgumentException(nameof(request), $"Score question '{id}' has {Count(levels, "level")}; it needs from {MinScoreLevels} to {MaxScoreLevels}."); + Throw.ArgumentException(nameof(request), $"Score question '{id}' has {Count(scores, "criterion", "criteria")}; it needs from {MinScoreCriteria} to {MaxScoreCriteria}."); } - foreach (string level in score.Criteria!) + // The SDK types allow null rubric entries, but the API answers 422 for them. + for (int index = 0; index < scores; index++) { - if (string.IsNullOrWhiteSpace(level)) + if (score.Criteria![index].IsNull) { - Throw.ArgumentException(nameof(request), $"Score question '{id}' has an empty level description."); + Throw.ArgumentException(nameof(request), $"Score question '{id}' has a null criterion at index {index.ToString(CultureInfo.InvariantCulture)}; every score needs text or JSON."); } } + break; + + case JevNoulQuestion noul: + if (IsEmpty(noul.Instructions) && IsEmpty(noul.Criteria?.True) && IsEmpty(noul.Criteria?.False)) + { + Throw.ArgumentException(nameof(request), $"Noul question '{id}' needs instructions or criteria."); + } + break; } } } - public static void ValidateResponse(JevRequest request, JevResponse response) + public static void ValidateResult(JevRequest request, JevResult result) { - if (response.Answers is null) + if (result.Answers is null) { - Throw.InvalidOperationException("The Jev response has no answers."); + Throw.InvalidOperationException("The Jev result has no answers."); } foreach (KeyValuePair entry in request.Questions) { string id = entry.Key; - if (!response.Answers.TryGetValue(id, out JevAnswer? answer) || answer is null) + if (!result.Answers.TryGetValue(id, out JevResponse? answer) || answer is null) { - Throw.InvalidOperationException($"The Jev response has no answer for question '{id}'."); + Throw.InvalidOperationException($"The Jev result has no answer for question '{id}'."); } switch (entry.Value, answer) { - case (JevChoiceQuestion question, JevChoiceAnswer choice): - // A choice outside the options would reach the model as a value it never offered, which is how a - // mapper that converts between option names and a client's own values would fail silently. + case (JevChoiceQuestion question, JevChoiceResponse choice): + // A label outside the criteria would reach the model as a value it never offered, which is how a + // mapping that converts between labels and a client's own values would fail silently. if (choice.Choice is null || !question.Criteria.ContainsKey(choice.Choice)) { - Throw.InvalidOperationException($"The answer to question '{id}' chose '{choice.Choice}', which is not one of its options."); + Throw.InvalidOperationException($"The answer to question '{id}' chose '{choice.Choice}', which is not one of its labels."); } EnsureInRange(choice.Confidence, 0, 1, id, "confidence"); - EnsureProbabilities(choice.Probabilities, id, key => question.Criteria.ContainsKey(key), "one of its options"); + EnsureProbabilities(choice.Probabilities, id, key => question.Criteria.ContainsKey(key), "one of its labels"); break; - case (JevScoreQuestion question, JevScoreAnswer score): + case (JevScoreQuestion question, JevScoreResponse score): EnsureInRange(score.Score, 0, question.Criteria.Count - 1, id, "score"); EnsureInRange(score.Confidence, 0, 1, id, "confidence"); - // Score answers key their probabilities and legend by level number ("0", "1", ...). - int levels = question.Criteria.Count; - bool IsLevel(string key) => int.TryParse(key, NumberStyles.None, CultureInfo.InvariantCulture, out int level) && level < levels; - string levelRange = $"a level number from 0 to {(levels - 1).ToString(CultureInfo.InvariantCulture)}"; - EnsureProbabilities(score.Probabilities, id, IsLevel, levelRange); - EnsureKeys(score.Legend?.Keys, id, "legend", IsLevel, levelRange); + // Score responses key their probabilities and legend by score ("0", "1", ...). + int scores = question.Criteria.Count; + bool IsScore(string key) => int.TryParse(key, NumberStyles.None, CultureInfo.InvariantCulture, out int value) && value < scores; + string scoreRange = $"a score from 0 to {(scores - 1).ToString(CultureInfo.InvariantCulture)}"; + EnsureProbabilities(score.Probabilities, id, IsScore, scoreRange); + EnsureKeys(score.Legend?.Keys, id, "legend", IsScore, scoreRange); break; - case (JevNoulQuestion, JevNoulAnswer noul): + case (JevNoulQuestion, JevNoulResponse noul): EnsureInRange(noul.Noul, 0, 1, id, "noul"); break; @@ -155,9 +145,12 @@ public static void ValidateResponse(JevRequest request, JevResponse response) } } + private static bool IsEmpty(JevEntry? entry) => + entry is not { } value || value.IsNull || (value.Kind == JsonValueKind.String && string.IsNullOrWhiteSpace(value.Text)); + /// - /// Checks that every probability is in range and is keyed by something the question offered. A mapper that fills - /// probabilities with a client's own labels would otherwise show the model options it never asked about. + /// Checks that every probability is in range and is keyed by something the question offered. A mapping that fills + /// probabilities with a client's own labels would otherwise show the model alternatives it never asked about. /// private static void EnsureProbabilities(IReadOnlyDictionary? probabilities, string id, Func isValidKey, string expected) { @@ -200,15 +193,15 @@ private static void EnsureInRange(double value, double min, double max, string i } } - private static string Count(int count, string noun) => - count == 1 ? $"1 {noun}" : $"{count.ToString(CultureInfo.InvariantCulture)} {noun}s"; + private static string Count(int count, string singular, string? plural = null) => + count == 1 ? $"1 {singular}" : $"{count.ToString(CultureInfo.InvariantCulture)} {plural ?? singular + "s"}"; private static string KindOf(object value) => value switch { - JevChoiceQuestion or JevChoiceAnswer => "Choice", - JevScoreQuestion or JevScoreAnswer => "Score", - JevNoulQuestion or JevNoulAnswer => "Noul", + JevChoiceQuestion or JevChoiceResponse => "Choice", + JevScoreQuestion or JevScoreResponse => "Score", + JevNoulQuestion or JevNoulResponse => "Noul", _ => value.GetType().Name, }; } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevEntryJsonConverter.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevEntryJsonConverter.cs new file mode 100644 index 00000000000..3287e38784f --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevEntryJsonConverter.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Reads and writes a as the JSON value it holds. +/// +internal sealed class JevEntryJsonConverter : JsonConverter +{ + // A JSON null is a valid entry (an undescribed criterion), so the converter reads it instead of the serializer. + public override bool HandleNull => true; + + public override JevEntry Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + JsonElement value = JsonElement.ParseValue(ref reader); + if (value.ValueKind is JsonValueKind.Number or JsonValueKind.True or JsonValueKind.False) + { + throw new JsonException("A Jev entry must be text, a JSON object or array, or null."); + } + + return new JevEntry(value); + } + + public override void Write(Utf8JsonWriter writer, JevEntry value, JsonSerializerOptions options) + { + if (value.Json is { } json) + { + json.WriteTo(writer); + } + else + { + writer.WriteNullValue(); + } + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpClient.cs similarity index 66% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs rename to dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpClient.cs index 02104e5fed6..c00c821cae2 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpClient.cs @@ -11,21 +11,19 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Evaluates a with the TypeSafe System One API through a System.ClientModel pipeline. +/// Sends a to POST /v1/systemone of the TypeSafe API through a System.ClientModel pipeline, +/// like systemOne of the official TypeSafe SDK. /// -internal sealed class JevHttpEvaluator +internal sealed class JevHttpClient { - private const string DefaultEndpoint = "https://api.typesafe.ai/v1/systemone"; private const int MaxErrorDetailLength = 500; private readonly ClientPipeline _pipeline; - private readonly Uri _endpoint; - private readonly string _modelId; - public JevHttpEvaluator(ApiKeyCredential credential, JevAIToolOptions options) + public JevHttpClient(ApiKeyCredential credential, ClientPipelineOptions options, Uri? endpoint, string modelId) { - this._endpoint = options.Endpoint ?? new Uri(DefaultEndpoint); - this._modelId = options.ModelId; + this.Endpoint = endpoint ?? new Uri(JevPipelineDefaults.DefaultEndpoint); + this.ModelId = modelId; // The credential is applied per try, like the OpenAI client does, so a retried request is signed again. // Creating the pipeline freezes the options, as with any other client built from them. @@ -36,7 +34,22 @@ public JevHttpEvaluator(ApiKeyCredential credential, JevAIToolOptions options) beforeTransportPolicies: []); } - public async Task EvaluateAsync(JevRequest request, IServiceProvider? services, CancellationToken cancellationToken) + /// Gets the System One endpoint that requests are sent to. + public Uri Endpoint { get; } + + /// Gets the model that answers requests unless a call selects another. + public string ModelId { get; } + + /// + /// Answers a request with the configured model. The signature matches the mapping of . + /// + public Task SystemOneAsync(JevRequest request, IServiceProvider? services, CancellationToken cancellationToken) => + this.SystemOneAsync(request, this.ModelId, cancellationToken); + + /// + /// Answers a request with the given model. + /// + public async Task SystemOneAsync(JevRequest request, string modelId, CancellationToken cancellationToken) { using PipelineMessage message = this._pipeline.CreateMessage(); message.ResponseClassifier = JevResponseClassifier.Instance; @@ -44,11 +57,11 @@ public async Task EvaluateAsync(JevRequest request, IServiceProvide PipelineRequest httpRequest = message.Request; httpRequest.Method = "POST"; - httpRequest.Uri = this._endpoint; + httpRequest.Uri = this.Endpoint; httpRequest.Headers.Set("Accept", "application/json"); httpRequest.Headers.Set("Content-Type", "application/json"); httpRequest.Content = BinaryContent.Create(BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes( - new JevApiRequest { State = request.State, Model = this._modelId, Questions = request.Questions }, + new JevApiRequest { State = request.State, Model = modelId, Questions = request.Questions }, JevJsonContext.Default.JevApiRequest))); // Transports leave the cancellation check to the HTTP handler, so a canceled call is stopped here before @@ -69,14 +82,14 @@ public async Task EvaluateAsync(JevRequest request, IServiceProvide response); } - return ReadResponse(response.Content); + return ReadResult(response.Content); } - private static JevResponse ReadResponse(BinaryData content) + private static JevResult ReadResult(BinaryData content) { try { - return JsonSerializer.Deserialize(content.ToMemory().Span, JevJsonContext.Default.JevResponse) + return JsonSerializer.Deserialize(content.ToMemory().Span, JevJsonContext.Default.JevResult) ?? throw new JsonException("The response body is null."); } catch (Exception ex) when (ex is JsonException or NotSupportedException) diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs index 4eb54bccb97..2a7c2e73b64 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs @@ -14,7 +14,8 @@ namespace Microsoft.Agents.AI.TypeSafe; /// [JsonSourceGenerationOptions(AllowOutOfOrderMetadataProperties = true)] [JsonSerializable(typeof(JevRequest))] -[JsonSerializable(typeof(JevResponse))] +[JsonSerializable(typeof(JevResult))] [JsonSerializable(typeof(JevApiRequest))] [JsonSerializable(typeof(Dictionary))] +[JsonSerializable(typeof(string))] internal sealed partial class JevJsonContext : JsonSerializerContext; diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs index 8e8ff710492..affd4f95ec6 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Buffers; +using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization.Metadata; @@ -9,12 +11,16 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Serializer options and JSON schemas shared by the Jev tool. +/// Serializer options, JSON schemas, and JSON value helpers shared by the Jev tool and chat client. /// internal static class JevJsonUtilities { - private static readonly Lazy s_requestSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevRequest), serializerOptions: Options)); - private static readonly Lazy s_responseSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevResponse), serializerOptions: Options)); + private static readonly JsonElement s_true = Parse("true"); + private static readonly JsonElement s_false = Parse("false"); + private static readonly JsonElement s_null = Parse("null"); + private static readonly AIJsonSchemaCreateOptions s_schemaOptions = new() { TransformSchemaNode = DescribeEntries }; + private static readonly Lazy s_requestSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevRequest), serializerOptions: Options, inferenceOptions: s_schemaOptions)); + private static readonly Lazy s_resultSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevResult), serializerOptions: Options, inferenceOptions: s_schemaOptions)); /// /// Gets the options that resolve the Jev contract types first and every other type through @@ -25,8 +31,8 @@ internal static class JevJsonUtilities /// Gets the JSON schema of the tool arguments, which is the schema of . public static JsonElement RequestSchema => s_requestSchema.Value; - /// Gets the JSON schema of the tool result, which is the schema of . - public static JsonElement ResponseSchema => s_responseSchema.Value; + /// Gets the JSON schema of the tool result, which is the schema of . + public static JsonElement ResultSchema => s_resultSchema.Value; /// /// Converts one tool argument into a JSON node. Chat clients pass arguments as values; @@ -47,6 +53,62 @@ internal static class JevJsonUtilities _ => JsonSerializer.SerializeToNode(value, Options.GetTypeInfo(value.GetType())), }; + /// + /// Converts a value from a chat message, such as a function argument or result, into a JSON node. A value that + /// cannot be serialized, for example a type without metadata under native AOT, becomes its text. + /// + public static JsonNode? ToNodeOrText(object? value) + { + try + { + return ToNode(value); + } + catch (Exception ex) when (ex is NotSupportedException or InvalidOperationException or JsonException or ArgumentException) + { + // Chat history is informational for Jev, so an unserializable value is described rather than failing + // the whole request. + return JsonValue.Create(value?.ToString()); + } + } + + /// Converts a value from a chat message into a detached , like . + public static JsonElement ToElement(object? value) => + value switch + { + JsonElement element => element.ValueKind == JsonValueKind.Undefined ? s_null : element.Clone(), + null => s_null, + _ => Parse(ToNodeOrText(value)?.ToJsonString() ?? "null"), + }; + + /// Gets the JSON or value. + public static JsonElement CreateBoolean(bool value) => value ? s_true : s_false; + + /// Writes values into a new JSON array. + public static JsonElement CreateArray(IEnumerable values) + { + var buffer = new ArrayBufferWriter(); + using (var writer = new Utf8JsonWriter(buffer)) + { + writer.WriteStartArray(); + foreach (JsonElement value in values) + { + value.WriteTo(writer); + } + + writer.WriteEndArray(); + } + + using JsonDocument document = JsonDocument.Parse(buffer.WrittenMemory); + return document.RootElement.Clone(); + } + + /// Parses JSON text into a detached . + public static JsonElement Parse(string json) + { + using JsonDocument document = JsonDocument.Parse(json); + return document.RootElement.Clone(); + } + private static JsonSerializerOptions CreateOptions() { var options = new JsonSerializerOptions(JevJsonContext.Default.Options) @@ -57,4 +119,62 @@ private static JsonSerializerOptions CreateOptions() options.MakeReadOnly(); return options; } + + /// + /// Gives the schema of the SDK's EntryType, narrowed to what the API accepts. Its + /// custom converter hides the shape from the schema generator, which would otherwise allow any JSON value, and + /// entries inside collections are only reachable from the collection's node. + /// + /// + /// The API rejects a state and Score rubric entries even though the + /// SDK types allow them, so those schemas leave out; the size limits tell the model how many + /// alternatives and scores it may send. + /// + private static JsonNode DescribeEntries(AIJsonSchemaCreateContext context, JsonNode schema) + { + Type type = context.TypeInfo.Type; + if (type == typeof(JevEntry)) + { + // JevRequest.State is the only non-nullable single entry. + return WithEntryType(schema, allowNull: false); + } + + if (type == typeof(JevEntry?)) + { + return WithEntryType(schema, allowNull: true); + } + + if (type == typeof(IReadOnlyList) && schema is JsonObject rubric) + { + rubric["items"] = WithEntryType(new JsonObject(), allowNull: false); + rubric["minItems"] = JevContractValidator.MinScoreCriteria; + rubric["maxItems"] = JevContractValidator.MaxScoreCriteria; + return rubric; + } + + if (type == typeof(IReadOnlyDictionary) && schema is JsonObject map) + { + // Choice criteria may leave an alternative undescribed; a Score legend echoes the rubric, which may not. + bool isChoiceCriteria = context.PropertyInfo?.Name == "criteria"; + map["additionalProperties"] = WithEntryType(new JsonObject(), allowNull: isChoiceCriteria); + if (isChoiceCriteria) + { + map["minProperties"] = 1; + map["maxProperties"] = JevContractValidator.MaxChoiceCriteria; + } + + return map; + } + + return schema; + } + + private static JsonObject WithEntryType(JsonNode schema, bool allowNull) + { + JsonObject entry = schema as JsonObject ?? []; + entry["type"] = allowNull + ? new JsonArray("string", "object", "array", "null") + : new JsonArray("string", "object", "array"); + return entry; + } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevPipelineDefaults.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevPipelineDefaults.cs new file mode 100644 index 00000000000..06476e6cec8 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevPipelineDefaults.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel.Primitives; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Assigns the pipeline defaults of the official TypeSafe SDK to the options of every Jev client. +/// +internal static class JevPipelineDefaults +{ + public const string DefaultEndpoint = "https://api.typesafe.ai/v1/systemone"; + public const string DefaultModelId = "jev-latest"; + + private static readonly TimeSpan s_defaultNetworkTimeout = TimeSpan.FromSeconds(10); + + /// + /// Sets the SDK retry policy and network timeout where the caller left them unset. + /// + /// + /// Call this from , not from a constructor, because + /// may be set after construction. ClientPipeline.Create + /// freezes the options before it reads them, so the defaults are in place when the pipeline is built. + /// + public static void Apply(ClientPipelineOptions options) + { + options.RetryPolicy ??= new JevRetryPolicy( + JevRetryPolicy.DefaultMaxRetries, + options.ClientLoggingOptions?.EnableLogging ?? true, + options.ClientLoggingOptions?.LoggerFactory); + options.NetworkTimeout ??= s_defaultNetworkTimeout; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs new file mode 100644 index 00000000000..f6fe3f278cf --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Globalization; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Counts the internal questions that tool routing adds to a request, and stops the request when they exceed +/// . +/// +/// +/// Each tool is compiled against a copy of the budget, which is committed only when the tool compiles. A tool that +/// is excluded for an unsupported schema therefore returns the questions it reserved to the tools after it. +/// +internal sealed class JevQuestionBudget +{ + public const int MaxInternalQuestions = 128; + + public JevQuestionBudget(int count = 0) + { + this.Count = count; + } + + public int Count { get; set; } + + public void Reserve(int count) + { + int next = this.Count + count; + if (next > MaxInternalQuestions) + { + // Too many questions is a property of the whole tool set, not of one tool, so it stops the request instead + // of excluding the tool that crossed the limit. + Throw.InvalidOperationException(string.Format( + CultureInfo.InvariantCulture, + "The tools need more than {0} internal Jev questions. Offer fewer tools, or tools with fewer arguments.", + MaxInternalQuestions)); + } + + this.Count = next; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs index adf42d6ac73..8677e243a61 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs @@ -6,17 +6,16 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Classifies System One responses: any 2xx status is a success, and 529 is retriable in addition to the statuses -/// that retries (408, 429, 500, 502, 503, and 504). +/// Classifies System One responses like the official TypeSafe SDK: any 2xx status is a success, and 408, 429, and +/// every 5xx status are retriable. /// /// -/// TypeSafe answers 529 when it is overloaded and documents it as transient, but the default classifier treats it as a -/// permanent error. Every other case, including network failures, is left to the default classifier. +/// only retries 500, 502, 503, and 504 among the 5xx statuses, which +/// leaves out 529, the status TypeSafe answers when it is overloaded. Network failures are left to the default +/// classifier, which retries them, as the SDK does. /// internal sealed class JevResponseClassifier : PipelineMessageClassifier { - private const int OverloadedStatus = 529; - public static JevResponseClassifier Instance { get; } = new(); public override bool TryClassify(PipelineMessage message, out bool isError) @@ -33,7 +32,13 @@ public override bool TryClassify(PipelineMessage message, out bool isError) public override bool TryClassify(PipelineMessage message, Exception? exception, out bool isRetriable) { - isRetriable = exception is null && message.Response?.Status == OverloadedStatus; - return isRetriable; + if (exception is not null || message.Response is not { } response) + { + isRetriable = false; + return false; + } + + isRetriable = response.Status is 408 or 429 or (>= 500 and <= 599); + return true; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs index b54b892ce09..0b5725db567 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs @@ -11,18 +11,30 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// The default retry policy of : the standard , except -/// that it never waits longer than . +/// The default retry policy of the Jev clients, with the defaults of the official TypeSafe SDK +/// (DEFAULT_RETRY_POLICY in retry.ts). /// /// -/// waits for whatever Retry-After the server sends. Inside a tool call that -/// wait blocks the agent run with no feedback, and a very large value makes the delay itself throw, so a longer -/// request fails at once with the service's status instead. +/// +/// Up to retries after the first attempt. +/// +/// A server delay from retry-after-ms or Retry-After is honored up to . +/// A longer delay falls back to backoff, because waiting it would block the agent run with no feedback. +/// +/// +/// Otherwise the delay is an exponential backoff from , capped at , +/// minus up to of random jitter so that concurrent callers do not retry in lockstep. +/// +/// +/// Which statuses are retried is decided by . /// internal sealed class JevRetryPolicy : ClientRetryPolicy { - public const int DefaultMaxRetries = 3; + public const int DefaultMaxRetries = 2; + public const double BackoffJitter = 0.25; + public static readonly TimeSpan InitialBackoff = TimeSpan.FromMilliseconds(500); + public static readonly TimeSpan MaxBackoff = TimeSpan.FromSeconds(5); public static readonly TimeSpan MaxRetryAfter = TimeSpan.FromMinutes(1); public JevRetryPolicy(int maxRetries = DefaultMaxRetries, bool enableLogging = true, ILoggerFactory? loggerFactory = null) @@ -36,11 +48,20 @@ public JevRetryPolicy(int maxRetries = DefaultMaxRetries, bool enableLogging = t /// internal List? RecordedWaitsForTests { get; set; } - protected override bool ShouldRetry(PipelineMessage message, Exception? exception) => - base.ShouldRetry(message, exception) && IsRetryAfterWithinLimit(message); + protected override TimeSpan GetNextDelay(PipelineMessage message, int tryCount) + { + if (TryGetServerDelayMilliseconds(message.Response, out double serverDelay) && serverDelay <= MaxRetryAfter.TotalMilliseconds) + { + return TimeSpan.FromMilliseconds(serverDelay); + } - protected override async ValueTask ShouldRetryAsync(PipelineMessage message, Exception? exception) => - await base.ShouldRetryAsync(message, exception).ConfigureAwait(false) && IsRetryAfterWithinLimit(message); + // tryCount is 1 for the first retry, matching attempt 0 of the SDK's retryDelayMs. + double exponential = Math.Min(InitialBackoff.TotalMilliseconds * Math.Pow(2, tryCount - 1), MaxBackoff.TotalMilliseconds); +#pragma warning disable CA5394 // Jitter only spreads retries over time; it needs no cryptographic randomness. + double jitter = Random.Shared.NextDouble() * BackoffJitter; +#pragma warning restore CA5394 + return TimeSpan.FromMilliseconds(Math.Round(exponential * (1 - jitter))); + } protected override Task WaitAsync(TimeSpan time, CancellationToken cancellationToken) { @@ -64,24 +85,53 @@ protected override void Wait(TimeSpan time, CancellationToken cancellationToken) base.Wait(time, cancellationToken); } - private static bool IsRetryAfterWithinLimit(PipelineMessage message) + /// + /// Reads retry-after-ms, preferred, or Retry-After in seconds or as an HTTP date, like the SDK's + /// parseRetryAfter, as a number of milliseconds. + /// + /// + /// The delay stays a until it is compared with : a huge value such as + /// 1e300 or the text NaN would make throw and fail the call, where the SDK + /// simply backs off. + /// + private static bool TryGetServerDelayMilliseconds(PipelineResponse? response, out double delay) { - if (message.Response?.Headers.TryGetValue("Retry-After", out string? value) is not true || string.IsNullOrWhiteSpace(value)) + delay = 0; + if (response is null) + { + return false; + } + + if (response.Headers.TryGetValue("retry-after-ms", out string? milliseconds) && + double.TryParse(milliseconds, NumberStyles.Float, CultureInfo.InvariantCulture, out double ms) && + double.IsFinite(ms) && ms >= 0) { + delay = ms; return true; } - // Retry-After is either a number of seconds or an HTTP date. - if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double seconds)) + if (!response.Headers.TryGetValue("Retry-After", out string? retryAfter) || string.IsNullOrWhiteSpace(retryAfter)) { - return seconds <= MaxRetryAfter.TotalSeconds; + return false; } - if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset date)) + if (double.TryParse(retryAfter, NumberStyles.Float, CultureInfo.InvariantCulture, out double seconds)) { - return date - DateTimeOffset.UtcNow <= MaxRetryAfter; + if (!double.IsFinite(seconds) || seconds < 0) + { + return false; + } + + delay = seconds * 1000; + return double.IsFinite(delay); + } + + if (DateTimeOffset.TryParse(retryAfter, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset date)) + { + delay = Math.Max(0, (date - DateTimeOffset.UtcNow).TotalMilliseconds); + return true; } - return true; + return false; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs new file mode 100644 index 00000000000..ef5e2807a84 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The tool call that Jev selected: the function and the arguments decoded from its answers. +/// +internal readonly record struct JevToolCall(AIFunctionDeclaration Function, Dictionary Arguments); diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs new file mode 100644 index 00000000000..4661cd6e096 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs @@ -0,0 +1,424 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Turns function tools into Jev questions, so that Jev, which cannot write arguments, can still call a tool. +/// +/// +/// +/// This is a port of the tool routing in the Python TypeSafe connector (_tool_calls.py), with the same +/// question IDs, limits, and wording, so both connectors ask Jev the same questions. +/// +/// +/// A route Choice selects the tool, or none. Each argument must come from a closed set: a constant, an enum +/// (a Choice), a Boolean (a Noul), or an array of enum values (one Noul per value). An optional argument also gets a +/// presence Noul, so that it is passed only when the user asked for it. A tool with any other argument, such as free +/// text or a number, cannot be expressed this way; it is logged and left out, or fails the request when the tool mode +/// requires a call and no tool remains. +/// +/// +internal static class JevToolCallCompiler +{ + public const string QuestionPrefix = "__af_tool__"; + public const string RouteQuestionId = QuestionPrefix + ".route"; + public const string RouteNone = "none"; + public const int MaxRoutableTools = 32; + public const int MaxToolProperties = 64; + public const int MaxEnumValues = 64; + + /// + /// Rejects caller question IDs that could collide with the internal tool questions. + /// + public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) + { + List reserved = [.. questionIds.Where(id => id.StartsWith(QuestionPrefix, StringComparison.Ordinal)).OrderBy(id => id, StringComparer.Ordinal)]; + if (reserved.Count > 0) + { + Throw.ArgumentException("questions", $"Question IDs that start with '{QuestionPrefix}' are reserved for tool calls: {string.Join(", ", reserved)}."); + } + } + + /// + /// Compiles the tools that the tool mode allows into a tool call plan. + /// + /// The function tools of the request. + /// The tool mode of the request; means . + /// + /// The argument sets of the tools already called in the current turn, by tool name, oldest first. They become + /// hints that keep Jev from repeating a call. + /// + /// Receives a warning for each tool that is left out. + /// The plan, or when no tool can be called. + public static JevToolCallPlan? Compile( + IReadOnlyList tools, + ChatToolMode? toolMode, + IReadOnlyDictionary>> previousCalls, + ILogger logger) + { + if (toolMode is NoneChatToolMode) + { + return null; + } + + bool required = toolMode is RequiredChatToolMode; + string? requiredName = (toolMode as RequiredChatToolMode)?.RequiredFunctionName; + if (tools.Count == 0) + { + if (required) + { + Throw.InvalidOperationException("The tool mode requires a tool call, but the request has no tools."); + } + + return null; + } + + List selected = [.. tools.Where(tool => requiredName is null || tool.Name == requiredName)]; + if (requiredName is not null && selected.Count == 0) + { + Throw.InvalidOperationException($"The tool mode requires the tool '{requiredName}', but the request does not have it."); + } + + if (selected.Count > MaxRoutableTools) + { + Throw.InvalidOperationException(string.Format( + CultureInfo.InvariantCulture, + "Jev can choose among at most {0} tools per request, but the request has {1}. Offer fewer tools.", + MaxRoutableTools, + selected.Count)); + } + + var compiled = new List(selected.Count); + var unsupported = new List>(); + var budget = new JevQuestionBudget(); + for (int index = 0; index < selected.Count; index++) + { + AIFunctionDeclaration tool = selected[index]; + + // Each tool reserves questions from a copy of the budget, which is committed only when the tool compiles. + var toolBudget = new JevQuestionBudget(budget.Count); + try + { + List> calls = previousCalls.TryGetValue(tool.Name, out List>? found) ? found : []; + compiled.Add(CompileTool(tool, index, calls, toolBudget)); + budget.Count = toolBudget.Count; + } + catch (JevUnsupportedToolSchemaException ex) + { + unsupported.Add(new(tool.Name, ex.Message)); + } + } + + foreach (KeyValuePair tool in unsupported) + { + logger.LogToolExcluded(tool.Key, tool.Value); + } + + if (required && compiled.Count == 0) + { + string details = string.Join("; ", unsupported.Select(tool => $"{tool.Key}: {tool.Value}")); + Throw.InvalidOperationException($"The tool mode requires a tool call, but no tool can be expressed as Jev questions. {details}".TrimEnd()); + } + + if (compiled.Count == 0) + { + return null; + } + + // With a single tool that must be called, there is nothing to route. + JevCompiledTool? requiredTool = required && compiled.Count == 1 ? compiled[0] : null; + var questions = new Dictionary(StringComparer.Ordinal); + if (requiredTool is null) + { + var criteria = new Dictionary(StringComparer.Ordinal); + foreach (JevCompiledTool tool in compiled) + { + string name = tool.Function.Name; + string description = string.IsNullOrWhiteSpace(tool.Function.Description) + ? $"Call the '{name}' tool." + : $"Tool '{name}': {tool.Function.Description}"; + if (previousCalls.TryGetValue(name, out List>? calls) && calls.Count > 0) + { + description += + " Call this tool again only if the latest user request still needs a distinct execution. " + + $"Do not repeat argument sets already called this turn: {JevToolSchema.Describe(calls)}."; + } + + criteria[tool.RouteLabel] = description; + } + + if (!required) + { + criteria[RouteNone] = "Do not call a tool because all requested tool actions are already satisfied, or because no tool is needed."; + } + + budget.Reserve(1); + questions[RouteQuestionId] = new JevChoiceQuestion + { + Instructions = "Which available tool, if any, should handle the latest user request?", + Criteria = criteria, + }; + } + + foreach (JevCompiledTool tool in compiled) + { + foreach (KeyValuePair question in tool.Questions) + { + questions[question.Key] = question.Value; + } + } + + return new JevToolCallPlan + { + Questions = questions, + Tools = compiled, + RouteQuestionId = requiredTool is null ? RouteQuestionId : null, + RequiredTool = requiredTool, + }; + } + + private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolIndex, List> previousCalls, JevQuestionBudget budget) + { + JsonElement schemaElement = tool.JsonSchema; + Dictionary schema = schemaElement.ValueKind switch + { + JsonValueKind.Object => JevToolSchema.ToMap(schemaElement), + JsonValueKind.Undefined or JsonValueKind.Null => [], + _ => throw new JevUnsupportedToolSchemaException("the top-level input schema must be an object"), + }; + + JevToolSchema.RejectUnsupportedKeys(schema, JevToolSchema.RootKeys, "root-level"); + if (schema.TryGetValue("type", out JsonElement type) && type.ValueKind != JsonValueKind.Null && !(type.ValueKind == JsonValueKind.String && type.ValueEquals("object"))) + { + throw new JevUnsupportedToolSchemaException("the top-level input schema must be an object"); + } + + List properties = []; + if (schema.TryGetValue("properties", out JsonElement propertiesElement) && propertiesElement.ValueKind != JsonValueKind.Null) + { + if (propertiesElement.ValueKind != JsonValueKind.Object) + { + throw new JevUnsupportedToolSchemaException("the input schema properties must be an object"); + } + + properties.AddRange(propertiesElement.EnumerateObject()); + } + + if (properties.Count > MaxToolProperties) + { + throw new JevUnsupportedToolSchemaException(string.Format( + CultureInfo.InvariantCulture, + "the input schema defines {0} properties; the supported maximum is {1}", + properties.Count, + MaxToolProperties)); + } + + var requiredNames = new HashSet(StringComparer.Ordinal); + if (schema.TryGetValue("required", out JsonElement requiredElement)) + { + if (requiredElement.ValueKind != JsonValueKind.Array || requiredElement.EnumerateArray().Any(item => item.ValueKind != JsonValueKind.String)) + { + throw new JevUnsupportedToolSchemaException("the input schema required field must be a string array"); + } + + requiredNames.UnionWith(requiredElement.EnumerateArray().Select(item => item.GetString()!)); + } + + List missing = [.. requiredNames.Where(name => !properties.Any(property => property.Name == name)).OrderBy(name => name, StringComparer.Ordinal)]; + if (missing.Count > 0) + { + throw new JevUnsupportedToolSchemaException($"required arguments are missing from properties: {string.Join(", ", missing)}"); + } + + var questions = new Dictionary(StringComparer.Ordinal); + var arguments = new List(properties.Count); + for (int argumentIndex = 0; argumentIndex < properties.Count; argumentIndex++) + { + JsonProperty property = properties[argumentIndex]; + bool isRequired = requiredNames.Contains(property.Name); + if (property.Value.ValueKind != JsonValueKind.Object) + { + throw new JevUnsupportedToolSchemaException($"argument '{property.Name}' has an invalid schema"); + } + + List previousValues = [.. previousCalls.Where(call => call.ContainsKey(property.Name)).Select(call => call[property.Name])]; + try + { + arguments.Add(CompileArgument(tool, schema, property.Name, JevToolSchema.ToMap(property.Value), isRequired, toolIndex, argumentIndex, previousValues, budget, questions)); + } + catch (JevUnsupportedToolSchemaException ex) + { + throw new JevUnsupportedToolSchemaException($"{(isRequired ? "required" : "optional")} argument '{property.Name}': {ex.Message}", ex); + } + } + + return new JevCompiledTool + { + Function = tool, + RouteLabel = string.Create(CultureInfo.InvariantCulture, $"t{toolIndex}"), + Questions = questions, + Arguments = arguments, + }; + } + + private static JevArgumentPlan CompileArgument( + AIFunctionDeclaration tool, + Dictionary root, + string name, + Dictionary rawSchema, + bool required, + int toolIndex, + int argumentIndex, + List previousValues, + JevQuestionBudget budget, + Dictionary questions) + { + (Dictionary schema, bool nullable) = JevToolSchema.Resolve(rawSchema, root); + JevToolSchema.RejectUnsupportedKeys(schema, JevToolSchema.ArgumentKeys, "argument"); + + // A required argument must be passed, and Jev cannot choose null among the closed-set values, so a required + // nullable argument would always have to be guessed. + if (nullable && required) + { + throw new JevUnsupportedToolSchemaException("required nullable arguments are not supported"); + } + + string prefix = string.Create(CultureInfo.InvariantCulture, $"{QuestionPrefix}.t{toolIndex}.a{argumentIndex}"); + string description = GetText(schema, "description") ?? GetText(rawSchema, "description") ?? GetText(schema, "title") ?? name; + string previousInstruction = previousValues.Count > 0 + ? $" Values already used for this argument in earlier calls this turn: {JevToolSchema.Describe(previousValues)}. Choose a different value when the user requested another distinct call." + : string.Empty; + string? presenceQuestionId = required ? null : prefix + ".present"; + + void AddPresenceQuestion() + { + if (presenceQuestionId is not null) + { + questions[presenceQuestionId] = new JevNoulQuestion + { + Instructions = $"For the {tool.Name} tool, did the user explicitly specify the {name} argument? Argument meaning: {description}.{previousInstruction}", + }; + } + } + + if (schema.TryGetValue("const", out JsonElement constValue)) + { + budget.Reserve(required ? 0 : 1); + AddPresenceQuestion(); + return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = constValue }; + } + + if (schema.TryGetValue("enum", out JsonElement enumElement) && enumElement.ValueKind == JsonValueKind.Array && enumElement.GetArrayLength() > 0) + { + List values = [.. enumElement.EnumerateArray()]; + if (values.Count > MaxEnumValues) + { + throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "enum defines {0} values; the supported maximum is {1}", values.Count, MaxEnumValues)); + } + + if (values.Count == 1) + { + budget.Reserve(required ? 0 : 1); + AddPresenceQuestion(); + return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = values[0] }; + } + + var labeled = new Dictionary(StringComparer.Ordinal); + var criteria = new Dictionary(StringComparer.Ordinal); + for (int index = 0; index < values.Count; index++) + { + string label = string.Create(CultureInfo.InvariantCulture, $"v{index}"); + labeled[label] = values[index]; + criteria[label] = JevToolSchema.Describe(values[index]); + } + + string questionId = prefix + ".value"; + budget.Reserve(required ? 1 : 2); + AddPresenceQuestion(); + questions[questionId] = new JevChoiceQuestion + { + Instructions = $"For the {tool.Name} tool, choose the {name} argument. Argument meaning: {description}.{previousInstruction}", + Criteria = criteria, + }; + return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Choice, PresenceQuestionId = presenceQuestionId, ValueQuestionId = questionId, Values = labeled }; + } + + string? schemaType = schema.TryGetValue("type", out JsonElement typeElement) && typeElement.ValueKind == JsonValueKind.String ? typeElement.GetString() : null; + if (schemaType == "boolean") + { + string questionId = prefix + ".value"; + budget.Reserve(required ? 1 : 2); + AddPresenceQuestion(); + questions[questionId] = new JevNoulQuestion + { + Instructions = $"For the {tool.Name} tool, should the {name} argument be true? Argument meaning: {description}.{previousInstruction}", + }; + return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Boolean, PresenceQuestionId = presenceQuestionId, ValueQuestionId = questionId }; + } + + if (schemaType == "array") + { + if (!schema.TryGetValue("items", out JsonElement itemsElement) || itemsElement.ValueKind != JsonValueKind.Object) + { + throw new JevUnsupportedToolSchemaException("array items must define an enum"); + } + + (Dictionary items, bool itemsNullable) = JevToolSchema.Resolve(JevToolSchema.ToMap(itemsElement), root); + JevToolSchema.RejectUnsupportedKeys(items, JevToolSchema.ArrayItemKeys, "array item"); + if (itemsNullable) + { + throw new JevUnsupportedToolSchemaException("nullable array members are not supported"); + } + + if (!items.TryGetValue("enum", out JsonElement membersElement) || membersElement.ValueKind != JsonValueKind.Array || membersElement.GetArrayLength() == 0) + { + throw new JevUnsupportedToolSchemaException("array items must define a non-empty enum"); + } + + List members = [.. membersElement.EnumerateArray()]; + if (members.Count > MaxEnumValues) + { + throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "array enum defines {0} values; the supported maximum is {1}", members.Count, MaxEnumValues)); + } + + IReadOnlyList itemTypes = JevToolSchema.GetTypes(items.TryGetValue("type", out JsonElement itemType) ? itemType : null, "array item"); + foreach (JsonElement member in members) + { + if (itemTypes.Count > 0 && !itemTypes.Any(candidate => JevToolSchema.MatchesType(member, candidate))) + { + string declared = itemTypes.Count == 1 ? $"\"{itemTypes[0]}\"" : JevToolSchema.Describe(itemTypes.Select(candidate => JevJsonUtilities.ToElement(candidate))); + throw new JevUnsupportedToolSchemaException($"array enum member {JevToolSchema.Describe(member)} does not match declared item type {declared}"); + } + } + + budget.Reserve(members.Count + (required ? 0 : 1)); + AddPresenceQuestion(); + var memberQuestions = new List>(members.Count); + for (int index = 0; index < members.Count; index++) + { + string questionId = string.Create(CultureInfo.InvariantCulture, $"{prefix}.m{index}"); + questions[questionId] = new JevNoulQuestion + { + Instructions = $"For the {tool.Name} tool, should the {name} argument include {JevToolSchema.Describe(members[index])}? Argument meaning: {description}.{previousInstruction}", + }; + memberQuestions.Add(new(questionId, members[index])); + } + + return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Set, PresenceQuestionId = presenceQuestionId, Members = memberQuestions }; + } + + throw new JevUnsupportedToolSchemaException("only const, enum, boolean, and arrays of enum values are supported"); + } + + private static string? GetText(Dictionary schema, string key) => + schema.TryGetValue(key, out JsonElement value) && value.ValueKind == JsonValueKind.String && value.GetString() is { Length: > 0 } text ? text : null; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs new file mode 100644 index 00000000000..e4f0335c806 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The internal questions that let Jev choose one tool call, and the decoders that turn their answers into the call. +/// +internal sealed class JevToolCallPlan +{ + /// + /// A Noul answer above this probability means yes, for presence, Boolean, and array member questions. + /// + public const double NoulYesThreshold = 0.5; + + public required IReadOnlyDictionary Questions { get; init; } + + public required IReadOnlyList Tools { get; init; } + + /// + /// Gets the ID of the Choice question that selects the tool, or when + /// is the only possible call. + /// + public string? RouteQuestionId { get; init; } + + /// + /// Gets the tool that must be called, when the tool mode requires a call and only one tool compiled. + /// + public JevCompiledTool? RequiredTool { get; init; } + + /// + /// Reads the selected tool call from Jev's answers. + /// + /// The tool call, or when Jev chose not to call a tool. + public JevToolCall? Decode(JevResult result) + { + JevCompiledTool? selected = this.RequiredTool; + if (selected is null) + { + string route = GetChoice(result, this.RouteQuestionId!); + if (route == JevToolCallCompiler.RouteNone) + { + return null; + } + + selected = this.Tools.FirstOrDefault(tool => tool.RouteLabel == route) + ?? throw new InvalidOperationException($"Jev returned the unknown tool route '{route}'."); + } + + return new JevToolCall(selected.Function, selected.DecodeArguments(result)); + } + + public static string GetChoice(JevResult result, string questionId) => + result.Answers.TryGetValue(questionId, out JevResponse? answer) && answer is JevChoiceResponse choice + ? choice.Choice + : throw new InvalidOperationException($"The Jev result has no Choice answer for the internal question '{questionId}'."); + + public static bool IsYes(JevResult result, string questionId) => + result.Answers.TryGetValue(questionId, out JevResponse? answer) && answer is JevNoulResponse noul + ? noul.Noul > NoulYesThreshold + : throw new InvalidOperationException($"The Jev result has no Noul answer for the internal question '{questionId}'."); +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs new file mode 100644 index 00000000000..07bcbb61309 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.Json; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Reads the JSON schemas of tool parameters for : which keywords are allowed where, +/// how references and nullable forms resolve, and how values are described to Jev. +/// +internal static class JevToolSchema +{ + // A schema definition may refer to itself; without a limit its resolution would never end. + private const int MaxResolutionDepth = 16; + + private static readonly string[] s_annotationKeys = ["$comment", "$id", "$schema", "default", "deprecated", "description", "examples", "readOnly", "title", "writeOnly"]; + + /// Gets the keywords that do not constrain a value, so they are allowed anywhere. + public static HashSet AnnotationKeys { get; } = new(s_annotationKeys, StringComparer.Ordinal); + + /// Gets the keywords allowed on the object schema of a tool's parameters. + public static HashSet RootKeys { get; } = new(s_annotationKeys.Concat(["$defs", "additionalProperties", "properties", "required", "type"]), StringComparer.Ordinal); + + /// Gets the keywords allowed on the schema of one argument. + public static HashSet ArgumentKeys { get; } = new(s_annotationKeys.Concat(["const", "enum", "items", "type"]), StringComparer.Ordinal); + + /// Gets the keywords allowed on the item schema of an array argument. + public static HashSet ArrayItemKeys { get; } = new(s_annotationKeys.Concat(["enum", "type"]), StringComparer.Ordinal); + + public static Dictionary ToMap(JsonElement schema) + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (JsonProperty property in schema.EnumerateObject()) + { + map[property.Name] = property.Value; + } + + return map; + } + + /// + /// Resolves a local $ref, an anyOf of one schema and null, and the nullable forms that + /// System.Text.Json writes, which are a type array that includes "null" and an enum that lists + /// . + /// + /// The schema without its alternative, and whether it allowed . + public static (Dictionary Schema, bool Nullable) Resolve(Dictionary schema, Dictionary root, int depth = 0) + { + if (depth > MaxResolutionDepth) + { + throw new JevUnsupportedToolSchemaException("schema references are nested too deeply"); + } + + var resolved = new Dictionary(schema, StringComparer.Ordinal); + if (resolved.Remove("$ref", out JsonElement reference)) + { + RejectCompositionSiblings(resolved, "$ref"); + string? path = reference.ValueKind == JsonValueKind.String ? reference.GetString() : null; + const string DefinitionsPrefix = "#/$defs/"; + if (path?.StartsWith(DefinitionsPrefix, StringComparison.Ordinal) is not true) + { + throw new JevUnsupportedToolSchemaException("only local $defs references are supported"); + } + + if (!root.TryGetValue("$defs", out JsonElement definitions) || + definitions.ValueKind != JsonValueKind.Object || + !definitions.TryGetProperty(path.Substring(DefinitionsPrefix.Length), out JsonElement definition) || + definition.ValueKind != JsonValueKind.Object) + { + throw new JevUnsupportedToolSchemaException($"unresolved schema reference '{path}'"); + } + + // Annotations next to the reference, such as a description, override those of the definition. An anyOf + // inside the definition is resolved below, together with the rest of the merged schema. + resolved = Merge(ToMap(definition), resolved); + } + + bool nullable = false; + if (resolved.Remove("anyOf", out JsonElement anyOf)) + { + RejectCompositionSiblings(resolved, "anyOf"); + if (anyOf.ValueKind != JsonValueKind.Array) + { + throw new JevUnsupportedToolSchemaException("anyOf must be an array"); + } + + List arms = [.. anyOf.EnumerateArray()]; + List nullArms = [.. arms.Where(arm => arm.ValueKind == JsonValueKind.Object && IsNullSchema(arm))]; + List valueArms = [.. arms.Where(arm => arm.ValueKind == JsonValueKind.Object && !IsNullSchema(arm))]; + if (arms.Count != 2 || nullArms.Count != 1 || valueArms.Count != 1) + { + throw new JevUnsupportedToolSchemaException("only a single schema combined with null is supported"); + } + + (Dictionary nested, _) = Resolve(ToMap(valueArms[0]), root, depth + 1); + resolved = Merge(nested, resolved); + nullable = true; + } + + if (resolved.TryGetValue("type", out JsonElement type) && type.ValueKind == JsonValueKind.Array && type.EnumerateArray().Any(IsNullName)) + { + List valueTypes = [.. type.EnumerateArray().Where(item => !IsNullName(item))]; + resolved["type"] = valueTypes.Count == 1 ? valueTypes[0] : JevJsonUtilities.CreateArray(valueTypes); + nullable = true; + } + + if (resolved.TryGetValue("enum", out JsonElement values) && values.ValueKind == JsonValueKind.Array && values.EnumerateArray().Any(value => value.ValueKind == JsonValueKind.Null)) + { + resolved["enum"] = JevJsonUtilities.CreateArray(values.EnumerateArray().Where(value => value.ValueKind != JsonValueKind.Null)); + nullable = true; + } + + return (resolved, nullable); + } + + public static void RejectUnsupportedKeys(Dictionary schema, HashSet supportedKeys, string location) + { + List unsupported = [.. schema.Keys.Where(key => !supportedKeys.Contains(key)).OrderBy(key => key, StringComparer.Ordinal)]; + if (unsupported.Count > 0) + { + throw new JevUnsupportedToolSchemaException($"unsupported {location} schema constraints: {string.Join(", ", unsupported)}"); + } + } + + /// + /// Reads a type keyword, which is a type name or a non-empty array of type names. + /// + public static IReadOnlyList GetTypes(JsonElement? type, string location) + { + if (type is not { } value || value.ValueKind == JsonValueKind.Null) + { + return []; + } + + if (value.ValueKind == JsonValueKind.String) + { + return [value.GetString()!]; + } + + if (value.ValueKind == JsonValueKind.Array && value.GetArrayLength() > 0 && value.EnumerateArray().All(item => item.ValueKind == JsonValueKind.String)) + { + return [.. value.EnumerateArray().Select(item => item.GetString()!)]; + } + + throw new JevUnsupportedToolSchemaException($"{location} type must be a string or non-empty string array"); + } + + public static bool MatchesType(JsonElement value, string type) => + type switch + { + "string" => value.ValueKind == JsonValueKind.String, + + // An integer has no fraction or exponent, as a JSON parser that separates integers from floats reads it. + "integer" => value.ValueKind == JsonValueKind.Number && value.TryGetInt64(out _), + "number" => value.ValueKind == JsonValueKind.Number, + "boolean" => value.ValueKind is JsonValueKind.True or JsonValueKind.False, + "null" => value.ValueKind == JsonValueKind.Null, + "array" => value.ValueKind == JsonValueKind.Array, + "object" => value.ValueKind == JsonValueKind.Object, + _ => false, + }; + + /// + /// Writes a value the way the Python connector does with json.dumps(value, ensure_ascii=False, sort_keys=True): + /// sorted object keys, ", " and ": " separators, and unescaped text. Jev reads these values in + /// criteria and instructions, so both connectors send the same words. + /// + public static string Describe(JsonElement value) + { + var builder = new StringBuilder(); + AppendValue(builder, value); + return builder.ToString(); + } + + /// Writes values as a JSON array, like . + public static string Describe(IEnumerable values) + { + var builder = new StringBuilder(); + AppendArray(builder, values, AppendValue); + return builder.ToString(); + } + + /// Writes the argument sets of earlier calls as a JSON array of objects, like . + public static string Describe(IEnumerable> calls) + { + var builder = new StringBuilder(); + AppendArray(builder, calls, static (target, call) => AppendObject(target, call.Select(argument => new KeyValuePair(argument.Key, argument.Value)))); + return builder.ToString(); + } + + private static void RejectCompositionSiblings(Dictionary schema, string keyword) + { + List siblings = [.. schema.Keys.Where(key => !AnnotationKeys.Contains(key)).OrderBy(key => key, StringComparer.Ordinal)]; + if (siblings.Count > 0) + { + throw new JevUnsupportedToolSchemaException($"{keyword} sibling constraints are not supported: {string.Join(", ", siblings)}"); + } + } + + private static Dictionary Merge(Dictionary target, Dictionary overrides) + { + foreach (KeyValuePair entry in overrides) + { + target[entry.Key] = entry.Value; + } + + return target; + } + + private static bool IsNullSchema(JsonElement schema) => + schema.TryGetProperty("type", out JsonElement type) && IsNullName(type); + + private static bool IsNullName(JsonElement type) => + type.ValueKind == JsonValueKind.String && type.ValueEquals("null"); + + private static void AppendValue(StringBuilder builder, JsonElement value) + { + switch (value.ValueKind) + { + case JsonValueKind.Object: + AppendObject(builder, value.EnumerateObject().Select(property => new KeyValuePair(property.Name, property.Value))); + break; + + case JsonValueKind.Array: + AppendArray(builder, value.EnumerateArray(), AppendValue); + break; + + case JsonValueKind.String: + AppendString(builder, value.GetString()!); + break; + + case JsonValueKind.Null or JsonValueKind.Undefined: + builder.Append("null"); + break; + + default: + // Numbers keep their original text, and true and false are already written as JSON. + builder.Append(value.GetRawText()); + break; + } + } + + private static void AppendObject(StringBuilder builder, IEnumerable> properties) + { + builder.Append('{'); + bool first = true; + foreach (KeyValuePair property in properties.OrderBy(property => property.Key, StringComparer.Ordinal)) + { + builder.Append(first ? string.Empty : ", "); + first = false; + AppendString(builder, property.Key); + builder.Append(": "); + AppendValue(builder, property.Value); + } + + builder.Append('}'); + } + + private static void AppendArray(StringBuilder builder, IEnumerable items, Action append) + { + builder.Append('['); + bool first = true; + foreach (T item in items) + { + builder.Append(first ? string.Empty : ", "); + first = false; + append(builder, item); + } + + builder.Append(']'); + } + + /// + /// Writes a JSON string with the escapes of Python's ensure_ascii=False: quotes, backslashes, and control + /// characters only. + /// + private static void AppendString(StringBuilder builder, string text) + { + builder.Append('"'); + foreach (char c in text) + { + switch (c) + { + case '"': builder.Append("\\\""); break; + case '\\': builder.Append("\\\\"); break; + case '\n': builder.Append("\\n"); break; + case '\r': builder.Append("\\r"); break; + case '\t': builder.Append("\\t"); break; + case '\b': builder.Append("\\b"); break; + case '\f': builder.Append("\\f"); break; + case < ' ': builder.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); break; + default: builder.Append(c); break; + } + } + + builder.Append('"'); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs new file mode 100644 index 00000000000..fe3ecfc72d9 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// A tool's JSON schema cannot be expressed as Jev questions, so the tool is left out of the request. +/// +/// +/// It is caught while a tool call plan is compiled and never leaves this package: an unsupported tool is logged and +/// excluded, or, when the tool mode requires a call and no tool remains, its reason becomes part of an +/// . +/// +internal sealed class JevUnsupportedToolSchemaException : NotSupportedException +{ + public JevUnsupportedToolSchemaException() + { + } + + public JevUnsupportedToolSchemaException(string message) + : base(message) + { + } + + public JevUnsupportedToolSchemaException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs index d1bb421f9c7..8dd97a2d0c8 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs @@ -18,16 +18,16 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// Jev does not write text. It answers typed questions about a state and returns calibrated probabilities, which /// makes it a fast and inexpensive classifier for an agent whose own model writes the replies. The function created by -/// is a thin wrapper around an evaluator: its arguments are a written by -/// the agent's model, and its result is the evaluator's , as JSON. +/// is a thin wrapper around a System One call: its arguments are a written by +/// the agent's model, and its result is the call's , as JSON. /// /// -/// With an , the evaluator is the TypeSafe System One API, called through a +/// With an , the call goes to the TypeSafe System One API through a /// System.ClientModel pipeline configured by . Any other client can be used instead with -/// or one of the UseClient methods. +/// or one of the UseClient methods. /// /// -/// Every evaluator receives the of the agent that calls the tool +/// Every mapping receives the of the agent that calls the tool /// (, which FunctionInvokingChatClient sets from the services /// the agent was created with), so it can resolve its dependencies per call. In a host, register the tool with /// services.AddAIAgent(...).WithAITool(sp => new JevAIToolBuilder(credential, options).Build()). @@ -37,10 +37,10 @@ public sealed class JevAIToolBuilder { private readonly ApiKeyCredential? _credential; private readonly JevAIToolOptions _options; - private Func>? _evaluateAsync; + private Func>? _systemOneAsync; /// - /// Initializes a new instance of the class that evaluates requests with the + /// Initializes a new instance of the class that answers requests with the /// TypeSafe System One API. /// /// The TypeSafe API key, sent as a bearer token. @@ -52,12 +52,12 @@ public JevAIToolBuilder(ApiKeyCredential credential, JevAIToolOptions? options = } /// - /// Initializes a new instance of the class for a custom evaluator, which must be set - /// with or a UseClient method before . + /// Initializes a new instance of the class for a custom mapping, which must be set + /// with or a UseClient method before . /// /// /// The tool options. Only and apply - /// to a custom evaluator, which uses its own client configuration. Defaults are used when . + /// to a custom mapping, which uses its own client configuration. Defaults are used when . /// public JevAIToolBuilder(JevAIToolOptions? options = null) { @@ -65,32 +65,33 @@ public JevAIToolBuilder(JevAIToolOptions? options = null) } /// - /// Sets the evaluator that answers each request, replacing the TypeSafe API. + /// Sets the function that maps each to its , replacing the + /// TypeSafe API. /// - /// - /// The evaluator. It receives the written by the model and the agent's services, which are + /// + /// The mapping. It receives the written by the model and the agent's services, which are /// when the caller provides none, and returns one answer per question. /// /// This builder. /// /// The agent's services are often the application's root provider, because hosted agents are singletons by - /// default. To use a scoped service, create a scope inside the evaluator, or use + /// default. To use a scoped service, create a scope inside the mapping, or use /// , which does so for every call. /// - public JevAIToolBuilder UseEvaluator(Func> evaluateAsync) + public JevAIToolBuilder UseMapping(Func> mapAsync) { - this._evaluateAsync = Throw.IfNull(evaluateAsync); + this._systemOneAsync = Throw.IfNull(mapAsync); return this; } /// - /// Evaluates requests with another client, converting between the Jev contract types and the client's own types. + /// Answers requests with another client, converting between the Jev contract types and the client's own types. /// /// The request type of the client. /// The response type of the client. - /// The client operation that evaluates a request. + /// The client operation that answers a request. /// Converts the written by the model into the client's request. - /// Converts the client's response into a . + /// Converts the client's response into a . /// This builder. /// /// The mapped response is checked against the request before it reaches the model: every question needs an answer @@ -99,51 +100,51 @@ public JevAIToolBuilder UseEvaluator(Func public JevAIToolBuilder UseClient( - Func> evaluateAsync, + Func> systemOneAsync, Func inputMapper, - Func outputMapper) + Func outputMapper) { - _ = Throw.IfNull(evaluateAsync); + _ = Throw.IfNull(systemOneAsync); _ = Throw.IfNull(inputMapper); _ = Throw.IfNull(outputMapper); - return this.UseEvaluator(async (request, _, cancellationToken) => + return this.UseMapping(async (request, _, cancellationToken) => { TClientRequest clientRequest = inputMapper(request); - TClientResponse clientResponse = await evaluateAsync(clientRequest, cancellationToken).ConfigureAwait(false); + TClientResponse clientResponse = await systemOneAsync(clientRequest, cancellationToken).ConfigureAwait(false); return outputMapper(clientResponse); }); } /// - /// Evaluates requests with a client registered in the agent's services, converting between the Jev contract types + /// Answers requests with a client registered in the agent's services, converting between the Jev contract types /// and the client's own types. /// /// The client type, resolved from the agent's services on every call. /// The request type of the client. /// The response type of the client. - /// The client operation that evaluates a request. + /// The client operation that answers a request. /// Converts the written by the model into the client's request. - /// Converts the client's response into a . + /// Converts the client's response into a . /// This builder. /// /// The client is resolved from a new service scope on every call, and the scope is disposed when the call ends. /// Scoped clients therefore live for one call, and disposable transient clients are released after it, even when /// the agent's services are the application's root provider. The call fails when the agent has no services or /// is not registered. The mapped response is checked as described for - /// . + /// . /// public JevAIToolBuilder UseClient( - Func> evaluateAsync, + Func> systemOneAsync, Func inputMapper, - Func outputMapper) + Func outputMapper) where TClient : notnull { - _ = Throw.IfNull(evaluateAsync); + _ = Throw.IfNull(systemOneAsync); _ = Throw.IfNull(inputMapper); _ = Throw.IfNull(outputMapper); - return this.UseEvaluator(async (request, services, cancellationToken) => + return this.UseMapping(async (request, services, cancellationToken) => { if (services is null) { @@ -157,16 +158,16 @@ public JevAIToolBuilder UseClient( // transient client until shutdown, so each call gets its own scope. if (services.GetService() is not { } scopeFactory) { - return await EvaluateWithClientAsync(services).ConfigureAwait(false); + return await CallClientAsync(services).ConfigureAwait(false); } AsyncServiceScope scope = scopeFactory.CreateAsyncScope(); await using (scope.ConfigureAwait(false)) { - return await EvaluateWithClientAsync(scope.ServiceProvider).ConfigureAwait(false); + return await CallClientAsync(scope.ServiceProvider).ConfigureAwait(false); } - async Task EvaluateWithClientAsync(IServiceProvider provider) + async Task CallClientAsync(IServiceProvider provider) { if (provider.GetService(typeof(TClient)) is not TClient client) { @@ -175,7 +176,7 @@ async Task EvaluateWithClientAsync(IServiceProvider provider) } TClientRequest clientRequest = inputMapper(request); - TClientResponse clientResponse = await evaluateAsync(client, clientRequest, cancellationToken).ConfigureAwait(false); + TClientResponse clientResponse = await systemOneAsync(client, clientRequest, cancellationToken).ConfigureAwait(false); return outputMapper(clientResponse); } }); @@ -186,10 +187,10 @@ async Task EvaluateWithClientAsync(IServiceProvider provider) /// /// /// An whose JSON schema is the schema of and whose result is the - /// evaluator's serialized as JSON. + /// call's serialized as JSON. /// /// - /// The builder has neither an nor a custom evaluator, or an option that it uses is + /// The builder has neither an nor a custom mapping, or an option that it uses is /// empty or invalid. /// /// @@ -204,15 +205,15 @@ public AIFunction Build() Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.Name)} and {nameof(JevAIToolOptions.Description)} must not be empty."); } - Func>? evaluateAsync = this._evaluateAsync; - if (evaluateAsync is null) + Func>? systemOneAsync = this._systemOneAsync; + if (systemOneAsync is null) { if (this._credential is null) { - Throw.InvalidOperationException($"Pass an {nameof(ApiKeyCredential)} to the constructor to use the TypeSafe API, or call {nameof(UseEvaluator)} or UseClient."); + Throw.InvalidOperationException($"Pass an {nameof(ApiKeyCredential)} to the constructor to use the TypeSafe API, or call {nameof(UseMapping)} or UseClient."); } - // The model and endpoint only matter to the TypeSafe API; a custom evaluator uses its own configuration. + // The model and endpoint only matter to the TypeSafe API; a custom mapping uses its own configuration. if (string.IsNullOrWhiteSpace(this._options.ModelId)) { Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.ModelId)} must not be empty."); @@ -223,9 +224,9 @@ public AIFunction Build() Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.Endpoint)} must be an absolute URI."); } - evaluateAsync = new JevHttpEvaluator(this._credential, this._options).EvaluateAsync; + systemOneAsync = new JevHttpClient(this._credential, this._options, this._options.Endpoint, this._options.ModelId).SystemOneAsync; } - return new JevAIFunction(this._options.Name, this._options.Description, evaluateAsync); + return new JevAIFunction(this._options.Name, this._options.Description, systemOneAsync); } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs index 7f08031b762..e0239af7324 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs @@ -19,13 +19,15 @@ namespace Microsoft.Agents.AI.TypeSafe; /// adds custom policies. The pipeline settings, /// , and apply only when the builder uses the TypeSafe API, that is, when /// it receives an ; and apply to every -/// evaluator. +/// mapping. /// /// -/// When is , the pipeline retries up to three -/// times on the statuses that TypeSafe documents as transient (429 and 529, besides the standard 408 and 5xx ones), and -/// fails at once when the API asks for a wait longer than one minute, because the wait would block the agent run -/// without any feedback. That default follows . +/// Unset pipeline settings get the defaults of the official TypeSafe SDK. When +/// is , up to two retries follow 408, 429, and +/// 5xx responses and network failures, waiting for the server's retry-after-ms or Retry-After delay up +/// to one minute, or else for an exponential backoff from half a second to five seconds; the policy follows +/// . When +/// is , each attempt times out after 10 seconds. /// /// /// Like other client options, an instance becomes read-only once a tool has been built with it. @@ -34,23 +36,17 @@ namespace Microsoft.Agents.AI.TypeSafe; public sealed class JevAIToolOptions : ClientPipelineOptions { private const string DefaultDescription = - "Evaluates content with Jev, a classification model. Jev does not write text: it answers typed questions about the " + - "given state and returns calibrated probabilities. Use it to classify, route, score, or check facts about text or " + - "JSON data, and ask all related questions in one call. A 'choice' question picks exactly one option from its " + - "criteria. A 'score' question rates the state against 2 to 10 ordered levels and can land between levels. A 'noul' " + - "question returns the probability, from 0 to 1, that its yes or no statement is true. Choice and score answers " + - "include a confidence from 0 to 1; treat a low confidence as uncertain."; + "Answers named questions about text or structured state with Jev, a classification model. Jev does not write " + + "text: it returns calibrated probabilities. Use it to classify, route, score, or check facts, and ask all related " + + "questions in one call. A 'choice' question selects one of the named alternatives in its criteria. A 'score' " + + "question assigns a score using an ordered rubric of 2 to 10 entries, and the score can fall between entries. A " + + "'noul' question returns the probability that the answer to a yes or no question is yes. Choice and score " + + "answers include a confidence from 0 to 1; treat a low confidence as uncertain."; /// public override void Freeze() { - // The default policy is assigned here, not in the constructor, because ClientLoggingOptions may be set after - // construction; ClientPipeline.Create freezes the options before it reads RetryPolicy, so the policy is in place. - this.RetryPolicy ??= new JevRetryPolicy( - JevRetryPolicy.DefaultMaxRetries, - this.ClientLoggingOptions?.EnableLogging ?? true, - this.ClientLoggingOptions?.LoggerFactory); - + JevPipelineDefaults.Apply(this); base.Freeze(); } @@ -78,10 +74,11 @@ public string ModelId this.AssertNotFrozen(); field = value; } - } = "jev-latest"; + } = JevPipelineDefaults.DefaultModelId; /// - /// Gets or sets the name of the function that the model sees. Defaults to evaluate_with_jev. + /// Gets or sets the name of the function that the model sees. Defaults to ask_system_one, after the SDK's + /// systemOne operation. /// public string Name { @@ -91,7 +88,7 @@ public string Name this.AssertNotFrozen(); field = value; } - } = "evaluate_with_jev"; + } = "ask_system_one"; /// /// Gets or sets the description of the function that the model sees. diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs deleted file mode 100644 index 77a1adf5e55..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json.Serialization; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Represents Jev's answer to one question of a . -/// -/// -/// The JSON type property matches the kind of the question: choice (), -/// score (), or noul (). -/// -[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] -[JsonDerivedType(typeof(JevChoiceAnswer), "choice")] -[JsonDerivedType(typeof(JevScoreAnswer), "score")] -[JsonDerivedType(typeof(JevNoulAnswer), "noul")] -public abstract class JevAnswer -{ - // Only the three answer kinds that the Jev API returns can exist. - private protected JevAnswer() - { - } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs new file mode 100644 index 00000000000..aa91948f13d --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Provides an for Jev, the TypeSafe System One model, which answers typed questions about +/// a conversation instead of writing text. +/// +/// +/// +/// Each request sends the conversation as the Jev state, together with the questions passed with +/// (or ). +/// The response text is the as JSON, and +/// returns it typed. This follows the TypeSafe connector for Python, so both languages behave alike. +/// +/// +/// Jev can also call function tools whose arguments come from a closed set: constants, enums, Booleans, and arrays of +/// enum values. The client adds internal questions that choose the tool and its arguments, and returns a +/// for a function-invoking client (which a ChatClientAgent adds) to run. +/// Once the turn has used calls, the client answers the +/// questions instead; the response text is then the tool results followed by one id: answer line per Choice +/// and Score answer. Tools with free-form arguments, such as text or numbers, are left out with a warning. +/// +/// +/// Features that need generated text fail with : streaming, sampling and output +/// options such as or a JSON , +/// server-side conversations and background responses, tools other than functions, and content other than text, +/// reasoning text, and function calls and results. Tool approval requests and responses are left out of the state, +/// because the call and result they lead to carry the information. +/// +/// +public sealed class JevChatClient : IChatClient +{ + private const string ProviderName = "typesafe.ai"; + + private readonly JevHttpClient _client; + private readonly IReadOnlyDictionary? _defaultQuestions; + private readonly int _maximumToolCallsPerTurn; + private readonly ILogger _logger; + private readonly ChatClientMetadata _metadata; + + /// + /// Initializes a new instance of the class. + /// + /// The TypeSafe API key, sent as a bearer token. + /// The client and pipeline options. Defaults are used when . + /// + /// is empty, or is not an + /// absolute URI. + /// + public JevChatClient(ApiKeyCredential credential, JevChatClientOptions? options = null) + { + _ = Throw.IfNull(credential); + options ??= new JevChatClientOptions(); + + if (string.IsNullOrWhiteSpace(options.ModelId)) + { + Throw.ArgumentException(nameof(options), $"{nameof(JevChatClientOptions)}.{nameof(JevChatClientOptions.ModelId)} must not be empty."); + } + + if (options.Endpoint?.IsAbsoluteUri is false) + { + Throw.ArgumentException(nameof(options), $"{nameof(JevChatClientOptions)}.{nameof(JevChatClientOptions.Endpoint)} must be an absolute URI."); + } + + // The questions are copied so that later changes to the caller's dictionary do not change this client. + this._defaultQuestions = options.DefaultQuestions?.ToDictionary(question => question.Key, question => question.Value, StringComparer.Ordinal); + this._maximumToolCallsPerTurn = options.MaximumToolCallsPerTurn; + this._logger = (options.ClientLoggingOptions?.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); + this._client = new JevHttpClient(credential, options, options.Endpoint, options.ModelId); + this._metadata = new ChatClientMetadata(ProviderName, this._client.Endpoint, this._client.ModelId); + } + + /// + /// + /// The request has no questions, a required tool call is impossible, or the Jev result does not answer every + /// question. + /// + /// The request uses an option, tool, or content that Jev does not support. + /// The TypeSafe API rejected the request. + public async Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + IReadOnlyList history = messages as IReadOnlyList ?? [.. messages]; + + EnsureSupported(options); + IReadOnlyDictionary questions = this.GetQuestions(options); + JevToolCallCompiler.EnsureNoReservedQuestionIds(questions.Keys); + List tools = GetFunctionTools(options); + + // Once the turn has used its tool calls, tools are no longer offered, so the next response answers the + // questions with the tool results in the state. This is what ends the function-invoking loop, like the + // max_function_calls default of one in the Python connector. + int turnStart = JevChatState.GetCurrentTurnStart(history); + ChatToolMode? toolMode = JevChatState.CountCurrentTurnCalls(history, turnStart) >= this._maximumToolCallsPerTurn + ? ChatToolMode.None + : options?.ToolMode; + JevToolCallPlan? plan = JevToolCallCompiler.Compile(tools, toolMode, JevChatState.GetCurrentTurnCalls(history, turnStart), this._logger); + + var allQuestions = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair question in questions) + { + allQuestions[question.Key] = question.Value; + } + + foreach (KeyValuePair question in plan?.Questions ?? new Dictionary()) + { + allQuestions[question.Key] = question.Value; + } + + var request = new JevRequest { State = JevChatState.Build(history, options?.Instructions), Questions = allQuestions }; + JevContractValidator.ValidateRequest(request); + + string modelId = string.IsNullOrWhiteSpace(options?.ModelId) ? this._client.ModelId : options!.ModelId!; + JevResult result = await this._client.SystemOneAsync(request, modelId, cancellationToken).ConfigureAwait(false); + + // Validation covers the internal questions too, so the decoders below only see labels they offered. + JevContractValidator.ValidateResult(request, result); + UsageDetails? usage = result.Usage is { } tokens + ? new UsageDetails { InputTokenCount = tokens.InputTokens, OutputTokenCount = tokens.OutputTokens, TotalTokenCount = tokens.InputTokens + tokens.OutputTokens } + : null; + + if (plan?.Decode(result) is { } call) + { + var callMessage = new ChatMessage(ChatRole.Assistant, [new FunctionCallContent($"typesafe-{Guid.NewGuid():N}", call.Function.Name, call.Arguments)]); + return new ChatResponse(callMessage) + { + ModelId = result.Model, + FinishReason = ChatFinishReason.ToolCalls, + Usage = usage, + CreatedAt = DateTimeOffset.UtcNow, + }; + } + + // The caller asked only its own questions, so the internal tool answers are removed from what it sees. + JevResult answers = WithoutInternalAnswers(result, questions); + var message = new ChatMessage(ChatRole.Assistant, BuildText(history, turnStart, answers)) { RawRepresentation = answers }; + return new ChatResponse(message) + { + ModelId = result.Model, + FinishReason = ChatFinishReason.Stop, + Usage = usage, + CreatedAt = DateTimeOffset.UtcNow, + RawRepresentation = answers, + }; + } + + /// + /// Always: Jev answers all questions at once and does not stream. + public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) => + throw new NotSupportedException($"Jev answers all questions at once and does not stream. Call {nameof(GetResponseAsync)}, or RunAsync on an agent."); + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + return serviceKey is not null ? null + : serviceType == typeof(ChatClientMetadata) ? this._metadata + : serviceType.IsInstanceOfType(this) ? this + : null; + } + + /// + public void Dispose() + { + // The pipeline owns no disposable resources; its default transport shares one HttpClient per process. + } + + private static void EnsureSupported(ChatOptions? options) + { + if (options is null) + { + return; + } + + var unsupported = new List(); + AddIf(options.Temperature is not null, nameof(ChatOptions.Temperature)); + AddIf(options.TopP is not null, nameof(ChatOptions.TopP)); + AddIf(options.TopK is not null, nameof(ChatOptions.TopK)); + AddIf(options.MaxOutputTokens is not null, nameof(ChatOptions.MaxOutputTokens)); + AddIf(options.FrequencyPenalty is not null, nameof(ChatOptions.FrequencyPenalty)); + AddIf(options.PresencePenalty is not null, nameof(ChatOptions.PresencePenalty)); + AddIf(options.Seed is not null, nameof(ChatOptions.Seed)); + AddIf(options.Reasoning is not null, nameof(ChatOptions.Reasoning)); + AddIf(options.StopSequences is { Count: > 0 }, nameof(ChatOptions.StopSequences)); + AddIf(options.ResponseFormat is not (null or ChatResponseFormatText), nameof(ChatOptions.ResponseFormat)); + AddIf(options.ConversationId is not null, nameof(ChatOptions.ConversationId)); + AddIf(options.AllowBackgroundResponses is true, nameof(ChatOptions.AllowBackgroundResponses)); +#pragma warning disable MEAI001 // ContinuationToken is experimental; it is only checked to reject it. + AddIf(options.ContinuationToken is not null, nameof(ChatOptions.ContinuationToken)); +#pragma warning restore MEAI001 + + if (unsupported.Count > 0) + { + throw new NotSupportedException( + $"Jev does not support these chat options: {string.Join(", ", unsupported)}. Jev does not generate text; it answers the questions passed with {nameof(JevChatOptionsExtensions.WithJevQuestions)}."); + } + + if (options.AllowMultipleToolCalls is true) + { + throw new NotSupportedException($"Jev requests one tool call per response. Leave {nameof(ChatOptions)}.{nameof(ChatOptions.AllowMultipleToolCalls)} unset or false."); + } + + void AddIf(bool condition, string name) + { + if (condition) + { + unsupported.Add(name); + } + } + } + + private IReadOnlyDictionary GetQuestions(ChatOptions? options) + { + IReadOnlyDictionary? questions = + (options?.RawRepresentationFactory?.Invoke(this) as JevChatRequestOptions)?.Questions ?? this._defaultQuestions; + if (questions is null) + { + Throw.InvalidOperationException( + $"Jev needs questions to answer. Pass them with {nameof(ChatOptions)}.{nameof(JevChatOptionsExtensions.WithJevQuestions)}, or set {nameof(JevChatClientOptions)}.{nameof(JevChatClientOptions.DefaultQuestions)}."); + } + + if (questions.Count == 0) + { + Throw.ArgumentException(nameof(options), "Jev needs at least one question."); + } + + return questions; + } + + private static List GetFunctionTools(ChatOptions? options) + { + if (options?.Tools is not { Count: > 0 } tools) + { + return []; + } + + List unsupported = [.. tools.Where(tool => tool is not AIFunctionDeclaration).Select(tool => tool.Name)]; + if (unsupported.Count > 0) + { + throw new NotSupportedException($"Jev can only call function tools; these tools are not supported: {string.Join(", ", unsupported)}."); + } + + return [.. tools.Cast()]; + } + + private static JevResult WithoutInternalAnswers(JevResult result, IReadOnlyDictionary questions) + { + var answers = new Dictionary(questions.Count, StringComparer.Ordinal); + foreach (string id in questions.Keys) + { + answers[id] = result.Answers[id]; + } + + return new JevResult { Model = result.Model, Answers = answers, Usage = result.Usage }; + } + + /// + /// Builds the response text: the result as JSON, or, after tool calls, the tool results followed by the Choice and + /// Score decisions, which is what a caller that reads only text needs to see. + /// + private static string BuildText(IReadOnlyList history, int turnStart, JevResult answers) + { + List lines = JevChatState.GetCurrentTurnResultTexts(history, turnStart); + if (lines.Count == 0) + { + return JsonSerializer.Serialize(answers, JevJsonContext.Default.JevResult); + } + + // Noul answers are probabilities rather than decisions, so, as in the Python connector, they are left to the + // typed result. + foreach (KeyValuePair answer in answers.Answers) + { + switch (answer.Value) + { + case JevChoiceResponse choice: + lines.Add($"{answer.Key}: {choice.Choice}"); + break; + + case JevScoreResponse score: + lines.Add($"{answer.Key}: {score.Score.ToString("G6", CultureInfo.InvariantCulture)}"); + break; + } + } + + return string.Join("\n", lines); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs new file mode 100644 index 00000000000..a8de9a73812 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Configures a and the client pipeline that sends its requests to the TypeSafe System +/// One API. +/// +/// +/// +/// This type follows the pattern of the OpenAI and Azure AI client libraries: +/// selects the HTTP stack, +/// controls retries, and adds custom +/// policies. Unset pipeline settings get the defaults of the official TypeSafe SDK, as described for +/// . also supplies the logger +/// that reports tools left out of a request. +/// +/// +/// Like other client options, an instance becomes read-only once a client has been created with it. +/// +/// +public sealed class JevChatClientOptions : ClientPipelineOptions +{ + /// + public override void Freeze() + { + JevPipelineDefaults.Apply(this); + base.Freeze(); + } + + /// + /// Gets or sets the System One endpoint. When , https://api.typesafe.ai/v1/systemone is used. + /// + public Uri? Endpoint + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } + + /// + /// Gets or sets the Jev model that answers the questions, unless + /// selects another for a request. Defaults to jev-latest. + /// + public string ModelId + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } = JevPipelineDefaults.DefaultModelId; + + /// + /// Gets or sets the questions to answer when a request does not pass its own with + /// . + /// + /// + /// Default questions let code that knows nothing about Jev, such as a framework component with a fixed task, + /// use the client as any other . The client copies the dictionary when it + /// is created. + /// + public IReadOnlyDictionary? DefaultQuestions + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } + + /// + /// Gets or sets how many tool calls Jev may request in one turn, that is, while answering one user request. + /// Defaults to 1. + /// + /// + /// The turn starts after the last user message, except that text sent together with a tool approval response + /// continues the turn of the call it approves. Once the turn has this many function calls, the client stops + /// offering tools and answers the questions, with the tool results in the state. The route question tells Jev + /// which calls the turn already made, but a closed-set choice can still repeat one, so the limit is what + /// guarantees that the tool loop ends. Raise it when one user request needs several distinct calls, such as the + /// weather in two cities; set it to 0 to never call tools. + /// + /// The value is negative. + public int MaximumToolCallsPerTurn + { + get; + set + { + this.AssertNotFrozen(); + field = Throw.IfLessThan(value, 0); + } + } = 1; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs new file mode 100644 index 00000000000..119269f117c --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Provides extension methods that pass Jev questions through . +/// +public static class JevChatOptionsExtensions +{ + /// + /// Sets the questions that a answers for a request. + /// + /// The chat options to configure. + /// The questions to answer, keyed by an ID that the caller chooses. + /// , for chaining. + /// + /// + /// This sets to return a + /// to a . For any other client it calls the factory that was set before, so the same + /// options can still carry provider options for another client. Calling this method again replaces the + /// questions. + /// + /// + /// With an agent, pass the options in a ChatClientAgentRunOptions for one run, or in the agent's + /// ChatClientAgentOptions.ChatOptions for every run; the agent prefers the run's questions. + /// + /// + public static ChatOptions WithJevQuestions(this ChatOptions options, IReadOnlyDictionary questions) + { + _ = Throw.IfNull(options); + _ = Throw.IfNull(questions); + + Func? previous = options.RawRepresentationFactory; + options.RawRepresentationFactory = client => + client is JevChatClient ? new JevChatRequestOptions { Questions = questions } : previous?.Invoke(client); + return options; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs new file mode 100644 index 00000000000..3701e3da708 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the Jev-specific options of one request. +/// +/// +/// A reads this type from , which is +/// how Microsoft.Extensions.AI passes options that only one provider understands. Other chat clients ignore it. +/// sets the factory for you. +/// +public sealed class JevChatRequestOptions +{ + /// + /// Gets or sets the questions to answer about the conversation, keyed by an ID that the caller chooses. + /// + /// + /// Each answer of the uses the ID of its question. IDs that start with + /// __af_tool__ are reserved for the questions that route tool calls. + /// + public required IReadOnlyDictionary Questions { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs deleted file mode 100644 index 9af3f283071..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.ComponentModel; -using System.Text.Json.Serialization; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Represents the answer to a . -/// -[Description("The answer to a Choice question.")] -public sealed class JevChoiceAnswer : JevAnswer -{ - /// - /// Gets or sets the option with the highest probability. - /// - [JsonPropertyName("choice")] - [Description("The option with the highest probability.")] - public required string Choice { get; set; } - - /// - /// Gets or sets how certain the answer is, from 0 to 1, derived from how the probability is spread across the options. - /// - [JsonPropertyName("confidence")] - [Description("How certain the answer is, from 0 to 1. Treat low confidence as uncertain.")] - public required double Confidence { get; set; } - - /// - /// Gets or sets the probability of each option. The probabilities sum to 1. - /// - [JsonPropertyName("probabilities")] - [Description("The probability of each option. The probabilities sum to 1.")] - public required IReadOnlyDictionary Probabilities { get; set; } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs index 7377a778036..a07918fa44d 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs @@ -7,22 +7,22 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents a Choice question, which picks exactly one option from a set that the caller defines. +/// Represents a Choice question, which selects one of the named alternatives in . /// /// -/// The answer is a . +/// The answer is a . This type mirrors ChoiceQuestion of the official TypeSafe SDK. /// -[Description("A Choice question: picks exactly one option from criteria.")] +[Description("A Choice question: selects one of the named alternatives in criteria.")] public sealed class JevChoiceQuestion : JevQuestion { /// - /// Gets or sets the options, keyed by the option name that the answer returns. + /// Gets or sets the alternatives, keyed by the label that the answer returns. /// /// - /// Each value describes when the option applies, or is when the name is enough. - /// Jev accepts from 1 to 255 options. + /// Each value describes the alternative as text or JSON, or is when the label is + /// enough. Jev accepts from 1 to 255 alternatives. /// [JsonPropertyName("criteria")] - [Description("The options to choose from, keyed by the option name that the answer returns. Each value describes when the option applies. From 1 to 255 options.")] - public required IReadOnlyDictionary Criteria { get; set; } + [Description("The alternatives, keyed by the label that the answer returns. Each value describes the alternative as text or JSON, or is null when the label is enough. From 1 to 255 alternatives.")] + public required IReadOnlyDictionary Criteria { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceResponse.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceResponse.cs new file mode 100644 index 00000000000..103fb2b12ef --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceResponse.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a : the selected label and its probabilities. +/// +/// +/// This type mirrors ChoiceResponse of the official TypeSafe SDK. +/// +[Description("The answer to a Choice question: the selected label and its probabilities.")] +public sealed class JevChoiceResponse : JevResponse +{ + /// + /// Gets or sets the selected label. + /// + [JsonPropertyName("choice")] + [Description("The selected label.")] + public required string Choice { get; set; } + + /// + /// Gets or sets the reported confidence in the selected label, from 0 to 1. + /// + [JsonPropertyName("confidence")] + [Description("Reported confidence in the selected label, from 0 to 1. Treat low confidence as uncertain.")] + public required double Confidence { get; set; } + + /// + /// Gets or sets the probabilities, keyed by label. They sum to 1. + /// + [JsonPropertyName("probabilities")] + [Description("Probabilities keyed by label. They sum to 1.")] + public required IReadOnlyDictionary Probabilities { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevEntry.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevEntry.cs new file mode 100644 index 00000000000..ae0999a26ad --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevEntry.cs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents a Jev entry: text, a JSON object or array, or . +/// +/// +/// This type mirrors EntryType of the official TypeSafe SDK. The state of a , the +/// instructions of a , and every criterion description are entries, so each can be plain +/// text or structured data that questions refer to by field name. A converts implicitly, a +/// converts explicitly because a JSON number or Boolean is not an entry, and +/// is the entry. +/// +[JsonConverter(typeof(JevEntryJsonConverter))] +public readonly struct JevEntry : IEquatable +{ + private readonly JsonElement _value; + + /// + /// Initializes a new instance of the struct from JSON. + /// + /// A JSON string, object, array, or null. + /// is a JSON number or Boolean. + public JevEntry(JsonElement value) + { + if (value.ValueKind is not (JsonValueKind.String or JsonValueKind.Object or JsonValueKind.Array or JsonValueKind.Null or JsonValueKind.Undefined)) + { + Throw.ArgumentException(nameof(value), "A Jev entry must be text, a JSON object or array, or null."); + } + + // Cloning detaches the value from the document it came from, which may be disposed after this call. + this._value = value.ValueKind is JsonValueKind.Null or JsonValueKind.Undefined ? default : value.Clone(); + } + + /// + /// Gets the entry, which leaves a criterion undescribed. + /// + public static JevEntry Null => default; + + /// + /// Gets the kind of the entry: , , + /// , or . + /// + public JsonValueKind Kind => this._value.ValueKind == JsonValueKind.Undefined ? JsonValueKind.Null : this._value.ValueKind; + + /// + /// Gets a value indicating whether the entry is . + /// + public bool IsNull => this.Kind == JsonValueKind.Null; + + /// + /// Gets the text of an entry whose is , or otherwise. + /// + public string? Text => this.Kind == JsonValueKind.String ? this._value.GetString() : null; + + /// + /// Gets the entry as JSON, or for the entry. + /// + public JsonElement? Json => this.IsNull ? null : this._value; + + /// + /// Creates an entry from text. + /// + /// The text, or for the entry. + /// The entry. + public static JevEntry FromString(string? text) => + text is null ? default : new JevEntry(JsonSerializer.SerializeToElement(text, JevJsonContext.Default.String)); + + /// + /// Creates an entry from JSON. + /// + /// A JSON string, object, array, or null. + /// The entry. + public static JevEntry FromJsonElement(JsonElement value) => new(value); + + /// + /// Converts text to an entry. + /// + /// The text, or for the entry. + public static implicit operator JevEntry(string? text) => FromString(text); + + /// + /// Converts JSON to an entry. + /// + /// A JSON string, object, array, or null. + /// is a JSON number or Boolean. + /// + /// The conversion is explicit because it fails for JSON numbers and Booleans, which are not entries. + /// + public static explicit operator JevEntry(JsonElement value) => FromJsonElement(value); + + /// Determines whether two entries have the same JSON value. + /// The first entry. + /// The second entry. + /// when the entries are equal. + public static bool operator ==(JevEntry left, JevEntry right) => left.Equals(right); + + /// Determines whether two entries have different JSON values. + /// The first entry. + /// The second entry. + /// when the entries differ. + public static bool operator !=(JevEntry left, JevEntry right) => !left.Equals(right); + + /// + public bool Equals(JevEntry other) => + this.Kind == other.Kind && (this.IsNull || JsonElement.DeepEquals(this._value, other._value)); + + /// + public override bool Equals(object? obj) => obj is JevEntry other && this.Equals(other); + + /// + /// + /// Objects are equal regardless of property order, so only the kind (and the text of a string) is hashed. + /// + public override int GetHashCode() => + this.Kind == JsonValueKind.String + ? HashCode.Combine(this.Kind, this._value.GetString()) + : this.Kind.GetHashCode(); + + /// + /// Returns the text of a string entry, the JSON of an object or array entry, or an empty string for the + /// entry. + /// + /// The entry as a string. + public override string ToString() => + this.Kind switch + { + JsonValueKind.String => this._value.GetString()!, + JsonValueKind.Null => string.Empty, + _ => this._value.GetRawText(), + }; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs deleted file mode 100644 index b737c69c977..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using System.Text.Json.Serialization; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Represents the answer to a . -/// -[Description("The answer to a Noul question.")] -public sealed class JevNoulAnswer : JevAnswer -{ - /// - /// Gets or sets the probability, from 0 to 1, that the answer is yes. - /// - [JsonPropertyName("noul")] - [Description("The probability, from 0 to 1, that the answer is yes.")] - public required double Noul { get; set; } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs index 4540e047ec0..609b9c0b90a 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs @@ -6,23 +6,23 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Describes what a yes and a no mean for a . +/// Describes the yes and no outcomes of a . /// public sealed class JevNoulCriteria { /// - /// Gets or sets what a yes, a probability near 1, means. + /// Gets or sets the description of the yes outcome, as text or JSON. /// [JsonPropertyName("true")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [Description("What a yes, a probability near 1, means.")] - public string? True { get; set; } + [Description("Description of the yes outcome, as text or JSON.")] + public JevEntry? True { get; set; } /// - /// Gets or sets what a no, a probability near 0, means. + /// Gets or sets the description of the no outcome, as text or JSON. /// [JsonPropertyName("false")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [Description("What a no, a probability near 0, means.")] - public string? False { get; set; } + [Description("Description of the no outcome, as text or JSON.")] + public JevEntry? False { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs index bee933254dd..cf54a32eefe 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs @@ -6,19 +6,20 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents a Noul question, which returns the probability that a yes or no statement is true. +/// Represents a Noul question, a yes or no question whose answer is the probability of yes. /// /// -/// The answer is a . +/// The answer is a . A Noul needs , +/// , or both. This type mirrors NoulQuestion of the official TypeSafe SDK. /// -[Description("A Noul question: returns the probability, from 0 to 1, that the yes or no statement in instructions is true.")] +[Description("A Noul question: a yes or no question whose answer is the probability of yes. Needs instructions, criteria, or both.")] public sealed class JevNoulQuestion : JevQuestion { /// - /// Gets or sets optional descriptions of what a yes and a no mean. + /// Gets or sets optional descriptions of the yes and no outcomes. /// [JsonPropertyName("criteria")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [Description("Optional descriptions of what a yes and a no mean.")] + [Description("Optional descriptions of the yes and no outcomes.")] public JevNoulCriteria? Criteria { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulResponse.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulResponse.cs new file mode 100644 index 00000000000..f67e5cab17b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulResponse.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a : the probability of yes. +/// +/// +/// This type mirrors NoulResponse of the official TypeSafe SDK. +/// +[Description("The answer to a Noul question: the probability of yes.")] +public sealed class JevNoulResponse : JevResponse +{ + /// + /// Gets or sets the probability of a yes answer, from 0 to 1. + /// + [JsonPropertyName("noul")] + [Description("Probability of a yes answer, from 0 to 1.")] + public required double Noul { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs index af3fe909da1..bbaa8c956c8 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs @@ -10,7 +10,8 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// /// The JSON type property selects the question kind: choice (), -/// score (), or noul (). +/// score (), or noul (). It mirrors +/// Question of the official TypeSafe SDK. /// [JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] [JsonDerivedType(typeof(JevChoiceQuestion), "choice")] @@ -24,9 +25,15 @@ private protected JevQuestion() } /// - /// Gets or sets what Jev should decide, rate, or check. + /// Gets or sets what Jev should decide, rate, or check, as text or JSON, or when the + /// criteria say enough. /// + /// + /// Structured instructions hold the question in one field and data it refers to in others, for example + /// { "potential_duplicate": { ... }, "question": "Is the resume for the same person as `potential_duplicate`?" }. + /// [JsonPropertyName("instructions")] - [Description("What Jev should decide, rate, or check. Describe the idea rather than exact words, because Jev matches on meaning.")] - public required string Instructions { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("What Jev should decide, rate, or check: text, or a JSON object that holds the question in one field and the data it refers to in others. Optional when the criteria say enough. Describe the idea rather than exact words, because Jev matches on meaning.")] + public JevEntry? Instructions { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs index 2899a89ccf8..b4529531d26 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs @@ -2,38 +2,38 @@ using System.Collections.Generic; using System.ComponentModel; -using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.AI; namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents the input of a Jev evaluation: the state to evaluate and the typed questions to answer about it. +/// Represents the input of a Jev call: the state and the typed questions to answer about it. /// /// -/// This type mirrors the body of a TypeSafe System One API request without the model, which the tool configuration -/// selects. Its JSON form is the argument shape of the that -/// creates. +/// This type mirrors SystemOneRequest of the official TypeSafe SDK without the model, which the tool +/// configuration selects. Its JSON form is the argument shape of the that +/// creates. /// public sealed class JevRequest { /// - /// Gets or sets the content to evaluate. + /// Gets or sets the content that the questions are about. /// /// - /// A JSON string for plain text, or a JSON object or array for structured data, such as a conversation or - /// related records. + /// Text, or a JSON object or array for structured data, such as a conversation or related records. It mirrors + /// state of SystemOneRequest in the official TypeSafe SDK, except that the API rejects + /// . /// [JsonPropertyName("state")] - [Description("The content to evaluate. Use a string for plain text, or a JSON object or array for structured data. Questions can refer to object fields by name in backticks, for example `order`.")] - public required JsonElement State { get; set; } + [Description("The content to answer questions about: text, or a JSON object or array for structured data. Questions can refer to object fields by name in backticks, for example `order`.")] + public required JevEntry State { get; set; } /// /// Gets or sets the questions to answer, keyed by an ID that the caller chooses. /// /// - /// Each answer in uses the ID of its question. + /// Each answer in uses the ID of its question. /// [JsonPropertyName("questions")] [Description("The questions to answer about the state, keyed by an ID you choose. Each answer comes back under the same ID. Ask all related questions in one call.")] diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs index 944240124d8..f04265209d5 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs @@ -1,41 +1,25 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.ComponentModel; using System.Text.Json.Serialization; -using Microsoft.Extensions.AI; namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents the result of a Jev evaluation: one typed answer per question of the . +/// Represents Jev's answer to one question of a . /// /// -/// This type mirrors the body of a TypeSafe System One API response. Its JSON form is the result of the -/// that creates. +/// The JSON type property matches the kind of the question: choice (), +/// score (), or noul (). The derived types +/// mirror ChoiceResponse, ScoreResponse, and NoulResponse of the official TypeSafe SDK. /// -public sealed class JevResponse +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(JevChoiceResponse), "choice")] +[JsonDerivedType(typeof(JevScoreResponse), "score")] +[JsonDerivedType(typeof(JevNoulResponse), "noul")] +public abstract class JevResponse { - /// - /// Gets or sets the model that performed the evaluation, when the evaluator reports it. - /// - [JsonPropertyName("model")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [Description("The model that performed the evaluation.")] - public string? Model { get; set; } - - /// - /// Gets or sets the answers, keyed by the ID of the question they answer. - /// - [JsonPropertyName("answers")] - [Description("One answer per question, keyed by the question ID.")] - public required IReadOnlyDictionary Answers { get; set; } - - /// - /// Gets or sets the token usage of the evaluation, when the evaluator reports it. - /// - [JsonPropertyName("usage")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [Description("The token usage of the evaluation.")] - public JevUsage? Usage { get; set; } + // Only the three response kinds that the Jev API returns can exist. + private protected JevResponse() + { + } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResult.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResult.cs new file mode 100644 index 00000000000..a847eb2f774 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResult.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the result of a Jev call: one typed answer per question of the , with the model +/// and token usage. +/// +/// +/// This type mirrors SystemOneResult of the official TypeSafe SDK. Its JSON form is the result of the +/// that creates. +/// +public sealed class JevResult +{ + /// + /// Gets or sets the model that answered the request. + /// + [JsonPropertyName("model")] + [Description("The model that answered the request.")] + public required string Model { get; set; } + + /// + /// Gets or sets the answers, keyed by the ID of the question they answer. + /// + [JsonPropertyName("answers")] + [Description("One answer per question, keyed by the question ID.")] + public required IReadOnlyDictionary Answers { get; set; } + + /// + /// Gets or sets the token usage of the request. + /// + [JsonPropertyName("usage")] + [Description("The token usage of the request.")] + public required JevUsage Usage { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs new file mode 100644 index 00000000000..a233cf7e45d --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Provides extension methods that read the of a response. +/// +public static class JevResultExtensions +{ + /// + /// Gets the typed answers of a response. + /// + /// The chat response. + /// + /// The answers to the request's questions, with their probabilities and token usage, or + /// when the response did not come from a or ended with a tool call. + /// + public static JevResult? GetJevResult(this ChatResponse response) + { + _ = Throw.IfNull(response); + return response.RawRepresentation as JevResult ?? FromMessages(response.Messages); + } + + /// + /// Gets the typed answers of an agent response produced with a . + /// + /// The agent response. + /// + /// The answers to the run's questions, with their probabilities and token usage, or when + /// the response has none. + /// + public static JevResult? GetJevResult(this AgentResponse response) + { + _ = Throw.IfNull(response); + + // A chat client agent keeps the chat response as the raw representation; agents that rebuild the response + // still keep the messages, which carry the result too. + return response.RawRepresentation switch + { + JevResult result => result, + ChatResponse chatResponse => chatResponse.GetJevResult(), + _ => FromMessages(response.Messages), + }; + } + + private static JevResult? FromMessages(IList messages) + { + for (int index = messages.Count - 1; index >= 0; index--) + { + if (messages[index].RawRepresentation is JevResult result) + { + return result; + } + } + + return null; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs deleted file mode 100644 index a4f4be565b0..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.ComponentModel; -using System.Text.Json.Serialization; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Represents the answer to a . -/// -[Description("The answer to a Score question.")] -public sealed class JevScoreAnswer : JevAnswer -{ - /// - /// Gets or sets the score, weighted by the probability of each level. It can fall between levels. - /// - [JsonPropertyName("score")] - [Description("The score, weighted by the probability of each level. It can fall between levels.")] - public required double Score { get; set; } - - /// - /// Gets or sets how certain the answer is, from 0 to 1, derived from how the probability is spread across the levels. - /// - [JsonPropertyName("confidence")] - [Description("How certain the answer is, from 0 to 1. Treat low confidence as uncertain.")] - public required double Confidence { get; set; } - - /// - /// Gets or sets the description of each level, keyed by the level number. - /// - [JsonPropertyName("legend")] - [Description("The description of each level, keyed by the level number.")] - public required IReadOnlyDictionary Legend { get; set; } - - /// - /// Gets or sets the probability of each level, keyed by the level number. The probabilities sum to 1. - /// - [JsonPropertyName("probabilities")] - [Description("The probability of each level, keyed by the level number. The probabilities sum to 1.")] - public required IReadOnlyDictionary Probabilities { get; set; } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs index fa8a7954b1a..fde0cc3f19e 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs @@ -7,21 +7,23 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents a Score question, which rates the state against ordered levels that the caller defines. +/// Represents a Score question, which assigns a score using the ordered rubric in . /// /// -/// The answer is a , whose score is weighted by probability and can fall between levels. +/// The answer is a , whose score is weighted by probability and can fall between +/// levels. This type mirrors ScoreQuestion of the official TypeSafe SDK. /// -[Description("A Score question: rates the state against ordered levels in criteria.")] +[Description("A Score question: assigns a score using the ordered rubric in criteria.")] public sealed class JevScoreQuestion : JevQuestion { /// - /// Gets or sets the level descriptions, ordered from level 0 (the lowest) to the highest level. + /// Gets or sets the rubric: one description per score, indexed from zero (the lowest score). /// /// - /// A Score needs from 2 to 10 levels. + /// Each description is text or JSON. The official SDK types also allow to leave a + /// score undescribed, but the API rejects it. A Score needs from 2 to 10 scores. /// [JsonPropertyName("criteria")] - [Description("The level descriptions, ordered from level 0 (the lowest) to the highest level. From 2 to 10 levels.")] - public required IReadOnlyList Criteria { get; set; } + [Description("The rubric: one description per score, as text or JSON, indexed from zero (the lowest score). From 2 to 10 scores.")] + public required IReadOnlyList Criteria { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreResponse.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreResponse.cs new file mode 100644 index 00000000000..30cec062631 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreResponse.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a : an expected score with its rubric and probabilities. +/// +/// +/// This type mirrors ScoreResponse of the official TypeSafe SDK. +/// +[Description("The answer to a Score question: an expected score with its rubric and probabilities.")] +public sealed class JevScoreResponse : JevResponse +{ + /// + /// Gets or sets the expected score, which may fall between integer rubric levels. + /// + [JsonPropertyName("score")] + [Description("Expected score, which may fall between integer rubric levels.")] + public required double Score { get; set; } + + /// + /// Gets or sets the reported confidence in the score, from 0 to 1. + /// + [JsonPropertyName("confidence")] + [Description("Reported confidence in the score, from 0 to 1. Treat low confidence as uncertain.")] + public required double Confidence { get; set; } + + /// + /// Gets or sets the rubric descriptions, keyed by score. They echo the rubric of the question. + /// + [JsonPropertyName("legend")] + [Description("Rubric descriptions keyed by score.")] + public required IReadOnlyDictionary Legend { get; set; } + + /// + /// Gets or sets the probabilities, keyed by score. They sum to 1. + /// + [JsonPropertyName("probabilities")] + [Description("Probabilities keyed by score. They sum to 1.")] + public required IReadOnlyDictionary Probabilities { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs index 1c004164070..34cbff9229f 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs @@ -6,7 +6,7 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents the token usage of a Jev evaluation. +/// Represents the token usage of a Jev call. This type mirrors Usage of the official TypeSafe SDK. /// public sealed class JevUsage { diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj index 203d77b4310..90159d6d784 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj @@ -15,15 +15,20 @@ Microsoft Agent Framework - TypeSafe (Jev) integration - Provides an AIFunction that lets agents classify, route, score, and check content with Jev, the TypeSafe System One model. + Provides an IChatClient and an AIFunction that let agents classify, route, score, and check content with Jev, the TypeSafe System One model. + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs index 9135880f01b..386c834b9a9 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs @@ -14,14 +14,14 @@ namespace Microsoft.Agents.AI.TypeSafe.UnitTests; public sealed class JevAIFunctionTests { [Fact] - public async Task InvokeAsync_ModelArguments_ReachTheEvaluatorAsTypedRequestAsync() + public async Task InvokeAsync_ModelArguments_ReachTheMappingAsTypedRequestAsync() { // Arrange JevRequest? received = null; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + AIFunction tool = new JevAIToolBuilder().UseMapping((request, _, _) => { received = request; - return Task.FromResult(JevTestData.CreateResponse()); + return Task.FromResult(JevTestData.CreateResult()); }).Build(); // Act @@ -29,27 +29,27 @@ public async Task InvokeAsync_ModelArguments_ReachTheEvaluatorAsTypedRequestAsyn // Assert Assert.NotNull(received); - Assert.Equal(JevTestData.TicketText, received.State.GetString()); + Assert.Equal(JevTestData.TicketText, received.State.Text); JevChoiceQuestion department = Assert.IsType(received.Questions["department"]); - Assert.Equal("Which team should handle this", department.Instructions); + Assert.Equal("Which team should handle this", department.Instructions?.Text); Assert.Equal(["billing", "technical", "sales"], department.Criteria.Keys); - Assert.Null(department.Criteria["sales"]); + Assert.True(department.Criteria["sales"].IsNull); JevScoreQuestion frustration = Assert.IsType(received.Questions["frustration"]); Assert.Equal(3, frustration.Criteria.Count); - Assert.Equal("Calm, just stating facts", frustration.Criteria[0]); + Assert.Equal("Calm, just stating facts", frustration.Criteria[0].Text); JevNoulQuestion urgent = Assert.IsType(received.Questions["is_urgent"]); - Assert.Equal("Explicitly time-sensitive", urgent.Criteria!.True); - Assert.Equal("No urgency expressed", urgent.Criteria.False); + Assert.Equal("Explicitly time-sensitive", urgent.Criteria!.True?.Text); + Assert.Equal("No urgency expressed", urgent.Criteria.False?.Text); } [Fact] - public async Task InvokeAsync_ReturnsTheResponseAsJsonWithWireNamesAsync() + public async Task InvokeAsync_ReturnsTheResultAsJsonWithWireNamesAsync() { // Arrange - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); // Act object? result = await tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); @@ -73,10 +73,10 @@ public async Task InvokeAsync_TypeWrittenLastAndObjectState_AreAcceptedAsync() { // Arrange JevRequest? received = null; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + AIFunction tool = new JevAIToolBuilder().UseMapping((request, _, _) => { received = request; - return Task.FromResult(new JevResponse { Answers = new Dictionary { ["paid"] = new JevNoulAnswer { Noul = 0.1 } } }); + return Task.FromResult(Result("paid", new JevNoulResponse { Noul = 0.1 })); }).Build(); // Act @@ -88,20 +88,65 @@ await tool.InvokeAsync(JevTestData.Arguments(""" """)); // Assert - Assert.Equal(JsonValueKind.Object, received!.State.ValueKind); - Assert.Equal("A-104", received.State.GetProperty("order").GetProperty("id").GetString()); + Assert.Equal(JsonValueKind.Object, received!.State.Kind); + Assert.Equal("A-104", received.State.Json!.Value.GetProperty("order").GetProperty("id").GetString()); Assert.IsType(received.Questions["paid"]); } + [Fact] + public async Task InvokeAsync_StructuredEntries_AreAcceptedLikeTheSdkAsync() + { + // Arrange + JevRequest? received = null; + AIFunction tool = new JevAIToolBuilder().UseMapping((request, _, _) => + { + received = request; + return Task.FromResult(new JevResult + { + Model = "m", + Answers = new Dictionary + { + ["same_person"] = new JevNoulResponse { Noul = 0.9 }, + ["team"] = new JevChoiceResponse { Choice = "billing", Confidence = 1, Probabilities = new Dictionary { ["billing"] = 1 } }, + }, + Usage = new JevUsage { InputTokens = 1, OutputTokens = 1 }, + }); + }).Build(); + + // Act + await tool.InvokeAsync(JevTestData.Arguments(""" + { + "state": ["Hi", "The payment failed."], + "questions": { + "same_person": { + "type": "noul", + "instructions": { "potential_duplicate": { "name": "John Smith" }, "question": "Is the resume for the same person as `potential_duplicate`?" } + }, + "team": { "type": "choice", "criteria": { "billing": { "covers": ["payments", "refunds"] } } } + } + } + """)); + + // Assert + Assert.Equal(JsonValueKind.Array, received!.State.Kind); + JevEntry instructions = Assert.IsType(received.Questions["same_person"]).Instructions!.Value; + Assert.Equal(JsonValueKind.Object, instructions.Kind); + Assert.Equal("John Smith", instructions.Json!.Value.GetProperty("potential_duplicate").GetProperty("name").GetString()); + + JevChoiceQuestion team = Assert.IsType(received.Questions["team"]); + Assert.Null(team.Instructions); + Assert.Equal(JsonValueKind.Object, team.Criteria["billing"].Kind); + } + [Fact] public async Task InvokeAsync_DotNetObjectArguments_AreAcceptedAsync() { // Arrange JevRequest? received = null; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + AIFunction tool = new JevAIToolBuilder().UseMapping((request, _, _) => { received = request; - return Task.FromResult(new JevResponse { Answers = new Dictionary { ["q"] = new JevNoulAnswer { Noul = 0.5 } } }); + return Task.FromResult(Result("q", new JevNoulResponse { Noul = 0.5 })); }).Build(); // Act @@ -112,31 +157,33 @@ await tool.InvokeAsync(new AIFunctionArguments }); // Assert - Assert.Equal("The payment failed.", received!.State.GetString()); - Assert.Equal("Did the payment fail?", Assert.IsType(received.Questions["q"]).Instructions); + Assert.Equal("The payment failed.", received!.State.Text); + Assert.Equal("Did the payment fail?", Assert.IsType(received.Questions["q"]).Instructions?.Text); } [Theory] [InlineData("""{"questions":{"q":{"type":"noul","instructions":"x"}}}""", "state")] - [InlineData("""{"state":42,"questions":{"q":{"type":"noul","instructions":"x"}}}""", "The state must be a string")] - [InlineData("""{"state":"x","questions":{}}""", "at least one question")] + [InlineData("""{"state":42,"questions":{"q":{"type":"noul","instructions":"x"}}}""", "must be text, a JSON object or array, or null")] + [InlineData("""{"state":null,"questions":{"q":{"type":"noul","instructions":"x"}}}""", "The state must be text or a JSON object or array")] + [InlineData("""{"state":"x","questions":{}}""", "At least one question is required")] [InlineData("""{"state":"x","questions":{"q":{"type":"rank","instructions":"x"}}}""", "not a valid Jev request")] [InlineData("""{"state":"x","questions":{"q":{"instructions":"x"}}}""", "not a valid Jev request")] [InlineData("""{"state":"x","questions":{"q":null}}""", "Question 'q' is null")] - [InlineData("""{"state":"x","questions":{"q":{"type":"noul","instructions":" "}}}""", "Question 'q' must have instructions")] - [InlineData("""{"state":"x","questions":{"q":{"type":"choice","instructions":"x","criteria":{}}}}""", "has 0 options; it needs from 1 to 255")] - [InlineData("""{"state":"x","questions":{"q":{"type":"choice","instructions":"x","criteria":{"":"empty"}}}}""", "empty option name")] - [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["only"]}}}""", "has 1 level; it needs from 2 to 10")] - [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["0","1","2","3","4","5","6","7","8","9","10"]}}}""", "has 11 levels")] - [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["low",""]}}}""", "empty level description")] - public async Task InvokeAsync_InvalidArguments_FailBeforeEvaluatingAsync(string arguments, string expectedMessage) + [InlineData("""{"state":"x","questions":{"q":{"type":"noul","instructions":" "}}}""", "Noul question 'q' needs instructions or criteria")] + [InlineData("""{"state":"x","questions":{"q":{"type":"noul","criteria":{"true":null}}}}""", "Noul question 'q' needs instructions or criteria")] + [InlineData("""{"state":"x","questions":{"q":{"type":"choice","instructions":"x","criteria":{}}}}""", "has 0 alternatives; it needs from 1 to 255")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["only"]}}}""", "has 1 criterion; it needs from 2 to 10")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["0","1","2","3","4","5","6","7","8","9","10"]}}}""", "has 11 criteria")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":[1,2]}}}""", "must be text, a JSON object or array, or null")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":[null,"High"]}}}""", "has a null criterion at index 0")] + public async Task InvokeAsync_InvalidArguments_FailBeforeCallingAsync(string arguments, string expectedMessage) { // Arrange int calls = 0; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => { calls++; - return Task.FromResult(JevTestData.CreateResponse()); + return Task.FromResult(JevTestData.CreateResult()); }).Build(); // Act @@ -148,42 +195,79 @@ public async Task InvokeAsync_InvalidArguments_FailBeforeEvaluatingAsync(string } [Fact] - public async Task InvokeAsync_TooManyChoiceOptions_FailBeforeEvaluatingAsync() + public async Task InvokeAsync_QuestionsWithoutInstructionsButWithCriteria_AreAcceptedAsync() { // Arrange - string options = string.Join(",", Enumerable.Range(0, 256).Select(static i => $"\"o{i}\":null")); - string arguments = "{\"state\":\"x\",\"questions\":{\"q\":{\"type\":\"choice\",\"instructions\":\"x\",\"criteria\":{" + options + "}}}}"; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + int calls = 0; + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => + { + calls++; + return Task.FromResult(new JevResult + { + Model = "m", + Answers = new Dictionary + { + ["urgent"] = new JevNoulResponse { Noul = 0.2 }, + ["level"] = new JevScoreResponse + { + Score = 0, + Confidence = 1, + Legend = new Dictionary { ["0"] = "Low", ["1"] = "High" }, + Probabilities = new Dictionary { ["0"] = 1, ["1"] = 0 }, + }, + }, + Usage = new JevUsage { InputTokens = 1, OutputTokens = 1 }, + }); + }).Build(); + + // Act + await tool.InvokeAsync(JevTestData.Arguments(""" + {"state":"x","questions":{ + "urgent":{"type":"noul","criteria":{"true":"Explicitly urgent"}}, + "level":{"type":"score","criteria":["Low",{"level":"high","signals":["outage"]}]}}} + """)); + + // Assert + Assert.Equal(1, calls); + } + + [Fact] + public async Task InvokeAsync_TooManyChoiceAlternatives_FailBeforeCallingAsync() + { + // Arrange + string alternatives = string.Join(",", Enumerable.Range(0, 256).Select(static i => $"\"o{i}\":null")); + string arguments = "{\"state\":\"x\",\"questions\":{\"q\":{\"type\":\"choice\",\"instructions\":\"x\",\"criteria\":{" + alternatives + "}}}}"; + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); // Act ArgumentException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(arguments)).AsTask()); // Assert - Assert.Contains("has 256 options", exception.Message, StringComparison.Ordinal); + Assert.Contains("has 256 alternatives", exception.Message, StringComparison.Ordinal); } - public static TheoryData, string> InvalidResponses => new() + public static TheoryData, string> InvalidResults => new() { { "no answer", static r => Without(r, "is_urgent"), "no answer for question 'is_urgent'" }, - { "wrong kind", static r => With(r, "department", new JevNoulAnswer { Noul = 0.5 }), "is a Noul answer, but the question is a Choice question" }, - { "unknown option", static r => With(r, "department", Choice("refunds", 0.9)), "chose 'refunds', which is not one of its options" }, + { "wrong kind", static r => With(r, "department", new JevNoulResponse { Noul = 0.5 }), "is a Noul answer, but the question is a Choice question" }, + { "unknown label", static r => With(r, "department", Choice("refunds", 0.9)), "chose 'refunds', which is not one of its labels" }, { "confidence out of range", static r => With(r, "department", Choice("technical", 1.2)), "has confidence 1.2, which is outside the range 0 to 1" }, { "score out of range", static r => With(r, "frustration", Score(3)), "has score 3, which is outside the range 0 to 2" }, - { "noul as percent", static r => With(r, "is_urgent", new JevNoulAnswer { Noul = 97 }), "has noul 97, which is outside the range 0 to 1" }, - { "no answers", static r => new JevResponse { Answers = null! }, "has no answers" }, - { "probability for an option not offered", static r => With(r, "department", ChoiceWith("technical", new() { ["TECHNICAL"] = 1 })), "has probabilities for 'TECHNICAL', which is not one of its options" }, + { "noul as percent", static r => With(r, "is_urgent", new JevNoulResponse { Noul = 97 }), "has noul 97, which is outside the range 0 to 1" }, + { "no answers", static r => new JevResult { Model = r.Model, Answers = null!, Usage = r.Usage }, "has no answers" }, + { "probability for a label not offered", static r => With(r, "department", ChoiceWith("technical", new() { ["TECHNICAL"] = 1 })), "has probabilities for 'TECHNICAL', which is not one of its labels" }, { "probability above one", static r => With(r, "department", ChoiceWith("technical", new() { ["technical"] = 42 })), "has probability 'technical' 42, which is outside the range 0 to 1" }, { "probability not a number", static r => With(r, "department", ChoiceWith("technical", new() { ["technical"] = double.NaN })), "has probability 'technical' NaN" }, - { "score probability for a missing level", static r => With(r, "frustration", ScoreWith(new() { ["3"] = 1 }, [])), "has probabilities for '3', which is not a level number from 0 to 2" }, - { "legend for a missing level", static r => With(r, "frustration", ScoreWith([], new() { ["high"] = "Very angry" })), "has legend for 'high', which is not a level number from 0 to 2" }, + { "score probability for a missing score", static r => With(r, "frustration", ScoreWith(new() { ["3"] = 1 }, [])), "has probabilities for '3', which is not a score from 0 to 2" }, + { "legend for a missing score", static r => With(r, "frustration", ScoreWith([], new() { ["high"] = "Very angry" })), "has legend for 'high', which is not a score from 0 to 2" }, }; [Theory] - [MemberData(nameof(InvalidResponses))] - public async Task InvokeAsync_InvalidEvaluatorResponse_FailsAsync(string scenario, Func corrupt, string expectedMessage) + [MemberData(nameof(InvalidResults))] + public async Task InvokeAsync_InvalidMappingResult_FailsAsync(string scenario, Func corrupt, string expectedMessage) { // Arrange - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(corrupt(JevTestData.CreateResponse()))).Build(); + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => Task.FromResult(corrupt(JevTestData.CreateResult()))).Build(); // Act InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); @@ -193,28 +277,28 @@ public async Task InvokeAsync_InvalidEvaluatorResponse_FailsAsync(string scenari } [Fact] - public async Task InvokeAsync_EvaluatorReturnsNull_FailsAsync() + public async Task InvokeAsync_MappingReturnsNull_FailsAsync() { // Arrange - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(null!)).Build(); + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => Task.FromResult(null!)).Build(); // Act InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); // Assert - Assert.Contains("returned no response", exception.Message, StringComparison.Ordinal); + Assert.Contains("returned no result", exception.Message, StringComparison.Ordinal); } [Fact] - public async Task InvokeAsync_PassesTheCancellationTokenToTheEvaluatorAsync() + public async Task InvokeAsync_PassesTheCancellationTokenToTheMappingAsync() { // Arrange using var cancellation = new CancellationTokenSource(); CancellationToken received = default; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, cancellationToken) => + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, cancellationToken) => { received = cancellationToken; - return Task.FromResult(JevTestData.CreateResponse()); + return Task.FromResult(JevTestData.CreateResult()); }).Build(); // Act @@ -234,15 +318,9 @@ public async Task UseClient_MapsBetweenTheContractAndTheClientTypesAsync() inputMapper: static request => { (string id, JevQuestion question) = request.Questions.Single(); - return new LabelRequest(id, request.State.GetString()!, ((JevChoiceQuestion)question).Criteria.Keys.ToList()); + return new LabelRequest(id, request.State.Text!, ((JevChoiceQuestion)question).Criteria.Keys.ToList()); }, - outputMapper: static response => new JevResponse - { - Answers = new Dictionary - { - [response.QuestionId] = Choice(response.Label, response.Score), - }, - }).Build(); + outputMapper: static response => Result(response.QuestionId, Choice(response.Label, response.Score))).Build(); // Act object? result = await tool.InvokeAsync(JevTestData.Arguments(""" @@ -256,14 +334,14 @@ public async Task UseClient_MapsBetweenTheContractAndTheClientTypesAsync() } [Fact] - public async Task UseClient_MapperReturnsAnOptionThatWasNotOffered_FailsAsync() + public async Task UseClient_MapperReturnsALabelThatWasNotOffered_FailsAsync() { // Arrange var client = new LabelClient(label: "TECHNICAL"); AIFunction tool = new JevAIToolBuilder().UseClient( client.ClassifyAsync, - static request => new LabelRequest("team", request.State.GetString()!, ["billing", "technical"]), - static response => new JevResponse { Answers = new Dictionary { ["team"] = Choice(response.Label, response.Score) } }).Build(); + static request => new LabelRequest("team", request.State.Text!, ["billing", "technical"]), + static response => Result("team", Choice(response.Label, response.Score))).Build(); // Act InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(""" @@ -275,15 +353,15 @@ public async Task UseClient_MapperReturnsAnOptionThatWasNotOffered_FailsAsync() } [Fact] - public async Task UseEvaluator_ReceivesTheServicesOfTheCallAsync() + public async Task UseMapping_ReceivesTheServicesOfTheCallAsync() { // Arrange using ServiceProvider services = new ServiceCollection().BuildServiceProvider(); IServiceProvider? received = null; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, callServices, _) => + AIFunction tool = new JevAIToolBuilder().UseMapping((_, callServices, _) => { received = callServices; - return Task.FromResult(JevTestData.CreateResponse()); + return Task.FromResult(JevTestData.CreateResult()); }).Build(); AIFunctionArguments arguments = JevTestData.Arguments(JevTestData.RequestJson); @@ -311,8 +389,8 @@ public async Task UseClient_RegisteredClient_IsResolvedFromTheServicesOnEveryCal AIFunction tool = new JevAIToolBuilder().UseClient( static (client, request, cancellationToken) => client.ClassifyAsync(request, cancellationToken), - static request => new LabelRequest("team", request.State.GetString()!, ["billing", "technical"]), - static response => new JevResponse { Answers = new Dictionary { ["team"] = Choice(response.Label, response.Score) } }).Build(); + static request => new LabelRequest("team", request.State.Text!, ["billing", "technical"]), + static response => Result("team", Choice(response.Label, response.Score))).Build(); AIFunctionArguments arguments = JevTestData.Arguments(""" {"state":"x","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} @@ -338,7 +416,7 @@ public async Task UseClient_ClientNotAvailable_FailsWithWhatIsMissingAsync(bool AIFunction tool = new JevAIToolBuilder().UseClient( static (client, request, cancellationToken) => client.ClassifyAsync(request, cancellationToken), static request => new LabelRequest("q", "x", []), - static response => JevTestData.CreateResponse()).Build(); + static response => JevTestData.CreateResult()).Build(); AIFunctionArguments arguments = JevTestData.Arguments(JevTestData.RequestJson); arguments.Services = withServices ? services : null; @@ -366,8 +444,8 @@ public async Task UseClient_ScopedClientFromTheRootProvider_GetsAScopePerCallAsy AIFunction tool = new JevAIToolBuilder().UseClient( static (client, text, _) => Task.FromResult(client.IsDisposed ? "disposed" : "technical"), - static request => request.State.GetString()!, - static label => new JevResponse { Answers = new Dictionary { ["team"] = Choice(label, 0.9) } }).Build(); + static request => request.State.Text!, + static label => Result("team", Choice(label, 0.9))).Build(); AIFunctionArguments arguments = JevTestData.Arguments(""" {"state":"x","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} @@ -383,29 +461,36 @@ public async Task UseClient_ScopedClientFromTheRootProvider_GetsAScopePerCallAsy Assert.All(created, static client => Assert.True(client.IsDisposed)); } - private static JevChoiceAnswer Choice(string choice, double confidence) => new() + private static JevResult Result(string id, JevResponse answer) => new() + { + Model = "custom-model", + Answers = new Dictionary { [id] = answer }, + Usage = new JevUsage { InputTokens = 1, OutputTokens = 1 }, + }; + + private static JevChoiceResponse Choice(string choice, double confidence) => new() { Choice = choice, Confidence = confidence, Probabilities = new Dictionary { [choice] = confidence }, }; - private static JevScoreAnswer Score(double score) => new() + private static JevScoreResponse Score(double score) => new() { Score = score, Confidence = 1, - Legend = new Dictionary(), + Legend = new Dictionary(), Probabilities = new Dictionary(), }; - private static JevChoiceAnswer ChoiceWith(string choice, Dictionary probabilities) => new() + private static JevChoiceResponse ChoiceWith(string choice, Dictionary probabilities) => new() { Choice = choice, Confidence = 0.5, Probabilities = probabilities, }; - private static JevScoreAnswer ScoreWith(Dictionary probabilities, Dictionary legend) => new() + private static JevScoreResponse ScoreWith(Dictionary probabilities, Dictionary legend) => new() { Score = 1, Confidence = 1, @@ -413,17 +498,17 @@ public async Task UseClient_ScopedClientFromTheRootProvider_GetsAScopePerCallAsy Probabilities = probabilities, }; - private static JevResponse With(JevResponse response, string id, JevAnswer answer) + private static JevResult With(JevResult result, string id, JevResponse answer) { - var answers = new Dictionary(response.Answers) { [id] = answer }; - return new JevResponse { Answers = answers }; + var answers = new Dictionary(result.Answers) { [id] = answer }; + return new JevResult { Model = result.Model, Answers = answers, Usage = result.Usage }; } - private static JevResponse Without(JevResponse response, string id) + private static JevResult Without(JevResult result, string id) { - var answers = new Dictionary(response.Answers); + var answers = new Dictionary(result.Answers); answers.Remove(id); - return new JevResponse { Answers = answers }; + return new JevResult { Model = result.Model, Answers = answers, Usage = result.Usage }; } /// The request type of a hypothetical third-party classification client. diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs index 1fd41c9ec41..9c7be795d9e 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs @@ -19,20 +19,32 @@ public void Build_Defaults_DescribeTheJevContract() AIFunction tool = new JevAIToolBuilder(new ApiKeyCredential("test-key")).Build(); // Assert - Assert.Equal("evaluate_with_jev", tool.Name); + Assert.Equal("ask_system_one", tool.Name); Assert.Contains("'choice'", tool.Description, StringComparison.Ordinal); Assert.Contains("'score'", tool.Description, StringComparison.Ordinal); Assert.Contains("'noul'", tool.Description, StringComparison.Ordinal); JsonElement schema = tool.JsonSchema; Assert.Equal(["state", "questions"], schema.GetProperty("required").EnumerateArray().Select(static r => r.GetString())); + Assert.Equal(["string", "object", "array"], schema.GetProperty("properties").GetProperty("state").GetProperty("type").EnumerateArray().Select(static t => t.GetString())); JsonElement question = schema.GetProperty("properties").GetProperty("questions").GetProperty("additionalProperties"); Assert.Equal( ["choice", "score", "noul"], question.GetProperty("anyOf").EnumerateArray().Select(static b => b.GetProperty("properties").GetProperty("type").GetProperty("const").GetString())); + JsonElement[] branches = [.. question.GetProperty("anyOf").EnumerateArray()]; + JsonElement choiceCriteria = branches[0].GetProperty("properties").GetProperty("criteria"); + Assert.Equal(["string", "object", "array", "null"], choiceCriteria.GetProperty("additionalProperties").GetProperty("type").EnumerateArray().Select(static t => t.GetString())); + Assert.Equal(1, choiceCriteria.GetProperty("minProperties").GetInt32()); + Assert.Equal(255, choiceCriteria.GetProperty("maxProperties").GetInt32()); + JsonElement scoreCriteria = branches[1].GetProperty("properties").GetProperty("criteria"); + Assert.Equal(["string", "object", "array"], scoreCriteria.GetProperty("items").GetProperty("type").EnumerateArray().Select(static t => t.GetString())); + Assert.Equal(2, scoreCriteria.GetProperty("minItems").GetInt32()); + Assert.Equal(10, scoreCriteria.GetProperty("maxItems").GetInt32()); + Assert.Equal(4, branches[2].GetProperty("properties").GetProperty("instructions").GetProperty("type").GetArrayLength()); + JsonElement returnSchema = tool.ReturnJsonSchema!.Value; - Assert.Equal(["answers"], returnSchema.GetProperty("required").EnumerateArray().Select(static r => r.GetString())); + Assert.Equal(["model", "answers", "usage"], returnSchema.GetProperty("required").EnumerateArray().Select(static r => r.GetString())); Assert.True(returnSchema.GetProperty("properties").GetProperty("usage").GetProperty("properties").TryGetProperty("input_tokens", out _)); } @@ -45,16 +57,20 @@ public void Options_Defaults_MatchTheTypeSafeApi() // Assert Assert.Null(options.Endpoint); Assert.Equal("jev-latest", options.ModelId); - Assert.Equal("evaluate_with_jev", options.Name); + Assert.Equal("ask_system_one", options.Name); Assert.Null(options.RetryPolicy); } [Fact] - public void Build_WithTheTypeSafeApi_AssignsTheDefaultRetryPolicyOnlyWhenNoneIsSet() + public void Build_WithTheTypeSafeApi_AssignsTheSdkDefaultsOnlyWhenUnset() { // Arrange var defaults = new JevAIToolOptions(); - var custom = new JevAIToolOptions { RetryPolicy = new System.ClientModel.Primitives.ClientRetryPolicy(maxRetries: 1) }; + var custom = new JevAIToolOptions + { + RetryPolicy = new System.ClientModel.Primitives.ClientRetryPolicy(maxRetries: 1), + NetworkTimeout = TimeSpan.FromSeconds(30), + }; // Act _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), defaults).Build(); @@ -62,20 +78,22 @@ public void Build_WithTheTypeSafeApi_AssignsTheDefaultRetryPolicyOnlyWhenNoneIsS // Assert Assert.IsType(defaults.RetryPolicy); + Assert.Equal(TimeSpan.FromSeconds(10), defaults.NetworkTimeout); Assert.IsType(custom.RetryPolicy); + Assert.Equal(TimeSpan.FromSeconds(30), custom.NetworkTimeout); } [Fact] - public void Build_WithACustomEvaluator_IgnoresTheApiOnlyOptions() + public void Build_WithACustomMapping_IgnoresTheApiOnlyOptions() { // Arrange var options = new JevAIToolOptions { ModelId = " ", Endpoint = new Uri("/relative", UriKind.Relative) }; // Act - AIFunction tool = new JevAIToolBuilder(options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + AIFunction tool = new JevAIToolBuilder(options).UseMapping(static (_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); // Assert - Assert.Equal("evaluate_with_jev", tool.Name); + Assert.Equal("ask_system_one", tool.Name); } [Fact] @@ -85,7 +103,7 @@ public void Build_WithNameAndDescription_UsesTheOptions() var options = new JevAIToolOptions { Name = "triage_ticket", Description = "Classifies support tickets." }; // Act - AIFunction tool = new JevAIToolBuilder(options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + AIFunction tool = new JevAIToolBuilder(options).UseMapping(static (_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); // Assert Assert.Equal("triage_ticket", tool.Name); @@ -107,13 +125,13 @@ public void Build_WithTheTypeSafeApi_MakesTheOptionsReadOnly() } [Fact] - public void Build_WithACustomEvaluator_LeavesTheOptionsWritable() + public void Build_WithACustomMapping_LeavesTheOptionsWritable() { // Arrange var options = new JevAIToolOptions(); // Act - _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), options).UseMapping(static (_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); // Assert options.ModelId = "jev-1.13"; @@ -121,7 +139,7 @@ public void Build_WithACustomEvaluator_LeavesTheOptionsWritable() } [Fact] - public void Build_WithoutCredentialOrEvaluator_Throws() + public void Build_WithoutCredentialOrMapping_Throws() { // Act InvalidOperationException exception = Assert.Throws(() => new JevAIToolBuilder().Build()); @@ -156,17 +174,17 @@ public void Methods_NullArguments_Throw() { // Arrange var builder = new JevAIToolBuilder(); - Task evaluateAsync(string request, CancellationToken _) => Task.FromResult(request); + Task systemOneAsync(string request, CancellationToken _) => Task.FromResult(request); string input(JevRequest _) => "request"; - JevResponse output(string _) => JevTestData.CreateResponse(); - Task evaluateWithClientAsync(object _1, string request, CancellationToken _2) => Task.FromResult(request); + JevResult output(string _) => JevTestData.CreateResult(); + Task systemOneWithClientAsync(object _1, string request, CancellationToken _2) => Task.FromResult(request); // Act & Assert Assert.Throws(() => new JevAIToolBuilder((ApiKeyCredential)null!)); - Assert.Throws(() => builder.UseEvaluator(null!)); - Assert.Throws(() => builder.UseClient(null!, input, (Func)output)); - Assert.Throws(() => builder.UseClient((Func>)evaluateAsync, null!, output)); - Assert.Throws(() => builder.UseClient(evaluateAsync, input, null!)); - Assert.Throws(() => builder.UseClient((Func>)evaluateWithClientAsync, null!, output)); + Assert.Throws(() => builder.UseMapping(null!)); + Assert.Throws(() => builder.UseClient(null!, input, (Func)output)); + Assert.Throws(() => builder.UseClient((Func>)systemOneAsync, null!, output)); + Assert.Throws(() => builder.UseClient(systemOneAsync, input, null!)); + Assert.Throws(() => builder.UseClient((Func>)systemOneWithClientAsync, null!, output)); } } diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs index 4a3582d140f..fee2448f03a 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs @@ -51,7 +51,7 @@ public async Task RunAsync_AgentCallsTheTool_ModelReceivesTheAnswersAsync() public async Task GetResponseAsync_InvalidArgumentsWithDetailedErrors_TellTheModelWhatToFixAsync() { // Arrange - AIFunction jev = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + AIFunction jev = new JevAIToolBuilder().UseMapping((_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); const string InvalidArguments = """{"state":"x","questions":{"q":{"type":"score","instructions":"How bad?","criteria":["only"]}}}"""; using var model = new ScriptedChatClient( @@ -64,7 +64,7 @@ public async Task GetResponseAsync_InvalidArgumentsWithDetailedErrors_TellTheMod // Assert FunctionResultContent result = model.Calls[1].SelectMany(static m => m.Contents).OfType().Single(); - Assert.Contains("has 1 level; it needs from 2 to 10", result.Result?.ToString(), StringComparison.Ordinal); + Assert.Contains("has 1 criterion; it needs from 2 to 10", result.Result?.ToString(), StringComparison.Ordinal); } [Fact] @@ -77,10 +77,12 @@ public async Task RunAsync_AgentWithServices_ToolResolvesTheClientFromThemAsync( AIFunction jev = new JevAIToolBuilder() .UseClient( static (client, text, cancellationToken) => client.ScoreAsync(text, cancellationToken), - inputMapper: static request => request.State.GetString()!, - outputMapper: static probability => new JevResponse + inputMapper: static request => request.State.Text!, + outputMapper: static probability => new JevResult { - Answers = new Dictionary { ["is_urgent"] = new JevNoulAnswer { Noul = probability } }, + Model = "urgency-classifier", + Answers = new Dictionary { ["is_urgent"] = new JevNoulResponse { Noul = probability } }, + Usage = new JevUsage { InputTokens = 0, OutputTokens = 0 }, }) .Build(); diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs new file mode 100644 index 00000000000..8b328f0a2de --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs @@ -0,0 +1,386 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevChatClientTests +{ + private const string Ticket = JevTestData.TicketText; + + [Fact] + public async Task GetResponseAsync_SendsTheConversationAndQuestions_ReturnsTheTypedResultAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + ChatMessage[] messages = [new(ChatRole.System, "You triage support tickets."), new(ChatRole.User, Ticket)]; + + // Act + ChatResponse response = await client.GetResponseAsync(messages, new ChatOptions { Instructions = "Be strict." }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: the request carries the conversation as the state, in the Python connector's shape. + JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); + Assert.Equal(new Uri("https://api.typesafe.ai/v1/systemone"), request.Uri); + Assert.Equal("test-key", request.Authorization!.Parameter); + JsonNode body = JsonNode.Parse(request.Body)!; + Assert.Equal("jev-latest", (string?)body["model"]); + JsonNode expectedState = JsonNode.Parse($$""" + { + "messages": [ + { "role": "system", "contents": [{ "type": "text", "text": "You triage support tickets." }] }, + { "role": "user", "contents": [{ "type": "text", "text": {{JsonSerializer.Serialize(Ticket)}} }] } + ], + "instructions": "Be strict." + } + """)!; + Assert.True(JsonNode.DeepEquals(expectedState, body["state"]), body["state"]!.ToJsonString()); + Assert.Equal(["department", "is_urgent"], body["questions"]!.AsObject().Select(question => question.Key)); + + // Assert: the text is the result as JSON, and the typed result, usage, and model are on the response. + JevResult result = Assert.IsType(response.GetJevResult()); + Assert.Equal("technical", Assert.IsType(result.Answers["department"]).Choice); + Assert.Equal(0.9, Assert.IsType(result.Answers["is_urgent"]).Noul); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(response.Text), JsonNode.Parse(JsonSerializer.Serialize(result, JevJsonUtilities.Options))), response.Text); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + Assert.Equal("jev-test", response.ModelId); + Assert.Equal(10, response.Usage!.InputTokenCount); + Assert.Equal(2, response.Usage.OutputTokenCount); + Assert.Equal(12, response.Usage.TotalTokenCount); + Assert.Same(result, Assert.Single(response.Messages).RawRepresentation); + } + + [Fact] + public async Task GetResponseAsync_ModelIdAndEndpoint_AreUsedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient, new JevChatClientOptions { ModelId = "jev-1.13", Endpoint = new Uri("https://proxy.contoso.com/jev") }); + + // Act + await client.GetResponseAsync(Ticket, new ChatOptions { ModelId = "jev-1.14" }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: a request's model wins over the client's. + JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); + Assert.Equal(new Uri("https://proxy.contoso.com/jev"), request.Uri); + Assert.Equal("jev-1.14", (string?)JsonNode.Parse(request.Body)!["model"]); + + ChatClientMetadata metadata = Assert.IsType(client.GetService(typeof(ChatClientMetadata))); + Assert.Equal("typesafe.ai", metadata.ProviderName); + Assert.Equal(new Uri("https://proxy.contoso.com/jev"), metadata.ProviderUri); + Assert.Equal("jev-1.13", metadata.DefaultModelId); + Assert.Same(client, client.GetService(typeof(JevChatClient))); + Assert.Null(client.GetService(typeof(JevChatClient), "key")); + } + + [Fact] + public async Task GetResponseAsync_DefaultQuestions_AnswerRequestsWithoutQuestionsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient, new JevChatClientOptions { DefaultQuestions = JevChatTestData.Questions() }); + + // Act + ChatResponse response = await client.GetResponseAsync(Ticket); + + // Assert + Assert.Equal(["department", "is_urgent"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); + Assert.NotNull(response.GetJevResult()); + } + + [Fact] + public async Task GetResponseAsync_RequestQuestions_ReplaceTheDefaultsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("spam", 0.1))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient, new JevChatClientOptions { DefaultQuestions = JevChatTestData.Questions() }); + var questions = new Dictionary { ["spam"] = new JevNoulQuestion { Instructions = "Is it spam?" } }; + + // Act + await client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions)); + + // Assert + Assert.Equal(["spam"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); + } + + [Fact] + public async Task GetResponseAsync_NoQuestions_ThrowsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket)); + Assert.Contains(nameof(JevChatOptionsExtensions.WithJevQuestions), exception.Message, StringComparison.Ordinal); + await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(new Dictionary()))); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_ReservedQuestionId_ThrowsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + var questions = new Dictionary { ["__af_tool__.mine"] = new JevNoulQuestion { Instructions = "Yes?" } }; + + // Act & Assert + ArgumentException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions))); + Assert.Contains("__af_tool__.mine", exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_InvalidQuestion_ThrowsBeforeSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + var questions = new Dictionary { ["level"] = new JevScoreQuestion { Instructions = "How bad?", Criteria = ["only one"] } }; + + // Act & Assert + await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions))); + Assert.Empty(handler.Requests); + } + + public static TheoryData UnsupportedOptions() => new() + { + { nameof(ChatOptions.Temperature), new ChatOptions { Temperature = 0.2f } }, + { nameof(ChatOptions.TopP), new ChatOptions { TopP = 0.5f } }, + { nameof(ChatOptions.TopK), new ChatOptions { TopK = 3 } }, + { nameof(ChatOptions.MaxOutputTokens), new ChatOptions { MaxOutputTokens = 100 } }, + { nameof(ChatOptions.FrequencyPenalty), new ChatOptions { FrequencyPenalty = 0.1f } }, + { nameof(ChatOptions.PresencePenalty), new ChatOptions { PresencePenalty = 0.1f } }, + { nameof(ChatOptions.Seed), new ChatOptions { Seed = 7 } }, + { nameof(ChatOptions.Reasoning), new ChatOptions { Reasoning = new ReasoningOptions() } }, + { nameof(ChatOptions.StopSequences), new ChatOptions { StopSequences = ["stop"] } }, + { nameof(ChatOptions.ResponseFormat), new ChatOptions { ResponseFormat = ChatResponseFormat.Json } }, + { nameof(ChatOptions.ConversationId), new ChatOptions { ConversationId = "conv_1" } }, + { nameof(ChatOptions.AllowBackgroundResponses), new ChatOptions { AllowBackgroundResponses = true } }, + { nameof(ChatOptions.AllowMultipleToolCalls), new ChatOptions { AllowMultipleToolCalls = true } }, + { "HostedWebSearchTool", new ChatOptions { Tools = [new HostedWebSearchTool()] } }, + }; + + [Theory] + [MemberData(nameof(UnsupportedOptions))] + public async Task GetResponseAsync_UnsupportedOption_ThrowsBeforeSendingAsync(string option, ChatOptions options) + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + NotSupportedException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, options.WithJevQuestions(JevChatTestData.Questions()))); + Assert.Contains(option == "HostedWebSearchTool" ? "web_search" : option, exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_TextResponseFormatAndSingleToolCalls_AreAllowedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Text, AllowMultipleToolCalls = false, AllowBackgroundResponses = false, StopSequences = [] }; + + // Act + ChatResponse response = await client.GetResponseAsync(Ticket, options.WithJevQuestions(JevChatTestData.Questions())); + + // Assert + Assert.NotNull(response.GetJevResult()); + } + + [Fact] + public async Task GetResponseAsync_NonTextContent_ThrowsBeforeSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + var message = new ChatMessage(ChatRole.User, [new TextContent("See the screenshot."), new DataContent(new byte[] { 1, 2, 3 }, "image/png")]); + + // Act & Assert + NotSupportedException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync([message], new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Contains(nameof(DataContent), exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_EmptyConversation_ThrowsBeforeSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + await Assert.ThrowsAsync(() => client.GetResponseAsync([new ChatMessage(ChatRole.User, "")], new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_ReasoningAndFunctionContent_AreSerializedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + ChatMessage[] messages = + [ + new(ChatRole.User, Ticket), + new(ChatRole.Assistant, [new TextReasoningContent("Check the order first.") { ProtectedData = "opaque" }, new FunctionCallContent("call_1", "lookup_order", new Dictionary { ["order"] = "A-1" })]), + new(ChatRole.Tool, [new FunctionResultContent("call_1", JsonSerializer.SerializeToElement("""{"result":"Order A-1 shipped."}"""))]), + new(ChatRole.Tool, [new FunctionResultContent("call_2", new TextContent("Refund issued."))]), + ]; + + // Act + await client.GetResponseAsync(messages, new ChatOptions().WithJevQuestions(JevChatTestData.Questions())); + + // Assert: protected reasoning stays out of the state, and rich results become text items. + JsonNode state = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["state"]!; + JsonNode expected = JsonNode.Parse(""" + [ + { "type": "text_reasoning", "text": "Check the order first." }, + { "type": "function_call", "call_id": "call_1", "name": "lookup_order", "arguments": { "order": "A-1" } } + ] + """)!; + Assert.True(JsonNode.DeepEquals(expected, state["messages"]![1]!["contents"]), state.ToJsonString()); + Assert.Equal("tool", (string?)state["messages"]![2]!["role"]); + Assert.Equal("""{"result":"Order A-1 shipped."}""", (string?)state["messages"]![2]!["contents"]![0]!["result"]); + JsonNode rich = JsonNode.Parse("""{ "type": "function_result", "call_id": "call_2", "result": "Refund issued.", "items": [{ "type": "text", "text": "Refund issued." }] }""")!; + Assert.True(JsonNode.DeepEquals(rich, state["messages"]![3]!["contents"]![0]), state.ToJsonString()); + } + + [Fact] + public async Task GetResponseAsync_RichFunctionResult_ThrowsBeforeSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + ChatMessage[] messages = + [ + new(ChatRole.User, Ticket), + new(ChatRole.Tool, [new FunctionResultContent("call_1", new DataContent(new byte[] { 1 }, "image/png"))]), + ]; + + // Act & Assert + await Assert.ThrowsAsync(() => client.GetResponseAsync(messages, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_ResultMissingAnAnswer_ThrowsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("is_urgent", 0.9))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Contains("department", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task GetResponseAsync_ApiError_ThrowsClientResultExceptionAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.UnprocessableEntity, """{"detail":"questions.department.criteria: Field required"}"""); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + ClientResultException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Equal(422, exception.Status); + Assert.Contains("Field required", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task GetStreamingResponseAsync_ThrowsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + await Assert.ThrowsAsync(async () => + { + await foreach (ChatResponseUpdate update in client.GetStreamingResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))) + { + } + }); + Assert.Empty(handler.Requests); + } + + [Fact] + public void Constructor_InvalidOptions_Throw() + { + Assert.Throws(() => new JevChatClient(new ApiKeyCredential("k"), new JevChatClientOptions { ModelId = " " })); + Assert.Throws(() => new JevChatClient(new ApiKeyCredential("k"), new JevChatClientOptions { Endpoint = new Uri("/relative", UriKind.Relative) })); + Assert.Throws(() => new JevChatClientOptions { MaximumToolCallsPerTurn = -1 }); + } + + [Fact] + public void Constructor_FreezesTheOptions() + { + // Arrange + var options = new JevChatClientOptions(); + + // Act + using var client = new JevChatClient(new ApiKeyCredential("k"), options); + + // Assert: like other client options, the options are read-only once used. + Assert.Throws(() => options.ModelId = "other"); + Assert.Throws(() => options.DefaultQuestions = JevChatTestData.Questions()); + Assert.IsType(options.RetryPolicy); + Assert.Equal(TimeSpan.FromSeconds(10), options.NetworkTimeout); + } + + [Fact] + public void WithJevQuestions_OtherClients_KeepThePreviousFactory() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient jev = CreateClient(httpClient); + using var other = new ScriptedChatClient(); + object otherOptions = new(); + var options = new ChatOptions { RawRepresentationFactory = _ => otherOptions }; + + // Act + options.WithJevQuestions(JevChatTestData.Questions()); + + // Assert + Assert.Same(otherOptions, options.RawRepresentationFactory!(other)); + Assert.Equal(["department", "is_urgent"], Assert.IsType(options.RawRepresentationFactory(jev)).Questions.Keys); + } + + internal static JevChatClient CreateClient(HttpClient httpClient, JevChatClientOptions? options = null) + { + options ??= new JevChatClientOptions(); + options.Transport = new HttpClientPipelineTransport(httpClient); + options.RetryPolicy = new JevRetryPolicy { RecordedWaitsForTests = [] }; + return new JevChatClient(new ApiKeyCredential("test-key"), options); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs new file mode 100644 index 00000000000..7ff09a33c35 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs @@ -0,0 +1,463 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevChatClientToolTests +{ + private const string Route = JevChatTestData.Route; + + public enum City + { + Seattle, + Paris, + } + + public enum Unit + { + Celsius, + Fahrenheit, + } + + public enum Feature + { + Wind, + Humidity, + Uv, + } + + [Fact] + public async Task GetResponseAsync_JevSelectsATool_ReturnsTheFunctionCallAsync() + { + // Arrange: city and detailed are required; unit is optional and nullable. + AIFunction weather = AIFunctionFactory.Create( + ([Description("The city")] City city, [Description("Include the hourly forecast")] bool detailed, Unit? unit = null) => "sunny", + "get_weather", + "Gets the weather for a supported city."); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a1.value", 0.9), + JevChatTestData.Noul("__af_tool__.t0.a2.present", 0.2), + JevChatTestData.Choice("__af_tool__.t0.a2.value", "v1", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + + // Act + ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: the call has the chosen enum value and Boolean, and leaves out the optional argument. + FunctionCallContent call = Assert.IsType(Assert.Single(Assert.Single(response.Messages).Contents)); + Assert.Equal("get_weather", call.Name); + Assert.StartsWith("typesafe-", call.CallId, StringComparison.Ordinal); + Assert.Equal(["city", "detailed"], call.Arguments!.Keys.Order()); + Assert.Equal("Paris", Assert.IsType(call.Arguments["city"]).GetString()); + Assert.True(Assert.IsType(call.Arguments["detailed"]).GetBoolean()); + Assert.Equal(ChatFinishReason.ToolCalls, response.FinishReason); + Assert.Null(response.GetJevResult()); + + // Assert: the request adds the internal routing questions after the caller's questions. + JsonObject questions = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject(); + Assert.Equal( + ["department", "is_urgent", Route, "__af_tool__.t0.a0.value", "__af_tool__.t0.a1.value", "__af_tool__.t0.a2.present", "__af_tool__.t0.a2.value"], + questions.Select(question => question.Key)); + Assert.Equal("Tool 'get_weather': Gets the weather for a supported city.", (string?)questions[Route]!["criteria"]!["t0"]); + Assert.NotNull(questions[Route]!["criteria"]!["none"]); + Assert.Equal("\"Paris\"", (string?)questions["__af_tool__.t0.a0.value"]!["criteria"]!["v1"]); + Assert.Equal("noul", (string?)questions["__af_tool__.t0.a1.value"]!["type"]); + Assert.Contains("did the user explicitly specify the unit argument", (string?)questions["__af_tool__.t0.a2.present"]!["instructions"], StringComparison.Ordinal); + } + + [Fact] + public async Task GetResponseAsync_OptionalArgumentPresent_IsPassedAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city, Unit? unit = null, Feature[]? features = null) => "sunny", "get_weather"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.8), + JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a2.present", 0.7), + JevChatTestData.Noul("__af_tool__.t0.a2.m0", 0.9), + JevChatTestData.Noul("__af_tool__.t0.a2.m1", 0.1), + JevChatTestData.Noul("__af_tool__.t0.a2.m2", 0.6))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + + // Act + ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: a nullable enum is a Choice without null, and an enum array is one Noul per member. + FunctionCallContent call = response.Messages.SelectMany(message => message.Contents).OfType().Single(); + Assert.Equal("Seattle", Assert.IsType(call.Arguments!["city"]).GetString()); + Assert.Equal("Fahrenheit", Assert.IsType(call.Arguments["unit"]).GetString()); + Assert.Equal(["Wind", "Uv"], Assert.IsType(call.Arguments["features"]).EnumerateArray().Select(item => item.GetString())); + + JsonObject questions = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject(); + Assert.Equal(["v0", "v1"], questions["__af_tool__.t0.a1.value"]!["criteria"]!.AsObject().Select(criterion => criterion.Key)); + } + + [Fact] + public async Task GetResponseAsync_JevChoosesNoTool_ReturnsOnlyTheCallersAnswersAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("none", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + + // Act + ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert + JevResult result = Assert.IsType(response.GetJevResult()); + Assert.Equal(["department", "is_urgent"], result.Answers.Keys); + Assert.DoesNotContain("__af_tool__", response.Text, StringComparison.Ordinal); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + } + + [Fact] + public async Task GetResponseAsync_AfterTheToolCallLimit_AnswersWithTheToolResultsAsync() + { + // Arrange: the turn already has one call and its result, which is the default limit. + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.Score("severity", 1.5, 3))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + Dictionary questions = JevChatTestData.Questions(); + questions["severity"] = new JevScoreQuestion { Instructions = "How severe?", Criteria = ["Low", "Medium", "High"] }; + ChatMessage[] messages = + [ + new(ChatRole.User, "Earlier question"), + new(ChatRole.Assistant, [new FunctionCallContent("call_0", "get_weather", new Dictionary { ["city"] = "Paris" })]), + new(ChatRole.Tool, [new FunctionResultContent("call_0", "Paris is cloudy.")]), + new(ChatRole.Assistant, "Paris is cloudy.\ndepartment: technical"), + new(ChatRole.User, "Weather in Seattle?"), + new(ChatRole.Assistant, [new FunctionCallContent("call_1", "get_weather", new Dictionary { ["city"] = "Seattle" })]), + new(ChatRole.Tool, [new FunctionResultContent("call_1", JsonSerializer.SerializeToElement("Seattle is sunny."))]), + ]; + + // Act + ChatResponse response = await client.GetResponseAsync(messages, new ChatOptions { Tools = [weather] }.WithJevQuestions(questions)); + + // Assert: no tool questions, and the text has this turn's tool result and the Choice and Score decisions. + Assert.Equal(["department", "is_urgent", "severity"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); + Assert.Equal("Seattle is sunny.\ndepartment: technical\nseverity: 1.5", response.Text); + Assert.NotNull(response.GetJevResult()); + } + + [Fact] + public async Task GetResponseAsync_HigherLimit_OffersToolsAgainWithHintsAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather", "Gets the weather."); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient, new JevChatClientOptions { MaximumToolCallsPerTurn = 2 }); + ChatMessage[] messages = + [ + new(ChatRole.User, "Weather in Seattle and Paris?"), + new(ChatRole.Assistant, [new FunctionCallContent("call_1", "get_weather", new Dictionary { ["city"] = JsonSerializer.SerializeToElement("Seattle") })]), + new(ChatRole.Tool, [new FunctionResultContent("call_1", "Seattle is sunny.")]), + ]; + + // Act + ChatResponse response = await client.GetResponseAsync(messages, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: the earlier call becomes a hint in the route and argument questions. + JsonObject questions = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject(); + Assert.EndsWith("""Do not repeat argument sets already called this turn: [{"city": "Seattle"}].""", (string?)questions[Route]!["criteria"]!["t0"], StringComparison.Ordinal); + Assert.Contains("""Values already used for this argument in earlier calls this turn: ["Seattle"].""", (string?)questions["__af_tool__.t0.a0.value"]!["instructions"], StringComparison.Ordinal); + Assert.Equal("Paris", Assert.IsType(response.Messages.SelectMany(message => message.Contents).OfType().Single().Arguments!["city"]).GetString()); + } + + [Fact] + public async Task GetResponseAsync_UnsupportedTool_IsLeftOutWithAWarningAsync() + { + // Arrange: free-form text cannot be chosen from a closed set. + AIFunction search = AIFunctionFactory.Create((string query) => "found", "search"); + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("none", "t1", "none"), + JevChatTestData.Choice("__af_tool__.t1.a0.value", "v0", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + var loggerFactory = new ListLoggerFactory(); + var options = new JevChatClientOptions(); + options.ClientLoggingOptions = new() { LoggerFactory = loggerFactory, EnableLogging = false }; + using JevChatClient client = JevChatClientTests.CreateClient(httpClient, options); + + // Act + await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [search, weather] }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: the tools keep their position, so the supported tool is still t1. + JsonObject criteria = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]![Route]!["criteria"]!.AsObject(); + Assert.Equal(["t1", "none"], criteria.Select(criterion => criterion.Key)); + (LogLevel level, string message) = Assert.Single(loggerFactory.Entries, entry => entry.Message.Contains("'search'", StringComparison.Ordinal)); + Assert.Equal(LogLevel.Warning, level); + Assert.Contains("required argument 'query'", message, StringComparison.Ordinal); + } + + [Fact] + public async Task GetResponseAsync_RequiredModeWithoutSupportedTools_ThrowsAsync() + { + // Arrange + AIFunction search = AIFunctionFactory.Create((string query) => "found", "search"); + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + ChatOptions options = new ChatOptions { Tools = [search], ToolMode = ChatToolMode.RequireAny }.WithJevQuestions(JevChatTestData.Questions()); + + // Act & Assert + InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(JevTestData.TicketText, options)); + Assert.Contains("search: required argument 'query'", exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_RequireSpecificTool_SkipsTheRouteQuestionAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + AIFunction forecast = AIFunctionFactory.Create((City city) => "rain", "get_forecast"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + ChatOptions options = new ChatOptions { Tools = [weather, forecast], ToolMode = ChatToolMode.RequireSpecific("get_forecast") }.WithJevQuestions(JevChatTestData.Questions()); + + // Act + ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, options); + + // Assert + Assert.Equal("get_forecast", response.Messages.SelectMany(message => message.Contents).OfType().Single().Name); + Assert.Null(JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]![Route]); + } + + [Fact] + public async Task GetResponseAsync_RequiredModeWithSeveralTools_OmitsTheNoneRouteAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + AIFunction forecast = AIFunctionFactory.Create((City city) => "rain", "get_forecast"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t1", "t0", "t1"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"), + JevChatTestData.Choice("__af_tool__.t1.a0.value", "v1", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + ChatOptions options = new ChatOptions { Tools = [weather, forecast], ToolMode = ChatToolMode.RequireAny }.WithJevQuestions(JevChatTestData.Questions()); + + // Act + ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, options); + + // Assert + Assert.Equal(["t0", "t1"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]![Route]!["criteria"]!.AsObject().Select(criterion => criterion.Key)); + Assert.Equal("get_forecast", response.Messages.SelectMany(message => message.Contents).OfType().Single().Name); + } + + [Fact] + public async Task GetResponseAsync_RequiredToolMissingOrNoneMode_BehaveAsExpectedAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + + // Act & Assert: a missing required tool fails, and the none mode sends only the caller's questions. + await Assert.ThrowsAsync(() => client.GetResponseAsync( + JevTestData.TicketText, + new ChatOptions { Tools = [weather], ToolMode = ChatToolMode.RequireSpecific("other") }.WithJevQuestions(JevChatTestData.Questions()))); + await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather], ToolMode = ChatToolMode.None }.WithJevQuestions(JevChatTestData.Questions())); + Assert.Equal(["department", "is_urgent"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); + } + + [Fact] + public async Task RunAsync_AgentExecutesTheToolOnce_ThenAnswersAsync() + { + // Arrange + var executions = new List<(City City, bool Detailed)>(); + AIFunction weather = AIFunctionFactory.Create( + (City city, bool detailed) => + { + executions.Add((city, detailed)); + return $"{city} is sunny{(detailed ? " with an hourly forecast" : string.Empty)}."; + }, + "get_weather", + "Gets the weather for a supported city."); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a1.value", 0.9))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(instructions: "Answer weather requests.", tools: [weather]); + + // Act + AgentResponse response = await agent.RunAsync( + "Give me a detailed weather report for Paris.", + options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + + // Assert: the tool ran once, and the second request answered the caller's questions with its result. + Assert.Equal([(City.Paris, true)], executions); + Assert.Equal(2, handler.Requests.Count); + JsonNode second = JsonNode.Parse(handler.Requests[1].Body)!; + Assert.Equal(["department", "is_urgent"], second["questions"]!.AsObject().Select(question => question.Key)); + Assert.Equal("Answer weather requests.", (string?)second["state"]!["instructions"]); + JsonNode toolResult = second["state"]!["messages"]!.AsArray().SelectMany(message => message!["contents"]!.AsArray()).Single(content => (string?)content!["type"] == "function_result")!; + Assert.Equal("Paris is sunny with an hourly forecast.", (string?)toolResult["result"]); + + Assert.Equal("Paris is sunny with an hourly forecast.\ndepartment: technical", response.Text); + JevResult result = Assert.IsType(response.GetJevResult()); + Assert.Equal(["department", "is_urgent"], result.Answers.Keys); + Assert.Equal(24, response.Usage!.TotalTokenCount); + } + + [Fact] + public async Task RunAsync_AgentLevelQuestions_AreUsedUnlessTheRunPassesItsOwnAsync() + { + // Arrange + var runQuestions = new Dictionary { ["spam"] = new JevNoulQuestion { Instructions = "Is it spam?" } }; + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("spam", 0.05))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions().WithJevQuestions(JevChatTestData.Questions()) }); + + // Act + AgentResponse agentLevel = await agent.RunAsync(JevTestData.TicketText); + AgentResponse runLevel = await agent.RunAsync(JevTestData.TicketText, options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(runQuestions))); + + // Assert + Assert.Equal(["department", "is_urgent"], agentLevel.GetJevResult()!.Answers.Keys); + Assert.Equal(["spam"], runLevel.GetJevResult()!.Answers.Keys); + } + + [Fact] + public async Task RunAsync_ApprovalRequiredTool_RunsOnlyAfterApprovalAsync() + { + // Arrange + int executions = 0; + AIFunction refund = new ApprovalRequiredAIFunction(AIFunctionFactory.Create( + (City city) => + { + executions++; + return "Refund issued."; + }, + "issue_refund")); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers("billing"))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers("billing"), + JevChatTestData.RouteTo("none", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions { Tools = [refund] }.WithJevQuestions(JevChatTestData.Questions()) }); + AgentSession session = await agent.CreateSessionAsync(); + + // Act + AgentResponse pending = await agent.RunAsync("Refund my Seattle order.", session); + ToolApprovalRequestContent approval = pending.Messages.SelectMany(message => message.Contents).OfType().Single(); + int executionsBeforeApproval = executions; + AgentResponse approved = await agent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(true)]), session); + AgentResponse later = await agent.RunAsync("Thanks. Anything else?", session); + + // Assert + Assert.Equal(0, executionsBeforeApproval); + Assert.Equal(1, executions); + Assert.Equal("Refund issued.\ndepartment: billing", approved.Text); + + // Assert: the next turn still works. The handled approval stays in the history, but Jev does not read it. + Assert.Equal(3, handler.Requests.Count); + Assert.DoesNotContain("approval", handler.Requests[2].Body, StringComparison.OrdinalIgnoreCase); + Assert.NotNull(later.GetJevResult()); + } + + [Fact] + public async Task RunAsync_ApprovalSentWithText_ContinuesTheTurnOfTheApprovedCallAsync() + { + // Arrange + int executions = 0; + AIFunction refund = new ApprovalRequiredAIFunction(AIFunctionFactory.Create( + (City city) => + { + executions++; + return "Refund issued."; + }, + "issue_refund")); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers("billing"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions { Tools = [refund] }.WithJevQuestions(JevChatTestData.Questions()) }); + AgentSession session = await agent.CreateSessionAsync(); + + // Act: FunctionInvokingChatClient moves the text after the approved call and its result. + AgentResponse pending = await agent.RunAsync("Refund my Seattle order.", session); + ToolApprovalRequestContent approval = pending.Messages.SelectMany(message => message.Contents).OfType().Single(); + AgentResponse approved = await agent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(true), new TextContent("Yes, go ahead.")]), session); + + // Assert: the approved call counts toward the limit, so the tool is not offered again. + Assert.Equal(1, executions); + Assert.Equal(2, handler.Requests.Count); + Assert.Equal(["department", "is_urgent"], JsonNode.Parse(handler.Requests[1].Body)!["questions"]!.AsObject().Select(question => question.Key)); + Assert.Equal("Refund issued.\ndepartment: billing", approved.Text); + } + + [Fact] + public async Task RunStreamingAsync_ThrowsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions().WithJevQuestions(JevChatTestData.Questions()) }); + + // Act & Assert + await Assert.ThrowsAsync(async () => + { + await foreach (AgentResponseUpdate update in agent.RunStreamingAsync(JevTestData.TicketText)) + { + } + }); + Assert.Empty(handler.Requests); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs new file mode 100644 index 00000000000..a20f99a0ff1 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// Builds TypeSafe API responses and questions for the chat client tests. +/// +internal static class JevChatTestData +{ + public const string Route = "__af_tool__.route"; + + /// Two questions about a support ticket: a Choice and a Noul. + public static Dictionary Questions() => new() + { + ["department"] = new JevChoiceQuestion + { + Instructions = "Which team should handle the request?", + Criteria = new Dictionary { ["billing"] = "Payments or refunds", ["technical"] = "Bugs or integrations" }, + }, + ["is_urgent"] = new JevNoulQuestion { Instructions = "Does the request need urgent attention?" }, + }; + + /// The answers to . + public static string[] UserAnswers(string department = "technical") => + [Choice("department", department, "billing", "technical"), Noul("is_urgent", 0.9)]; + + public static string Result(params string[] answers) => + "{\"model\":\"jev-test\",\"answers\":{" + string.Join(",", answers) + "},\"usage\":{\"input_tokens\":10,\"output_tokens\":2}}"; + + public static string Result(IEnumerable answers, params string[] more) => Result([.. answers, .. more]); + + public static string Choice(string id, string choice, params string[] labels) + { + string probabilities = string.Join(",", labels.Select(label => $"\"{label}\":{(label == choice ? "1.0" : "0.0")}")); + return $"\"{id}\":{{\"type\":\"choice\",\"choice\":\"{choice}\",\"confidence\":0.9,\"probabilities\":{{{probabilities}}}}}"; + } + + public static string Noul(string id, double noul) => + $"\"{id}\":{{\"type\":\"noul\",\"noul\":{noul.ToString(CultureInfo.InvariantCulture)}}}"; + + public static string Score(string id, double score, int levels) + { + string legend = string.Join(",", Enumerable.Range(0, levels).Select(level => $"\"{level}\":\"Level {level}\"")); + string probabilities = string.Join(",", Enumerable.Range(0, levels).Select(level => $"\"{level}\":{(level == (int)score ? "1.0" : "0.0")}")); + return $"\"{id}\":{{\"type\":\"score\",\"score\":{score.ToString(CultureInfo.InvariantCulture)},\"confidence\":0.8,\"legend\":{{{legend}}},\"probabilities\":{{{probabilities}}}}}"; + } + + /// A route answer that selects one of the route labels. + public static string RouteTo(string label, params string[] labels) => Choice(Route, label, labels); +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevEntryTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevEntryTests.cs new file mode 100644 index 00000000000..532d670a315 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevEntryTests.cs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevEntryTests +{ + [Fact] + public void Conversions_FromTextAndJson_KeepTheValue() + { + // Arrange + using JsonDocument document = JsonDocument.Parse("""{ "question": "Is `order` paid?" }"""); + + // Act + JevEntry text = "Is this urgent?"; + JevEntry nullText = (string?)null; + var json = (JevEntry)document.RootElement; + + // Assert + Assert.Equal(JsonValueKind.String, text.Kind); + Assert.Equal("Is this urgent?", text.Text); + Assert.Equal("Is this urgent?", text.ToString()); + Assert.True(nullText.IsNull); + Assert.Equal(JevEntry.Null, nullText); + Assert.Equal(JsonValueKind.Object, json.Kind); + Assert.Null(json.Text); + Assert.Equal("""{ "question": "Is `order` paid?" }""", json.ToString()); + } + + [Fact] + public void Default_IsTheNullEntry() + { + // Act + JevEntry entry = default; + + // Assert + Assert.True(entry.IsNull); + Assert.Equal(JsonValueKind.Null, entry.Kind); + Assert.Null(entry.Json); + Assert.Equal(string.Empty, entry.ToString()); + } + + [Fact] + public void Json_OutlivesTheDocumentItCameFrom() + { + // Arrange + JevEntry entry; + using (JsonDocument document = JsonDocument.Parse("""["a", "b"]""")) + { + entry = JevEntry.FromJsonElement(document.RootElement); + } + + // Act & Assert + Assert.Equal(2, entry.Json!.Value.GetArrayLength()); + } + + [Theory] + [InlineData("42")] + [InlineData("true")] + [InlineData("false")] + public void FromJsonElement_NumberOrBoolean_Throws(string json) + { + // Arrange + using JsonDocument document = JsonDocument.Parse(json); + JsonElement element = document.RootElement; + + // Act & Assert + Assert.Throws(() => (JevEntry)element); + Assert.Throws(() => JevEntry.FromJsonElement(element)); + } + + [Fact] + public void Equality_ComparesJsonValuesRegardlessOfPropertyOrder() + { + // Arrange + using JsonDocument first = JsonDocument.Parse("""{ "a": 1, "b": [true] }"""); + using JsonDocument second = JsonDocument.Parse("""{ "b": [true], "a": 1 }"""); + using JsonDocument other = JsonDocument.Parse("""{ "a": 2 }"""); + var left = (JevEntry)first.RootElement; + var right = (JevEntry)second.RootElement; + + // Act & Assert + Assert.True(left == right); + Assert.Equal(left.GetHashCode(), right.GetHashCode()); + Assert.True(left != (JevEntry)other.RootElement); + Assert.Equal((JevEntry)"same", (JevEntry)"same"); + Assert.NotEqual((JevEntry)"same", JevEntry.Null); + } + + [Theory] + [InlineData("\"text\"", JsonValueKind.String)] + [InlineData("{\"k\":\"v\"}", JsonValueKind.Object)] + [InlineData("[\"x\"]", JsonValueKind.Array)] + [InlineData("null", JsonValueKind.Null)] + public void Serialization_RoundTripsTheJsonValue(string json, JsonValueKind kind) + { + // Act + var entry = (JevEntry)JsonSerializer.Deserialize(json, JevJsonUtilities.Options.GetTypeInfo(typeof(JevEntry)))!; + string written = JsonSerializer.Serialize(entry, JevJsonUtilities.Options.GetTypeInfo(typeof(JevEntry))); + + // Assert + Assert.Equal(kind, entry.Kind); + Assert.Equal(json, written); + } + + [Theory] + [InlineData("42")] + [InlineData("true")] + public void Deserialization_NumberOrBoolean_Throws(string json) + { + // Act & Assert + Assert.Throws(() => JsonSerializer.Deserialize(json, JevJsonUtilities.Options.GetTypeInfo(typeof(JevEntry)))); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpClientTests.cs similarity index 76% rename from dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs rename to dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpClientTests.cs index 7ec9e0ab56c..a1e0d699931 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpClientTests.cs @@ -15,7 +15,7 @@ namespace Microsoft.Agents.AI.TypeSafe.UnitTests; -public sealed class JevHttpEvaluatorTests +public sealed class JevHttpClientTests { private const string ApiKey = "test-key-9f2c"; @@ -79,7 +79,7 @@ public async Task InvokeAsync_CustomPolicy_RunsInThePipelineAsync() } [Fact] - public async Task InvokeAsync_RateLimitedOrOverloaded_RetriesAsync() + public async Task InvokeAsync_RateLimitedOrOverloaded_RetriesWithBackoffAsync() { // Arrange using var handler = new JevTestHttpHandler() @@ -92,12 +92,33 @@ public async Task InvokeAsync_RateLimitedOrOverloaded_RetriesAsync() // Act object? result = await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); - // Assert + // Assert: the SDK backoff is 500 ms doubled per attempt, minus up to 25% jitter. Assert.Equal(3, handler.Requests.Count); Assert.Equal(2, waits.Count); + Assert.InRange(waits[0], TimeSpan.FromMilliseconds(375), TimeSpan.FromMilliseconds(500)); + Assert.InRange(waits[1], TimeSpan.FromMilliseconds(750), TimeSpan.FromMilliseconds(1000)); Assert.Equal("technical", Assert.IsType(result).GetProperty("answers").GetProperty("department").GetProperty("choice").GetString()); } + [Theory] + [InlineData(503)] + [InlineData(520)] + [InlineData(408)] + public async Task InvokeAsync_TransientStatus_IsRetriedLikeTheSdkAsync(int status) + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply((HttpStatusCode)status, "{}") + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + + // Act + await CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal(2, handler.Requests.Count); + } + [Fact] public async Task InvokeAsync_StillRateLimitedAfterRetries_FailsWithTheStatusAsync() { @@ -109,9 +130,9 @@ public async Task InvokeAsync_StillRateLimitedAfterRetries_FailsWithTheStatusAsy ClientResultException exception = await Assert.ThrowsAsync(() => CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); - // Assert + // Assert: the first attempt plus the SDK's two retries. Assert.Equal(429, exception.Status); - Assert.Equal(4, handler.Requests.Count); + Assert.Equal(3, handler.Requests.Count); } [Fact] @@ -132,6 +153,23 @@ public async Task InvokeAsync_RetryAfterSeconds_IsWaitedAsync() Assert.Equal(TimeSpan.FromSeconds(5), Assert.Single(waits)); } + [Fact] + public async Task InvokeAsync_RetryAfterMilliseconds_IsPreferredAsync() + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.TooManyRequests, "{}", new RetryConditionHeaderValue(TimeSpan.FromSeconds(30)), retryAfterMs: "1500") + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal(TimeSpan.FromMilliseconds(1500), Assert.Single(waits)); + } + [Fact] public async Task InvokeAsync_RetryAfterDate_IsWaitedAsync() { @@ -150,27 +188,54 @@ public async Task InvokeAsync_RetryAfterDate_IsWaitedAsync() Assert.InRange(Assert.Single(waits), TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(21)); } + [Theory] + [InlineData("Retry-After", "1e300")] + [InlineData("Retry-After", "99999999999999")] + [InlineData("Retry-After", "NaN")] + [InlineData("retry-after-ms", "1e30")] + [InlineData("retry-after-ms", "NaN")] + public async Task InvokeAsync_MalformedOrHugeServerDelay_FallsBackToBackoffAsync(string header, string value) + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply( + HttpStatusCode.TooManyRequests, + "{}", + retryAfterMs: header == "retry-after-ms" ? value : null, + retryAfterRaw: header == "Retry-After" ? value : null) + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert: like the SDK, an unusable server delay is ignored instead of failing the call. + Assert.Equal(2, handler.Requests.Count); + Assert.InRange(Assert.Single(waits), TimeSpan.FromMilliseconds(375), TimeSpan.FromMilliseconds(500)); + } + [Theory] [InlineData(false)] [InlineData(true)] - public async Task InvokeAsync_RetryAfterBeyondTheLimit_FailsAtOnceAsync(bool asDate) + public async Task InvokeAsync_RetryAfterBeyondTheLimit_FallsBackToBackoffAsync(bool asDate) { // Arrange RetryConditionHeaderValue retryAfter = asDate ? new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddHours(1)) : new RetryConditionHeaderValue(TimeSpan.FromHours(1)); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.TooManyRequests, """{"detail":"Slow down"}""", retryAfter); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.TooManyRequests, """{"detail":"Slow down"}""", retryAfter) + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); using var httpClient = new HttpClient(handler); List waits = []; // Act - ClientResultException exception = await Assert.ThrowsAsync(() => - CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); - // Assert - Assert.Equal(429, exception.Status); - Assert.Single(handler.Requests); - Assert.Empty(waits); + // Assert: waiting an hour would block the agent run, so the SDK's backoff is used instead. + Assert.Equal(2, handler.Requests.Count); + Assert.InRange(Assert.Single(waits), TimeSpan.FromMilliseconds(375), TimeSpan.FromMilliseconds(500)); } [Fact] diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs new file mode 100644 index 00000000000..88d99d27b80 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +#pragma warning disable MAAI001 // LoopAgent and its evaluators are experimental. + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// Tests Jev as the judge of a , as the loop judge sample does. +/// +public sealed class JevLoopJudgeTests +{ + private static readonly Dictionary s_criteria = new() + { + ["blue_sky"] = new JevNoulQuestion { Instructions = "Does the assistant's latest response explain why the sky is blue?" }, + ["red_sunsets"] = new JevNoulQuestion { Instructions = "Does the assistant's latest response explain why sunsets are red?" }, + }; + + [Fact] + public async Task RunAsync_DelegateEvaluatorWithJev_LoopsUntilEveryCriterionIsMetAsync() + { + // Arrange: the first answer misses the sunsets, the second covers both. + using var answerer = new ScriptedChatClient( + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Air scatters blue light.")), + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Air scatters blue light, and sunsets are red because light crosses more air."))); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("blue_sky", 0.98), JevChatTestData.Noul("red_sunsets", 0.03))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("blue_sky", 0.97), JevChatTestData.Noul("red_sunsets", 0.95))); + using var httpClient = new HttpClient(handler); + using JevChatClient judge = JevChatClientTests.CreateClient(httpClient); + + var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => + { + List conversation = [.. context.InitialMessages, new ChatMessage(ChatRole.Assistant, context.LastResponse.Text)]; + ChatResponse verdict = await judge.GetResponseAsync(conversation, new ChatOptions().WithJevQuestions(s_criteria), cancellationToken); + List missing = [.. verdict.GetJevResult()!.Answers.Where(answer => ((JevNoulResponse)answer.Value).Noul <= 0.5).Select(answer => answer.Key)]; + return missing.Count == 0 ? LoopEvaluation.Stop() : LoopEvaluation.Continue($"Missing: {string.Join(", ", missing)}."); + }); + var loop = new LoopAgent(answerer.AsAIAgent(), evaluator, new LoopAgentOptions { MaxIterations = 3, NonStreamingReturnsLastResponseOnly = true }); + + // Act + AgentResponse response = await loop.RunAsync("Explain why the sky is blue and why sunsets are red."); + + // Assert: two runs, and the second run received the missing criterion as feedback. + Assert.Equal(2, answerer.Calls.Count); + Assert.Contains(answerer.Calls[1], message => message.Role == ChatRole.User && message.Text == "Missing: red_sunsets."); + Assert.Contains("sunsets are red", response.Text, StringComparison.Ordinal); + + // Assert: Jev read the original request and the latest answer, and only the criteria questions. + Assert.Equal(2, handler.Requests.Count); + JsonNode first = JsonNode.Parse(handler.Requests[0].Body)!; + Assert.Equal(["blue_sky", "red_sunsets"], first["questions"]!.AsObject().Select(question => question.Key)); + JsonArray messages = first["state"]!["messages"]!.AsArray(); + Assert.Equal(["user", "assistant"], messages.Select(message => (string?)message!["role"])); + Assert.Equal("Air scatters blue light.", (string?)messages[1]!["contents"]![0]!["text"]); + } + + [Fact] + public async Task EvaluateAsync_AIJudgeLoopEvaluatorWithJev_IsNotSupportedAsync() + { + // Arrange: AIJudgeLoopEvaluator asks for a JudgeVerdict as JSON structured output, which Jev cannot write. + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient judge = JevChatClientTests.CreateClient(httpClient, new JevChatClientOptions { DefaultQuestions = s_criteria }); + using var answerer = new ScriptedChatClient(_ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Air scatters blue light."))); + var loop = new LoopAgent(answerer.AsAIAgent(), new AIJudgeLoopEvaluator(judge)); + + // Act & Assert: the request fails before anything is sent, which is why the sample uses a delegate evaluator. + NotSupportedException exception = await Assert.ThrowsAsync(() => loop.RunAsync("Why is the sky blue?")); + Assert.Contains(nameof(ChatOptions.ResponseFormat), exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs index 3afde9ae642..e0c045c97e2 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs @@ -79,26 +79,26 @@ public static AIFunctionArguments Arguments(string json) return arguments; } - /// A correct response to , built from the contract types. - public static JevResponse CreateResponse() => new() + /// A correct result for , built from the contract types. + public static JevResult CreateResult() => new() { Model = "custom-model", - Answers = new Dictionary + Answers = new Dictionary { - ["department"] = new JevChoiceAnswer + ["department"] = new JevChoiceResponse { Choice = "technical", Confidence = 0.78, Probabilities = new Dictionary { ["technical"] = 0.85, ["billing"] = 0.15, ["sales"] = 0 }, }, - ["frustration"] = new JevScoreAnswer + ["frustration"] = new JevScoreResponse { Score = 1, Confidence = 1, - Legend = new Dictionary { ["0"] = "Calm", ["1"] = "Frustrated", ["2"] = "Very angry" }, + Legend = new Dictionary { ["0"] = "Calm", ["1"] = "Frustrated", ["2"] = "Very angry" }, Probabilities = new Dictionary { ["0"] = 0, ["1"] = 1, ["2"] = 0 }, }, - ["is_urgent"] = new JevNoulAnswer { Noul = 0.97 }, + ["is_urgent"] = new JevNoulResponse { Noul = 0.97 }, }, Usage = new JevUsage { InputTokens = 10, OutputTokens = 5 }, }; diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs index bc53c152cd1..178285578f0 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs @@ -16,14 +16,14 @@ namespace Microsoft.Agents.AI.TypeSafe.UnitTests; /// internal sealed class JevTestHttpHandler : HttpMessageHandler { - private readonly Queue<(HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter)> _responses = new(); - private (HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter) _last = (HttpStatusCode.OK, JevTestData.ResponseJson, null); + private readonly Queue<(HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter, string? RetryAfterMs, string? RetryAfterRaw)> _responses = new(); + private (HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter, string? RetryAfterMs, string? RetryAfterRaw) _last = (HttpStatusCode.OK, JevTestData.ResponseJson, null, null, null); public List Requests { get; } = []; - public JevTestHttpHandler Reply(HttpStatusCode status, string body, RetryConditionHeaderValue? retryAfter = null) + public JevTestHttpHandler Reply(HttpStatusCode status, string body, RetryConditionHeaderValue? retryAfter = null, string? retryAfterMs = null, string? retryAfterRaw = null) { - this._responses.Enqueue((status, body, retryAfter)); + this._responses.Enqueue((status, body, retryAfter, retryAfterMs, retryAfterRaw)); return this; } @@ -48,6 +48,17 @@ protected override async Task SendAsync(HttpRequestMessage Content = new StringContent(this._last.Body, Encoding.UTF8, "application/json"), }; response.Headers.RetryAfter = this._last.RetryAfter; + if (this._last.RetryAfterMs is { } retryAfterMs) + { + response.Headers.Add("retry-after-ms", retryAfterMs); + } + + // A raw value lets tests send Retry-After headers that HttpClient would refuse to parse. + if (this._last.RetryAfterRaw is { } raw) + { + response.Headers.TryAddWithoutValidation("Retry-After", raw); + } + return response; } diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs new file mode 100644 index 00000000000..e78d148c492 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevToolCallCompilerTests +{ + private static readonly Dictionary>> s_noCalls = []; + + [Fact] + public void Compile_ReferencesAndAnyOfNull_ResolveAndDecode() + { + // Arrange: a $defs reference, and an optional anyOf of an enum and null, as Pydantic writes them. + AIFunctionDeclaration tool = Declaration("set_mode", """ + { + "type": "object", + "properties": { + "mode": { "$ref": "#/$defs/Mode", "description": "The mode" }, + "level": { "anyOf": [{ "enum": ["low", "high"] }, { "type": "null" }], "default": null }, + "kind": { "const": "fixed" } + }, + "required": ["mode", "kind"], + "$defs": { "Mode": { "type": "string", "enum": ["fast", "safe"] } } + } + """); + + // Act + JevToolCallPlan plan = Compile([tool])!; + JevToolCall? call = plan.Decode(Result( + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.9), + JevChatTestData.Choice("__af_tool__.t0.a1.value", "v0", "v0", "v1"))); + + // Assert + Assert.Contains("The mode", plan.Questions["__af_tool__.t0.a0.value"].Instructions!.Value.Text, StringComparison.Ordinal); + Assert.Equal(["route", "t0.a0.value", "t0.a1.present", "t0.a1.value"], plan.Questions.Keys.Select(key => key["__af_tool__.".Length..])); + Assert.Equal("safe", ((JsonElement)call!.Value.Arguments["mode"]!).GetString()); + Assert.Equal("low", ((JsonElement)call.Value.Arguments["level"]!).GetString()); + Assert.Equal("fixed", ((JsonElement)call.Value.Arguments["kind"]!).GetString()); + } + + [Theory] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "boolean" } }, "minProperties": 1 }""", "unsupported root-level schema constraints: minProperties")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "enum": ["x", "y"], "minLength": 1 } }, "required": ["a"] }""", "required argument 'a': unsupported argument schema constraints: minLength")] + [InlineData("""{ "type": "object", "properties": { "a": { "$ref": "#/$defs/A", "enum": ["x"] } }, "$defs": { "A": { "type": "boolean" } } }""", "optional argument 'a': $ref sibling constraints are not supported: enum")] + [InlineData("""{ "type": "object", "properties": { "a": { "anyOf": [{ "type": "boolean" }, { "type": "string", "enum": ["x"] }] } } }""", "only a single schema combined with null is supported")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": ["boolean", "null"] } }, "required": ["a"] }""", "required nullable arguments are not supported")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": "string", "enum": ["x"], "maxLength": 3 } } } }""", "unsupported array item schema constraints: maxLength")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": "integer", "enum": [1, 2.5] } } } }""", "array enum member 2.5 does not match declared item type \"integer\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": ["string", "null"], "enum": ["x", null] } } } }""", "nullable array members are not supported")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer" } } }""", "only const, enum, boolean, and arrays of enum values are supported")] + [InlineData("""{ "type": "object", "properties": {}, "required": ["missing"] }""", "required arguments are missing from properties: missing")] + [InlineData("""{ "type": "array" }""", "the top-level input schema must be an object")] + public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema, string reason) + { + // Arrange + var logger = new ListLoggerFactory(); + + // Act + JevToolCallPlan? plan = Compile([Declaration("tool", schema)], logger: logger.CreateLogger("test")); + + // Assert + Assert.Null(plan); + Assert.Contains(reason, Assert.Single(logger.Entries).Message, StringComparison.Ordinal); + } + + [Fact] + public void Compile_SingleValueEnumAndNoArguments_NeedNoArgumentQuestions() + { + // Arrange + AIFunctionDeclaration refresh = Declaration("refresh", """{ "type": "object", "properties": {} }"""); + AIFunctionDeclaration single = Declaration("single", """{ "type": "object", "properties": { "scope": { "enum": ["all"] } }, "required": ["scope"] }"""); + + // Act + JevToolCallPlan plan = Compile([refresh, single])!; + JevToolCall? call = plan.Decode(Result(JevChatTestData.RouteTo("t1", "t0", "t1", "none"))); + + // Assert + Assert.Equal([JevChatTestData.Route], plan.Questions.Keys); + Assert.Equal("single", call!.Value.Function.Name); + Assert.Equal("all", ((JsonElement)call.Value.Arguments["scope"]!).GetString()); + } + + [Fact] + public void Compile_TooManyTools_Throws() + { + // Arrange + List tools = [.. Enumerable.Range(0, JevToolCallCompiler.MaxRoutableTools + 1).Select(index => Declaration($"tool{index}", """{ "type": "object", "properties": {} }"""))]; + + // Act & Assert + Assert.Throws(() => Compile(tools)); + } + + [Fact] + public void Compile_RequiredToolFilter_AppliesBeforeTheToolLimit() + { + // Arrange + List tools = [.. Enumerable.Range(0, JevToolCallCompiler.MaxRoutableTools + 1).Select(index => Declaration($"tool{index}", """{ "type": "object", "properties": {} }"""))]; + + // Act + JevToolCallPlan plan = Compile(tools, ChatToolMode.RequireSpecific("tool3"))!; + + // Assert + Assert.Equal("tool3", plan.RequiredTool!.Function.Name); + Assert.Null(plan.RouteQuestionId); + } + + [Fact] + public void Compile_TooManyInternalQuestions_Throws() + { + // Arrange: each tool needs 40 member questions, so the fourth exceeds 128. + string members = string.Join(",", Enumerable.Range(0, 40).Select(index => $"\"m{index}\"")); + List tools = [.. Enumerable.Range(0, 4).Select(index => Declaration($"tool{index}", $$"""{ "type": "object", "properties": { "tags": { "type": "array", "items": { "enum": [{{members}}] } } }, "required": ["tags"] }"""))]; + + // Act & Assert + InvalidOperationException exception = Assert.Throws(() => Compile(tools)); + Assert.Contains("128", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void Compile_UnsupportedTool_ReturnsItsQuestionBudget() + { + // Arrange: the first tool fails only after its array reserved 64 questions; they must be released. + string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); + AIFunctionDeclaration failing = Declaration("failing", $$"""{ "type": "object", "properties": { "tags": { "type": "array", "items": { "enum": [{{members}}] } }, "text": { "type": "string" } }, "required": ["tags"] }"""); + AIFunctionDeclaration large = Declaration("large", $$"""{ "type": "object", "properties": { "a": { "type": "array", "items": { "enum": [{{members}}] } }, "b": { "type": "array", "items": { "enum": [{{string.Join(",", Enumerable.Range(0, 60).Select(index => $"\"n{index}\""))}}] } } }, "required": ["a", "b"] }"""); + + // Act + JevToolCallPlan plan = Compile([failing, large])!; + + // Assert + Assert.Equal(125, plan.Questions.Count); + } + + [Fact] + public void Compile_TooManyEnumValuesOrProperties_ExcludeTheTool() + { + // Arrange + string values = string.Join(",", Enumerable.Range(0, JevToolCallCompiler.MaxEnumValues + 1).Select(index => $"\"v{index}\"")); + string properties = string.Join(",", Enumerable.Range(0, JevToolCallCompiler.MaxToolProperties + 1).Select(index => $"\"p{index}\": {{ \"type\": \"boolean\" }}")); + + // Act + JevToolCallPlan? plan = Compile( + [ + Declaration("enum", $$"""{ "type": "object", "properties": { "a": { "enum": [{{values}}] } } }"""), + Declaration("wide", $$"""{ "type": "object", "properties": { {{properties}} } }"""), + ]); + + // Assert + Assert.Null(plan); + } + + [Fact] + public void Decode_UnknownRouteOrMissingAnswer_Throws() + { + // Arrange + JevToolCallPlan plan = Compile([Declaration("flag", """{ "type": "object", "properties": { "on": { "type": "boolean" } }, "required": ["on"] }""")])!; + + // Act & Assert + Assert.Throws(() => plan.Decode(Result(JevChatTestData.RouteTo("t9", "t9")))); + Assert.Throws(() => plan.Decode(Result(JevChatTestData.RouteTo("t0", "t0")))); + } + + [Fact] + public void Compile_DuplicateDescriptions_KeepToolNamesInTheRoute() + { + // Arrange + AIFunctionDeclaration first = Declaration("first", """{ "type": "object", "properties": {} }""", "Does it."); + AIFunctionDeclaration second = Declaration("second", """{ "type": "object", "properties": {} }""", "Does it."); + + // Act + JevChoiceQuestion route = Assert.IsType(Compile([first, second])!.Questions[JevChatTestData.Route]); + + // Assert + Assert.Equal("Tool 'first': Does it.", route.Criteria["t0"].Text); + Assert.Equal("Tool 'second': Does it.", route.Criteria["t1"].Text); + } + + [Fact] + public void Describe_MatchesPythonJsonDumps() + { + // Arrange: json.dumps(value, ensure_ascii=False, sort_keys=True) in the Python connector. + JsonElement value = JsonDocument.Parse("""{"city":"São Paulo","b":[1,2.5,true,null],"q":"say \"hi\"\n"}""").RootElement; + + // Act + string described = JevToolSchema.Describe(value); + + // Assert + Assert.Equal("""{"b": [1, 2.5, true, null], "city": "São Paulo", "q": "say \"hi\"\n"}""", described); + } + + private static AIFunctionDeclaration Declaration(string name, string schema, string? description = null) => + AIFunctionFactory.CreateDeclaration(name, description, JsonDocument.Parse(schema).RootElement.Clone()); + + private static JevToolCallPlan? Compile(IReadOnlyList tools, ChatToolMode? mode = null, ILogger? logger = null) => + JevToolCallCompiler.Compile(tools, mode, s_noCalls, logger ?? new ListLoggerFactory().CreateLogger("test")); + + private static JevResult Result(params string[] answers) => + JsonSerializer.Deserialize(JevChatTestData.Result(answers), JevJsonContext.Default.JevResult)!; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs new file mode 100644 index 00000000000..df54fb10202 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// A logger factory that records every message, so tests can check what a client logs. +/// +internal sealed class ListLoggerFactory : ILoggerFactory +{ + public List<(LogLevel Level, string Message)> Entries { get; } = []; + + public ILogger CreateLogger(string categoryName) => new ListLogger(this.Entries); + + public void AddProvider(ILoggerProvider provider) + { + } + + public void Dispose() + { + } + + private sealed class ListLogger(List<(LogLevel Level, string Message)> entries) : ILogger + { + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + entries.Add((logLevel, formatter(state, exception))); + } +} From 3e7756ee2b53259b17065e716ce78e851cb677d1 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:46:36 +0100 Subject: [PATCH 03/16] .NET: Address TypeSafe review feedback on schema checks and the loop judge sample - Follow chains of local $ref aliases when compiling tool schemas, bounded by the existing resolution depth, so an alias of a definition is routed instead of being excluded and a reference cycle excludes the tool. - Check const and enum values against their declared type, as array members already were, so a tool whose values could only fail when called is excluded before Jev routes a call to it. - Name the criterion threshold in the loop judge sample and document how to choose it, and how to word criteria for a judge. --- .../Program.cs | 10 ++- .../Agent_TypeSafe_Step01_LoopJudge/README.md | 20 +++++- .../Core/JevToolCallCompiler.cs | 16 ++--- .../Core/JevToolSchema.cs | 46 ++++++++++++- .../JevToolCallCompilerTests.cs | 66 +++++++++++++++++++ 5 files changed, 142 insertions(+), 16 deletions(-) diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs index 4bfe9cb9c7c..5310dbff117 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs @@ -18,7 +18,8 @@ var typeSafeApiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); // 1. The criteria. Each one becomes a Noul question, whose answer is the probability that the answer meets it, so the -// judge reports exactly which criteria are missing instead of a single verdict. +// judge reports exactly which criteria are missing instead of a single verdict. The questions describe what a good +// answer contains; wording that steers the judge, such as "only say yes if", tends to push every answer one way. Dictionary criteria = new() { ["blue_sky"] = "explains why the daytime sky appears blue", @@ -29,6 +30,11 @@ criterion => criterion.Key, criterion => (JevQuestion)new JevNoulQuestion { Instructions = $"Does the assistant's latest response {criterion.Value}?" }); +// A criterion is met when its probability is above this threshold. 0.5 is not calibrated for any task: a higher value +// accepts fewer answers that still miss a criterion, at the cost of more loop runs. Choose it on labeled examples of +// your own answers. +const double CriterionThreshold = 0.5; + // 2. The agent that writes the answers. Its instructions hold back part of the first answer, so the judge has a gap // to find and the loop runs more than once. // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. @@ -89,7 +95,7 @@ async Task> JudgeAsync(IEnumerable request, string ans foreach ((string id, string description) in criteria) { double probability = ((JevNoulResponse)result.Answers[id]).Noul; - bool met = probability > 0.5; + bool met = probability > CriterionThreshold; Console.WriteLine($" {id}: {(met ? "met" : "missing")} (probability {probability:F2})"); if (!met) { diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md index be92f66704a..12c563f986b 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md @@ -39,12 +39,30 @@ var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => ``` `JudgeAsync` sends Jev the original request and the answer as a conversation, with -`new ChatOptions().WithJevQuestions(questions)`, and returns the criteria whose Noul is 0.5 or lower. +`new ChatOptions().WithJevQuestions(questions)`, and returns the criteria whose Noul is at or below +`CriterionThreshold`. Jev reads the original request and the latest answer as a conversation, and each question refers to "the assistant's latest response". The same pattern works with a `JevScoreQuestion` for a graded rubric, or a `JevChoiceQuestion` to classify what kind of revision is needed. +## Writing criteria for a judge + +- **Choose the threshold on your own data.** The sample treats a criterion as met when its probability is above + `CriterionThreshold`, which is 0.5. That value is not calibrated for any task. The threshold trades two errors + against each other: a higher value accepts fewer answers that still miss a criterion, and costs more loop runs, while + a lower value stops sooner and lets more incomplete answers through. Pick it from labeled examples of the answers your + agent writes, based on which error costs you more. +- **Describe the criterion, do not steer the judge.** Ask what a good answer contains, as the sample does ("Does the + assistant's latest response explain why sunsets appear red?"). Directional wording such as "only say yes if" or + "lean towards yes" tends to push every answer the same way instead of making the judge more accurate. When a + criterion is hard to judge, a neutral description of what is being judged helps more than stronger wording. +- **Ask about the response, not the topic.** When the conversation contains harmful or sensitive content, a question + such as "Is this safe?" can be read as a question about that content rather than about the answer. A correct + refusal could then look like a missing criterion, and the loop would keep revising an answer that was already right. + Ask how the response handles the request, for example "Does the assistant's latest response decline to give + dangerous instructions?", and check such criteria on labeled examples, including refusals, before relying on them. + ## Prerequisites - .NET 10 SDK or later diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs index 4661cd6e096..e731c1692c6 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs @@ -17,7 +17,9 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// /// This is a port of the tool routing in the Python TypeSafe connector (_tool_calls.py), with the same -/// question IDs, limits, and wording, so both connectors ask Jev the same questions. +/// question IDs, limits, and wording, so both connectors ask Jev the same questions. Two schema checks are stricter +/// than in Python: chains of $ref aliases are followed instead of excluding the tool, and const and +/// enum values must match their declared type, so a tool that could only fail when called is excluded. /// /// /// A route Choice selects the tool, or none. Each argument must come from a closed set: a constant, an enum @@ -312,6 +314,7 @@ void AddPresenceQuestion() if (schema.TryGetValue("const", out JsonElement constValue)) { + JevToolSchema.EnsureValuesMatchType(schema, [constValue], "argument", "const value", "type"); budget.Reserve(required ? 0 : 1); AddPresenceQuestion(); return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = constValue }; @@ -325,6 +328,7 @@ void AddPresenceQuestion() throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "enum defines {0} values; the supported maximum is {1}", values.Count, MaxEnumValues)); } + JevToolSchema.EnsureValuesMatchType(schema, values, "argument", "enum value", "type"); if (values.Count == 1) { budget.Reserve(required ? 0 : 1); @@ -390,15 +394,7 @@ void AddPresenceQuestion() throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "array enum defines {0} values; the supported maximum is {1}", members.Count, MaxEnumValues)); } - IReadOnlyList itemTypes = JevToolSchema.GetTypes(items.TryGetValue("type", out JsonElement itemType) ? itemType : null, "array item"); - foreach (JsonElement member in members) - { - if (itemTypes.Count > 0 && !itemTypes.Any(candidate => JevToolSchema.MatchesType(member, candidate))) - { - string declared = itemTypes.Count == 1 ? $"\"{itemTypes[0]}\"" : JevToolSchema.Describe(itemTypes.Select(candidate => JevJsonUtilities.ToElement(candidate))); - throw new JevUnsupportedToolSchemaException($"array enum member {JevToolSchema.Describe(member)} does not match declared item type {declared}"); - } - } + JevToolSchema.EnsureValuesMatchType(items, members, "array item", "array enum member", "item type"); budget.Reserve(members.Count + (required ? 0 : 1)); AddPresenceQuestion(); diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs index 07bcbb61309..52209e687a8 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs @@ -57,8 +57,16 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di } var resolved = new Dictionary(schema, StringComparer.Ordinal); - if (resolved.Remove("$ref", out JsonElement reference)) + + // A definition can itself be a reference, such as an alias of another definition, so references are followed + // until a schema without one. A cycle would never end, so the chain is bounded like nested resolution. + for (int hops = 0; resolved.Remove("$ref", out JsonElement reference); hops++) { + if (depth + hops >= MaxResolutionDepth) + { + throw new JevUnsupportedToolSchemaException("schema references are nested too deeply"); + } + RejectCompositionSiblings(resolved, "$ref"); string? path = reference.ValueKind == JsonValueKind.String ? reference.GetString() : null; const string DefinitionsPrefix = "#/$defs/"; @@ -75,8 +83,9 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di throw new JevUnsupportedToolSchemaException($"unresolved schema reference '{path}'"); } - // Annotations next to the reference, such as a description, override those of the definition. An anyOf - // inside the definition is resolved below, together with the rest of the merged schema. + // Annotations next to the reference, such as a description, override those of the definition, so the + // annotations closest to the argument win along a chain. An anyOf inside the definition is resolved below, + // together with the rest of the merged schema. resolved = Merge(ToMap(definition), resolved); } @@ -165,6 +174,37 @@ public static bool MatchesType(JsonElement value, string type) => _ => false, }; + /// + /// Checks that every const or enum value of a schema matches its declared type. + /// + /// + /// A value of another type can never be a valid argument, for example "fast" for an integer. Jev would + /// still choose it, and the call would fail only when the function binds its arguments, so such a tool is left out + /// of routing instead. A schema without a type is not checked. + /// + /// The resolved schema whose type applies. + /// The values to check. + /// Where the schema is, for the message about an invalid type keyword. + /// How a value is named in the message, such as enum value. + /// How the type is named in the message, such as type. + public static void EnsureValuesMatchType(Dictionary schema, IEnumerable values, string location, string valueName, string typeName) + { + IReadOnlyList types = GetTypes(schema.TryGetValue("type", out JsonElement type) ? type : null, location); + if (types.Count == 0) + { + return; + } + + foreach (JsonElement value in values) + { + if (!types.Any(candidate => MatchesType(value, candidate))) + { + string declared = types.Count == 1 ? $"\"{types[0]}\"" : Describe(types.Select(candidate => JevJsonUtilities.ToElement(candidate))); + throw new JevUnsupportedToolSchemaException($"{valueName} {Describe(value)} does not match declared {typeName} {declared}"); + } + } + } + /// /// Writes a value the way the Python connector does with json.dumps(value, ensure_ascii=False, sort_keys=True): /// sorted object keys, ", " and ": " separators, and unescaped text. Jev reads these values in diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs index e78d148c492..3c2abeeaddb 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs @@ -46,6 +46,66 @@ public void Compile_ReferencesAndAnyOfNull_ResolveAndDecode() Assert.Equal("fixed", ((JsonElement)call.Value.Arguments["kind"]!).GetString()); } + [Fact] + public void Compile_ChainedReferences_ResolveAndDecode() + { + // Arrange: Speed aliases Mode, and the nullable Level aliases a nullable enum, so each reference leads to another. + AIFunctionDeclaration tool = Declaration("set_speed", """ + { + "type": "object", + "properties": { + "speed": { "$ref": "#/$defs/Speed", "description": "How fast to go" }, + "level": { "$ref": "#/$defs/Level" } + }, + "required": ["speed"], + "$defs": { + "Speed": { "$ref": "#/$defs/Mode" }, + "Mode": { "type": "string", "enum": ["fast", "safe"] }, + "Level": { "$ref": "#/$defs/NullableLevel" }, + "NullableLevel": { "anyOf": [{ "type": "string", "enum": ["low", "high"] }, { "type": "null" }] } + } + } + """); + + // Act + JevToolCallPlan plan = Compile([tool])!; + JevToolCall? call = plan.Decode(Result( + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.9), + JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"))); + + // Assert: the description next to the first reference wins, and the nullable alias still gets a presence question. + Assert.Contains("How fast to go", plan.Questions["__af_tool__.t0.a0.value"].Instructions!.Value.Text, StringComparison.Ordinal); + Assert.Equal("safe", ((JsonElement)call!.Value.Arguments["speed"]!).GetString()); + Assert.Equal("high", ((JsonElement)call.Value.Arguments["level"]!).GetString()); + } + + [Fact] + public void Compile_EnumAndConstValuesMatchingTheirType_Compile() + { + // Arrange: numbers, mixed types, and a nullable enum whose null was removed before the check. + AIFunctionDeclaration tool = Declaration("tune", """ + { + "type": "object", + "properties": { + "ratio": { "type": "number", "enum": [1, 2.5] }, + "mode": { "type": ["string", "integer"], "enum": ["auto", 3] }, + "unit": { "type": ["string", "null"], "enum": ["c", "f", null] }, + "version": { "type": "integer", "const": 2 } + }, + "required": ["ratio", "mode", "version"] + } + """); + + // Act + JevToolCallPlan? plan = Compile([tool]); + + // Assert + Assert.NotNull(plan); + Assert.Single(plan.Tools); + } + [Theory] [InlineData("""{ "type": "object", "properties": { "a": { "type": "boolean" } }, "minProperties": 1 }""", "unsupported root-level schema constraints: minProperties")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "enum": ["x", "y"], "minLength": 1 } }, "required": ["a"] }""", "required argument 'a': unsupported argument schema constraints: minLength")] @@ -56,6 +116,12 @@ public void Compile_ReferencesAndAnyOfNull_ResolveAndDecode() [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": "integer", "enum": [1, 2.5] } } } }""", "array enum member 2.5 does not match declared item type \"integer\"")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": ["string", "null"], "enum": ["x", null] } } } }""", "nullable array members are not supported")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer" } } }""", "only const, enum, boolean, and arrays of enum values are supported")] + [InlineData("""{ "type": "object", "properties": { "a": { "$ref": "#/$defs/A" } }, "$defs": { "A": { "$ref": "#/$defs/B" }, "B": { "$ref": "#/$defs/A" } } }""", "schema references are nested too deeply")] + [InlineData("""{ "type": "object", "properties": { "a": { "$ref": "#/$defs/A" } }, "$defs": { "A": { "$ref": "#/$defs/Missing" } } }""", "unresolved schema reference '#/$defs/Missing'")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": ["fast", "safe"] } } }""", "enum value \"fast\" does not match declared type \"integer\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": ["fast"] } } }""", "enum value \"fast\" does not match declared type \"integer\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": ["integer", "boolean"], "enum": [1, "x"] } } }""", "enum value \"x\" does not match declared type [\"integer\", \"boolean\"]")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "const": 5 } }, "required": ["a"] }""", "required argument 'a': const value 5 does not match declared type \"string\"")] [InlineData("""{ "type": "object", "properties": {}, "required": ["missing"] }""", "required arguments are missing from properties: missing")] [InlineData("""{ "type": "array" }""", "the top-level input schema must be an object")] public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema, string reason) From 3ac445d587fc845b9956c2e44d9f19eabb5f155e Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:58:55 +0100 Subject: [PATCH 04/16] .NET: Link the TypeSafe quickstart instead of the sign-in console in the Jev sample The console page returns 403 to unauthenticated link checkers, which failed markdown-link-check. The quickstart explains how to get an API key and is publicly reachable. --- .../Agents/Agent_Step25_ClassificationWithJev/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md index cd289614e72..e4753e04d0e 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md @@ -110,7 +110,7 @@ the container or configuration. - .NET 10 SDK or later - A Microsoft Foundry project with a model deployment - Azure CLI installed and authenticated (for Azure credential authentication) -- A TypeSafe API key from the [TypeSafe console](https://console.typesafe.ai/keys) +- A TypeSafe API key (see the [TypeSafe quickstart](https://docs.typesafe.ai/introduction/quickstart)) Set the following environment variables: From 9894d5bea739e3724a6fb1b1ae217ba00e309298 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:43:37 +0100 Subject: [PATCH 05/16] .NET: Accept integers of any size in TypeSafe tool schema checks The integer type check used TryGetInt64, so const, enum, and array values beyond the long range, such as unsigned 64-bit values, excluded otherwise valid tools. An integer is now any JSON number written without a fraction or exponent, like a Python int, so 1.0 and 1e2 are still rejected. --- .../Core/JevToolSchema.cs | 5 +-- .../JevToolCallCompilerTests.cs | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs index 52209e687a8..dc0d97e5c40 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs @@ -164,8 +164,9 @@ public static bool MatchesType(JsonElement value, string type) => { "string" => value.ValueKind == JsonValueKind.String, - // An integer has no fraction or exponent, as a JSON parser that separates integers from floats reads it. - "integer" => value.ValueKind == JsonValueKind.Number && value.TryGetInt64(out _), + // An integer has no fraction or exponent, as a JSON parser that separates integers from floats reads it, + // and may be of any size, like a Python int; a long would reject valid unsigned 64-bit values. + "integer" => value.ValueKind == JsonValueKind.Number && value.GetRawText().AsSpan().IndexOfAny('.', 'e', 'E') < 0, "number" => value.ValueKind == JsonValueKind.Number, "boolean" => value.ValueKind is JsonValueKind.True or JsonValueKind.False, "null" => value.ValueKind == JsonValueKind.Null, diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs index 3c2abeeaddb..e32f3fd8538 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs @@ -81,6 +81,36 @@ public void Compile_ChainedReferences_ResolveAndDecode() Assert.Equal("high", ((JsonElement)call.Value.Arguments["level"]!).GetString()); } + [Fact] + public void Compile_IntegersOutsideTheInt64Range_Compile() + { + // Arrange: an unsigned 64-bit contract, whose largest values do not fit in a long. + AIFunctionDeclaration tool = Declaration("set_mask", """ + { + "type": "object", + "properties": { + "mask": { "type": "integer", "enum": [0, 18446744073709551615] }, + "all": { "type": "integer", "const": 18446744073709551615 }, + "bits": { "type": "array", "items": { "type": "integer", "enum": [1, 9223372036854775808] } } + }, + "required": ["mask", "all", "bits"] + } + """); + + // Act + JevToolCallPlan plan = Compile([tool])!; + JevToolCall? call = plan.Decode(Result( + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a2.m0", 0.1), + JevChatTestData.Noul("__af_tool__.t0.a2.m1", 0.9))); + + // Assert: the values reach the function unchanged. + Assert.Equal(ulong.MaxValue, ((JsonElement)call!.Value.Arguments["mask"]!).GetUInt64()); + Assert.Equal(ulong.MaxValue, ((JsonElement)call.Value.Arguments["all"]!).GetUInt64()); + Assert.Equal([9223372036854775808UL], ((JsonElement)call.Value.Arguments["bits"]!).EnumerateArray().Select(item => item.GetUInt64())); + } + [Fact] public void Compile_EnumAndConstValuesMatchingTheirType_Compile() { @@ -122,6 +152,8 @@ public void Compile_EnumAndConstValuesMatchingTheirType_Compile() [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": ["fast"] } } }""", "enum value \"fast\" does not match declared type \"integer\"")] [InlineData("""{ "type": "object", "properties": { "a": { "type": ["integer", "boolean"], "enum": [1, "x"] } } }""", "enum value \"x\" does not match declared type [\"integer\", \"boolean\"]")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "const": 5 } }, "required": ["a"] }""", "required argument 'a': const value 5 does not match declared type \"string\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": [1, 2.0] } } }""", "enum value 2.0 does not match declared type \"integer\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "const": 1e2 } }, "required": ["a"] }""", "const value 1e2 does not match declared type \"integer\"")] [InlineData("""{ "type": "object", "properties": {}, "required": ["missing"] }""", "required arguments are missing from properties: missing")] [InlineData("""{ "type": "array" }""", "the top-level input schema must be an object")] public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema, string reason) From 78d2948474e2ec594cf6ec107c6e4b7c92a0ffcc Mon Sep 17 00:00:00 2001 From: Roger Barreto Date: Tue, 20 Oct 2026 15:44:15 -0700 Subject: [PATCH 06/16] .NET: Fix TypeSafe turn boundaries and tool schema compilation --- .../typesafe/Agent_With_TypeSafe/README.md | 7 + .../Core/JevChatState.cs | 36 ++++- .../Core/JevQuestionBudget.cs | 14 +- .../Core/JevToolCallCompiler.cs | 75 +++++----- .../Core/JevToolSchema.cs | 75 ++++++++++ .../JevChatClientOptions.cs | 4 + .../JevChatClientToolTests.cs | 137 ++++++++++++++++++ .../JevToolCallCompilerTests.cs | 106 ++++++++++++++ 8 files changed, 409 insertions(+), 45 deletions(-) diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md index 77ac532b541..0d5dd3a98e4 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md @@ -51,6 +51,8 @@ The client follows the TypeSafe connector for Python (`agent-framework-typesafe` mean nothing. In this sample, saying that `detailed` is true "when the user asks for a detailed report" is what makes Jev choose it reliably. Tools with free-form arguments, such as text or numbers, are left out of the request with a warning in the log. + Constant and enum values must satisfy their other schema constraints too, including array item constraints; + tools with conflicting constraints are excluded rather than called with invalid arguments. 5. **Tool approval.** A tool wrapped in `ApprovalRequiredAIFunction` runs only after the user approves the call that Jev chose. The first run returns a `ToolApprovalRequestContent`; the sample asks for approval in the console and @@ -67,6 +69,11 @@ Jev reads, because the call and result they lead to carry the information. Because streaming is not supported, run agents with `RunAsync`. A function-invoking client runs approved tools before it calls the chat client, so a streaming run that resumes a tool approval would run the tool and then fail. +Agents tag stored messages as chat history, which lets Jev distinguish a new user request from text moved after a +resumed approval's tool result. When managing history yourself with `FunctionInvokingChatClient`, tag prior messages +with `message.WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory)` before appending new input. +Without that attribution, a user message starts a new turn. + ## Configuring the client `JevChatClientOptions` follows the `ClientPipelineOptions` pattern of the OpenAI and Azure AI client libraries, with the diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs index bbbc5a84582..27c121da184 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs @@ -97,10 +97,9 @@ public static JevEntry Build(IReadOnlyList messages, string? instru /// request. /// /// - /// User messages that directly follow a function result continue the turn instead of starting one. - /// FunctionInvokingChatClient moves text that was sent together with an approval response, such as "Yes, go - /// ahead", after the approved call and its result. Starting the turn there would hide the approved call from the - /// tool call limit and offer the tool again with no hint that it already ran. + /// FunctionInvokingChatClient moves text sent with an approval response after the resumed call and its result. + /// The agent tags the reconstructed call as chat history, while its new result is external. That pairing + /// distinguishes an approval continuation from a new user request after a run that ended with a tool result. /// public static int GetCurrentTurnStart(IReadOnlyList messages) { @@ -120,7 +119,7 @@ public static int GetCurrentTurnStart(IReadOnlyList messages) runStart--; } - if (runStart > 0 && messages[runStart - 1].Contents.Any(content => content is FunctionResultContent)) + if (runStart > 0 && IsResumedToolResult(messages, runStart - 1)) { index = runStart - 1; continue; @@ -132,6 +131,33 @@ public static int GetCurrentTurnStart(IReadOnlyList messages) return 0; } + private static bool IsResumedToolResult(IReadOnlyList messages, int resultIndex) + { + ChatMessage resultMessage = messages[resultIndex]; + if (resultMessage.GetAgentRequestMessageSourceType() != AgentRequestMessageSourceType.External) + { + return false; + } + + var callIds = new HashSet(resultMessage.Contents.OfType().Select(result => result.CallId), StringComparer.Ordinal); + for (int index = resultIndex - 1; callIds.Count > 0 && index >= 0; index--) + { + ChatMessage message = messages[index]; + if (message.Role == ChatRole.User) + { + break; + } + + if (message.GetAgentRequestMessageSourceType() == AgentRequestMessageSourceType.ChatHistory && + message.Contents.OfType().Any(call => callIds.Contains(call.CallId))) + { + return true; + } + } + + return false; + } + /// Gets the argument sets of the calls in the current turn, by tool name, oldest first. public static Dictionary>> GetCurrentTurnCalls(IReadOnlyList messages, int turnStart) { diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs index f6fe3f278cf..96e995e4911 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs @@ -12,6 +12,7 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// Each tool is compiled against a copy of the budget, which is committed only when the tool compiles. A tool that /// is excluded for an unsupported schema therefore returns the questions it reserved to the tools after it. +/// Exceeding the budget stops question materialization, but is reported only after the whole tool schema is checked. /// internal sealed class JevQuestionBudget { @@ -24,10 +25,15 @@ public JevQuestionBudget(int count = 0) public int Count { get; set; } - public void Reserve(int count) + public bool TryReserve(int count) { - int next = this.Count + count; - if (next > MaxInternalQuestions) + this.Count += count; + return this.Count <= MaxInternalQuestions; + } + + public void Validate() + { + if (this.Count > MaxInternalQuestions) { // Too many questions is a property of the whole tool set, not of one tool, so it stops the request instead // of excluding the tool that crossed the limit. @@ -36,7 +42,5 @@ public void Reserve(int count) "The tools need more than {0} internal Jev questions. Offer fewer tools, or tools with fewer arguments.", MaxInternalQuestions)); } - - this.Count = next; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs index e731c1692c6..1f420dbf877 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs @@ -19,7 +19,7 @@ namespace Microsoft.Agents.AI.TypeSafe; /// This is a port of the tool routing in the Python TypeSafe connector (_tool_calls.py), with the same /// question IDs, limits, and wording, so both connectors ask Jev the same questions. Two schema checks are stricter /// than in Python: chains of $ref aliases are followed instead of excluding the tool, and const and -/// enum values must match their declared type, so a tool that could only fail when called is excluded. +/// enum values must satisfy their declared constraints, so a tool that could fail when called is excluded. /// /// /// A route Choice selects the tool, or none. Each argument must come from a closed set: a constant, an enum @@ -163,7 +163,8 @@ public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) criteria[RouteNone] = "Do not call a tool because all requested tool actions are already satisfied, or because no tool is needed."; } - budget.Reserve(1); + _ = budget.TryReserve(1); + budget.Validate(); questions[RouteQuestionId] = new JevChoiceQuestion { Instructions = "Which available tool, if any, should handle the latest user request?", @@ -255,7 +256,10 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI List previousValues = [.. previousCalls.Where(call => call.ContainsKey(property.Name)).Select(call => call[property.Name])]; try { - arguments.Add(CompileArgument(tool, schema, property.Name, JevToolSchema.ToMap(property.Value), isRequired, toolIndex, argumentIndex, previousValues, budget, questions)); + if (CompileArgument(tool, schema, property.Name, JevToolSchema.ToMap(property.Value), isRequired, toolIndex, argumentIndex, previousValues, budget, questions) is { } argument) + { + arguments.Add(argument); + } } catch (JevUnsupportedToolSchemaException ex) { @@ -263,6 +267,8 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI } } + // Unsupported tools are excluded even if their earlier arguments exceeded the question budget. + budget.Validate(); return new JevCompiledTool { Function = tool, @@ -272,7 +278,7 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI }; } - private static JevArgumentPlan CompileArgument( + private static JevArgumentPlan? CompileArgument( AIFunctionDeclaration tool, Dictionary root, string name, @@ -314,28 +320,36 @@ void AddPresenceQuestion() if (schema.TryGetValue("const", out JsonElement constValue)) { - JevToolSchema.EnsureValuesMatchType(schema, [constValue], "argument", "const value", "type"); - budget.Reserve(required ? 0 : 1); + JevToolSchema.EnsureValuesMatchSchema(schema, [constValue], root, "const value"); + if (!budget.TryReserve(required ? 0 : 1)) + { + return null; + } + AddPresenceQuestion(); return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = constValue }; } - if (schema.TryGetValue("enum", out JsonElement enumElement) && enumElement.ValueKind == JsonValueKind.Array && enumElement.GetArrayLength() > 0) + if (schema.TryGetValue("enum", out JsonElement enumElement)) { - List values = [.. enumElement.EnumerateArray()]; - if (values.Count > MaxEnumValues) - { - throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "enum defines {0} values; the supported maximum is {1}", values.Count, MaxEnumValues)); - } - - JevToolSchema.EnsureValuesMatchType(schema, values, "argument", "enum value", "type"); + List values = JevToolSchema.ReadEnumValues(enumElement, "enum"); + JevToolSchema.EnsureValuesMatchSchema(schema, values, root, "enum value"); if (values.Count == 1) { - budget.Reserve(required ? 0 : 1); + if (!budget.TryReserve(required ? 0 : 1)) + { + return null; + } + AddPresenceQuestion(); return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = values[0] }; } + if (!budget.TryReserve(required ? 1 : 2)) + { + return null; + } + var labeled = new Dictionary(StringComparer.Ordinal); var criteria = new Dictionary(StringComparer.Ordinal); for (int index = 0; index < values.Count; index++) @@ -346,7 +360,6 @@ void AddPresenceQuestion() } string questionId = prefix + ".value"; - budget.Reserve(required ? 1 : 2); AddPresenceQuestion(); questions[questionId] = new JevChoiceQuestion { @@ -360,7 +373,11 @@ void AddPresenceQuestion() if (schemaType == "boolean") { string questionId = prefix + ".value"; - budget.Reserve(required ? 1 : 2); + if (!budget.TryReserve(required ? 1 : 2)) + { + return null; + } + AddPresenceQuestion(); questions[questionId] = new JevNoulQuestion { @@ -371,32 +388,20 @@ void AddPresenceQuestion() if (schemaType == "array") { - if (!schema.TryGetValue("items", out JsonElement itemsElement) || itemsElement.ValueKind != JsonValueKind.Object) - { - throw new JevUnsupportedToolSchemaException("array items must define an enum"); - } - - (Dictionary items, bool itemsNullable) = JevToolSchema.Resolve(JevToolSchema.ToMap(itemsElement), root); - JevToolSchema.RejectUnsupportedKeys(items, JevToolSchema.ArrayItemKeys, "array item"); - if (itemsNullable) - { - throw new JevUnsupportedToolSchemaException("nullable array members are not supported"); - } - + Dictionary items = JevToolSchema.ResolveArrayItems(schema, root); if (!items.TryGetValue("enum", out JsonElement membersElement) || membersElement.ValueKind != JsonValueKind.Array || membersElement.GetArrayLength() == 0) { throw new JevUnsupportedToolSchemaException("array items must define a non-empty enum"); } - List members = [.. membersElement.EnumerateArray()]; - if (members.Count > MaxEnumValues) + List members = JevToolSchema.ReadEnumValues(membersElement, "array enum"); + JevToolSchema.EnsureValuesMatchType(items, members, "array item", "array enum member", "item type"); + + if (!budget.TryReserve(members.Count + (required ? 0 : 1))) { - throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "array enum defines {0} values; the supported maximum is {1}", members.Count, MaxEnumValues)); + return null; } - JevToolSchema.EnsureValuesMatchType(items, members, "array item", "array enum member", "item type"); - - budget.Reserve(members.Count + (required ? 0 : 1)); AddPresenceQuestion(); var memberQuestions = new List>(members.Count); for (int index = 0; index < members.Count; index++) diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs index dc0d97e5c40..59454182438 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs @@ -206,6 +206,81 @@ public static void EnsureValuesMatchType(Dictionary schema, } } + public static List ReadEnumValues(JsonElement values, string name) + { + if (values.ValueKind != JsonValueKind.Array || values.GetArrayLength() == 0) + { + throw new JevUnsupportedToolSchemaException($"{name} must be a non-empty array"); + } + + if (values.GetArrayLength() > JevToolCallCompiler.MaxEnumValues) + { + throw new JevUnsupportedToolSchemaException(string.Format( + CultureInfo.InvariantCulture, + "{0} defines {1} values; the supported maximum is {2}", + name, + values.GetArrayLength(), + JevToolCallCompiler.MaxEnumValues)); + } + + return [.. values.EnumerateArray()]; + } + + public static Dictionary ResolveArrayItems(Dictionary schema, Dictionary root) + { + if (!schema.TryGetValue("items", out JsonElement element) || element.ValueKind != JsonValueKind.Object) + { + throw new JevUnsupportedToolSchemaException("array items must define an object schema"); + } + + (Dictionary items, bool nullable) = Resolve(ToMap(element), root); + RejectUnsupportedKeys(items, ArrayItemKeys, "array item"); + if (nullable) + { + throw new JevUnsupportedToolSchemaException("nullable array members are not supported"); + } + + return items; + } + + /// Checks all supported constraints before offering a constant or enum value to Jev. + public static void EnsureValuesMatchSchema(Dictionary schema, IReadOnlyList values, Dictionary root, string valueName) + { + EnsureValuesMatchType(schema, values, "argument", valueName, "type"); + EnsureValuesMatchEnum(schema, values, valueName); + if (!schema.ContainsKey("items")) + { + return; + } + + Dictionary items = ResolveArrayItems(schema, root); + if (items.TryGetValue("enum", out JsonElement itemEnum)) + { + EnsureValuesMatchType(items, ReadEnumValues(itemEnum, "array enum"), "array item", "array enum member", "item type"); + } + + foreach (JsonElement value in values.Where(value => value.ValueKind == JsonValueKind.Array)) + { + EnsureValuesMatchType(items, value.EnumerateArray(), "array item", "array member", "item type"); + EnsureValuesMatchEnum(items, value.EnumerateArray(), "array member"); + } + } + + private static void EnsureValuesMatchEnum(Dictionary schema, IEnumerable values, string valueName) + { + if (schema.TryGetValue("enum", out JsonElement element)) + { + List alternatives = ReadEnumValues(element, "enum"); + foreach (JsonElement value in values) + { + if (!alternatives.Any(alternative => JsonElement.DeepEquals(value, alternative))) + { + throw new JevUnsupportedToolSchemaException($"{valueName} {Describe(value)} does not match declared enum {Describe(alternatives)}"); + } + } + } + } + /// /// Writes a value the way the Python connector does with json.dumps(value, ensure_ascii=False, sort_keys=True): /// sorted object keys, ", " and ": " separators, and unescaped text. Jev reads these values in diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs index a8de9a73812..61d365557b5 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs @@ -90,6 +90,10 @@ public IReadOnlyDictionary? DefaultQuestions /// which calls the turn already made, but a closed-set choice can still repeat one, so the limit is what /// guarantees that the tool loop ends. Raise it when one user request needs several distinct calls, such as the /// weather in two cities; set it to 0 to never call tools. + /// When managing history without an agent, tag prior messages with + /// and + /// so a newly produced approval result can be distinguished + /// from an older tool result. Without that attribution, a user message starts a new turn. /// /// The value is negative. public int MaximumToolCallsPerTurn diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs index 7ff09a33c35..7774a0dac01 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs @@ -442,6 +442,143 @@ public async Task RunAsync_ApprovalSentWithText_ContinuesTheTurnOfTheApprovedCal Assert.Equal("Refund issued.\ndepartment: billing", approved.Text); } + [Fact] + public async Task GetResponseAsync_ApprovalWithManuallyAttributedHistory_ContinuesTheTurnAsync() + { + // Arrange + int executions = 0; + AIFunction refresh = new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => + { + executions++; + return "Refreshed."; + }, "refresh")); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers(), JevChatTestData.RouteTo("t0", "t0", "none"))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + using var invoking = new FunctionInvokingChatClient(client); + ChatOptions options = new ChatOptions { Tools = [refresh] }.WithJevQuestions(JevChatTestData.Questions()); + List messages = [new(ChatRole.User, "Refresh the data.")]; + + // Act + ChatResponse pending = await invoking.GetResponseAsync(messages, options); + ToolApprovalRequestContent approval = pending.Messages.SelectMany(message => message.Contents).OfType().Single(); + messages = [.. messages.Concat(pending.Messages).Select(message => message.WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory))]; + messages.Add(new ChatMessage(ChatRole.User, [approval.CreateResponse(true), new TextContent("Yes, go ahead.")])); + ChatResponse response = await invoking.GetResponseAsync(messages, options); + + // Assert + Assert.Equal(1, executions); + Assert.Equal(2, handler.Requests.Count); + Assert.Equal(["department", "is_urgent"], JsonNode.Parse(handler.Requests[1].Body)!["questions"]!.AsObject().Select(question => question.Key)); + Assert.Equal("Refreshed.\ndepartment: technical", response.Text); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RunAsync_NewRequestAfterTerminatedToolCall_StartsANewTurnAsync(bool requiresApproval) + { + // Arrange + int executions = 0; + AIFunction refresh = AIFunctionFactory.Create(() => $"Execution {++executions}", "refresh"); + if (requiresApproval) + { + refresh = new ApprovalRequiredAIFunction(refresh); + } + + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers(), JevChatTestData.RouteTo("t0", "t0", "none"))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers(), JevChatTestData.RouteTo("t0", "t0", "none"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions { Tools = [refresh] }.WithJevQuestions(JevChatTestData.Questions()) }) + .AsBuilder() + .Use(async (_, context, next, cancellationToken) => + { + context.Terminate = true; + return await next(context, cancellationToken); + }) + .Build(); + AgentSession session = await agent.CreateSessionAsync(); + + // Act + await RunAsync("Refresh the data."); + AgentResponse response = await RunAsync("Refresh it again."); + + // Assert + Assert.Equal(2, executions); + Assert.Equal(2, handler.Requests.Count); + JsonObject questions = JsonNode.Parse(handler.Requests[1].Body)!["questions"]!.AsObject(); + Assert.NotNull(questions[Route]); + Assert.DoesNotContain("already called this turn", (string?)questions[Route]!["criteria"]!["t0"], StringComparison.Ordinal); + Assert.Equal("Execution 2", Assert.Single(response.Messages.SelectMany(message => message.Contents).OfType()).Result?.ToString()); + + async Task RunAsync(string request) + { + AgentResponse response = await agent.RunAsync(request, session); + if (response.Messages.SelectMany(message => message.Contents).OfType().SingleOrDefault() is { } approval) + { + return await agent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(true), new TextContent("Yes, go ahead.")]), session); + } + + return response; + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void GetCurrentTurnStart_NewUserAfterFunctionResult_StartsANewTurn(bool withHistoryAttribution) + { + // Arrange + ChatMessage[] messages = + [ + new(ChatRole.User, "Refresh the data."), + new(ChatRole.Assistant, [new FunctionCallContent("call_1", "refresh")]), + new(ChatRole.Tool, [new FunctionResultContent("call_1", "Old result")]), + new(ChatRole.User, "Refresh it again."), + ]; + if (withHistoryAttribution) + { + for (int index = 0; index < messages.Length - 1; index++) + { + messages[index] = messages[index].WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory); + } + } + + // Act + int turnStart = JevChatState.GetCurrentTurnStart(messages); + + // Assert + Assert.Equal(messages.Length, turnStart); + Assert.Equal(0, JevChatState.CountCurrentTurnCalls(messages, turnStart)); + Assert.Empty(JevChatState.GetCurrentTurnResultTexts(messages, turnStart)); + } + + [Theory] + [InlineData("call_1", 1)] + [InlineData("unrelated_call", 5)] + public void GetCurrentTurnStart_ResumedResultBeforeSeveralUserMessages_MatchesTheHistoricalCall(string resultCallId, int expectedTurnStart) + { + // Arrange + ChatMessage[] messages = + [ + new ChatMessage(ChatRole.User, "Refresh the data.").WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", "refresh")]).WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory), + new(ChatRole.Tool, [new FunctionResultContent(resultCallId, "New result")]), + new(ChatRole.User, "Yes, go ahead."), + new(ChatRole.User, "Thanks."), + ]; + + // Act + int turnStart = JevChatState.GetCurrentTurnStart(messages); + + // Assert + Assert.Equal(expectedTurnStart, turnStart); + } + [Fact] public async Task RunStreamingAsync_ThrowsAsync() { diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs index e32f3fd8538..4c0442c78a5 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs @@ -154,6 +154,13 @@ public void Compile_EnumAndConstValuesMatchingTheirType_Compile() [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "const": 5 } }, "required": ["a"] }""", "required argument 'a': const value 5 does not match declared type \"string\"")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": [1, 2.0] } } }""", "enum value 2.0 does not match declared type \"integer\"")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "const": 1e2 } }, "required": ["a"] }""", "const value 1e2 does not match declared type \"integer\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "const": "x", "enum": ["y"] } } }""", "const value \"x\" does not match declared enum")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "enum": [["allowed"], ["forbidden"]], "items": { "type": "string", "enum": ["allowed"] } } } }""", "array member \"forbidden\" does not match declared enum")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "enum": [["forbidden"]], "items": { "enum": ["allowed"] } } } }""", "array member \"forbidden\" does not match declared enum")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "const": ["forbidden"], "items": { "enum": ["allowed"] } } } }""", "array member \"forbidden\" does not match declared enum")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "const": [1], "items": { "type": "string" } } } }""", "array member 1 does not match declared item type \"string\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "enum": [["x"]], "items": { "type": "string", "maxLength": 3 } } } }""", "unsupported array item schema constraints: maxLength")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "boolean", "enum": [] } } }""", "enum must be a non-empty array")] [InlineData("""{ "type": "object", "properties": {}, "required": ["missing"] }""", "required arguments are missing from properties: missing")] [InlineData("""{ "type": "array" }""", "the top-level input schema must be an object")] public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema, string reason) @@ -169,6 +176,33 @@ public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema Assert.Contains(reason, Assert.Single(logger.Entries).Message, StringComparison.Ordinal); } + [Fact] + public void Compile_ConstAndEnumWithArrayItemConstraints_ResolveAndDecode() + { + // Arrange + AIFunctionDeclaration tool = Declaration("set_tags", """ + { + "type": "object", + "properties": { + "fixed": { "type": "array", "const": ["x"], "enum": [["x"], ["y"]], "items": { "$ref": "#/$defs/Tag" } }, + "chosen": { "type": "array", "enum": [["x"], ["x", "y"]], "items": { "$ref": "#/$defs/Tag" } } + }, + "required": ["fixed", "chosen"], + "$defs": { "Tag": { "type": "string", "enum": ["x", "y"] } } + } + """); + + // Act + JevToolCallPlan plan = Compile([tool])!; + JevToolCall? call = plan.Decode(Result( + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"))); + + // Assert + Assert.Equal(["x"], ((JsonElement)call!.Value.Arguments["fixed"]!).EnumerateArray().Select(item => item.GetString())); + Assert.Equal(["x", "y"], ((JsonElement)call.Value.Arguments["chosen"]!).EnumerateArray().Select(item => item.GetString())); + } + [Fact] public void Compile_SingleValueEnumAndNoArguments_NeedNoArgumentQuestions() { @@ -222,6 +256,37 @@ public void Compile_TooManyInternalQuestions_Throws() Assert.Contains("128", exception.Message, StringComparison.Ordinal); } + [Fact] + public void Compile_QuestionBudget_IncludesTheRouteQuestion() + { + // Arrange + string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); + string array = "{ \"type\": \"array\", \"items\": { \"enum\": [" + members + "] } }"; + AIFunctionDeclaration tool = Declaration("large", "{ \"type\": \"object\", \"properties\": { \"a\": " + array + ", \"b\": " + array + " }, \"required\": [\"a\", \"b\"] }"); + + // Act & Assert + JevToolCallPlan plan = Compile([tool], ChatToolMode.RequireAny)!; + Assert.Equal(128, plan.Questions.Count); + Assert.Null(plan.RouteQuestionId); + Assert.Throws(() => Compile([tool])); + } + + [Fact] + public void Compile_QuestionBudgetWithRouteAtTheLimit_Compiles() + { + // Arrange + string first = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); + string second = string.Join(",", Enumerable.Range(0, 63).Select(index => $"\"n{index}\"")); + string schema = "{ \"type\": \"object\", \"properties\": { \"a\": { \"type\": \"array\", \"items\": { \"enum\": [" + first + "] } }, \"b\": { \"type\": \"array\", \"items\": { \"enum\": [" + second + "] } } }, \"required\": [\"a\", \"b\"] }"; + + // Act + JevToolCallPlan plan = Compile([Declaration("large", schema)])!; + + // Assert + Assert.Equal(128, plan.Questions.Count); + Assert.Equal(JevChatTestData.Route, plan.RouteQuestionId); + } + [Fact] public void Compile_UnsupportedTool_ReturnsItsQuestionBudget() { @@ -237,6 +302,47 @@ public void Compile_UnsupportedTool_ReturnsItsQuestionBudget() Assert.Equal(125, plan.Questions.Count); } + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void Compile_UnsupportedToolExceedingQuestionBudget_IsExcludedRegardlessOfPropertyOrder(bool unsupportedFirst, bool textFirst) + { + // Arrange + string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); + string array = "\"tags\": { \"type\": \"array\", \"items\": { \"enum\": [" + members + "] } }"; + string properties = textFirst ? "\"text\": { \"type\": \"string\" }, " + array : array + ", \"text\": { \"type\": \"string\" }"; + AIFunctionDeclaration supported = Declaration("supported", "{ \"type\": \"object\", \"properties\": { " + array + " }, \"required\": [\"tags\"] }"); + AIFunctionDeclaration unsupported = Declaration("unsupported", "{ \"type\": \"object\", \"properties\": { " + properties + " } }"); + var logger = new ListLoggerFactory(); + + // Act + JevToolCallPlan plan = Compile(unsupportedFirst ? [unsupported, supported] : [supported, unsupported], logger: logger.CreateLogger("test"))!; + + // Assert + Assert.Equal("supported", Assert.Single(plan.Tools).Function.Name); + Assert.Equal(65, plan.Questions.Count); + Assert.Contains("argument 'text'", Assert.Single(logger.Entries).Message, StringComparison.Ordinal); + } + + [Fact] + public void Compile_SingleUnsupportedToolExceedingQuestionBudget_IsExcluded() + { + // Arrange + string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); + string array = "{ \"type\": \"array\", \"items\": { \"enum\": [" + members + "] } }"; + AIFunctionDeclaration tool = Declaration("unsupported", "{ \"type\": \"object\", \"properties\": { \"a\": " + array + ", \"b\": " + array + ", \"text\": { \"type\": \"string\" } } }"); + var logger = new ListLoggerFactory(); + + // Act + JevToolCallPlan? plan = Compile([tool], logger: logger.CreateLogger("test")); + + // Assert + Assert.Null(plan); + Assert.Contains("argument 'text'", Assert.Single(logger.Entries).Message, StringComparison.Ordinal); + } + [Fact] public void Compile_TooManyEnumValuesOrProperties_ExcludeTheTool() { From 80e78f2200eea1123ffd906237ecf7afc9669a97 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:51:31 -0800 Subject: [PATCH 07/16] .NET: Add an internal copy of the proposed MEAI decision API Copy the provider-neutral decision API proposed in dotnet/extensions#7795 (Layer 1) and #7796 (Layer 2) into src/Shared/Decisions, with every top-level type made internal. Projects opt in with InjectSharedDecisions. The namespace and the MEAI001 attributes are unchanged, so code built on the copy compiles against the shipped types once the folder is deleted. Microsoft.Agents.AI.TypeSafe compiles the copy. --- dotnet/eng/MSBuild/Shared.props | 3 + .../Microsoft.Agents.AI.TypeSafe.csproj | 2 + dotnet/src/Shared/Decisions/.editorconfig | 20 + .../src/Shared/Decisions/DecisionAnswers.cs | 404 ++++++++++ .../src/Shared/Decisions/DecisionBinding.cs | 191 +++++ dotnet/src/Shared/Decisions/DecisionClient.cs | 546 +++++++++++++ .../Shared/Decisions/DecisionDefinitions.cs | 739 ++++++++++++++++++ .../src/Shared/Decisions/DecisionFeatures.cs | 243 ++++++ .../src/Shared/Decisions/DecisionQuestions.cs | 281 +++++++ dotnet/src/Shared/Decisions/README.md | 54 ++ .../Shared/DiagnosticIds/DiagnosticsIds.cs | 6 + 11 files changed, 2489 insertions(+) create mode 100644 dotnet/src/Shared/Decisions/.editorconfig create mode 100644 dotnet/src/Shared/Decisions/DecisionAnswers.cs create mode 100644 dotnet/src/Shared/Decisions/DecisionBinding.cs create mode 100644 dotnet/src/Shared/Decisions/DecisionClient.cs create mode 100644 dotnet/src/Shared/Decisions/DecisionDefinitions.cs create mode 100644 dotnet/src/Shared/Decisions/DecisionFeatures.cs create mode 100644 dotnet/src/Shared/Decisions/DecisionQuestions.cs create mode 100644 dotnet/src/Shared/Decisions/README.md diff --git a/dotnet/eng/MSBuild/Shared.props b/dotnet/eng/MSBuild/Shared.props index 3c8f9670284..59e6e6dbdce 100644 --- a/dotnet/eng/MSBuild/Shared.props +++ b/dotnet/eng/MSBuild/Shared.props @@ -44,4 +44,7 @@ + + + diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj index 90159d6d784..b840a3b1292 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj @@ -8,6 +8,8 @@ true + true + true diff --git a/dotnet/src/Shared/Decisions/.editorconfig b/dotnet/src/Shared/Decisions/.editorconfig new file mode 100644 index 00000000000..25c6b56d3bc --- /dev/null +++ b/dotnet/src/Shared/Decisions/.editorconfig @@ -0,0 +1,20 @@ +# These files are a verbatim copy of the proposed Microsoft.Extensions.AI decision API, with every top-level type made +# internal. The rules below keep them compiling unchanged, so the copy can be diffed against the upstream source and +# deleted when Microsoft.Extensions.AI ships the API. See README.md. + +[*.cs] +# The upstream code uses the upstream style. +dotnet_style_qualification_for_field = false:none +dotnet_style_qualification_for_property = false:none +dotnet_style_qualification_for_method = false:none +dotnet_style_qualification_for_event = false:none +dotnet_diagnostic.IDE0009.severity = none +dotnet_diagnostic.IDE0073.severity = none +dotnet_diagnostic.RCS1146.severity = none +dotnet_diagnostic.RCS1173.severity = none + +# Upstream marks every type experimental, and the internal helper types that use them do not repeat the attribute. +dotnet_diagnostic.MEAI001.severity = none + +# The exceptions are public upstream; they are internal here only until the API ships. +dotnet_diagnostic.CA1064.severity = none diff --git a/dotnet/src/Shared/Decisions/DecisionAnswers.cs b/dotnet/src/Shared/Decisions/DecisionAnswers.cs new file mode 100644 index 00000000000..2908f66aebc --- /dev/null +++ b/dotnet/src/Shared/Decisions/DecisionAnswers.cs @@ -0,0 +1,404 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable SA1402 // File may only contain a single type — decision answer types are co-located as one immutable model. +#pragma warning disable SA1649 // File name should match first type name — decision answer types are co-located as one immutable model. +#pragma warning disable CA1032 // Implement standard exception constructors — protocol exceptions expose the protocol-specific constructors. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Identifies the precision represented by provider-reported probabilities. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal enum DecisionPrecision +{ + /// The provider reports values without a declared decimal rounding. + HighPrecision, + + /// The provider reports values rounded to four decimal places. + FourDecimalPlaces, + + /// The provider reports values rounded to two decimal places. + TwoDecimalPlaces, +} + +/// Represents one named probability observation. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionProbability +{ + /// Initializes a new instance of the class. + /// The exact opaque candidate or level ID. + /// The probability value. + [JsonConstructor] + public DecisionProbability(string id, double value) + { + Id = Throw.IfNullOrWhitespace(id); + Value = value; + } + + /// Gets the exact opaque candidate or level ID. + public string Id { get; } + + /// Gets the native double probability value. + public double Value { get; } +} + +/// Represents a provider result for one decision question. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] +[JsonDerivedType(typeof(BinaryDecisionAnswer), "binary")] +[JsonDerivedType(typeof(ChoiceDecisionAnswer), "choice")] +[JsonDerivedType(typeof(ScoreDecisionAnswer), "score")] +internal abstract class DecisionAnswer +{ + internal const double DistributionTolerance = 1e-6; + + private protected DecisionAnswer( + string questionId, + DecisionKind kind, + DecisionPrecision precision, + object? rawRepresentation, + IReadOnlyDictionary? additionalProperties) + { + QuestionId = Throw.IfNullOrWhitespace(questionId); + if (!Enum.IsDefined(typeof(DecisionPrecision), precision)) + { + Throw.ArgumentOutOfRangeException(nameof(precision)); + } + + Kind = kind; + Precision = precision; + RawRepresentation = rawRepresentation; + AdditionalProperties = DecisionSnapshots.Properties(additionalProperties); + } + + /// Gets the exact opaque question ID this answer belongs to. + public string QuestionId { get; } + + /// Gets the semantic kind of this answer. + public DecisionKind Kind { get; } + + /// Gets the precision declared for the numeric observations. + public DecisionPrecision Precision { get; } + + /// + /// Gets the provider-owned raw representation, if available. + /// This value is not cloned or used for semantic data and must be treated as diagnostic-only. + /// + [JsonIgnore] + public object? RawRepresentation { get; } + + /// + /// Gets an owned read-only snapshot of provider extension data. + /// Values are shallow-copied references and must be independently safe to share. + /// + public IReadOnlyDictionary? AdditionalProperties { get; } + + internal static void ValidateProbability(double value) + { + if (double.IsNaN(value) || double.IsInfinity(value) || value < 0 || value > 1) + { + throw new DecisionProtocolException("Probability must be finite and in the range [0, 1]."); + } + } + + internal static void ValidateDistribution( + IReadOnlyList probabilities, + DecisionPrecision precision) + { + double sum = 0; + HashSet ids = new(StringComparer.Ordinal); + + foreach (DecisionProbability probability in probabilities) + { + if (probability is null) + { + throw new DecisionProtocolException("A probability distribution cannot contain null entries."); + } + + if (!ids.Add(probability.Id)) + { + throw new DecisionProtocolException("A probability distribution contains a duplicate ID."); + } + + ValidateProbability(probability.Value); + sum += probability.Value; + } + + double tolerance = GetDistributionTolerance(precision, probabilities.Count); + if (Math.Abs(sum - 1) > tolerance) + { + throw new DecisionProtocolException( + "A probability distribution must sum to 1 within the bound allowed by its declared rounding."); + } + + double roundingHalfUnit = GetRoundingHalfUnit(precision); + double lowerBound = 0; + double upperBound = 0; + foreach (DecisionProbability probability in probabilities) + { + lowerBound += Math.Max(0, probability.Value - roundingHalfUnit); + upperBound += Math.Min(1, probability.Value + roundingHalfUnit); + } + + if (lowerBound > 1 + DistributionTolerance || upperBound < 1 - DistributionTolerance) + { + throw new DecisionProtocolException( + "A probability distribution has no normalized latent values consistent with its declared rounding."); + } + } + + internal static double GetDistributionTolerance(DecisionPrecision precision, int count) + { + return precision switch + { + DecisionPrecision.HighPrecision => DistributionTolerance, + DecisionPrecision.FourDecimalPlaces => (count * 0.00005) + DistributionTolerance, + DecisionPrecision.TwoDecimalPlaces => (count * 0.005) + DistributionTolerance, + _ => throw new ArgumentOutOfRangeException(nameof(precision)), + }; + } + + internal static (double Minimum, double Maximum) GetFeasibleExpectedScoreBounds( + IReadOnlyList probabilities, + DecisionPrecision precision) + { + double roundingHalfUnit = GetRoundingHalfUnit(precision); + double lowerMass = 0; + double upperMass = 0; + for (int i = 0; i < probabilities.Count; i++) + { + double probability = probabilities[i].Value; + lowerMass += Math.Max(0, probability - roundingHalfUnit); + upperMass += Math.Min(1, probability + roundingHalfUnit); + } + + if (lowerMass > 1 + DistributionTolerance || upperMass < 1 - DistributionTolerance) + { + throw new DecisionProtocolException( + "A probability distribution has no normalized latent values consistent with its declared rounding."); + } + + double remaining = Math.Max(0, 1 - lowerMass); + double minimum = 0; + for (int i = 0; i < probabilities.Count; i++) + { + double lower = Math.Max(0, probabilities[i].Value - roundingHalfUnit); + double capacity = Math.Min(1, probabilities[i].Value + roundingHalfUnit) - lower; + double allocated = Math.Min(remaining, capacity); + minimum += (lower + allocated) * i; + remaining -= allocated; + } + + remaining = Math.Max(0, 1 - lowerMass); + double maximum = 0; + double[] lowerValues = new double[probabilities.Count]; + for (int i = 0; i < probabilities.Count; i++) + { + lowerValues[i] = Math.Max(0, probabilities[i].Value - roundingHalfUnit); + maximum += lowerValues[i] * i; + } + + for (int i = probabilities.Count - 1; i >= 0; i--) + { + double capacity = Math.Min(1, probabilities[i].Value + roundingHalfUnit) - lowerValues[i]; + double allocated = Math.Min(remaining, capacity); + maximum += allocated * i; + remaining -= allocated; + } + + if (remaining > DistributionTolerance) + { + throw new DecisionProtocolException( + "A probability distribution has no normalized latent values consistent with its declared rounding."); + } + + return (minimum, maximum); + } + + internal static double GetRoundingHalfUnit(DecisionPrecision precision) => + precision switch + { + DecisionPrecision.HighPrecision => 0, + DecisionPrecision.FourDecimalPlaces => 0.00005, + DecisionPrecision.TwoDecimalPlaces => 0.005, + _ => throw new ArgumentOutOfRangeException(nameof(precision)), + }; +} + +/// Represents the probability that a binary proposition is true. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class BinaryDecisionAnswer : DecisionAnswer +{ + /// Initializes a new instance of the class. + [JsonConstructor] + public BinaryDecisionAnswer( + string questionId, + double trueProbability, + DecisionPrecision precision = DecisionPrecision.HighPrecision, + object? rawRepresentation = null, + IReadOnlyDictionary? additionalProperties = null) + : base(questionId, DecisionKind.Binary, precision, rawRepresentation, additionalProperties) + { + ValidateProbability(trueProbability); + TrueProbability = trueProbability; + } + + /// Gets the probability that the proposition is true. + public double TrueProbability { get; } +} + +/// Represents a selected candidate and its complete probability distribution. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class ChoiceDecisionAnswer : DecisionAnswer +{ + /// Initializes a new instance of the class. + [JsonConstructor] + public ChoiceDecisionAnswer( + string questionId, + string selectedCandidateId, + IReadOnlyList probabilities, + DecisionPrecision precision = DecisionPrecision.HighPrecision, + object? rawRepresentation = null, + IReadOnlyDictionary? additionalProperties = null) + : base(questionId, DecisionKind.Choice, precision, rawRepresentation, additionalProperties) + { + SelectedCandidateId = Throw.IfNullOrWhitespace(selectedCandidateId); + _ = Throw.IfNull(probabilities); + + DecisionProbability[] copy = probabilities.ToArray(); + if (copy.Length == 0) + { + throw new DecisionProtocolException("A choice answer requires a complete distribution."); + } + + ValidateDistribution(copy, precision); + if (!Array.Exists(copy, probability => string.Equals(probability.Id, SelectedCandidateId, StringComparison.Ordinal))) + { + throw new DecisionProtocolException("The selected candidate is not present in the distribution."); + } + + Probabilities = Array.AsReadOnly(copy); + } + + /// Gets the exact selected candidate ID. + public string SelectedCandidateId { get; } + + /// Gets the complete ordered candidate distribution. + public IReadOnlyList Probabilities { get; } +} + +/// +/// Represents an ordinal score and its complete level distribution. +/// A reported score is checked against normalized latent values within the declared clipped rounding intervals; observed probabilities and the derived expectation are never repaired. +/// +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class ScoreDecisionAnswer : DecisionAnswer +{ + /// Initializes a new instance of the class. + [JsonConstructor] + public ScoreDecisionAnswer( + string questionId, + double? score, + IReadOnlyList probabilities, + DecisionPrecision precision = DecisionPrecision.HighPrecision, + object? rawRepresentation = null, + IReadOnlyDictionary? additionalProperties = null) + : base(questionId, DecisionKind.Score, precision, rawRepresentation, additionalProperties) + { + _ = Throw.IfNull(probabilities); + + DecisionProbability[] copy = probabilities.ToArray(); + if (copy.Length < 2) + { + throw new DecisionProtocolException("A score answer requires at least two levels."); + } + + ValidateDistribution(copy, precision); + if (score is double reportedScore && + (double.IsNaN(reportedScore) || double.IsInfinity(reportedScore) || reportedScore < 0 || reportedScore > copy.Length - 1)) + { + throw new DecisionProtocolException("A score must be finite and within the ordinal level range."); + } + + double expectedScore = copy.Select((probability, index) => probability.Value * index).Sum(); + if (score is double nativeScore) + { + (double minimum, double maximum) = GetFeasibleExpectedScoreBounds(copy, precision); + double scalarHalfUnit = GetRoundingHalfUnit(precision); + if (nativeScore + scalarHalfUnit + DistributionTolerance < minimum || + nativeScore - scalarHalfUnit - DistributionTolerance > maximum) + { + throw new DecisionProtocolException("The score does not agree with its probability distribution."); + } + } + + Score = score; + ExpectedScore = expectedScore; + Probabilities = Array.AsReadOnly(copy); + } + + /// Gets the provider-reported score, retaining any declared rounding, or when it was not reported. + public double? Score { get; } + + /// Gets the score derived from the observed distribution without repairing it, which may fall outside the ordinal level bounds after rounding. + public double ExpectedScore { get; } + + /// Gets the complete ordered level distribution. + public IReadOnlyList Probabilities { get; } +} + +/// Represents a protocol-invalid provider result. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionProtocolException : Exception +{ + /// Initializes a new instance of the class. + public DecisionProtocolException() + { + } + + /// Initializes a new instance of the class with a message. + public DecisionProtocolException(string message) + : base(message) + { + } + + /// Initializes a new instance of the class with a message and inner exception. + public DecisionProtocolException(string message, Exception innerException) + : base(message, innerException) + { + } +} + +internal static class DecisionSnapshots +{ + internal static IReadOnlyDictionary? Properties( + IReadOnlyDictionary? entries) + { + if (entries is null) + { + return null; + } + + Dictionary copy = new(StringComparer.Ordinal); + foreach (KeyValuePair property in entries) + { + if (copy.ContainsKey(property.Key)) + { + Throw.ArgumentException(nameof(entries), "Extension property names must be unique."); + } + + copy.Add(property.Key, property.Value); + } + + return new ReadOnlyDictionary(copy); + } +} diff --git a/dotnet/src/Shared/Decisions/DecisionBinding.cs b/dotnet/src/Shared/Decisions/DecisionBinding.cs new file mode 100644 index 00000000000..82d41a63a01 --- /dev/null +++ b/dotnet/src/Shared/Decisions/DecisionBinding.cs @@ -0,0 +1,191 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable SA1402 // File may only contain a single type — binding proof types are co-located as one API surface. +#pragma warning disable SA1649 // File name should match first type name — binding proof types are co-located as one API surface. +#pragma warning disable SA1204 // Static extension type is intentionally grouped with the binding types. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Text.Json.Serialization.Metadata; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides an explicit mapping from a choice candidate ID to an enum value. +/// The application-owned enum type. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionEnumDefinition + where TEnum : struct, Enum +{ + private readonly ReadOnlyDictionary _values; + + /// Initializes a new instance of the class. + public DecisionEnumDefinition(IEnumerable> values) + { + _ = Throw.IfNull(values); + + DecisionEnumValue[] copy = values.ToArray(); + if (copy.Length == 0) + { + Throw.ArgumentException(nameof(values), "An enum definition requires at least one value."); + } + + Dictionary byId = new(StringComparer.Ordinal); + HashSet enumValues = new(); + DecisionCandidate[] candidates = new DecisionCandidate[copy.Length]; + + for (int i = 0; i < copy.Length; i++) + { + DecisionEnumValue value = copy[i]; + if (!enumValues.Add(value.Value)) + { + Throw.ArgumentException(nameof(values), "Enum aliases or duplicate enum values are not allowed."); + } + + if (byId.ContainsKey(value.Id)) + { + Throw.ArgumentException( + nameof(values), + "Enum candidate IDs must be unique using ordinal comparison."); + } + + byId.Add(value.Id, value.Value); + candidates[i] = new DecisionCandidate(value.Id, value.Description); + } + + _values = new ReadOnlyDictionary(byId); + Candidates = Array.AsReadOnly(candidates); + } + + /// Gets the ordered candidate snapshot for a choice question. + public IReadOnlyList Candidates { get; } + + /// Creates a choice question using this exact enum mapping. + /// The choice question with the exact mapped candidate IDs. + public ChoiceDecisionQuestion CreateQuestion(string id, string? instructions) => + new(id, instructions, Candidates); + + /// Maps an exact selected candidate ID to its enum value. + /// The mapped enum value. + public TEnum Parse(string candidateId) + { + _ = Throw.IfNullOrWhitespace(candidateId); + if (_values.TryGetValue(candidateId, out TEnum value)) + { + return value; + } + + throw new DecisionProtocolException($"Unknown candidate ID '{candidateId}'."); + } +} + +/// Describes one enum mapping entry. +/// The application-owned enum type. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionEnumValue + where TEnum : struct, Enum +{ + /// Initializes a new instance of the class. + public DecisionEnumValue(TEnum value, string id, string description) + { + Value = value; + Id = Throw.IfNullOrWhitespace(id); + Description = Throw.IfNullOrWhitespace(description); + } + + /// Gets the application enum value. + public TEnum Value { get; } + + /// Gets the exact candidate ID. + public string Id { get; } + + /// Gets the candidate description. + public string Description { get; } +} + +/// Provides typed access to a complete decision response. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionAnswerSet +{ + internal DecisionAnswerSet(DecisionResponse response) + { + Response = response; + } + + /// Gets the complete unmodified response, including probability evidence. + public DecisionResponse Response { get; } + + /// Gets a binary answer by exact question ID. + /// The complete binary answer. + public BinaryDecisionAnswer GetBinary(string questionId) => + Response.GetAnswer(questionId) as BinaryDecisionAnswer ?? + throw new DecisionProtocolException($"Question '{questionId}' is not binary."); + + /// Gets a choice answer by exact question ID. + /// The complete choice answer. + public ChoiceDecisionAnswer GetChoice(string questionId) => + Response.GetAnswer(questionId) as ChoiceDecisionAnswer ?? + throw new DecisionProtocolException($"Question '{questionId}' is not a choice."); + + /// Gets a choice answer and maps its exact selected ID to an enum. + /// The application-owned enum type. + /// The mapped enum value. + public TEnum GetChoice(string questionId, DecisionEnumDefinition definition) + where TEnum : struct, Enum + { + _ = Throw.IfNull(definition); + return definition.Parse(GetChoice(questionId).SelectedCandidateId); + } + + /// Gets a score answer by exact question ID. + /// The complete score answer. + public ScoreDecisionAnswer GetScore(string questionId) => + Response.GetAnswer(questionId) as ScoreDecisionAnswer ?? + throw new DecisionProtocolException($"Question '{questionId}' is not a score."); +} + +/// Maps one complete response to an application-owned result without inference or reflection. +/// The application-owned result type. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionResultBinding +{ + private readonly Func _map; + + /// Initializes a new instance of the class. + /// The source-generated JSON contract for the application result. + /// The application-owned mapping delegate. + public DecisionResultBinding( + JsonTypeInfo resultTypeInfo, + Func map) + { + ResultTypeInfo = Throw.IfNull(resultTypeInfo); + _map = Throw.IfNull(map); + } + + /// Gets the explicit source-generated result contract. + public JsonTypeInfo ResultTypeInfo { get; } + + internal TResult Map(DecisionResponse response) => _map(new DecisionAnswerSet(response)); +} + +/// Provides response binding extensions. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal static class DecisionBindingExtensions +{ + /// Maps a complete response locally using the supplied explicit binding. + /// The application-owned result type. + /// The application-owned result. + public static TResult Bind( + this DecisionResponse response, + DecisionResultBinding binding) + { + _ = Throw.IfNull(response); + _ = Throw.IfNull(binding); + return binding.Map(response); + } +} diff --git a/dotnet/src/Shared/Decisions/DecisionClient.cs b/dotnet/src/Shared/Decisions/DecisionClient.cs new file mode 100644 index 00000000000..2c982a7bb24 --- /dev/null +++ b/dotnet/src/Shared/Decisions/DecisionClient.cs @@ -0,0 +1,546 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable SA1402 // File may only contain a single type — decision client contracts are co-located as one API surface. +#pragma warning disable SA1649 // File name should match first type name — decision client contracts are co-located as one API surface. +#pragma warning disable SA1204 // Static extension types are intentionally grouped with the client contracts. +#pragma warning disable S1128 // Unused "using" should be removed — ExperimentalAttribute is supplied by the shared API contract. +#pragma warning disable S2302 // Protocol diagnostics intentionally mention the opaque question concept. +#pragma warning disable CA1032 // Implement standard exception constructors — client exceptions expose the client-specific constructors. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides provider-neutral decision inference over a heterogeneous question batch. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal interface IDecisionClient : IDisposable +{ + /// Evaluates all questions against the same request state. + /// The owned decision request. + /// Optional per-request options. + /// The cancellation token to monitor. + /// A complete, correlated decision response or an exception. + Task GetResponseAsync( + DecisionRequest request, + DecisionOptions? options = null, + CancellationToken cancellationToken = default); + + /// Retrieves a provider service for the exact requested type and key. + /// The requested provider service, or when unavailable. + object? GetService(Type serviceType, object? serviceKey = null); +} + +/// Provides per-request decision options. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal class DecisionOptions +{ + /// Initializes a new instance of the class. + public DecisionOptions() + { + } + + /// Initializes a new instance of the class by copying another instance. + protected DecisionOptions(DecisionOptions? other) + { + if (other is null) + { + return; + } + + ModelId = other.ModelId; + AdditionalProperties = other.AdditionalProperties?.Clone(); + RawRepresentationFactory = other.RawRepresentationFactory; + } + + /// Gets or sets the requested model ID. + public string? ModelId { get; set; } + + /// Gets or sets provider-specific extension properties. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } + + /// + /// Gets or sets a callback that creates provider-specific raw options. + /// The callback should return a new instance for each invocation; the provider may mutate it. + /// + [JsonIgnore] + public Func? RawRepresentationFactory { get; set; } + + /// Creates a deliberate shallow clone of these options. + /// A cloned options instance. + public virtual DecisionOptions Clone() => new(this); +} + +/// Provides metadata about an . +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionClientMetadata +{ + /// Initializes a new instance of the class. + public DecisionClientMetadata( + string? providerName = null, + Uri? providerUri = null, + string? defaultModelId = null, + IReadOnlyList? supportedKinds = null) + { + ProviderName = providerName; + ProviderUri = providerUri; + DefaultModelId = defaultModelId; + + if (supportedKinds is not null) + { + DecisionKind[] copy = supportedKinds.Distinct().ToArray(); + foreach (DecisionKind kind in copy) + { + if (!Enum.IsDefined(typeof(DecisionKind), kind)) + { + Throw.ArgumentException(nameof(supportedKinds), "Supported kinds must be defined decision kinds."); + } + } + + SupportedKinds = Array.AsReadOnly(copy); + } + } + + /// Gets the provider name, when known. + public string? ProviderName { get; } + + /// Gets the provider URI, when known. + public Uri? ProviderUri { get; } + + /// Gets the default model ID, when known. + public string? DefaultModelId { get; } + + /// Gets the owned supported-kind snapshot, or when unknown. + public IReadOnlyList? SupportedKinds { get; } +} + +/// Classifies a provider failure without prescribing retry behavior. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionClientException : Exception +{ + /// Initializes a new instance of the class. + public DecisionClientException() + : this(string.Empty, isTransient: false) + { + } + + /// Initializes a new instance of the class with a message. + public DecisionClientException(string message) + : this(message, isTransient: false) + { + } + + /// Initializes a new instance of the class with provider classification. + public DecisionClientException(string message, bool isTransient, Exception? innerException = null) + : base(message, innerException) + { + IsTransient = isTransient; + } + + /// Initializes a new instance of the class with a message and inner exception. + public DecisionClientException(string message, Exception innerException) + : this(message, isTransient: false, innerException) + { + } + + /// Gets a value indicating whether the provider classified the failure as transient. + public bool IsTransient { get; } +} + +/// Provides extensions for decision clients. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal static class DecisionClientExtensions +{ + /// Gets a typed provider service, or when it is unavailable. + /// The provider service type. + /// The requested provider service, or when unavailable. + public static TService? GetService(this IDecisionClient client, object? serviceKey = null) + { + _ = Throw.IfNull(client); + return client.GetService(typeof(TService), serviceKey) is TService service ? service : default; + } + + /// Evaluates typed state using an explicit source-generated JSON contract. + /// The application state type. + /// A complete, correlated decision response. + public static async Task GetResponseAsync( + this IDecisionClient client, + TState state, + JsonTypeInfo stateTypeInfo, + IEnumerable questions, + DecisionOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(stateTypeInfo); + + DecisionRequest request = DecisionRequest.Create(state, stateTypeInfo, questions); + DecisionResponse response = await client.GetResponseAsync(request, options, cancellationToken).ConfigureAwait(false); + response.ValidateAgainst(request); + return response; + } + + /// Evaluates a JSON state using a reusable typed decision definition. + /// The application-owned result type. + /// The decision client. + /// The explicit JSON state evaluated by every declared question. + /// The immutable typed decision definition. + /// Optional per-request options. + /// The cancellation token to monitor. + /// The typed result and complete neutral evidence. + public static async Task> GetResponseAsync( + this IDecisionClient client, + JsonElement state, + DecisionDefinition definition, + DecisionOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(definition); + + DecisionRequest request = new(state, definition.Questions); + DecisionResponse response = await client.GetResponseAsync( + request, + options, + cancellationToken).ConfigureAwait(false); + return definition.Bind(response, request); + } + + /// Evaluates typed application state using a reusable typed decision definition. + /// The application state type. + /// The application-owned result type. + /// The decision client. + /// The application-owned state. + /// The immutable typed decision definition. + /// Optional per-request options. + /// The cancellation token to monitor. + /// The typed result and complete neutral evidence. + public static Task> GetResponseAsync( + this IDecisionClient client, + TState state, + DecisionDefinition definition, + DecisionOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(definition); + return GetResponseAsync( + client, + state, + definition, + definition.SerializerOptions, + options, + cancellationToken); + } + + /// Evaluates typed application state using explicit source-generated state metadata. + /// The application state type. + /// The application-owned result type. + /// The decision client. + /// The application-owned state. + /// The JSON contract for . + /// The immutable typed decision definition. + /// Optional per-request options. + /// The cancellation token to monitor. + /// The typed result and complete neutral evidence. + public static Task> GetResponseAsync( + this IDecisionClient client, + TState state, + JsonTypeInfo stateTypeInfo, + DecisionDefinition definition, + DecisionOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(stateTypeInfo); + return GetResponseAsync( + client, + state, + definition, + stateTypeInfo, + options, + cancellationToken); + } + + private static async Task> GetResponseAsync( + IDecisionClient client, + TState state, + DecisionDefinition definition, + JsonSerializerOptions serializerOptions, + DecisionOptions? options, + CancellationToken cancellationToken) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(definition); + + JsonTypeInfo stateTypeInfo = serializerOptions.GetTypeInfo(typeof(TState)); + JsonElement stateJson = JsonSerializer.SerializeToElement(state, stateTypeInfo); + return await GetResponseAsync(client, stateJson, definition, options, cancellationToken).ConfigureAwait(false); + } + + private static async Task> GetResponseAsync( + IDecisionClient client, + TState state, + DecisionDefinition definition, + JsonTypeInfo stateTypeInfo, + DecisionOptions? options, + CancellationToken cancellationToken) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(definition); + + JsonElement stateJson = JsonSerializer.SerializeToElement(state, stateTypeInfo); + return await GetResponseAsync(client, stateJson, definition, options, cancellationToken).ConfigureAwait(false); + } +} + +/// Identifies provider/model provenance without inventing missing identity. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionProvenance +{ + /// Initializes a new instance of the class. + public DecisionProvenance( + string? providerName = null, + Uri? providerUri = null, + string? modelId = null, + string? responseId = null, + DecisionPrecision? precision = null) + { + ProviderName = providerName; + ProviderUri = providerUri; + ModelId = modelId; + ResponseId = responseId; + Precision = precision; + } + + /// Gets the provider name, when reported. + public string? ProviderName { get; } + + /// Gets the provider URI, when reported. + public Uri? ProviderUri { get; } + + /// Gets the model ID, when reported. + public string? ModelId { get; } + + /// Gets the provider response ID, when reported. + public string? ResponseId { get; } + + /// Gets the reported probability precision, when available. + public DecisionPrecision? Precision { get; } +} + +/// Represents a complete response correlated to one request. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionResponse +{ + /// Initializes a new instance of the class and validates it. + [JsonConstructor] + public DecisionResponse( + DecisionRequest request, + IReadOnlyList answers, + DecisionProvenance? provenance = null, + UsageDetails? usage = null, + object? rawRepresentation = null, + IReadOnlyDictionary? additionalProperties = null) + { + Request = Throw.IfNull(request); + _ = Throw.IfNull(answers); + + DecisionAnswer[] copy = answers.ToArray(); + if (copy.Length != request.Questions.Count) + { + throw new DecisionProtocolException("A response must contain one answer for every question."); + } + + for (int i = 0; i < copy.Length; i++) + { + ValidateAnswer(request.Questions[i], copy[i]); + } + + Answers = Array.AsReadOnly(copy); + Provenance = provenance; + Usage = usage; + RawRepresentation = rawRepresentation; + AdditionalProperties = DecisionSnapshots.Properties(additionalProperties); + } + + /// Gets the request snapshot evaluated by the provider. + public DecisionRequest Request { get; } + + /// Gets complete answers in the exact request order. + public IReadOnlyList Answers { get; } + + /// Gets provider/model provenance, when reported. + public DecisionProvenance? Provenance { get; } + + /// Gets usage details reported by the provider. + public UsageDetails? Usage { get; } + + /// + /// Gets the provider-owned raw response, if available. + /// This value is not cloned or used for semantic data and must be treated as diagnostic-only. + /// + [JsonIgnore] + public object? RawRepresentation { get; } + + /// + /// Gets an owned read-only snapshot of provider extension data. + /// Values are shallow-copied references and must be independently safe to share. + /// + public IReadOnlyDictionary? AdditionalProperties { get; } + + /// Gets an answer by exact ordinal question ID. + /// The answer associated with the exact question ID. + public DecisionAnswer GetAnswer(string questionId) + { + _ = Throw.IfNullOrWhitespace(questionId); + foreach (DecisionAnswer answer in Answers) + { + if (string.Equals(answer.QuestionId, questionId, StringComparison.Ordinal)) + { + return answer; + } + } + + throw new KeyNotFoundException($"No answer exists for question ID '{questionId}'."); + } + + internal void ValidateAgainst(DecisionRequest expectedRequest) + { + _ = Throw.IfNull(expectedRequest); + + if (!JsonElement.DeepEquals(Request.State, expectedRequest.State) || + Request.Questions.Count != expectedRequest.Questions.Count) + { + throw new DecisionProtocolException("The response does not correspond to the request sent by the client."); + } + + for (int i = 0; i < Request.Questions.Count; i++) + { + if (!QuestionsAreEquivalent(Request.Questions[i], expectedRequest.Questions[i])) + { + throw new DecisionProtocolException("The response does not correspond to the request sent by the client."); + } + } + } + + private static bool QuestionsAreEquivalent(DecisionQuestion actual, DecisionQuestion expected) + { + if (actual.Kind != expected.Kind || + !string.Equals(actual.Id, expected.Id, StringComparison.Ordinal) || + !string.Equals(actual.Instructions, expected.Instructions, StringComparison.Ordinal)) + { + return false; + } + + return (actual, expected) switch + { + (BinaryDecisionQuestion actualBinary, BinaryDecisionQuestion expectedBinary) => + string.Equals(actualBinary.TrueDescription, expectedBinary.TrueDescription, StringComparison.Ordinal) && + string.Equals(actualBinary.FalseDescription, expectedBinary.FalseDescription, StringComparison.Ordinal), + + (ChoiceDecisionQuestion actualChoice, ChoiceDecisionQuestion expectedChoice) => + CandidatesAreEquivalent(actualChoice.Candidates, expectedChoice.Candidates), + + (ScoreDecisionQuestion actualScore, ScoreDecisionQuestion expectedScore) => + LevelsAreEquivalent(actualScore.Levels, expectedScore.Levels), + + _ => false, + }; + } + + private static bool CandidatesAreEquivalent( + IReadOnlyList actual, + IReadOnlyList expected) + { + if (actual.Count != expected.Count) + { + return false; + } + + for (int i = 0; i < actual.Count; i++) + { + if (!string.Equals(actual[i].Id, expected[i].Id, StringComparison.Ordinal) || + !string.Equals(actual[i].Description, expected[i].Description, StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private static bool LevelsAreEquivalent( + IReadOnlyList actual, + IReadOnlyList expected) + { + if (actual.Count != expected.Count) + { + return false; + } + + for (int i = 0; i < actual.Count; i++) + { + if (!string.Equals(actual[i].Id, expected[i].Id, StringComparison.Ordinal) || + !string.Equals(actual[i].Description, expected[i].Description, StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private static void ValidateAnswer(DecisionQuestion question, DecisionAnswer answer) + { + if (answer is null || + !string.Equals(question.Id, answer.QuestionId, StringComparison.Ordinal) || + question.Kind != answer.Kind) + { + throw new DecisionProtocolException("An answer does not exactly match its question."); + } + + switch (question) + { + case BinaryDecisionQuestion when answer is BinaryDecisionAnswer: + return; + + case ChoiceDecisionQuestion choice when answer is ChoiceDecisionAnswer choiceAnswer: + if (choiceAnswer.Probabilities.Count != choice.Candidates.Count || + !choice.Candidates.Select(static candidate => candidate.Id) + .SequenceEqual( + choiceAnswer.Probabilities.Select(static probability => probability.Id), + StringComparer.Ordinal)) + { + throw new DecisionProtocolException( + "Choice probabilities must contain exactly the requested candidate IDs in order."); + } + + return; + + case ScoreDecisionQuestion score when answer is ScoreDecisionAnswer scoreAnswer: + if (scoreAnswer.Probabilities.Count != score.Levels.Count || + !score.Levels.Select(static level => level.Id) + .SequenceEqual( + scoreAnswer.Probabilities.Select(static probability => probability.Id), + StringComparer.Ordinal)) + { + throw new DecisionProtocolException( + "Score probabilities must contain exactly the requested level IDs in order."); + } + + return; + + default: + throw new DecisionProtocolException("An answer kind does not match its question."); + } + } +} diff --git a/dotnet/src/Shared/Decisions/DecisionDefinitions.cs b/dotnet/src/Shared/Decisions/DecisionDefinitions.cs new file mode 100644 index 00000000000..8979cced54b --- /dev/null +++ b/dotnet/src/Shared/Decisions/DecisionDefinitions.cs @@ -0,0 +1,739 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable SA1402 // Definition and nested builder are one immutable API surface. +#pragma warning disable SA1649 // File name identifies the primary definition type. +#pragma warning disable CA1000 // The generic type is the natural home for the typed factory methods. +#pragma warning disable CA1032 // Decision materialization exposes protocol-specific failures. +#pragma warning disable S2302 // Protocol exception messages intentionally describe the response and configured options. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Declares the decision questions and application-owned result properties for a typed decision flow. +/// +/// The application-owned result type. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionDefinition +{ + private readonly ReadOnlyCollection> _questions; + + private DecisionDefinition( + JsonTypeInfo resultTypeInfo, + JsonSerializerOptions serializerOptions, + IReadOnlyList> questions) + { + ResultTypeInfo = resultTypeInfo; + SerializerOptions = serializerOptions; + _questions = new ReadOnlyCollection>(questions.ToArray()); + Questions = new ReadOnlyCollection(_questions.Select(static question => question.Question).ToArray()); + } + + /// Gets the explicit JSON contract used to materialize the application result. + public JsonTypeInfo ResultTypeInfo { get; } + + /// Gets the serializer options associated with this definition. + [JsonIgnore] + public JsonSerializerOptions SerializerOptions { get; } + + /// Gets the immutable ordered questions declared by this definition. + public IReadOnlyList Questions { get; } + + /// + /// Creates a typed definition using the configured default JSON contract for . + /// + /// The explicit question declaration callback. + /// + /// Optional JSON options used for property and enum metadata. A missing resolver is supplied from the MEAI defaults; + /// explicit caller resolvers and converters are preserved without mutating the caller's options. + /// + /// An immutable decision definition. + public static DecisionDefinition Create( + Action configure, + JsonSerializerOptions? serializerOptions = null) + { + _ = Throw.IfNull(configure); + serializerOptions = new JsonSerializerOptions(serializerOptions ?? AIJsonUtilities.DefaultOptions); + serializerOptions.TypeInfoResolver ??= AIJsonUtilities.DefaultOptions.TypeInfoResolver; + serializerOptions.MakeReadOnly(); + + JsonTypeInfo resultTypeInfo = GetResultTypeInfo(serializerOptions); + return Create(resultTypeInfo, configure); + } + + /// Creates a typed definition using an application-supplied JSON contract. + /// The JSON contract for . + /// The explicit question declaration callback. + /// An immutable decision definition. + public static DecisionDefinition Create( + JsonTypeInfo resultTypeInfo, + Action configure) + { + _ = Throw.IfNull(resultTypeInfo); + _ = Throw.IfNull(configure); + + Builder builder = new(resultTypeInfo); + configure(builder); + return builder.Build(); + } + + private static AIFunctionFactoryOptions GetFunctionOptions( + AIFunctionFactoryOptions? functionOptions, + JsonSerializerOptions serializerOptions) + { + if (functionOptions is null) + { + return new AIFunctionFactoryOptions { SerializerOptions = serializerOptions }; + } + + if (functionOptions.SerializerOptions is not null) + { + return functionOptions; + } + + return new AIFunctionFactoryOptions + { + SerializerOptions = serializerOptions, + JsonSchemaCreateOptions = functionOptions.JsonSchemaCreateOptions, + Name = functionOptions.Name, + Description = functionOptions.Description, + AdditionalProperties = functionOptions.AdditionalProperties, + ConfigureParameterBinding = functionOptions.ConfigureParameterBinding, + MarshalResult = functionOptions.MarshalResult, + ExcludeResultSchema = functionOptions.ExcludeResultSchema, + }; + } + + /// Creates an that evaluates this definition for a caller-owned state type. + /// The application-owned state type exposed as the function's input. + /// The caller-owned decision client. + /// Optional standard function factory options. + /// + /// Optional decision options shallow-copied during construction and cloned for each invocation. Nested values and + /// the raw-options factory remain caller/provider-owned. + /// + /// + /// Optional explicit JSON contract for . When omitted, the definition's serializer + /// options must contain the state contract. + /// + /// A function whose only model-facing input is and whose result is . + public AIFunction AsAIFunction( + IDecisionClient client, + AIFunctionFactoryOptions? functionOptions = null, + DecisionOptions? decisionOptions = null, + JsonTypeInfo? stateTypeInfo = null) + { + _ = Throw.IfNull(client); + + stateTypeInfo ??= GetStateTypeInfo(); + DecisionOptions? decisionOptionsSnapshot = decisionOptions?.Clone(); + DecisionFunctionInvoker invoker = new(client, this, stateTypeInfo, decisionOptionsSnapshot); + + return AIFunctionFactory.Create( + invoker.InvokeAsync, + GetFunctionOptions(functionOptions, stateTypeInfo.Options)); + } + + internal DecisionResponse Bind(DecisionResponse response, DecisionRequest expectedRequest) + { + _ = Throw.IfNull(response); + response.ValidateAgainst(expectedRequest); + + using MemoryStream stream = new(); + using (Utf8JsonWriter writer = new(stream)) + { + writer.WriteStartObject(); + + foreach (DecisionDefinitionQuestion question in _questions) + { + DecisionAnswer answer = response.GetAnswer(question.Question.Id); + switch (question.Kind) + { + case DecisionKind.Binary: + writer.WriteNumber(question.PropertyName, ((BinaryDecisionAnswer)answer).TrueProbability); + break; + + case DecisionKind.Choice: + ChoiceDecisionAnswer choice = (ChoiceDecisionAnswer)answer; + writer.WriteString(question.PropertyName, choice.SelectedCandidateId); + break; + + case DecisionKind.Score: + ScoreDecisionAnswer score = (ScoreDecisionAnswer)answer; + if (question.UsesExpectedScore) + { + writer.WriteNumber(question.PropertyName, score.ExpectedScore); + } + else if (score.Score is double reportedScore) + { + writer.WriteNumber(question.PropertyName, reportedScore); + } + else + { + throw new DecisionProtocolException( + $"Question '{question.Question.Id}' does not contain a provider-reported score."); + } + break; + + default: + throw new DecisionProtocolException($"Unsupported decision kind '{question.Kind}'."); + } + } + + writer.WriteEndObject(); + } + + byte[] resultJson = stream.ToArray(); + TResult result; + try + { + result = JsonSerializer.Deserialize(resultJson, ResultTypeInfo)!; + } + catch (Exception exception) when (exception is JsonException or NotSupportedException) + { + throw new DecisionProtocolException( + "The decision response could not be materialized as the declared result type.", + exception); + } + + if (result is null) + { + throw new DecisionProtocolException("The decision response produced a null result."); + } + + foreach (DecisionDefinitionQuestion question in _questions) + { + DecisionAnswer answer = response.GetAnswer(question.Question.Id); + object? actualValue = question.ValueAccessor(result); + bool preserved = question.Kind switch + { + DecisionKind.Binary => actualValue is double actualBinary && + actualBinary.Equals(((BinaryDecisionAnswer)answer).TrueProbability), + DecisionKind.Choice => question.ChoiceValueMatches is not null && + question.ChoiceValueMatches(actualValue, ((ChoiceDecisionAnswer)answer).SelectedCandidateId), + DecisionKind.Score => actualValue is double actualScore && + actualScore.Equals(question.UsesExpectedScore + ? ((ScoreDecisionAnswer)answer).ExpectedScore + : ((ScoreDecisionAnswer)answer).Score ?? double.NaN), + _ => false, + }; + + if (!preserved) + { + throw new DecisionProtocolException( + $"The decision response value for result property '{question.Property.Name}' was not preserved by the configured JSON contract."); + } + } + + return new DecisionResponse(result, response, this); + } + + private JsonTypeInfo GetStateTypeInfo() + { + return SerializerOptions.GetTypeInfo(typeof(TState)) is JsonTypeInfo stateTypeInfo + ? stateTypeInfo + : throw new NotSupportedException( + $"The configured JSON options do not contain a contract for state type '{typeof(TState)}'."); + } + + private sealed class DecisionFunctionInvoker( + IDecisionClient client, + DecisionDefinition definition, + JsonTypeInfo stateTypeInfo, + DecisionOptions? decisionOptionsSnapshot) + { + private readonly IDecisionClient _client = client; + private readonly DecisionDefinition _definition = definition; + private readonly JsonTypeInfo _stateTypeInfo = stateTypeInfo; + private readonly DecisionOptions? _decisionOptionsSnapshot = decisionOptionsSnapshot; + + [DisplayName("evaluate_decision")] + [Description("Evaluate the declared decision for one application state.")] + public async Task InvokeAsync( + [Description("The application state for the declared decision.")] TState state, + CancellationToken cancellationToken) + { + DecisionResponse response = await _client.GetResponseAsync( + state, + _stateTypeInfo, + _definition, + _decisionOptionsSnapshot?.Clone(), + cancellationToken).ConfigureAwait(false); + return response.Result; + } + } + + internal IReadOnlyDictionary GetDistribution( + DecisionResponse response, + Expression> selector) + where TEnum : struct, Enum + { + DecisionDefinitionQuestion question = FindQuestion(selector); + if (question.Kind != DecisionKind.Choice || + question.EnumDefinition is not DecisionEnumDefinition definition) + { + throw new DecisionProtocolException("The selected result property is not a choice enum in this definition."); + } + + ChoiceDecisionAnswer answer = (ChoiceDecisionAnswer)response.Evidence.GetAnswer(question.Question.Id); + Dictionary values = new(); + foreach (DecisionProbability probability in answer.Probabilities) + { + values.Add(definition.Parse(probability.Id), probability.Value); + } + + return new ReadOnlyDictionary(values); + } + + private static JsonTypeInfo GetResultTypeInfo(JsonSerializerOptions options) + { + if (options.GetTypeInfo(typeof(TResult)) is not JsonTypeInfo resultTypeInfo) + { + throw new NotSupportedException( + $"The configured JSON options do not contain a contract for result type '{typeof(TResult)}'."); + } + + return resultTypeInfo; + } + + private static PropertyInfo GetDirectProperty(Expression> selector) + { + _ = Throw.IfNull(selector); + + Expression body = selector.Body; + if (body is UnaryExpression { NodeType: ExpressionType.Convert } conversion) + { + body = conversion.Operand; + } + + if (body is not MemberExpression { Member: PropertyInfo property, Expression: ParameterExpression parameter } || + selector.Parameters.Count != 1 || + selector.Parameters[0] != parameter) + { + Throw.ArgumentException( + nameof(selector), + "The selector must access one direct instance property of the result type."); + return null!; + } + + return property; + } + + private DecisionDefinitionQuestion FindQuestion(Expression> selector) + { + PropertyInfo property = GetDirectProperty(selector); + return _questions.FirstOrDefault(question => question.Property == property) ?? + throw new DecisionProtocolException($"The result property '{property.Name}' is not declared in this definition."); + } + + /// Builds a through explicit property declarations. + [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] + public sealed class Builder + { + private readonly JsonTypeInfo _resultTypeInfo; + private readonly List> _questions = []; + + private static DecisionEnumDefinition CreateEnumDefinition<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicFields)] TEnum>( + PropertyInfo property, + JsonTypeInfo resultTypeInfo, + JsonPropertyInfo jsonProperty) + where TEnum : struct, Enum + { + if (property.PropertyType.IsDefined(typeof(FlagsAttribute), inherit: false)) + { + Throw.ArgumentException("selector", "Flags enums are not supported as closed decision choices."); + } + + if (jsonProperty.CustomConverter is not null) + { + throw new NotSupportedException( + $"Property-specific JSON converters are not supported for decision enum property '{property.Name}'. " + + "Configure the enum converter at the JSON type/options level so candidate IDs and result materialization share one contract."); + } + + JsonTypeInfo enumTypeInfo; + try + { + enumTypeInfo = resultTypeInfo.Options.GetTypeInfo(typeof(TEnum)); + } + catch (Exception exception) when (exception is NotSupportedException or InvalidOperationException) + { + throw new NotSupportedException( + $"The configured JSON options do not contain a contract for enum type '{typeof(TEnum)}'.", + exception); + } + +#if NET5_0_OR_GREATER + TEnum[] enumValues = Enum.GetValues(); +#else + TEnum[] enumValues = Enum.GetValues(typeof(TEnum)).Cast().ToArray(); +#endif + List> values = new(enumValues.Length); + foreach (TEnum value in enumValues) + { + JsonElement serialized = JsonSerializer.SerializeToElement(value, enumTypeInfo); + if (serialized.ValueKind != JsonValueKind.String || serialized.GetString() is not string id) + { + throw new NotSupportedException( + $"Enum type '{typeof(TEnum)}' must use string JSON serialization for decision choices."); + } + + string enumName = Enum.GetName(typeof(TEnum), value)!; + string description = typeof(TEnum).GetField(enumName)?.GetCustomAttribute()?.Description ?? + enumName; + + values.Add(new DecisionEnumValue(value, id, description)); + } + + return new DecisionEnumDefinition(values); + } + + private static JsonPropertyInfo GetJsonProperty(JsonTypeInfo resultTypeInfo, PropertyInfo property) + { + JsonPropertyInfo? jsonProperty = resultTypeInfo.Properties.FirstOrDefault( + candidate => candidate.AttributeProvider is PropertyInfo propertyInfo && propertyInfo == property); + + if (jsonProperty is null) + { + string expectedName = property.GetCustomAttribute()?.Name ?? + resultTypeInfo.Options.PropertyNamingPolicy?.ConvertName(property.Name) ?? + property.Name; + jsonProperty = resultTypeInfo.Properties.FirstOrDefault( + candidate => string.Equals(candidate.Name, expectedName, StringComparison.Ordinal)); + } + + if (jsonProperty is null) + { + Throw.ArgumentException( + nameof(property), + $"The selected property '{property.Name}' is not present in the configured JSON contract."); + } + + return jsonProperty; + } + + private static string GetInstructions(JsonPropertyInfo jsonProperty, PropertyInfo property, string? explicitInstructions) + { + if (explicitInstructions is not null) + { + return Throw.IfNullOrWhitespace(explicitInstructions); + } + + string? description = (jsonProperty.AttributeProvider as PropertyInfo)?.GetCustomAttribute(inherit: true)?.Description; + if (description is not null) + { + return description; + } + +#if NET9_0_OR_GREATER + description = jsonProperty.AssociatedParameter?.AttributeProvider?.GetCustomAttributes(typeof(DescriptionAttribute), inherit: true) + .OfType() + .FirstOrDefault() + ?.Description; +#else + description = GetConstructorDescription(property); +#endif + if (description is not null) + { + return description; + } + + return property.PropertyType.GetCustomAttribute(inherit: true)?.Description ?? + property.Name; + } + + private static void EnsurePropertyCanBeDeclared(PropertyInfo property, Type expectedType) + { + if (property.GetMethod is null || + property.GetMethod.IsStatic || + property.PropertyType != expectedType) + { + Throw.ArgumentException("selector", $"The selected property must be a readable instance property of type '{expectedType}'."); + } + } + +#if !NET9_0_OR_GREATER + [UnconditionalSuppressMessage("Trimming", "IL2075", Justification = "The reflection convenience path checks preserved constructor binding metadata; source-generated metadata is used on newer TFMs.")] +#endif + private static void EnsurePropertyCanBeMaterialized( + PropertyInfo property, + JsonPropertyInfo jsonProperty) + { + if (jsonProperty.Set is not null) + { + return; + } + +#if NET9_0_OR_GREATER + if (jsonProperty.AssociatedParameter is not null) + { + return; + } +#else + ConstructorInfo[] constructors = property.DeclaringType?.GetConstructors() ?? []; + if (Array.Exists( + constructors, + candidate => Array.Exists( + candidate.GetParameters(), + parameter => string.Equals(parameter.Name, property.Name, StringComparison.OrdinalIgnoreCase)))) + { + return; + } +#endif + + Throw.ArgumentException( + "selector", + $"The selected property '{property.Name}' is not writable or bound to a JSON constructor parameter."); + } + +#if !NET9_0_OR_GREATER + [UnconditionalSuppressMessage("Trimming", "IL2075", Justification = "The reflection convenience path intentionally reads preserved constructor parameter metadata; source-generated metadata is used on newer TFMs.")] + private static string? GetConstructorDescription(PropertyInfo property) + { + ConstructorInfo[] constructors = property.DeclaringType?.GetConstructors() ?? []; + ConstructorInfo? constructor = Array.Find( + constructors, + candidate => Array.Exists(candidate.GetParameters(), parameter => + string.Equals(parameter.Name, property.Name, StringComparison.OrdinalIgnoreCase))); + return Array.Find( + constructor?.GetParameters() ?? [], + parameter => string.Equals(parameter.Name, property.Name, StringComparison.OrdinalIgnoreCase)) + ?.GetCustomAttribute(inherit: true) + ?.Description; + } +#endif + + private static Func CreateValueAccessor(Expression> selector) + { +#if NETFRAMEWORK + Func accessor = selector.Compile(); +#else + Func accessor = selector.Compile(true); +#endif + return result => accessor(result); + } + + private static Func CreateChoiceValueMatcher( + DecisionEnumDefinition enumDefinition) + where TEnum : struct, Enum => + (actualValue, candidateId) => + actualValue is TEnum actual && + EqualityComparer.Default.Equals(actual, enumDefinition.Parse(candidateId)); + + internal Builder(JsonTypeInfo resultTypeInfo) + { + _resultTypeInfo = resultTypeInfo; + } + + /// Declares an enum-backed choice question. + /// The result property enum type. + /// A direct result property selector. + /// Optional explicit question instructions. + public void Choice<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicFields)] TEnum>( + Expression> selector, + string? instructions = null) + where TEnum : struct, Enum + { + PropertyInfo property = GetDirectProperty(selector); + EnsurePropertyCanBeDeclared(property, typeof(TEnum)); + + JsonPropertyInfo jsonProperty = GetJsonProperty(_resultTypeInfo, property); + EnsurePropertyCanBeMaterialized(property, jsonProperty); + DecisionEnumDefinition enumDefinition = CreateEnumDefinition(property, _resultTypeInfo, jsonProperty); + AddQuestion( + property, + jsonProperty.Name, + new ChoiceDecisionQuestion(jsonProperty.Name, GetInstructions(jsonProperty, property, instructions), enumDefinition.Candidates), + enumDefinition, + DecisionKind.Choice, + valueAccessor: CreateValueAccessor(selector), + choiceValueMatches: CreateChoiceValueMatcher(enumDefinition)); + } + + /// Declares a binary probability question backed by a result property. + /// A direct result property selector. + /// Optional explicit question instructions. + public void BinaryProbability( + Expression> selector, + string? instructions = null) + { + PropertyInfo property = GetDirectProperty(selector); + EnsurePropertyCanBeDeclared(property, typeof(double)); + + JsonPropertyInfo jsonProperty = GetJsonProperty(_resultTypeInfo, property); + EnsurePropertyCanBeMaterialized(property, jsonProperty); + AddQuestion( + property, + jsonProperty.Name, + new BinaryDecisionQuestion(jsonProperty.Name, GetInstructions(jsonProperty, property, instructions)), + enumDefinition: null, + DecisionKind.Binary, + valueAccessor: CreateValueAccessor(selector)); + } + + /// Declares an ordinal score question with an explicit ordered rubric. + /// A direct result property selector. + /// The caller-defined ordered score levels. + /// Optional explicit question instructions. + public void Score( + Expression> selector, + IReadOnlyList levels, + string? instructions = null) + { + AddScore(selector, levels, instructions, usesExpectedScore: false); + } + + /// Declares an ordinal score question projected from the complete observed distribution. + /// A direct result property selector. + /// The caller-defined ordered score levels. + /// Optional explicit question instructions. + public void ExpectedScore( + Expression> selector, + IReadOnlyList levels, + string? instructions = null) + { + AddScore(selector, levels, instructions, usesExpectedScore: true); + } + + private void AddScore( + Expression> selector, + IReadOnlyList levels, + string? instructions, + bool usesExpectedScore) + { + PropertyInfo property = GetDirectProperty(selector); + EnsurePropertyCanBeDeclared(property, typeof(double)); + _ = Throw.IfNull(levels); + + JsonPropertyInfo jsonProperty = GetJsonProperty(_resultTypeInfo, property); + EnsurePropertyCanBeMaterialized(property, jsonProperty); + AddQuestion( + property, + jsonProperty.Name, + new ScoreDecisionQuestion(jsonProperty.Name, GetInstructions(jsonProperty, property, instructions), levels), + enumDefinition: null, + DecisionKind.Score, + usesExpectedScore, + valueAccessor: CreateValueAccessor(selector)); + } + + internal DecisionDefinition Build() + { + if (_questions.Count == 0) + { + Throw.ArgumentException("configure", "A decision definition must declare at least one question."); + } + + return new DecisionDefinition( + _resultTypeInfo, + _resultTypeInfo.Options, + _questions); + } + + private void AddQuestion( + PropertyInfo property, + string propertyName, + DecisionQuestion question, + object? enumDefinition, + DecisionKind kind, + bool usesExpectedScore = false, + Func? valueAccessor = null, + Func? choiceValueMatches = null) + { + if (_questions.Exists(existing => existing.Property == property || string.Equals(existing.PropertyName, propertyName, StringComparison.Ordinal))) + { + Throw.ArgumentException("selector", "A result property or JSON property name can only be declared once."); + } + + _questions.Add(new DecisionDefinitionQuestion( + property, + propertyName, + question, + enumDefinition, + kind, + usesExpectedScore, + valueAccessor ?? throw new InvalidOperationException("A direct property accessor is required."), + choiceValueMatches)); + } + } + +} + +/// Contains a typed result and the complete neutral evidence that produced it. +/// The application-owned result type. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionResponse +{ + private readonly DecisionDefinition _definition; + + internal DecisionResponse(TResult result, DecisionResponse evidence, DecisionDefinition definition) + { + Result = result; + Evidence = evidence; + _definition = definition; + } + + /// Gets the application-owned materialized result. + public TResult Result { get; } + + /// Gets the complete original response, including distributions and metadata. + public DecisionResponse Evidence { get; } + + /// Gets the complete probability distribution for an enum-backed choice property. + /// The enum property type. + /// The direct result property selector. + /// An immutable enum-to-probability snapshot. + public IReadOnlyDictionary GetDistribution( + Expression> selector) + where TEnum : struct, Enum => + _definition.GetDistribution(this, selector); +} + +internal sealed class DecisionDefinitionQuestion +{ + internal DecisionDefinitionQuestion( + PropertyInfo property, + string propertyName, + DecisionQuestion question, + object? enumDefinition, + DecisionKind kind, + bool usesExpectedScore, + Func valueAccessor, + Func? choiceValueMatches) + { + Property = property; + PropertyName = propertyName; + Question = question; + EnumDefinition = enumDefinition; + Kind = kind; + UsesExpectedScore = usesExpectedScore; + ValueAccessor = valueAccessor; + ChoiceValueMatches = choiceValueMatches; + } + + internal PropertyInfo Property { get; } + internal string PropertyName { get; } + internal DecisionQuestion Question { get; } + internal object? EnumDefinition { get; } + internal DecisionKind Kind { get; } + internal bool UsesExpectedScore { get; } + internal Func ValueAccessor { get; } + internal Func? ChoiceValueMatches { get; } +} diff --git a/dotnet/src/Shared/Decisions/DecisionFeatures.cs b/dotnet/src/Shared/Decisions/DecisionFeatures.cs new file mode 100644 index 00000000000..fbe68784d6e --- /dev/null +++ b/dotnet/src/Shared/Decisions/DecisionFeatures.cs @@ -0,0 +1,243 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable SA1402 // File may only contain a single type — feature projection types are co-located as one API surface. +#pragma warning disable SA1649 // File name should match first type name — feature projection types are co-located as one API surface. +#pragma warning disable SA1204 // Static projection type is intentionally grouped with the feature model. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Identifies the semantic value projected by a feature coordinate. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal enum DecisionFeatureValueKind +{ + /// The probability that a binary proposition is true. + BinaryTrueProbability, + + /// The probability that a binary proposition is false. + BinaryFalseProbability, + + /// The probability of one choice candidate. + ChoiceProbability, + + /// The probability of one score level. + ScoreProbability, + + /// The provider-reported ordinal score. + Score, + + /// The ordinal score calculated from the complete observed distribution. + ExpectedScore, +} + +/// Names one stable semantic coordinate in a feature schema. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionFeatureCoordinate +{ + /// Initializes a new instance of the class. + public DecisionFeatureCoordinate( + string name, + string questionId, + DecisionFeatureValueKind valueKind, + string? memberId = null) + { + Name = Throw.IfNullOrWhitespace(name); + QuestionId = Throw.IfNullOrWhitespace(questionId); + if (!Enum.IsDefined(typeof(DecisionFeatureValueKind), valueKind)) + { + Throw.ArgumentOutOfRangeException(nameof(valueKind)); + } + + ValueKind = valueKind; + + bool requiresMemberId = valueKind is DecisionFeatureValueKind.ChoiceProbability or DecisionFeatureValueKind.ScoreProbability; + if (requiresMemberId != (memberId is not null)) + { + Throw.ArgumentException( + nameof(memberId), + requiresMemberId ? "This coordinate kind requires a member ID." : "This coordinate kind does not accept a member ID."); + } + + if (memberId is not null) + { + MemberId = Throw.IfNullOrWhitespace(memberId); + } + } + + /// Gets the stable coordinate name. + public string Name { get; } + + /// Gets the exact question ID supplying this coordinate. + public string QuestionId { get; } + + /// Gets the coordinate semantic kind. + public DecisionFeatureValueKind ValueKind { get; } + + /// Gets the exact candidate or level ID, when applicable. + public string? MemberId { get; } +} + +/// Identifies a named feature schema and its version. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionFeatureSchema +{ + /// Initializes a new instance of the class. + public DecisionFeatureSchema(string id, int version, IEnumerable coordinates) + { + Id = Throw.IfNullOrWhitespace(id); + if (version < 1) + { + Throw.ArgumentOutOfRangeException(nameof(version)); + } + + _ = Throw.IfNull(coordinates); + DecisionFeatureCoordinate[] copy = coordinates.ToArray(); + if (copy.Length == 0 || Array.Exists(copy, static coordinate => coordinate is null)) + { + Throw.ArgumentException(nameof(coordinates), "Coordinates must be nonempty."); + } + + HashSet names = new(StringComparer.Ordinal); + foreach (DecisionFeatureCoordinate coordinate in copy) + { + if (!names.Add(coordinate.Name)) + { + Throw.ArgumentException(nameof(coordinates), "Feature names must be unique using ordinal comparison."); + } + } + + Version = version; + Coordinates = Array.AsReadOnly(copy); + } + + /// Gets the stable schema identity. + public string Id { get; } + + /// Gets the schema version. + public int Version { get; } + + /// Gets the ordered coordinate snapshot. + public IReadOnlyList Coordinates { get; } +} + +/// Represents one named native-double feature value. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionFeatureValue +{ + /// Initializes a new instance of the class. + public DecisionFeatureValue(string name, double value) + { + Name = Throw.IfNullOrWhitespace(name); + if (double.IsNaN(value) || double.IsInfinity(value)) + { + Throw.ArgumentOutOfRangeException(nameof(value)); + } + + Value = value; + } + + /// Gets the stable coordinate name. + public string Name { get; } + + /// Gets the native double value. + public double Value { get; } +} + +/// Represents a named feature projection with schema and provenance. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionFeatureVector +{ + internal DecisionFeatureVector( + DecisionFeatureSchema schema, + IEnumerable values, + DecisionProvenance? provenance) + { + Schema = schema; + Values = Array.AsReadOnly(values.ToArray()); + Provenance = provenance; + } + + /// Gets the explicit feature schema. + public DecisionFeatureSchema Schema { get; } + + /// Gets named values in schema order. + public IReadOnlyList Values { get; } + + /// Gets provider/model provenance, when reported by the response. + public DecisionProvenance? Provenance { get; } +} + +/// Projects complete decision observations into named native-double coordinates. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal static class DecisionFeatureProjection +{ + /// Projects a response without an inference call or probability repair. + /// The named feature vector projected from the complete response. + public static DecisionFeatureVector Project( + DecisionResponse response, + DecisionFeatureSchema schema) + { + _ = Throw.IfNull(response); + _ = Throw.IfNull(schema); + + List values = new(schema.Coordinates.Count); + foreach (DecisionFeatureCoordinate coordinate in schema.Coordinates) + { + DecisionAnswer answer = response.GetAnswer(coordinate.QuestionId); + double value = coordinate.ValueKind switch + { + DecisionFeatureValueKind.BinaryTrueProbability => + GetBinary(answer, coordinate).TrueProbability, + DecisionFeatureValueKind.BinaryFalseProbability => + 1 - GetBinary(answer, coordinate).TrueProbability, + DecisionFeatureValueKind.ChoiceProbability => + GetProbability(GetChoice(answer, coordinate).Probabilities, coordinate.MemberId!), + DecisionFeatureValueKind.ScoreProbability => + GetProbability(GetScore(answer, coordinate).Probabilities, coordinate.MemberId!), + DecisionFeatureValueKind.Score => + GetScore(answer, coordinate).Score ?? + throw new DecisionProtocolException( + $"Question '{coordinate.QuestionId}' does not contain a provider-reported score."), + DecisionFeatureValueKind.ExpectedScore => + GetScore(answer, coordinate).ExpectedScore, + _ => throw new ArgumentOutOfRangeException(nameof(schema)), + }; + + values.Add(new DecisionFeatureValue(coordinate.Name, value)); + } + + return new DecisionFeatureVector(schema, values, response.Provenance); + } + + private static BinaryDecisionAnswer GetBinary(DecisionAnswer answer, DecisionFeatureCoordinate coordinate) => + answer as BinaryDecisionAnswer ?? + throw new ArgumentException($"Question '{coordinate.QuestionId}' is not binary.", nameof(coordinate)); + + private static ChoiceDecisionAnswer GetChoice(DecisionAnswer answer, DecisionFeatureCoordinate coordinate) => + answer as ChoiceDecisionAnswer ?? + throw new ArgumentException($"Question '{coordinate.QuestionId}' is not a choice.", nameof(coordinate)); + + private static ScoreDecisionAnswer GetScore(DecisionAnswer answer, DecisionFeatureCoordinate coordinate) => + answer as ScoreDecisionAnswer ?? + throw new ArgumentException($"Question '{coordinate.QuestionId}' is not a score.", nameof(coordinate)); + + private static double GetProbability(IReadOnlyList probabilities, string memberId) + { + foreach (DecisionProbability probability in probabilities) + { + if (string.Equals(probability.Id, memberId, StringComparison.Ordinal)) + { + return probability.Value; + } + } + + throw new DecisionProtocolException($"Unknown feature member ID '{memberId}'."); + } +} diff --git a/dotnet/src/Shared/Decisions/DecisionQuestions.cs b/dotnet/src/Shared/Decisions/DecisionQuestions.cs new file mode 100644 index 00000000000..86e1eef5671 --- /dev/null +++ b/dotnet/src/Shared/Decisions/DecisionQuestions.cs @@ -0,0 +1,281 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable SA1402 // File may only contain a single type — decision question types are co-located as one immutable model. +#pragma warning disable SA1649 // File name should match first type name — decision question types are co-located as one immutable model. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Identifies the semantic kind of a decision question or answer. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal enum DecisionKind +{ + /// A two-outcome proposition. + Binary, + + /// A choice from a caller-defined unordered set. + Choice, + + /// A position on a caller-defined ordered scale. + Score, +} + +/// Describes a decision question. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] +[JsonDerivedType(typeof(BinaryDecisionQuestion), "binary")] +[JsonDerivedType(typeof(ChoiceDecisionQuestion), "choice")] +[JsonDerivedType(typeof(ScoreDecisionQuestion), "score")] +internal abstract class DecisionQuestion +{ + private protected DecisionQuestion(string id, DecisionKind kind, string? instructions) + { + Id = Throw.IfNullOrWhitespace(id); + Kind = kind; + Instructions = instructions ?? string.Empty; + } + + /// Gets the opaque, caller-owned correlation ID for this question. + public string Id { get; } + + /// Gets the semantic kind of this question. + public DecisionKind Kind { get; } + + /// Gets the explicit instructions that define the question. + public string Instructions { get; } +} + +/// Describes a binary decision question. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class BinaryDecisionQuestion : DecisionQuestion +{ + /// Initializes a new instance of the class. + /// The opaque, caller-owned question ID. + /// The instructions defining the proposition. + /// Optional criteria describing a true result. + /// Optional criteria describing a false result. + [JsonConstructor] + public BinaryDecisionQuestion( + string id, + string? instructions = null, + string? trueDescription = null, + string? falseDescription = null) + : base(id, DecisionKind.Binary, instructions) + { + if (string.IsNullOrWhiteSpace(instructions) && + (string.IsNullOrWhiteSpace(trueDescription) || string.IsNullOrWhiteSpace(falseDescription))) + { + Throw.ArgumentException( + nameof(instructions), + "Binary questions require instructions or both true and false descriptions."); + } + + if (trueDescription is not null) + { + _ = Throw.IfNullOrWhitespace(trueDescription); + } + + if (falseDescription is not null) + { + _ = Throw.IfNullOrWhitespace(falseDescription); + } + + TrueDescription = trueDescription; + FalseDescription = falseDescription; + } + + /// Gets the optional criteria describing a true result. + public string? TrueDescription { get; } + + /// Gets the optional criteria describing a false result. + public string? FalseDescription { get; } +} + +/// Describes one caller-defined choice candidate. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionCandidate +{ + /// Initializes a new instance of the class. + /// The exact opaque candidate ID to round-trip in results. + /// The human-readable candidate criteria. + [JsonConstructor] + public DecisionCandidate(string id, string description) + { + Id = Throw.IfNullOrWhitespace(id); + Description = Throw.IfNullOrWhitespace(description); + } + + /// Gets the exact opaque, caller-owned candidate ID. + public string Id { get; } + + /// Gets the human-readable candidate criteria. + public string Description { get; } +} + +/// Describes a choice decision question. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class ChoiceDecisionQuestion : DecisionQuestion +{ + /// Initializes a new instance of the class. + /// The opaque, caller-owned question ID. + /// The instructions defining the choice. + /// The ordered, caller-defined candidates. + [JsonConstructor] + public ChoiceDecisionQuestion(string id, string? instructions, IReadOnlyList candidates) + : base(id, DecisionKind.Choice, instructions) + { + _ = Throw.IfNull(candidates); + + DecisionCandidate[] copy = candidates.ToArray(); + if (copy.Length == 0 || Array.Exists(copy, static candidate => candidate is null)) + { + Throw.ArgumentException(nameof(candidates), "Candidates must be nonempty."); + } + + HashSet ids = new(StringComparer.Ordinal); + foreach (DecisionCandidate candidate in copy) + { + if (!ids.Add(candidate.Id)) + { + Throw.ArgumentException( + nameof(candidates), + "Candidate IDs must be unique using ordinal comparison."); + } + } + + Candidates = Array.AsReadOnly(copy); + } + + /// Gets the ordered, owned candidate snapshot. + public IReadOnlyList Candidates { get; } +} + +/// Describes one level in an ordered score rubric. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionScoreLevel +{ + /// Initializes a new instance of the class. + /// The exact opaque, caller-owned level ID. + /// The criteria for this level. + [JsonConstructor] + public DecisionScoreLevel(string id, string description) + { + Id = Throw.IfNullOrWhitespace(id); + Description = Throw.IfNullOrWhitespace(description); + } + + /// Gets the exact opaque level ID. + public string Id { get; } + + /// Gets the criteria for this level. + public string Description { get; } +} + +/// Describes a score decision question. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class ScoreDecisionQuestion : DecisionQuestion +{ + /// Initializes a new instance of the class. + /// The opaque, caller-owned question ID. + /// The instructions defining the score. + /// The ordered, caller-defined score levels. + [JsonConstructor] + public ScoreDecisionQuestion(string id, string? instructions, IReadOnlyList levels) + : base(id, DecisionKind.Score, instructions) + { + _ = Throw.IfNull(levels); + + DecisionScoreLevel[] copy = levels.ToArray(); + if (copy.Length < 2 || Array.Exists(copy, static level => level is null)) + { + Throw.ArgumentException(nameof(levels), "A score question requires at least two levels."); + } + + HashSet ids = new(StringComparer.Ordinal); + foreach (DecisionScoreLevel level in copy) + { + if (!ids.Add(level.Id)) + { + Throw.ArgumentException( + nameof(levels), + "Score level IDs must be unique using ordinal comparison."); + } + } + + Levels = Array.AsReadOnly(copy); + } + + /// Gets the ordered, owned score-level snapshot. + public IReadOnlyList Levels { get; } +} + +/// Represents a state and a nonempty, ordered batch of questions. +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +internal sealed class DecisionRequest +{ + /// Initializes a new instance of the class and snapshots its state and questions. + /// The explicit JSON state evaluated by every question. + /// The ordered heterogeneous question batch. + [JsonConstructor] + public DecisionRequest(JsonElement state, IReadOnlyList questions) + { + if (state.ValueKind == JsonValueKind.Undefined) + { + Throw.ArgumentException(nameof(state), "State must be explicit JSON."); + } + + _ = Throw.IfNull(questions); + + DecisionQuestion[] copy = questions.ToArray(); + if (copy.Length == 0 || Array.Exists(copy, static question => question is null)) + { + Throw.ArgumentException(nameof(questions), "Questions must be nonempty."); + } + + HashSet ids = new(StringComparer.Ordinal); + foreach (DecisionQuestion question in copy) + { + if (!ids.Add(question.Id)) + { + Throw.ArgumentException( + nameof(questions), + "Question IDs must be unique using ordinal comparison."); + } + } + + State = state.Clone(); + Questions = Array.AsReadOnly(copy); + } + + /// Gets the owned JSON state snapshot. + public JsonElement State { get; } + + /// Gets the ordered, owned question snapshot. + public IReadOnlyList Questions { get; } + + /// Creates a request from an application-owned state using an explicit source-generated contract. + /// The application state type. + /// The application-owned state. + /// The source-generated JSON contract for . + /// The ordered heterogeneous question batch. + /// An owned decision request. + public static DecisionRequest Create( + TState state, + JsonTypeInfo stateTypeInfo, + IEnumerable questions) + { + _ = Throw.IfNull(stateTypeInfo); + _ = Throw.IfNull(questions); + return new(JsonSerializer.SerializeToElement(state, stateTypeInfo), questions.ToArray()); + } +} diff --git a/dotnet/src/Shared/Decisions/README.md b/dotnet/src/Shared/Decisions/README.md new file mode 100644 index 00000000000..7914b9b90a9 --- /dev/null +++ b/dotnet/src/Shared/Decisions/README.md @@ -0,0 +1,54 @@ +# Decisions + +An internal copy of the provider-neutral decision API that is proposed for `Microsoft.Extensions.AI.Abstractions`: +`IDecisionClient`, Binary, Choice, and Score questions and answers, and the typed `DecisionDefinition`. +Decision models (Jev, Azure OpenAI Decisions, and others) answer typed questions about a state with calibrated +probabilities, instead of writing text. + +The copy lets Agent Framework packages build on the proposed contract before it ships, without exposing it. When +`Microsoft.Extensions.AI` ships the API, this folder is deleted and the same code compiles against the shipped types. + +## Source + +| Proposal | Pull request | Commit | +|---|---|---| +| Layer 1: provider-neutral decision abstractions | [dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795) | `02e54f023bdf3c8c2729353b110c22f5f8593957` | +| Layer 2: decision interoperability (`AsAIFunction`) | [dotnet/extensions#7796](https://github.com/dotnet/extensions/pull/7796) | `e87f1126cf22108fb27f6391c7826fb91c25138e` | + +The files are `src/Libraries/Microsoft.Extensions.AI.Abstractions/Decisions/*.cs` at the Layer 2 commit, which contains +Layer 1. The only change is that every top-level type is `internal` instead of `public`. The namespace is still +`Microsoft.Extensions.AI` and the types keep their `[Experimental("MEAI001")]` attributes, so code that uses them looks +the same before and after the API ships. `.editorconfig` keeps the upstream style from failing the build. + +Not copied: the upstream tests, the README, and the registration of the decision types in +`AIJsonUtilities.DefaultOptions`. The upstream tests cover the copied code; the tests of the packages that use it +cover their own use. + +## Usage + +```xml + + true + true + true + +``` + +The project also needs a package reference to `Microsoft.Extensions.AI.Abstractions`. Code that uses the types +suppresses `MEAI001`, as it will have to once they ship. + +Each project that injects this folder compiles its own copy, so the types are distinct per assembly. Do not expose them +in a public API, and do not pass an `IDecisionClient` from one package to another; expose provider-specific types +instead. + +## When Microsoft.Extensions.AI ships the API + +1. Update `Microsoft.Extensions.AI.Abstractions` in `Directory.Packages.props`. From that version on, the copied types + conflict with the shipped ones (CS0436), so the next two steps go in the same change. +2. Delete this folder and the `InjectSharedDecisions` item group in `eng/MSBuild/Shared.props`, and remove + `` from the projects that set it. +3. Fix any difference between this copy and the shipped API, then decide which internal decision clients become + public. + +To follow upstream changes before then, copy the new files over these ones and make the top-level types internal again +(`sed -E 's/^public /internal /'`). diff --git a/dotnet/src/Shared/DiagnosticIds/DiagnosticsIds.cs b/dotnet/src/Shared/DiagnosticIds/DiagnosticsIds.cs index f9e0d4b15e3..9106273ef1f 100644 --- a/dotnet/src/Shared/DiagnosticIds/DiagnosticsIds.cs +++ b/dotnet/src/Shared/DiagnosticIds/DiagnosticsIds.cs @@ -7,6 +7,11 @@ namespace Microsoft.Shared.DiagnosticIds; /// internal static class DiagnosticIds { + /// + /// The help link format of the MEAI diagnostics, which reuses for MEAI experiment IDs. + /// + internal const string UrlFormat = "https://aka.ms/dotnet-extensions-warnings/{0}"; + /// /// Experiments supported by this repo. /// @@ -25,6 +30,7 @@ internal static class Experiments internal const string AIMcpServers = MEAIExperiments; internal const string AIFunctionApprovals = MEAIExperiments; internal const string AIOpenAIRequestPolicies = MEAIExperiments; + internal const string AIDecisions = MEAIExperiments; // These diagnostic IDs are defined by the OpenAI package for its experimental APIs. // We use the same IDs so consumers do not need to suppress additional diagnostics From 0f39b6a4dec321c739b91527d6bf287de86f809e Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:10:14 -0800 Subject: [PATCH 08/16] .NET: Add an internal Jev client for the decision API JevDecisionClient implements the internal IDecisionClient copy over the TypeSafe System One API. Binary, Choice, and Score questions map to Noul, Choice, and Score questions; answers map back with complete, ordered distributions, Jev's confidence as an additional property, and the JevResult as the raw representation. Callers that hold Jev questions can pass them through DecisionOptions.RawRepresentationFactory so JSON instructions and criteria reach the API unchanged. The Score test fixture now reports probabilities that agree with a fractional score, as the decision API requires. --- .../Core/JevContractValidator.cs | 26 ++- .../Core/JevDecisionClient.cs | 99 +++++++++ .../Core/JevDecisionMapping.cs | 209 ++++++++++++++++++ .../Core/JevDecisionRequestOptions.cs | 23 ++ .../Microsoft.Agents.AI.TypeSafe.csproj | 2 + .../JevChatTestData.cs | 13 +- .../JevDecisionClientTests.cs | 206 +++++++++++++++++ ...rosoft.Agents.AI.TypeSafe.UnitTests.csproj | 1 + 8 files changed, 570 insertions(+), 9 deletions(-) create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionClient.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionRequestOptions.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs index c27a2db5f0d..2738cdca56a 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs @@ -41,25 +41,35 @@ public static void ValidateRequest(JevRequest request) Throw.ArgumentException(nameof(request), "The state must be text or a JSON object or array."); } - if (request.Questions is null || request.Questions.Count == 0) + ValidateQuestions(request.Questions, nameof(request)); + } + + /// + /// Checks the questions of a request, which does too. + /// + /// The questions, keyed by ID. + /// The parameter that the questions came from, for the exception. + public static void ValidateQuestions(IReadOnlyDictionary? questions, string paramName) + { + if (questions is null || questions.Count == 0) { - Throw.ArgumentException(nameof(request), "At least one question is required."); + Throw.ArgumentException(paramName, "At least one question is required."); } - foreach (KeyValuePair entry in request.Questions) + foreach (KeyValuePair entry in questions) { string id = entry.Key; switch (entry.Value) { case null: - Throw.ArgumentException(nameof(request), $"Question '{id}' is null."); + Throw.ArgumentException(paramName, $"Question '{id}' is null."); break; case JevChoiceQuestion choice: int alternatives = choice.Criteria?.Count ?? 0; if (alternatives is < 1 or > MaxChoiceCriteria) { - Throw.ArgumentException(nameof(request), $"Choice question '{id}' has {Count(alternatives, "alternative")}; it needs from 1 to {MaxChoiceCriteria}."); + Throw.ArgumentException(paramName, $"Choice question '{id}' has {Count(alternatives, "alternative")}; it needs from 1 to {MaxChoiceCriteria}."); } break; @@ -68,7 +78,7 @@ public static void ValidateRequest(JevRequest request) int scores = score.Criteria?.Count ?? 0; if (scores is < MinScoreCriteria or > MaxScoreCriteria) { - Throw.ArgumentException(nameof(request), $"Score question '{id}' has {Count(scores, "criterion", "criteria")}; it needs from {MinScoreCriteria} to {MaxScoreCriteria}."); + Throw.ArgumentException(paramName, $"Score question '{id}' has {Count(scores, "criterion", "criteria")}; it needs from {MinScoreCriteria} to {MaxScoreCriteria}."); } // The SDK types allow null rubric entries, but the API answers 422 for them. @@ -76,7 +86,7 @@ public static void ValidateRequest(JevRequest request) { if (score.Criteria![index].IsNull) { - Throw.ArgumentException(nameof(request), $"Score question '{id}' has a null criterion at index {index.ToString(CultureInfo.InvariantCulture)}; every score needs text or JSON."); + Throw.ArgumentException(paramName, $"Score question '{id}' has a null criterion at index {index.ToString(CultureInfo.InvariantCulture)}; every score needs text or JSON."); } } @@ -85,7 +95,7 @@ public static void ValidateRequest(JevRequest request) case JevNoulQuestion noul: if (IsEmpty(noul.Instructions) && IsEmpty(noul.Criteria?.True) && IsEmpty(noul.Criteria?.False)) { - Throw.ArgumentException(nameof(request), $"Noul question '{id}' needs instructions or criteria."); + Throw.ArgumentException(paramName, $"Noul question '{id}' needs instructions or criteria."); } break; diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionClient.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionClient.cs new file mode 100644 index 00000000000..f0f925266dc --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionClient.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Answers provider-neutral decision requests with Jev, through POST /v1/systemone of the TypeSafe API. +/// +/// +/// +/// Binary, Choice, and Score questions become Noul, Choice, and Score questions (see ). +/// A request is checked with the TypeSafe rules before it is sent, and the result is checked against the request before +/// it is converted, as the chat client and the tool already do. The response keeps the as its +/// raw representation, each answer keeps its , and Choice and Score answers carry Jev's +/// confidence as the confidence additional property. +/// +/// +/// is a proposed Microsoft.Extensions.AI API that this package compiles from an internal +/// copy, so this client is internal; is built on it. Once the API ships, this client can +/// become public. +/// +/// +internal sealed class JevDecisionClient : IDecisionClient +{ + public const string ProviderName = "typesafe.ai"; + + private readonly JevHttpClient _client; + private readonly DecisionClientMetadata _metadata; + + public JevDecisionClient(JevHttpClient client) + { + this._client = Throw.IfNull(client); + this._metadata = new DecisionClientMetadata(ProviderName, client.Endpoint, client.ModelId, [DecisionKind.Binary, DecisionKind.Choice, DecisionKind.Score]); + } + + /// + /// The request breaks a TypeSafe rule, such as the number of Choice alternatives. + /// The Jev result does not answer every question as asked. + /// An answer is not a complete, correlated decision answer. + /// The TypeSafe API rejected the request. + public async Task GetResponseAsync(DecisionRequest request, DecisionOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(request); + + IReadOnlyDictionary? jevQuestions = (options?.RawRepresentationFactory?.Invoke(this) as JevDecisionRequestOptions)?.Questions; + var questions = new Dictionary(request.Questions.Count, StringComparer.Ordinal); + foreach (DecisionQuestion question in request.Questions) + { + questions[question.Id] = jevQuestions is not null && jevQuestions.TryGetValue(question.Id, out JevQuestion? jevQuestion) + ? jevQuestion + : JevDecisionMapping.ToJevQuestion(question); + } + + var jevRequest = new JevRequest { State = JevDecisionMapping.ToEntry(request.State), Questions = questions }; + JevContractValidator.ValidateRequest(jevRequest); + + string modelId = string.IsNullOrWhiteSpace(options?.ModelId) ? this._client.ModelId : options!.ModelId!; + JevResult result = await this._client.SystemOneAsync(jevRequest, modelId, cancellationToken).ConfigureAwait(false); + + // The TypeSafe checks come first, so a result that answers the wrong labels fails as it does for the tool. + JevContractValidator.ValidateResult(jevRequest, result); + + var answers = new List(request.Questions.Count); + foreach (DecisionQuestion question in request.Questions) + { + answers.Add(JevDecisionMapping.ToDecisionAnswer(question, result.Answers[question.Id])); + } + + return new DecisionResponse( + request, + answers, + new DecisionProvenance(ProviderName, this._client.Endpoint, result.Model, precision: JevDecisionMapping.Precision), + JevDecisionMapping.ToUsage(result.Usage), + rawRepresentation: result); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + return serviceKey is not null ? null + : serviceType == typeof(DecisionClientMetadata) ? this._metadata + : serviceType.IsInstanceOfType(this) ? this + : null; + } + + /// + public void Dispose() + { + // The pipeline owns no disposable resources; its default transport shares one HttpClient per process. + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs new file mode 100644 index 00000000000..3ce5dc1be75 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Converts between the Jev contract types and the provider-neutral decision types. +/// +/// +/// +/// A Binary question is a Noul, a Choice question a Choice whose labels are the candidate IDs, and a Score question a +/// Score whose rubric is the level descriptions in order. Jev keys Score probabilities by level index, so answers are +/// mapped back to the level IDs by position. +/// +/// +/// The decision types require complete, correlated answers: every candidate and level needs a probability, and each +/// distribution must sum to 1. Jev computes its values in floating point, so a value that exceeds its bounds by less +/// than is moved onto the bound, as already accepts it. +/// Nothing else is repaired. +/// +/// +internal static class JevDecisionMapping +{ + /// + /// Gets the precision of Jev probabilities. Jev does not declare a rounding, so they are taken at face value. + /// + public const DecisionPrecision Precision = DecisionPrecision.HighPrecision; + + /// The name of the entry that holds Jev's confidence. + public const string ConfidenceProperty = "confidence"; + + private const double Tolerance = 1e-6; + + /// Converts a decision state into a Jev state, which is text or a JSON object or array. + public static JevEntry ToEntry(JsonElement state) + { + if (state.ValueKind is not (JsonValueKind.String or JsonValueKind.Object or JsonValueKind.Array)) + { + Throw.ArgumentException(nameof(state), $"Jev answers questions about text or a JSON object or array, but the state is a JSON {state.ValueKind}."); + } + + return new JevEntry(state); + } + + /// Converts a decision question into the Jev question that asks it. + public static JevQuestion ToJevQuestion(DecisionQuestion question) => + question switch + { + BinaryDecisionQuestion binary => new JevNoulQuestion + { + Instructions = ToEntry(binary.Instructions), + Criteria = binary.TrueDescription is null && binary.FalseDescription is null + ? null + : new JevNoulCriteria { True = ToEntry(binary.TrueDescription), False = ToEntry(binary.FalseDescription) }, + }, + ChoiceDecisionQuestion choice => new JevChoiceQuestion + { + Instructions = ToEntry(choice.Instructions), + Criteria = ToCriteria(choice.Candidates), + }, + ScoreDecisionQuestion score => new JevScoreQuestion + { + Instructions = ToEntry(score.Instructions), + Criteria = [.. score.Levels.Select(level => (JevEntry)level.Description)], + }, + _ => throw new NotSupportedException($"Jev does not answer {question.Kind} questions."), + }; + + /// + /// Converts a Jev question into the decision question that correlates its answer. + /// + /// + /// JSON instructions and criteria become their JSON text, and an undescribed Choice label is described by the + /// label itself. The Jev question itself is what is sent (see ), so these + /// texts never reach the API. Call first, so that invalid + /// questions fail with the Jev rule they break. + /// + public static DecisionQuestion ToDecisionQuestion(string id, JevQuestion question) => + question switch + { + JevNoulQuestion noul => ToBinaryQuestion(id, noul), + JevChoiceQuestion choice => new ChoiceDecisionQuestion( + id, + ToText(choice.Instructions), + [.. choice.Criteria.Select(criterion => new DecisionCandidate(criterion.Key, ToText(criterion.Value) ?? criterion.Key))]), + JevScoreQuestion score => new ScoreDecisionQuestion( + id, + ToText(score.Instructions), + [.. score.Criteria.Select((criterion, index) => new DecisionScoreLevel(ToKey(index), ToText(criterion) ?? ToKey(index)))]), + _ => throw new ArgumentException($"Question '{id}' is not a Jev Choice, Score, or Noul question.", nameof(question)), + }; + + /// + /// Converts Jev's answer to a question into the decision answer. + /// + /// The answer is not a complete answer to the question. + public static DecisionAnswer ToDecisionAnswer(DecisionQuestion question, JevResponse response) => + (question, response) switch + { + (BinaryDecisionQuestion, JevNoulResponse noul) => new BinaryDecisionAnswer( + question.Id, + OntoBounds(noul.Noul, 1), + Precision, + rawRepresentation: noul), + (ChoiceDecisionQuestion choice, JevChoiceResponse answer) => new ChoiceDecisionAnswer( + question.Id, + answer.Choice, + [.. choice.Candidates.Select(candidate => new DecisionProbability(candidate.Id, GetProbability(answer.Probabilities, candidate.Id, question.Id)))], + Precision, + rawRepresentation: answer, + additionalProperties: Confidence(answer.Confidence)), + (ScoreDecisionQuestion score, JevScoreResponse answer) => new ScoreDecisionAnswer( + question.Id, + OntoBounds(answer.Score, score.Levels.Count - 1), + [.. score.Levels.Select((level, index) => new DecisionProbability(level.Id, GetProbability(answer.Probabilities, ToKey(index), question.Id)))], + Precision, + rawRepresentation: answer, + additionalProperties: Confidence(answer.Confidence)), + _ => throw new DecisionProtocolException($"The Jev answer to question '{question.Id}' does not match its {question.Kind} question."), + }; + + /// Converts Jev token usage into usage details. + public static UsageDetails? ToUsage(JevUsage? usage) => + usage is { } tokens + ? new UsageDetails { InputTokenCount = tokens.InputTokens, OutputTokenCount = tokens.OutputTokens, TotalTokenCount = tokens.InputTokens + tokens.OutputTokens } + : null; + + /// + /// Gets the Jev result of a decision response from , with only the answers that the + /// response holds. + /// + public static JevResult ToJevResult(DecisionResponse response) + { + if (response.RawRepresentation is not JevResult result) + { + throw new InvalidOperationException("The decision response does not come from Jev."); + } + + var answers = new Dictionary(response.Answers.Count, StringComparer.Ordinal); + foreach (DecisionAnswer answer in response.Answers) + { + answers[answer.QuestionId] = result.Answers[answer.QuestionId]; + } + + return new JevResult { Model = result.Model, Answers = answers, Usage = result.Usage }; + } + + private static BinaryDecisionQuestion ToBinaryQuestion(string id, JevNoulQuestion noul) + { + string? instructions = ToText(noul.Instructions); + string? trueDescription = ToText(noul.Criteria?.True); + string? falseDescription = ToText(noul.Criteria?.False); + + // A Noul may describe only one outcome, but a Binary question needs instructions or both descriptions, so the + // one description also stands in for the instructions. + return new BinaryDecisionQuestion( + id, + instructions ?? (trueDescription is null || falseDescription is null ? trueDescription ?? falseDescription : null), + trueDescription, + falseDescription); + } + + private static Dictionary ToCriteria(IReadOnlyList candidates) + { + var criteria = new Dictionary(candidates.Count, StringComparer.Ordinal); + foreach (DecisionCandidate candidate in candidates) + { + criteria[candidate.Id] = candidate.Description; + } + + return criteria; + } + + private static JevEntry? ToEntry(string? text) + { + // A null string converts to JevEntry.Null, which is written as JSON null, so an absent text stays absent. + if (string.IsNullOrEmpty(text)) + { + return null; + } + + return JevEntry.FromString(text); + } + + private static string? ToText(JevEntry? entry) => + entry is { IsNull: false } value && value.ToString() is { } text && !string.IsNullOrWhiteSpace(text) ? text : null; + + private static string ToKey(int index) => index.ToString(CultureInfo.InvariantCulture); + + private static double GetProbability(IReadOnlyDictionary probabilities, string key, string questionId) => + probabilities.TryGetValue(key, out double probability) + ? OntoBounds(probability, 1) + : throw new DecisionProtocolException($"The Jev answer to question '{questionId}' has no probability for '{key}'."); + + private static double OntoBounds(double value, double maximum) => + value < 0 && value >= -Tolerance ? 0 + : value > maximum && value <= maximum + Tolerance ? maximum + : value; + + private static Dictionary Confidence(double confidence) => + new(StringComparer.Ordinal) { [ConfidenceProperty] = confidence }; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionRequestOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionRequestOptions.cs new file mode 100644 index 00000000000..e8b2d92ad61 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionRequestOptions.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The raw options of a request, returned by +/// . +/// +internal sealed class JevDecisionRequestOptions +{ + /// + /// Gets the Jev questions to send instead of the decision questions with the same IDs. + /// + /// + /// Jev questions can hold JSON instructions and criteria, which decision questions carry only as text. A caller + /// that has the Jev questions passes them here, so the API receives them unchanged, and the decision questions + /// only correlate the answers. + /// + public IReadOnlyDictionary? Questions { get; init; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj index b840a3b1292..9bee88a5508 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj @@ -4,6 +4,8 @@ $(TargetFrameworksCore) Microsoft.Agents.AI.TypeSafe alpha + + $(NoWarn);MEAI001 diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs index a20f99a0ff1..b73c95cbcd2 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -42,10 +43,20 @@ public static string Choice(string id, string choice, params string[] labels) public static string Noul(string id, double noul) => $"\"{id}\":{{\"type\":\"noul\",\"noul\":{noul.ToString(CultureInfo.InvariantCulture)}}}"; + /// + /// A Score answer whose probabilities agree with the score, which is their expectation: a fractional score splits + /// the probability between the two levels around it. + /// public static string Score(string id, double score, int levels) { string legend = string.Join(",", Enumerable.Range(0, levels).Select(level => $"\"{level}\":\"Level {level}\"")); - string probabilities = string.Join(",", Enumerable.Range(0, levels).Select(level => $"\"{level}\":{(level == (int)score ? "1.0" : "0.0")}")); + int lower = (int)Math.Floor(score); + double upperShare = score - lower; + string probabilities = string.Join(",", Enumerable.Range(0, levels).Select(level => + { + double probability = level == lower ? 1 - upperShare : level == lower + 1 ? upperShare : 0; + return $"\"{level}\":{probability.ToString(CultureInfo.InvariantCulture)}"; + })); return $"\"{id}\":{{\"type\":\"score\",\"score\":{score.ToString(CultureInfo.InvariantCulture)},\"confidence\":0.8,\"legend\":{{{legend}}},\"probabilities\":{{{probabilities}}}}}"; } diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs new file mode 100644 index 00000000000..ab36d781eb3 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// Tests , which answers provider-neutral decision requests with Jev. +/// +public sealed class JevDecisionClientTests +{ + private static readonly JsonElement s_state = JsonDocument.Parse("""{ "ticket": "I was charged twice." }""").RootElement.Clone(); + + [Fact] + public async Task GetResponseAsync_DecisionQuestions_AreAskedAsJevQuestionsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.Noul("refund", 0.95), + "\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":0.8,\"probabilities\":{\"technical\":0.1,\"billing\":0.9}}", + JevChatTestData.Score("severity", 1.5, 3))); + using var httpClient = new HttpClient(handler); + using JevDecisionClient client = CreateClient(httpClient); + var request = new DecisionRequest(s_state, [ + new BinaryDecisionQuestion("refund", "Does the customer ask for a refund?", trueDescription: "A refund or reversal is requested."), + new ChoiceDecisionQuestion("department", "Which team should handle it?", [new("billing", "Payments"), new("technical", "Bugs")]), + new ScoreDecisionQuestion("severity", null, [new("cosmetic", "Cosmetic"), new("workaround", "Workaround available"), new("blocked", "Fully blocked")]), + ]); + + // Act + DecisionResponse response = await client.GetResponseAsync(request); + + // Assert: the request is the Jev wire shape, with the Score rubric in level order. + JsonNode expected = JsonNode.Parse(""" + { + "state": { "ticket": "I was charged twice." }, + "model": "jev-test", + "questions": { + "refund": { "type": "noul", "instructions": "Does the customer ask for a refund?", "criteria": { "true": "A refund or reversal is requested." } }, + "department": { "type": "choice", "instructions": "Which team should handle it?", "criteria": { "billing": "Payments", "technical": "Bugs" } }, + "severity": { "type": "score", "criteria": ["Cosmetic", "Workaround available", "Fully blocked"] } + } + } + """)!; + string body = Assert.Single(handler.Requests).Body; + Assert.True(JsonNode.DeepEquals(expected, JsonNode.Parse(body)), body); + + // Assert: the answers are in request order, keyed by the caller's IDs, with Jev's values and confidence. + Assert.Equal(0.95, Assert.IsType(response.Answers[0]).TrueProbability); + ChoiceDecisionAnswer department = Assert.IsType(response.Answers[1]); + Assert.Equal("billing", department.SelectedCandidateId); + Assert.Equal([("billing", 0.9), ("technical", 0.1)], department.Probabilities.Select(probability => (probability.Id, probability.Value))); + Assert.Equal(0.8, department.AdditionalProperties![JevDecisionMapping.ConfidenceProperty]); + Assert.IsType(department.RawRepresentation); + ScoreDecisionAnswer severity = Assert.IsType(response.Answers[2]); + Assert.Equal(1.5, severity.Score); + Assert.Equal(1.5, severity.ExpectedScore, 6); + Assert.Equal(["cosmetic", "workaround", "blocked"], severity.Probabilities.Select(probability => probability.Id)); + Assert.Equal([0.0, 0.5, 0.5], severity.Probabilities.Select(probability => probability.Value)); + + // Assert: the provenance, usage, and raw result. + Assert.Equal(JevDecisionClient.ProviderName, response.Provenance!.ProviderName); + Assert.Equal("jev-test", response.Provenance.ModelId); + Assert.Equal(DecisionPrecision.HighPrecision, response.Provenance.Precision); + Assert.Equal(12, response.Usage!.TotalTokenCount); + Assert.IsType(response.RawRepresentation); + } + + [Fact] + public async Task GetResponseAsync_JevQuestionsInRawOptions_AreSentUnchangedAsync() + { + // Arrange: JSON instructions and an undescribed label, which decision questions cannot carry as they are. + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Choice("duplicate", "yes", "yes", "no"))); + using var httpClient = new HttpClient(handler); + using JevDecisionClient client = CreateClient(httpClient); + JsonElement instructions = JsonDocument.Parse("""{ "question": "Is the resume for the same person as `candidate`?", "candidate": { "name": "Ana" } }""").RootElement.Clone(); + var jevQuestions = new Dictionary + { + ["duplicate"] = new JevChoiceQuestion { Instructions = (JevEntry)instructions, Criteria = new Dictionary { ["yes"] = JevEntry.Null, ["no"] = "A different person" } }, + }; + DecisionQuestion question = JevDecisionMapping.ToDecisionQuestion("duplicate", jevQuestions["duplicate"]); + var options = new DecisionOptions { RawRepresentationFactory = _ => new JevDecisionRequestOptions { Questions = jevQuestions } }; + + // Act + DecisionResponse response = await client.GetResponseAsync(new DecisionRequest(s_state, [question]), options); + + // Assert + JsonNode sent = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!["duplicate"]!; + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(instructions.GetRawText()), sent["instructions"]), sent.ToJsonString()); + Assert.Null(sent["criteria"]!["yes"]); + Assert.Equal("yes", Assert.IsType(Assert.Single(response.Answers)).SelectedCandidateId); + } + + [Fact] + public async Task GetResponseAsync_IncompleteDistribution_ThrowsDecisionProtocolExceptionAsync() + { + // Arrange: the answer leaves out a label, so its distribution is not complete. + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + "\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":0.9,\"probabilities\":{\"billing\":1.0}}")); + using var httpClient = new HttpClient(handler); + using JevDecisionClient client = CreateClient(httpClient); + var request = new DecisionRequest(s_state, [new ChoiceDecisionQuestion("department", "Which team?", [new("billing", "Payments"), new("technical", "Bugs")])]); + + // Act & Assert + DecisionProtocolException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(request)); + Assert.Contains("'technical'", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task GetResponseAsync_ValueWithinRoundingErrorOfItsBound_IsMovedOntoTheBoundAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("refund", 1.0000001))); + using var httpClient = new HttpClient(handler); + using JevDecisionClient client = CreateClient(httpClient); + + // Act + DecisionResponse response = await client.GetResponseAsync(new DecisionRequest(s_state, [new BinaryDecisionQuestion("refund", "Refund?")])); + + // Assert + Assert.Equal(1.0, Assert.IsType(Assert.Single(response.Answers)).TrueProbability); + } + + [Fact] + public async Task GetResponseAsync_StateThatIsNotTextObjectOrArray_ThrowsBeforeSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevDecisionClient client = CreateClient(httpClient); + JsonElement number = JsonDocument.Parse("5").RootElement.Clone(); + + // Act & Assert + await Assert.ThrowsAsync(() => client.GetResponseAsync(new DecisionRequest(number, [new BinaryDecisionQuestion("refund", "Refund?")]))); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_ModelInOptions_WinsOverTheClientModelAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("refund", 0.5))); + using var httpClient = new HttpClient(handler); + using JevDecisionClient client = CreateClient(httpClient); + + // Act + await client.GetResponseAsync(new DecisionRequest(s_state, [new BinaryDecisionQuestion("refund", "Refund?")]), new DecisionOptions { ModelId = "jev-1.14" }); + + // Assert + Assert.Equal("jev-1.14", (string?)JsonNode.Parse(Assert.Single(handler.Requests).Body)!["model"]); + } + + [Fact] + public void GetService_ReturnsTheMetadataAndTheClient() + { + // Arrange + using var httpClient = new HttpClient(new JevTestHttpHandler()); + using JevDecisionClient client = CreateClient(httpClient); + + // Act + DecisionClientMetadata? metadata = client.GetService(); + + // Assert + Assert.Equal(JevDecisionClient.ProviderName, metadata!.ProviderName); + Assert.Equal("jev-test", metadata.DefaultModelId); + Assert.Equal([DecisionKind.Binary, DecisionKind.Choice, DecisionKind.Score], metadata.SupportedKinds); + Assert.Same(client, client.GetService()); + Assert.Null(client.GetService("key")); + } + + [Fact] + public void ToDecisionQuestion_NoulWithOneCriterion_UsesItAsTheInstructions() + { + // Arrange: Jev accepts a Noul with only one outcome described, which a Binary question cannot express directly. + var noul = new JevNoulQuestion { Criteria = new JevNoulCriteria { True = "The customer is angry." } }; + + // Act + BinaryDecisionQuestion question = Assert.IsType(JevDecisionMapping.ToDecisionQuestion("angry", noul)); + + // Assert + Assert.Equal("The customer is angry.", question.Instructions); + Assert.Equal("The customer is angry.", question.TrueDescription); + Assert.Null(question.FalseDescription); + } + + private static JevDecisionClient CreateClient(HttpClient httpClient) + { + var options = new JevChatClientOptions + { + Transport = new HttpClientPipelineTransport(httpClient), + RetryPolicy = new JevRetryPolicy { RecordedWaitsForTests = [] }, + }; + + return new JevDecisionClient(new JevHttpClient(new ApiKeyCredential("test-key"), options, endpoint: null, modelId: "jev-test")); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj index 76656380a93..ee41effe009 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj @@ -2,6 +2,7 @@ $(TargetFrameworksCore) + $(NoWarn);MEAI001 From 296675e8ed5d6e2ac95a99e998d9878a5f382d24 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:10:23 -0800 Subject: [PATCH 09/16] .NET: Build JevChatClient on a provider-neutral decision chat engine Move the conversation state, closed-set tool compiler, and turn logic of JevChatClient into src/Shared/DecisionChat, built on the internal IDecisionClient copy instead of the Jev contract types. Projects opt in with InjectSharedDecisionChat, so other decision models (Azure OpenAI Decisions, Gemini) can reuse the same tool calling and turn handling. JevChatClient keeps its public surface and wire format: it validates and projects its Jev questions, runs the engine over JevDecisionClient, and describes the answers as the JevResult. Question IDs, limits, and wording are unchanged. A response that is not a complete answer to the questions now fails with InvalidOperationException. --- dotnet/agent-framework-dotnet.slnx | 27 ++ dotnet/eng/MSBuild/Shared.props | 3 + .../Core/JevArgumentKind.cs | 21 -- .../Core/JevArgumentPlan.cs | 79 ------ .../Core/JevChatClientLog.cs | 15 -- .../Core/JevJsonUtilities.cs | 62 +---- .../Core/JevToolCall.cs | 11 - .../Core/JevToolCallPlan.cs | 65 ----- .../Core/JevUnsupportedToolSchemaException.cs | 30 --- .../JevChatClient.cs | 196 +++----------- .../Microsoft.Agents.AI.TypeSafe.csproj | 1 + .../DecisionChat/DecisionArgumentKind.cs | 21 ++ .../DecisionChat/DecisionArgumentPlan.cs | 80 ++++++ .../Shared/DecisionChat/DecisionChatEngine.cs | 254 ++++++++++++++++++ .../DecisionChat/DecisionChatEngineOptions.cs | 45 ++++ .../Shared/DecisionChat/DecisionChatLog.cs | 15 ++ .../DecisionChat/DecisionChatState.cs} | 35 +-- .../DecisionChat/DecisionCompiledTool.cs} | 16 +- .../src/Shared/DecisionChat/DecisionJson.cs | 96 +++++++ .../DecisionChat/DecisionQuestionBudget.cs} | 8 +- .../Shared/DecisionChat/DecisionToolCall.cs | 11 + .../DecisionChat/DecisionToolCallCompiler.cs} | 183 +++++++------ .../DecisionChat/DecisionToolCallPlan.cs | 81 ++++++ .../DecisionChat/DecisionToolSchema.cs} | 56 ++-- .../DecisionUnsupportedToolSchemaException.cs | 30 +++ dotnet/src/Shared/DecisionChat/README.md | 39 +++ dotnet/src/Shared/Decisions/.editorconfig | 1 + dotnet/src/Shared/Decisions/README.md | 4 +- ...ts.cs => DecisionToolCallCompilerTests.cs} | 124 ++++++--- .../JevChatClientTests.cs | 39 +++ .../JevChatClientToolTests.cs | 9 +- 31 files changed, 1019 insertions(+), 638 deletions(-) delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs create mode 100644 dotnet/src/Shared/DecisionChat/DecisionArgumentKind.cs create mode 100644 dotnet/src/Shared/DecisionChat/DecisionArgumentPlan.cs create mode 100644 dotnet/src/Shared/DecisionChat/DecisionChatEngine.cs create mode 100644 dotnet/src/Shared/DecisionChat/DecisionChatEngineOptions.cs create mode 100644 dotnet/src/Shared/DecisionChat/DecisionChatLog.cs rename dotnet/src/{Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs => Shared/DecisionChat/DecisionChatState.cs} (87%) rename dotnet/src/{Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs => Shared/DecisionChat/DecisionCompiledTool.cs} (50%) create mode 100644 dotnet/src/Shared/DecisionChat/DecisionJson.cs rename dotnet/src/{Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs => Shared/DecisionChat/DecisionQuestionBudget.cs} (83%) create mode 100644 dotnet/src/Shared/DecisionChat/DecisionToolCall.cs rename dotnet/src/{Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs => Shared/DecisionChat/DecisionToolCallCompiler.cs} (59%) create mode 100644 dotnet/src/Shared/DecisionChat/DecisionToolCallPlan.cs rename dotnet/src/{Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs => Shared/DecisionChat/DecisionToolSchema.cs} (86%) create mode 100644 dotnet/src/Shared/DecisionChat/DecisionUnsupportedToolSchemaException.cs create mode 100644 dotnet/src/Shared/DecisionChat/README.md rename dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/{JevToolCallCompilerTests.cs => DecisionToolCallCompilerTests.cs} (77%) diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index c60b7e4dded..d7dfec732dc 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -554,6 +554,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/eng/MSBuild/Shared.props b/dotnet/eng/MSBuild/Shared.props index 59e6e6dbdce..9ca0317bb11 100644 --- a/dotnet/eng/MSBuild/Shared.props +++ b/dotnet/eng/MSBuild/Shared.props @@ -47,4 +47,7 @@ + + + diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs deleted file mode 100644 index a99e841d048..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// How the value of one tool argument is read from Jev's answers. -/// -internal enum JevArgumentKind -{ - /// The schema allows one value, so no question is asked about it. - Const, - - /// A Choice question selects one value of an enum. - Choice, - - /// A Noul question decides a Boolean. - Boolean, - - /// One Noul question per enum member decides which members an array includes. - Set, -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs deleted file mode 100644 index 825e535cf36..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// The questions asked about one tool argument and how its value is read from the answers. -/// -internal sealed class JevArgumentPlan -{ - public required string Name { get; init; } - - public required JevArgumentKind Kind { get; init; } - - /// - /// Gets the ID of the Noul question that decides whether an optional argument is passed at all, or - /// for a required argument. - /// - public string? PresenceQuestionId { get; init; } - - /// Gets the ID of the Choice or Noul question of a or argument. - public string? ValueQuestionId { get; init; } - - /// Gets the enum values of a argument, keyed by their Choice label. - public IReadOnlyDictionary Values { get; init; } = new Dictionary(); - - /// Gets the Noul question ID and the value of each member of a argument. - public IReadOnlyList> Members { get; init; } = []; - - /// Gets the value of a argument. - public JsonElement ConstValue { get; init; } - - /// - /// Reads the argument value from Jev's answers. - /// - /// - /// when an optional argument should be left out, so that the function uses its default. - /// - public bool TryDecode(JevResult result, out JsonElement value) - { - value = default; - if (this.PresenceQuestionId is not null && !JevToolCallPlan.IsYes(result, this.PresenceQuestionId)) - { - return false; - } - - switch (this.Kind) - { - case JevArgumentKind.Const: - value = this.ConstValue; - return true; - - case JevArgumentKind.Boolean: - value = JevJsonUtilities.CreateBoolean(JevToolCallPlan.IsYes(result, this.ValueQuestionId!)); - return true; - - case JevArgumentKind.Choice: - string label = JevToolCallPlan.GetChoice(result, this.ValueQuestionId!); - if (!this.Values.TryGetValue(label, out value)) - { - Throw.InvalidOperationException($"Jev returned the unknown choice '{label}' for the tool argument '{this.Name}'."); - } - - return true; - - case JevArgumentKind.Set: - value = JevJsonUtilities.CreateArray(this.Members.Where(member => JevToolCallPlan.IsYes(result, member.Key)).Select(member => member.Value)); - return true; - - default: - Throw.InvalidOperationException($"The argument plan of '{this.Name}' has the unknown kind {this.Kind}."); - return false; - } - } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs deleted file mode 100644 index 3d093ad6a39..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.Logging; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// Source-generated log messages of . -internal static partial class JevChatClientLog -{ - [LoggerMessage( - EventId = 1, - Level = LogLevel.Warning, - Message = "The tool '{ToolName}' is left out of the Jev request because its parameters cannot be expressed as Jev questions: {Reason}")] - public static partial void LogToolExcluded(this ILogger logger, string toolName, string reason); -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs index affd4f95ec6..57d20767fa9 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Buffers; using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Nodes; @@ -11,13 +10,10 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Serializer options, JSON schemas, and JSON value helpers shared by the Jev tool and chat client. +/// Serializer options and JSON schemas of the Jev contract types, shared by the Jev tool and chat client. /// internal static class JevJsonUtilities { - private static readonly JsonElement s_true = Parse("true"); - private static readonly JsonElement s_false = Parse("false"); - private static readonly JsonElement s_null = Parse("null"); private static readonly AIJsonSchemaCreateOptions s_schemaOptions = new() { TransformSchemaNode = DescribeEntries }; private static readonly Lazy s_requestSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevRequest), serializerOptions: Options, inferenceOptions: s_schemaOptions)); private static readonly Lazy s_resultSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevResult), serializerOptions: Options, inferenceOptions: s_schemaOptions)); @@ -53,62 +49,6 @@ internal static class JevJsonUtilities _ => JsonSerializer.SerializeToNode(value, Options.GetTypeInfo(value.GetType())), }; - /// - /// Converts a value from a chat message, such as a function argument or result, into a JSON node. A value that - /// cannot be serialized, for example a type without metadata under native AOT, becomes its text. - /// - public static JsonNode? ToNodeOrText(object? value) - { - try - { - return ToNode(value); - } - catch (Exception ex) when (ex is NotSupportedException or InvalidOperationException or JsonException or ArgumentException) - { - // Chat history is informational for Jev, so an unserializable value is described rather than failing - // the whole request. - return JsonValue.Create(value?.ToString()); - } - } - - /// Converts a value from a chat message into a detached , like . - public static JsonElement ToElement(object? value) => - value switch - { - JsonElement element => element.ValueKind == JsonValueKind.Undefined ? s_null : element.Clone(), - null => s_null, - _ => Parse(ToNodeOrText(value)?.ToJsonString() ?? "null"), - }; - - /// Gets the JSON or value. - public static JsonElement CreateBoolean(bool value) => value ? s_true : s_false; - - /// Writes values into a new JSON array. - public static JsonElement CreateArray(IEnumerable values) - { - var buffer = new ArrayBufferWriter(); - using (var writer = new Utf8JsonWriter(buffer)) - { - writer.WriteStartArray(); - foreach (JsonElement value in values) - { - value.WriteTo(writer); - } - - writer.WriteEndArray(); - } - - using JsonDocument document = JsonDocument.Parse(buffer.WrittenMemory); - return document.RootElement.Clone(); - } - - /// Parses JSON text into a detached . - public static JsonElement Parse(string json) - { - using JsonDocument document = JsonDocument.Parse(json); - return document.RootElement.Clone(); - } - private static JsonSerializerOptions CreateOptions() { var options = new JsonSerializerOptions(JevJsonContext.Default.Options) diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs deleted file mode 100644 index ef5e2807a84..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using Microsoft.Extensions.AI; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// The tool call that Jev selected: the function and the arguments decoded from its answers. -/// -internal readonly record struct JevToolCall(AIFunctionDeclaration Function, Dictionary Arguments); diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs deleted file mode 100644 index e4f0335c806..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// The internal questions that let Jev choose one tool call, and the decoders that turn their answers into the call. -/// -internal sealed class JevToolCallPlan -{ - /// - /// A Noul answer above this probability means yes, for presence, Boolean, and array member questions. - /// - public const double NoulYesThreshold = 0.5; - - public required IReadOnlyDictionary Questions { get; init; } - - public required IReadOnlyList Tools { get; init; } - - /// - /// Gets the ID of the Choice question that selects the tool, or when - /// is the only possible call. - /// - public string? RouteQuestionId { get; init; } - - /// - /// Gets the tool that must be called, when the tool mode requires a call and only one tool compiled. - /// - public JevCompiledTool? RequiredTool { get; init; } - - /// - /// Reads the selected tool call from Jev's answers. - /// - /// The tool call, or when Jev chose not to call a tool. - public JevToolCall? Decode(JevResult result) - { - JevCompiledTool? selected = this.RequiredTool; - if (selected is null) - { - string route = GetChoice(result, this.RouteQuestionId!); - if (route == JevToolCallCompiler.RouteNone) - { - return null; - } - - selected = this.Tools.FirstOrDefault(tool => tool.RouteLabel == route) - ?? throw new InvalidOperationException($"Jev returned the unknown tool route '{route}'."); - } - - return new JevToolCall(selected.Function, selected.DecodeArguments(result)); - } - - public static string GetChoice(JevResult result, string questionId) => - result.Answers.TryGetValue(questionId, out JevResponse? answer) && answer is JevChoiceResponse choice - ? choice.Choice - : throw new InvalidOperationException($"The Jev result has no Choice answer for the internal question '{questionId}'."); - - public static bool IsYes(JevResult result, string questionId) => - result.Answers.TryGetValue(questionId, out JevResponse? answer) && answer is JevNoulResponse noul - ? noul.Noul > NoulYesThreshold - : throw new InvalidOperationException($"The Jev result has no Noul answer for the internal question '{questionId}'."); -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs deleted file mode 100644 index fe3ecfc72d9..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// A tool's JSON schema cannot be expressed as Jev questions, so the tool is left out of the request. -/// -/// -/// It is caught while a tool call plan is compiled and never leaves this package: an unsupported tool is logged and -/// excluded, or, when the tool mode requires a call and no tool remains, its reason becomes part of an -/// . -/// -internal sealed class JevUnsupportedToolSchemaException : NotSupportedException -{ - public JevUnsupportedToolSchemaException() - { - } - - public JevUnsupportedToolSchemaException(string message) - : base(message) - { - } - - public JevUnsupportedToolSchemaException(string message, Exception innerException) - : base(message, innerException) - { - } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs index aa91948f13d..b11e85b8f45 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs @@ -3,11 +3,11 @@ using System; using System.ClientModel; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using Microsoft.Agents.AI.Decisions; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -44,12 +44,9 @@ namespace Microsoft.Agents.AI.TypeSafe; /// public sealed class JevChatClient : IChatClient { - private const string ProviderName = "typesafe.ai"; - private readonly JevHttpClient _client; + private readonly DecisionChatEngine _engine; private readonly IReadOnlyDictionary? _defaultQuestions; - private readonly int _maximumToolCallsPerTurn; - private readonly ILogger _logger; private readonly ChatClientMetadata _metadata; /// @@ -78,10 +75,20 @@ public JevChatClient(ApiKeyCredential credential, JevChatClientOptions? options // The questions are copied so that later changes to the caller's dictionary do not change this client. this._defaultQuestions = options.DefaultQuestions?.ToDictionary(question => question.Key, question => question.Value, StringComparer.Ordinal); - this._maximumToolCallsPerTurn = options.MaximumToolCallsPerTurn; - this._logger = (options.ClientLoggingOptions?.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); this._client = new JevHttpClient(credential, options, options.Endpoint, options.ModelId); - this._metadata = new ChatClientMetadata(ProviderName, this._client.Endpoint, this._client.ModelId); + this._metadata = new ChatClientMetadata(JevDecisionClient.ProviderName, this._client.Endpoint, this._client.ModelId); + + // The conversation, tool calling, and turn logic are provider-neutral; this client supplies Jev as the decision + // client and keeps the Jev result as what callers see. + this._engine = new DecisionChatEngine(new JevDecisionClient(this._client), new DecisionChatEngineOptions + { + ModelName = "Jev", + QuestionsDescription = $"the questions passed with {nameof(JevChatOptionsExtensions.WithJevQuestions)}", + CallIdPrefix = "typesafe", + DescribeAnswers = DescribeAnswers, + MaximumToolCallsPerTurn = options.MaximumToolCallsPerTurn, + Logger = (options.ClientLoggingOptions?.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(), + }); } /// @@ -94,68 +101,19 @@ public JevChatClient(ApiKeyCredential credential, JevChatClientOptions? options public async Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(messages); - IReadOnlyList history = messages as IReadOnlyList ?? [.. messages]; - EnsureSupported(options); + // The Jev questions are sent unchanged, so JSON instructions and criteria keep their form; the decision + // questions made from them correlate the answers. IReadOnlyDictionary questions = this.GetQuestions(options); - JevToolCallCompiler.EnsureNoReservedQuestionIds(questions.Keys); - List tools = GetFunctionTools(options); - - // Once the turn has used its tool calls, tools are no longer offered, so the next response answers the - // questions with the tool results in the state. This is what ends the function-invoking loop, like the - // max_function_calls default of one in the Python connector. - int turnStart = JevChatState.GetCurrentTurnStart(history); - ChatToolMode? toolMode = JevChatState.CountCurrentTurnCalls(history, turnStart) >= this._maximumToolCallsPerTurn - ? ChatToolMode.None - : options?.ToolMode; - JevToolCallPlan? plan = JevToolCallCompiler.Compile(tools, toolMode, JevChatState.GetCurrentTurnCalls(history, turnStart), this._logger); - - var allQuestions = new Dictionary(StringComparer.Ordinal); - foreach (KeyValuePair question in questions) - { - allQuestions[question.Key] = question.Value; - } - - foreach (KeyValuePair question in plan?.Questions ?? new Dictionary()) + JevContractValidator.ValidateQuestions(questions, nameof(options)); + List decisionQuestions = [.. questions.Select(question => JevDecisionMapping.ToDecisionQuestion(question.Key, question.Value))]; + var decisionOptions = new DecisionOptions { - allQuestions[question.Key] = question.Value; - } - - var request = new JevRequest { State = JevChatState.Build(history, options?.Instructions), Questions = allQuestions }; - JevContractValidator.ValidateRequest(request); - - string modelId = string.IsNullOrWhiteSpace(options?.ModelId) ? this._client.ModelId : options!.ModelId!; - JevResult result = await this._client.SystemOneAsync(request, modelId, cancellationToken).ConfigureAwait(false); - - // Validation covers the internal questions too, so the decoders below only see labels they offered. - JevContractValidator.ValidateResult(request, result); - UsageDetails? usage = result.Usage is { } tokens - ? new UsageDetails { InputTokenCount = tokens.InputTokens, OutputTokenCount = tokens.OutputTokens, TotalTokenCount = tokens.InputTokens + tokens.OutputTokens } - : null; - - if (plan?.Decode(result) is { } call) - { - var callMessage = new ChatMessage(ChatRole.Assistant, [new FunctionCallContent($"typesafe-{Guid.NewGuid():N}", call.Function.Name, call.Arguments)]); - return new ChatResponse(callMessage) - { - ModelId = result.Model, - FinishReason = ChatFinishReason.ToolCalls, - Usage = usage, - CreatedAt = DateTimeOffset.UtcNow, - }; - } - - // The caller asked only its own questions, so the internal tool answers are removed from what it sees. - JevResult answers = WithoutInternalAnswers(result, questions); - var message = new ChatMessage(ChatRole.Assistant, BuildText(history, turnStart, answers)) { RawRepresentation = answers }; - return new ChatResponse(message) - { - ModelId = result.Model, - FinishReason = ChatFinishReason.Stop, - Usage = usage, - CreatedAt = DateTimeOffset.UtcNow, - RawRepresentation = answers, + ModelId = options?.ModelId, + RawRepresentationFactory = _ => new JevDecisionRequestOptions { Questions = questions }, }; + + return await this._engine.GetResponseAsync(messages, options, decisionQuestions, decisionOptions, cancellationToken).ConfigureAwait(false); } /// @@ -180,48 +138,14 @@ public void Dispose() // The pipeline owns no disposable resources; its default transport shares one HttpClient per process. } - private static void EnsureSupported(ChatOptions? options) + /// + /// Describes the answers to the caller's questions as the that callers read, in JSON for the + /// response text and typed for . + /// + private static DecisionChatAnswers DescribeAnswers(DecisionResponse answers) { - if (options is null) - { - return; - } - - var unsupported = new List(); - AddIf(options.Temperature is not null, nameof(ChatOptions.Temperature)); - AddIf(options.TopP is not null, nameof(ChatOptions.TopP)); - AddIf(options.TopK is not null, nameof(ChatOptions.TopK)); - AddIf(options.MaxOutputTokens is not null, nameof(ChatOptions.MaxOutputTokens)); - AddIf(options.FrequencyPenalty is not null, nameof(ChatOptions.FrequencyPenalty)); - AddIf(options.PresencePenalty is not null, nameof(ChatOptions.PresencePenalty)); - AddIf(options.Seed is not null, nameof(ChatOptions.Seed)); - AddIf(options.Reasoning is not null, nameof(ChatOptions.Reasoning)); - AddIf(options.StopSequences is { Count: > 0 }, nameof(ChatOptions.StopSequences)); - AddIf(options.ResponseFormat is not (null or ChatResponseFormatText), nameof(ChatOptions.ResponseFormat)); - AddIf(options.ConversationId is not null, nameof(ChatOptions.ConversationId)); - AddIf(options.AllowBackgroundResponses is true, nameof(ChatOptions.AllowBackgroundResponses)); -#pragma warning disable MEAI001 // ContinuationToken is experimental; it is only checked to reject it. - AddIf(options.ContinuationToken is not null, nameof(ChatOptions.ContinuationToken)); -#pragma warning restore MEAI001 - - if (unsupported.Count > 0) - { - throw new NotSupportedException( - $"Jev does not support these chat options: {string.Join(", ", unsupported)}. Jev does not generate text; it answers the questions passed with {nameof(JevChatOptionsExtensions.WithJevQuestions)}."); - } - - if (options.AllowMultipleToolCalls is true) - { - throw new NotSupportedException($"Jev requests one tool call per response. Leave {nameof(ChatOptions)}.{nameof(ChatOptions.AllowMultipleToolCalls)} unset or false."); - } - - void AddIf(bool condition, string name) - { - if (condition) - { - unsupported.Add(name); - } - } + JevResult result = JevDecisionMapping.ToJevResult(answers); + return new DecisionChatAnswers(JsonSerializer.Serialize(result, JevJsonContext.Default.JevResult), result); } private IReadOnlyDictionary GetQuestions(ChatOptions? options) @@ -241,62 +165,4 @@ private IReadOnlyDictionary GetQuestions(ChatOptions? optio return questions; } - - private static List GetFunctionTools(ChatOptions? options) - { - if (options?.Tools is not { Count: > 0 } tools) - { - return []; - } - - List unsupported = [.. tools.Where(tool => tool is not AIFunctionDeclaration).Select(tool => tool.Name)]; - if (unsupported.Count > 0) - { - throw new NotSupportedException($"Jev can only call function tools; these tools are not supported: {string.Join(", ", unsupported)}."); - } - - return [.. tools.Cast()]; - } - - private static JevResult WithoutInternalAnswers(JevResult result, IReadOnlyDictionary questions) - { - var answers = new Dictionary(questions.Count, StringComparer.Ordinal); - foreach (string id in questions.Keys) - { - answers[id] = result.Answers[id]; - } - - return new JevResult { Model = result.Model, Answers = answers, Usage = result.Usage }; - } - - /// - /// Builds the response text: the result as JSON, or, after tool calls, the tool results followed by the Choice and - /// Score decisions, which is what a caller that reads only text needs to see. - /// - private static string BuildText(IReadOnlyList history, int turnStart, JevResult answers) - { - List lines = JevChatState.GetCurrentTurnResultTexts(history, turnStart); - if (lines.Count == 0) - { - return JsonSerializer.Serialize(answers, JevJsonContext.Default.JevResult); - } - - // Noul answers are probabilities rather than decisions, so, as in the Python connector, they are left to the - // typed result. - foreach (KeyValuePair answer in answers.Answers) - { - switch (answer.Value) - { - case JevChoiceResponse choice: - lines.Add($"{answer.Key}: {choice.Choice}"); - break; - - case JevScoreResponse score: - lines.Add($"{answer.Key}: {score.Score.ToString("G6", CultureInfo.InvariantCulture)}"); - break; - } - } - - return string.Join("\n", lines); - } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj index 9bee88a5508..2645c86fab7 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj @@ -12,6 +12,7 @@ true true true + true diff --git a/dotnet/src/Shared/DecisionChat/DecisionArgumentKind.cs b/dotnet/src/Shared/DecisionChat/DecisionArgumentKind.cs new file mode 100644 index 00000000000..cbca2a6a202 --- /dev/null +++ b/dotnet/src/Shared/DecisionChat/DecisionArgumentKind.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Agents.AI.Decisions; + +/// +/// How the value of one tool argument is read from the answers of a decision model. +/// +internal enum DecisionArgumentKind +{ + /// The schema allows one value, so no question is asked about it. + Const, + + /// A Choice question selects one value of an enum. + Choice, + + /// A Binary question decides a Boolean. + Boolean, + + /// One Binary question per enum member decides which members an array includes. + Set, +} diff --git a/dotnet/src/Shared/DecisionChat/DecisionArgumentPlan.cs b/dotnet/src/Shared/DecisionChat/DecisionArgumentPlan.cs new file mode 100644 index 00000000000..021f4fafc22 --- /dev/null +++ b/dotnet/src/Shared/DecisionChat/DecisionArgumentPlan.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.Decisions; + +/// +/// The questions asked about one tool argument and how its value is read from the answers. +/// +internal sealed class DecisionArgumentPlan +{ + public required string Name { get; init; } + + public required DecisionArgumentKind Kind { get; init; } + + /// + /// Gets the ID of the Binary question that decides whether an optional argument is passed at all, or + /// for a required argument. + /// + public string? PresenceQuestionId { get; init; } + + /// Gets the ID of the Choice or Binary question of a or argument. + public string? ValueQuestionId { get; init; } + + /// Gets the enum values of a argument, keyed by their Choice label. + public IReadOnlyDictionary Values { get; init; } = new Dictionary(); + + /// Gets the Binary question ID and the value of each member of a argument. + public IReadOnlyList> Members { get; init; } = []; + + /// Gets the value of a argument. + public JsonElement ConstValue { get; init; } + + /// + /// Reads the argument value from the answers. + /// + /// + /// when an optional argument should be left out, so that the function uses its default. + /// + public bool TryDecode(DecisionResponse response, out JsonElement value) + { + value = default; + if (this.PresenceQuestionId is not null && !DecisionToolCallPlan.IsYes(response, this.PresenceQuestionId)) + { + return false; + } + + switch (this.Kind) + { + case DecisionArgumentKind.Const: + value = this.ConstValue; + return true; + + case DecisionArgumentKind.Boolean: + value = DecisionJson.CreateBoolean(DecisionToolCallPlan.IsYes(response, this.ValueQuestionId!)); + return true; + + case DecisionArgumentKind.Choice: + string label = DecisionToolCallPlan.GetChoice(response, this.ValueQuestionId!); + if (!this.Values.TryGetValue(label, out value)) + { + Throw.InvalidOperationException($"The decision model returned the unknown choice '{label}' for the tool argument '{this.Name}'."); + } + + return true; + + case DecisionArgumentKind.Set: + value = DecisionJson.CreateArray(this.Members.Where(member => DecisionToolCallPlan.IsYes(response, member.Key)).Select(member => member.Value)); + return true; + + default: + Throw.InvalidOperationException($"The argument plan of '{this.Name}' has the unknown kind {this.Kind}."); + return false; + } + } +} diff --git a/dotnet/src/Shared/DecisionChat/DecisionChatEngine.cs b/dotnet/src/Shared/DecisionChat/DecisionChatEngine.cs new file mode 100644 index 00000000000..4e86e3c08e3 --- /dev/null +++ b/dotnet/src/Shared/DecisionChat/DecisionChatEngine.cs @@ -0,0 +1,254 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.Decisions; + +/// +/// Answers chat requests with a decision model, which answers typed questions about a state instead of writing text. +/// +/// +/// +/// This is the provider-neutral part of a decision model chat client. Each request sends the conversation as the state +/// (see ), together with the caller's questions and the internal questions that let the +/// model call a function tool whose arguments come from a closed set (see ). The +/// response is a for a function-invoking client to run, or the answers to the +/// caller's questions. Once the turn has used calls, +/// tools are no longer offered, which ends the function-invoking loop. +/// +/// +/// A provider's public chat client supplies the , converts its own question types into +/// values, and describes the answers in its own result type through +/// . The engine uses only the public surface of the decision +/// API, so it compiles unchanged against Microsoft.Extensions.AI once that API ships. +/// +/// +internal sealed class DecisionChatEngine +{ + private readonly IDecisionClient _client; + private readonly DecisionChatEngineOptions _options; + + public DecisionChatEngine(IDecisionClient client, DecisionChatEngineOptions options) + { + this._client = Throw.IfNull(client); + this._options = Throw.IfNull(options); + } + + /// + /// Answers one chat request. + /// + /// The conversation. + /// The chat options, which supply the tools, the tool mode, and the instructions. + /// The caller's questions, which the answer text and raw representation describe. + /// The options passed to the decision client, such as the model ID. + /// The cancellation token to monitor. + /// A tool call, or the answers to . + /// A message or question is invalid. + /// + /// A required tool call is impossible, or the decision response does not answer every question as asked. + /// + /// The request uses an option, tool, or content that the model does not support. + public async Task GetResponseAsync( + IEnumerable messages, + ChatOptions? options, + IReadOnlyList questions, + DecisionOptions? decisionOptions, + CancellationToken cancellationToken) + { + _ = Throw.IfNull(messages); + _ = Throw.IfNull(questions); + IReadOnlyList history = messages as IReadOnlyList ?? [.. messages]; + string modelName = this._options.ModelName; + + this.EnsureSupported(options); + if (questions.Count == 0) + { + Throw.ArgumentException(nameof(questions), $"{modelName} needs at least one question."); + } + + DecisionToolCallCompiler.EnsureNoReservedQuestionIds(questions.Select(question => question.Id)); + List tools = this.GetFunctionTools(options); + + // Once the turn has used its tool calls, tools are no longer offered, so the next response answers the + // questions with the tool results in the state. This is what ends the function-invoking loop, like the + // max_function_calls default of one in the Python TypeSafe connector. + int turnStart = DecisionChatState.GetCurrentTurnStart(history); + ChatToolMode? toolMode = DecisionChatState.CountCurrentTurnCalls(history, turnStart) >= this._options.MaximumToolCallsPerTurn + ? ChatToolMode.None + : options?.ToolMode; + DecisionToolCallPlan? plan = DecisionToolCallCompiler.Compile( + tools, + toolMode, + DecisionChatState.GetCurrentTurnCalls(history, turnStart), + modelName, + this._options.Logger); + + // The reserved prefix keeps the internal question IDs apart from the caller's. + List allQuestions = [.. questions, .. plan?.Questions.Values ?? []]; + var request = new DecisionRequest(DecisionChatState.Build(history, options?.Instructions, modelName), allQuestions); + + DecisionResponse response; + try + { + response = await this._client.GetResponseAsync(request, decisionOptions, cancellationToken).ConfigureAwait(false); + } + catch (DecisionProtocolException ex) + { + // A response that does not answer the questions as asked is a provider failure, which callers of a chat + // client see as an InvalidOperationException. + throw new InvalidOperationException($"The {modelName} response is invalid: {ex.Message}", ex); + } + + if (plan?.Decode(response) is { } call) + { + var callMessage = new ChatMessage( + ChatRole.Assistant, + [new FunctionCallContent($"{this._options.CallIdPrefix}-{Guid.NewGuid():N}", call.Function.Name, call.Arguments)]); + return new ChatResponse(callMessage) + { + ModelId = response.Provenance?.ModelId, + FinishReason = ChatFinishReason.ToolCalls, + Usage = response.Usage, + CreatedAt = DateTimeOffset.UtcNow, + }; + } + + // The caller asked only its own questions, so the internal tool answers are removed from what it sees. + DecisionResponse answers = OnlyAnswersTo(response, questions); + DecisionChatAnswers described = this._options.DescribeAnswers(answers); + var message = new ChatMessage(ChatRole.Assistant, BuildText(history, turnStart, answers, described.Text)) + { + RawRepresentation = described.RawRepresentation, + }; + + return new ChatResponse(message) + { + ModelId = response.Provenance?.ModelId, + FinishReason = ChatFinishReason.Stop, + Usage = response.Usage, + CreatedAt = DateTimeOffset.UtcNow, + RawRepresentation = described.RawRepresentation, + }; + } + + private void EnsureSupported(ChatOptions? options) + { + if (options is null) + { + return; + } + + var unsupported = new List(); + AddIf(options.Temperature is not null, nameof(ChatOptions.Temperature)); + AddIf(options.TopP is not null, nameof(ChatOptions.TopP)); + AddIf(options.TopK is not null, nameof(ChatOptions.TopK)); + AddIf(options.MaxOutputTokens is not null, nameof(ChatOptions.MaxOutputTokens)); + AddIf(options.FrequencyPenalty is not null, nameof(ChatOptions.FrequencyPenalty)); + AddIf(options.PresencePenalty is not null, nameof(ChatOptions.PresencePenalty)); + AddIf(options.Seed is not null, nameof(ChatOptions.Seed)); + AddIf(options.Reasoning is not null, nameof(ChatOptions.Reasoning)); + AddIf(options.StopSequences is { Count: > 0 }, nameof(ChatOptions.StopSequences)); + AddIf(options.ResponseFormat is not (null or ChatResponseFormatText), nameof(ChatOptions.ResponseFormat)); + AddIf(options.ConversationId is not null, nameof(ChatOptions.ConversationId)); + AddIf(options.AllowBackgroundResponses is true, nameof(ChatOptions.AllowBackgroundResponses)); +#pragma warning disable MEAI001 // ContinuationToken is experimental; it is only checked to reject it. + AddIf(options.ContinuationToken is not null, nameof(ChatOptions.ContinuationToken)); +#pragma warning restore MEAI001 + + string modelName = this._options.ModelName; + if (unsupported.Count > 0) + { + throw new NotSupportedException( + $"{modelName} does not support these chat options: {string.Join(", ", unsupported)}. {modelName} does not generate text; it answers {this._options.QuestionsDescription}."); + } + + if (options.AllowMultipleToolCalls is true) + { + throw new NotSupportedException($"{modelName} requests one tool call per response. Leave {nameof(ChatOptions)}.{nameof(ChatOptions.AllowMultipleToolCalls)} unset or false."); + } + + void AddIf(bool condition, string name) + { + if (condition) + { + unsupported.Add(name); + } + } + } + + private List GetFunctionTools(ChatOptions? options) + { + if (options?.Tools is not { Count: > 0 } tools) + { + return []; + } + + List unsupported = [.. tools.Where(tool => tool is not AIFunctionDeclaration).Select(tool => tool.Name)]; + if (unsupported.Count > 0) + { + throw new NotSupportedException($"{this._options.ModelName} can only call function tools; these tools are not supported: {string.Join(", ", unsupported)}."); + } + + return [.. tools.Cast()]; + } + + private static DecisionResponse OnlyAnswersTo(DecisionResponse response, IReadOnlyList questions) + { + if (response.Answers.Count == questions.Count) + { + return response; + } + + var answers = new Dictionary(response.Answers.Count, StringComparer.Ordinal); + foreach (DecisionAnswer answer in response.Answers) + { + answers[answer.QuestionId] = answer; + } + + return new DecisionResponse( + new DecisionRequest(response.Request.State, questions), + [.. questions.Select(question => answers[question.Id])], + response.Provenance, + response.Usage, + response.RawRepresentation, + response.AdditionalProperties); + } + + /// + /// Builds the response text: the described answers, or, after tool calls, the tool results followed by the Choice + /// and Score decisions, which is what a caller that reads only text needs to see. + /// + private static string BuildText(IReadOnlyList history, int turnStart, DecisionResponse answers, string describedAnswers) + { + List lines = DecisionChatState.GetCurrentTurnResultTexts(history, turnStart); + if (lines.Count == 0) + { + return describedAnswers; + } + + // Binary answers are probabilities rather than decisions, so, as in the Python TypeSafe connector, they are left + // to the typed result. + foreach (DecisionAnswer answer in answers.Answers) + { + switch (answer) + { + case ChoiceDecisionAnswer choice: + lines.Add($"{answer.QuestionId}: {choice.SelectedCandidateId}"); + break; + + case ScoreDecisionAnswer score: + lines.Add($"{answer.QuestionId}: {(score.Score ?? score.ExpectedScore).ToString("G6", CultureInfo.InvariantCulture)}"); + break; + } + } + + return string.Join("\n", lines); + } +} diff --git a/dotnet/src/Shared/DecisionChat/DecisionChatEngineOptions.cs b/dotnet/src/Shared/DecisionChat/DecisionChatEngineOptions.cs new file mode 100644 index 00000000000..c026953bd3b --- /dev/null +++ b/dotnet/src/Shared/DecisionChat/DecisionChatEngineOptions.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Microsoft.Agents.AI.Decisions; + +/// +/// Configures a for one provider. +/// +internal sealed class DecisionChatEngineOptions +{ + /// Gets the name of the decision model in messages, such as Jev. + public required string ModelName { get; init; } + + /// + /// Gets how the provider's questions are passed, which completes the sentence "it answers ..." in the message about + /// unsupported chat options, such as the questions passed with WithJevQuestions. + /// + public required string QuestionsDescription { get; init; } + + /// Gets the prefix of the IDs of the function calls the engine returns, such as typesafe. + public required string CallIdPrefix { get; init; } + + /// + /// Gets the function that describes the answers to the caller's questions in the provider's own terms: the response + /// text when no tool ran in the turn, and the raw representation of the response and its message. + /// + public required Func DescribeAnswers { get; init; } + + /// Gets the number of tool calls after which a turn answers the questions instead. + public int MaximumToolCallsPerTurn { get; init; } = 1; + + /// Gets the logger that receives a warning for each tool that cannot be expressed as questions. + public ILogger Logger { get; init; } = NullLogger.Instance; +} + +/// +/// The answers to the caller's questions, described by the provider. +/// +/// The response text when no tool ran in the turn. +/// The raw representation of the response and its message. +internal readonly record struct DecisionChatAnswers(string Text, object? RawRepresentation); diff --git a/dotnet/src/Shared/DecisionChat/DecisionChatLog.cs b/dotnet/src/Shared/DecisionChat/DecisionChatLog.cs new file mode 100644 index 00000000000..c44160d92ed --- /dev/null +++ b/dotnet/src/Shared/DecisionChat/DecisionChatLog.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.Decisions; + +/// Source-generated log messages of . +internal static partial class DecisionChatLog +{ + [LoggerMessage( + EventId = 1, + Level = LogLevel.Warning, + Message = "The tool '{ToolName}' is left out of the {ModelName} request because its parameters cannot be expressed as closed-set questions: {Reason}")] + public static partial void LogToolExcluded(this ILogger logger, string modelName, string toolName, string reason); +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs b/dotnet/src/Shared/DecisionChat/DecisionChatState.cs similarity index 87% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs rename to dotnet/src/Shared/DecisionChat/DecisionChatState.cs index 27c121da184..3ff3cdac341 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs +++ b/dotnet/src/Shared/DecisionChat/DecisionChatState.cs @@ -8,10 +8,11 @@ using Microsoft.Extensions.AI; using Microsoft.Shared.Diagnostics; -namespace Microsoft.Agents.AI.TypeSafe; +namespace Microsoft.Agents.AI.Decisions; /// -/// Converts a conversation into the state that Jev answers questions about, and reads the current turn of it. +/// Converts a conversation into the state that a decision model answers questions about, and reads the current turn of +/// it. /// /// /// The state has the shape of the Python TypeSafe connector: {"messages": [{"role", "contents": [...]}], @@ -20,9 +21,13 @@ namespace Microsoft.Agents.AI.TypeSafe; /// and result they lead to carry the information. The current turn is everything after the last user message that /// starts a request: the tool calls and results that answer it. /// -internal static class JevChatState +internal static class DecisionChatState { - public static JevEntry Build(IReadOnlyList messages, string? instructions) + /// Builds the JSON state of a conversation. + /// The conversation. + /// The chat instructions, which become the instructions field. + /// The name of the decision model, such as Jev, for messages. + public static JsonElement Build(IReadOnlyList messages, string? instructions, string modelName) { var stateMessages = new JsonArray(); for (int index = 0; index < messages.Count; index++) @@ -36,7 +41,7 @@ public static JevEntry Build(IReadOnlyList messages, string? instru if (unsupported.Count > 0) { throw new NotSupportedException( - $"Jev supports text, reasoning text, and function call and result content only; message {index} contains {string.Join(", ", unsupported)}."); + $"{modelName} supports text, reasoning text, and function call and result content only; message {index} contains {string.Join(", ", unsupported)}."); } // Approval content is not written: FunctionInvokingChatClient keeps handled approvals in the history of @@ -67,7 +72,7 @@ public static JevEntry Build(IReadOnlyList messages, string? instru break; case FunctionResultContent result: - contents.Add((JsonNode)SerializeResult(result)); + contents.Add((JsonNode)SerializeResult(result, modelName)); break; } } @@ -80,7 +85,7 @@ public static JevEntry Build(IReadOnlyList messages, string? instru if (stateMessages.Count == 0) { - Throw.ArgumentException(nameof(messages), "Jev needs at least one message with text or function call content."); + Throw.ArgumentException(nameof(messages), $"{modelName} needs at least one message with text or function call content."); } var state = new JsonObject { ["messages"] = stateMessages }; @@ -89,7 +94,7 @@ public static JevEntry Build(IReadOnlyList messages, string? instru state["instructions"] = instructions; } - return new JevEntry(JevJsonUtilities.Parse(state.ToJsonString())); + return DecisionJson.Parse(state.ToJsonString()); } /// @@ -172,7 +177,7 @@ public static Dictionary>> GetCurre var arguments = new Dictionary(StringComparer.Ordinal); foreach (KeyValuePair argument in call.Arguments ?? new Dictionary()) { - arguments[argument.Key] = JevJsonUtilities.ToElement(argument.Value); + arguments[argument.Key] = DecisionJson.ToElement(argument.Value); } if (!calls.TryGetValue(call.Name, out List>? list)) @@ -214,13 +219,13 @@ private static JsonObject ToObject(IDictionary arguments) var json = new JsonObject(); foreach (KeyValuePair argument in arguments) { - json[argument.Key] = JevJsonUtilities.ToNodeOrText(argument.Value); + json[argument.Key] = DecisionJson.ToNodeOrText(argument.Value); } return json; } - private static JsonObject SerializeResult(FunctionResultContent result) + private static JsonObject SerializeResult(FunctionResultContent result, string modelName) { var json = new JsonObject { ["type"] = "function_result", ["call_id"] = result.CallId }; IEnumerable? items = result.Result switch @@ -232,17 +237,17 @@ private static JsonObject SerializeResult(FunctionResultContent result) if (items is null) { - json["result"] = JevJsonUtilities.ToNodeOrText(result.Result); + json["result"] = DecisionJson.ToNodeOrText(result.Result); return json; } - // Rich results are sent as text items, like the Python connector; other media cannot be described to Jev. + // Rich results are sent as text items, like the Python connector; other media cannot be described to the model. var texts = new List(); foreach (AIContent item in items) { if (item is not TextContent text) { - throw new NotSupportedException($"Jev supports text function results only; a result of the call '{result.CallId}' contains {item.GetType().Name}."); + throw new NotSupportedException($"{modelName} supports text function results only; a result of the call '{result.CallId}' contains {item.GetType().Name}."); } texts.Add(text.Text ?? string.Empty); @@ -269,7 +274,7 @@ private static JsonObject SerializeResult(FunctionResultContent result) JsonElement element => Unwrap(element.GetRawText()), TextContent content => content.Text, IEnumerable contents => string.Concat(contents.OfType().Select(content => content.Text)), - _ => JevJsonUtilities.ToNodeOrText(result)?.ToJsonString(), + _ => DecisionJson.ToNodeOrText(result)?.ToJsonString(), }; /// diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs b/dotnet/src/Shared/DecisionChat/DecisionCompiledTool.cs similarity index 50% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs rename to dotnet/src/Shared/DecisionChat/DecisionCompiledTool.cs index cd4a568618b..82de0016094 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs +++ b/dotnet/src/Shared/DecisionChat/DecisionCompiledTool.cs @@ -4,28 +4,28 @@ using System.Text.Json; using Microsoft.Extensions.AI; -namespace Microsoft.Agents.AI.TypeSafe; +namespace Microsoft.Agents.AI.Decisions; /// -/// One tool that can be expressed as Jev questions: its route label, its argument questions, and their decoders. +/// One tool that can be expressed as decision questions: its route label, its argument questions, and their decoders. /// -internal sealed class JevCompiledTool +internal sealed class DecisionCompiledTool { public required AIFunctionDeclaration Function { get; init; } /// Gets the label of the tool in the route question, such as t0. public required string RouteLabel { get; init; } - public required IReadOnlyDictionary Questions { get; init; } + public required IReadOnlyDictionary Questions { get; init; } - public required IReadOnlyList Arguments { get; init; } + public required IReadOnlyList Arguments { get; init; } - public Dictionary DecodeArguments(JevResult result) + public Dictionary DecodeArguments(DecisionResponse response) { var arguments = new Dictionary(this.Arguments.Count); - foreach (JevArgumentPlan argument in this.Arguments) + foreach (DecisionArgumentPlan argument in this.Arguments) { - if (argument.TryDecode(result, out JsonElement value)) + if (argument.TryDecode(response, out JsonElement value)) { arguments[argument.Name] = value; } diff --git a/dotnet/src/Shared/DecisionChat/DecisionJson.cs b/dotnet/src/Shared/DecisionChat/DecisionJson.cs new file mode 100644 index 00000000000..da33f1b6f82 --- /dev/null +++ b/dotnet/src/Shared/DecisionChat/DecisionJson.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Decisions; + +/// +/// JSON value helpers for converting chat messages and tool arguments into decision state and back. +/// +internal static class DecisionJson +{ + private static readonly JsonElement s_true = Parse("true"); + private static readonly JsonElement s_false = Parse("false"); + private static readonly JsonElement s_null = Parse("null"); + + /// + /// Converts one tool argument into a JSON node. Chat clients pass arguments as values; + /// callers that invoke the function directly may pass .NET objects instead, which are serialized with + /// . + /// + public static JsonNode? ToNode(object? value) => + value switch + { + null => null, + JsonNode node => node.DeepClone(), + JsonElement element => element.ValueKind switch + { + JsonValueKind.Object => JsonObject.Create(element), + JsonValueKind.Array => JsonArray.Create(element), + JsonValueKind.Null or JsonValueKind.Undefined => null, + _ => JsonValue.Create(element), + }, + _ => JsonSerializer.SerializeToNode(value, AIJsonUtilities.DefaultOptions.GetTypeInfo(value.GetType())), + }; + + /// + /// Converts a value from a chat message, such as a function argument or result, into a JSON node. A value that + /// cannot be serialized, for example a type without metadata under native AOT, becomes its text. + /// + public static JsonNode? ToNodeOrText(object? value) + { + try + { + return ToNode(value); + } + catch (Exception ex) when (ex is NotSupportedException or InvalidOperationException or JsonException or ArgumentException) + { + // Chat history is informational for the decision model, so an unserializable value is described rather + // than failing the whole request. + return JsonValue.Create(value?.ToString()); + } + } + + /// Converts a value from a chat message into a detached , like . + public static JsonElement ToElement(object? value) => + value switch + { + JsonElement element => element.ValueKind == JsonValueKind.Undefined ? s_null : element.Clone(), + null => s_null, + _ => Parse(ToNodeOrText(value)?.ToJsonString() ?? "null"), + }; + + /// Gets the JSON or value. + public static JsonElement CreateBoolean(bool value) => value ? s_true : s_false; + + /// Writes values into a new JSON array. + public static JsonElement CreateArray(IEnumerable values) + { + var buffer = new ArrayBufferWriter(); + using (var writer = new Utf8JsonWriter(buffer)) + { + writer.WriteStartArray(); + foreach (JsonElement value in values) + { + value.WriteTo(writer); + } + + writer.WriteEndArray(); + } + + using JsonDocument document = JsonDocument.Parse(buffer.WrittenMemory); + return document.RootElement.Clone(); + } + + /// Parses JSON text into a detached . + public static JsonElement Parse(string json) + { + using JsonDocument document = JsonDocument.Parse(json); + return document.RootElement.Clone(); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs b/dotnet/src/Shared/DecisionChat/DecisionQuestionBudget.cs similarity index 83% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs rename to dotnet/src/Shared/DecisionChat/DecisionQuestionBudget.cs index 96e995e4911..09acd7fa2ba 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs +++ b/dotnet/src/Shared/DecisionChat/DecisionQuestionBudget.cs @@ -3,7 +3,7 @@ using System.Globalization; using Microsoft.Shared.Diagnostics; -namespace Microsoft.Agents.AI.TypeSafe; +namespace Microsoft.Agents.AI.Decisions; /// /// Counts the internal questions that tool routing adds to a request, and stops the request when they exceed @@ -14,11 +14,11 @@ namespace Microsoft.Agents.AI.TypeSafe; /// is excluded for an unsupported schema therefore returns the questions it reserved to the tools after it. /// Exceeding the budget stops question materialization, but is reported only after the whole tool schema is checked. /// -internal sealed class JevQuestionBudget +internal sealed class DecisionQuestionBudget { public const int MaxInternalQuestions = 128; - public JevQuestionBudget(int count = 0) + public DecisionQuestionBudget(int count = 0) { this.Count = count; } @@ -39,7 +39,7 @@ public void Validate() // of excluding the tool that crossed the limit. Throw.InvalidOperationException(string.Format( CultureInfo.InvariantCulture, - "The tools need more than {0} internal Jev questions. Offer fewer tools, or tools with fewer arguments.", + "The tools need more than {0} internal decision questions. Offer fewer tools, or tools with fewer arguments.", MaxInternalQuestions)); } } diff --git a/dotnet/src/Shared/DecisionChat/DecisionToolCall.cs b/dotnet/src/Shared/DecisionChat/DecisionToolCall.cs new file mode 100644 index 00000000000..fba6b6f5b79 --- /dev/null +++ b/dotnet/src/Shared/DecisionChat/DecisionToolCall.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Decisions; + +/// +/// The tool call that a decision model selected: the function and the arguments decoded from its answers. +/// +internal readonly record struct DecisionToolCall(AIFunctionDeclaration Function, Dictionary Arguments); diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs b/dotnet/src/Shared/DecisionChat/DecisionToolCallCompiler.cs similarity index 59% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs rename to dotnet/src/Shared/DecisionChat/DecisionToolCallCompiler.cs index 1f420dbf877..3acec5e433f 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs +++ b/dotnet/src/Shared/DecisionChat/DecisionToolCallCompiler.cs @@ -9,27 +9,28 @@ using Microsoft.Extensions.Logging; using Microsoft.Shared.Diagnostics; -namespace Microsoft.Agents.AI.TypeSafe; +namespace Microsoft.Agents.AI.Decisions; /// -/// Turns function tools into Jev questions, so that Jev, which cannot write arguments, can still call a tool. +/// Turns function tools into decision questions, so that a decision model, which cannot write arguments, can still call +/// a tool. /// /// /// /// This is a port of the tool routing in the Python TypeSafe connector (_tool_calls.py), with the same -/// question IDs, limits, and wording, so both connectors ask Jev the same questions. Two schema checks are stricter +/// question IDs, limits, and wording, so both connectors ask the same questions. Two schema checks are stricter /// than in Python: chains of $ref aliases are followed instead of excluding the tool, and const and /// enum values must satisfy their declared constraints, so a tool that could fail when called is excluded. /// /// /// A route Choice selects the tool, or none. Each argument must come from a closed set: a constant, an enum -/// (a Choice), a Boolean (a Noul), or an array of enum values (one Noul per value). An optional argument also gets a -/// presence Noul, so that it is passed only when the user asked for it. A tool with any other argument, such as free -/// text or a number, cannot be expressed this way; it is logged and left out, or fails the request when the tool mode -/// requires a call and no tool remains. +/// (a Choice), a Boolean (a Binary question), or an array of enum values (one Binary question per value). An optional +/// argument also gets a presence Binary question, so that it is passed only when the user asked for it. A tool with any +/// other argument, such as free text or a number, cannot be expressed this way; it is logged and left out, or fails the +/// request when the tool mode requires a call and no tool remains. /// /// -internal static class JevToolCallCompiler +internal static class DecisionToolCallCompiler { public const string QuestionPrefix = "__af_tool__"; public const string RouteQuestionId = QuestionPrefix + ".route"; @@ -57,14 +58,16 @@ public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) /// The tool mode of the request; means . /// /// The argument sets of the tools already called in the current turn, by tool name, oldest first. They become - /// hints that keep Jev from repeating a call. + /// hints that keep the model from repeating a call. /// + /// The name of the decision model, such as Jev, for messages. /// Receives a warning for each tool that is left out. /// The plan, or when no tool can be called. - public static JevToolCallPlan? Compile( + public static DecisionToolCallPlan? Compile( IReadOnlyList tools, ChatToolMode? toolMode, IReadOnlyDictionary>> previousCalls, + string modelName, ILogger logger) { if (toolMode is NoneChatToolMode) @@ -84,7 +87,7 @@ public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) return null; } - List selected = [.. tools.Where(tool => requiredName is null || tool.Name == requiredName)]; + List selected = [.. tools.Where(tool => requiredName is null || string.Equals(tool.Name, requiredName, StringComparison.Ordinal))]; if (requiredName is not null && selected.Count == 0) { Throw.InvalidOperationException($"The tool mode requires the tool '{requiredName}', but the request does not have it."); @@ -94,27 +97,28 @@ public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) { Throw.InvalidOperationException(string.Format( CultureInfo.InvariantCulture, - "Jev can choose among at most {0} tools per request, but the request has {1}. Offer fewer tools.", + "{0} can choose among at most {1} tools per request, but the request has {2}. Offer fewer tools.", + modelName, MaxRoutableTools, selected.Count)); } - var compiled = new List(selected.Count); + var compiled = new List(selected.Count); var unsupported = new List>(); - var budget = new JevQuestionBudget(); + var budget = new DecisionQuestionBudget(); for (int index = 0; index < selected.Count; index++) { AIFunctionDeclaration tool = selected[index]; // Each tool reserves questions from a copy of the budget, which is committed only when the tool compiles. - var toolBudget = new JevQuestionBudget(budget.Count); + var toolBudget = new DecisionQuestionBudget(budget.Count); try { List> calls = previousCalls.TryGetValue(tool.Name, out List>? found) ? found : []; compiled.Add(CompileTool(tool, index, calls, toolBudget)); budget.Count = toolBudget.Count; } - catch (JevUnsupportedToolSchemaException ex) + catch (DecisionUnsupportedToolSchemaException ex) { unsupported.Add(new(tool.Name, ex.Message)); } @@ -122,13 +126,13 @@ public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) foreach (KeyValuePair tool in unsupported) { - logger.LogToolExcluded(tool.Key, tool.Value); + logger.LogToolExcluded(modelName, tool.Key, tool.Value); } if (required && compiled.Count == 0) { string details = string.Join("; ", unsupported.Select(tool => $"{tool.Key}: {tool.Value}")); - Throw.InvalidOperationException($"The tool mode requires a tool call, but no tool can be expressed as Jev questions. {details}".TrimEnd()); + Throw.InvalidOperationException($"The tool mode requires a tool call, but no tool can be expressed as {modelName} questions. {details}".TrimEnd()); } if (compiled.Count == 0) @@ -137,12 +141,12 @@ public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) } // With a single tool that must be called, there is nothing to route. - JevCompiledTool? requiredTool = required && compiled.Count == 1 ? compiled[0] : null; - var questions = new Dictionary(StringComparer.Ordinal); + DecisionCompiledTool? requiredTool = required && compiled.Count == 1 ? compiled[0] : null; + var questions = new Dictionary(StringComparer.Ordinal); if (requiredTool is null) { - var criteria = new Dictionary(StringComparer.Ordinal); - foreach (JevCompiledTool tool in compiled) + var candidates = new List(compiled.Count + 1); + foreach (DecisionCompiledTool tool in compiled) { string name = tool.Function.Name; string description = string.IsNullOrWhiteSpace(tool.Function.Description) @@ -152,35 +156,34 @@ public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) { description += " Call this tool again only if the latest user request still needs a distinct execution. " + - $"Do not repeat argument sets already called this turn: {JevToolSchema.Describe(calls)}."; + $"Do not repeat argument sets already called this turn: {DecisionToolSchema.Describe(calls)}."; } - criteria[tool.RouteLabel] = description; + candidates.Add(new DecisionCandidate(tool.RouteLabel, description)); } if (!required) { - criteria[RouteNone] = "Do not call a tool because all requested tool actions are already satisfied, or because no tool is needed."; + candidates.Add(new DecisionCandidate(RouteNone, "Do not call a tool because all requested tool actions are already satisfied, or because no tool is needed.")); } _ = budget.TryReserve(1); budget.Validate(); - questions[RouteQuestionId] = new JevChoiceQuestion - { - Instructions = "Which available tool, if any, should handle the latest user request?", - Criteria = criteria, - }; + questions[RouteQuestionId] = new ChoiceDecisionQuestion( + RouteQuestionId, + "Which available tool, if any, should handle the latest user request?", + candidates); } - foreach (JevCompiledTool tool in compiled) + foreach (DecisionCompiledTool tool in compiled) { - foreach (KeyValuePair question in tool.Questions) + foreach (KeyValuePair question in tool.Questions) { questions[question.Key] = question.Value; } } - return new JevToolCallPlan + return new DecisionToolCallPlan { Questions = questions, Tools = compiled, @@ -189,20 +192,20 @@ public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) }; } - private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolIndex, List> previousCalls, JevQuestionBudget budget) + private static DecisionCompiledTool CompileTool(AIFunctionDeclaration tool, int toolIndex, List> previousCalls, DecisionQuestionBudget budget) { JsonElement schemaElement = tool.JsonSchema; Dictionary schema = schemaElement.ValueKind switch { - JsonValueKind.Object => JevToolSchema.ToMap(schemaElement), + JsonValueKind.Object => DecisionToolSchema.ToMap(schemaElement), JsonValueKind.Undefined or JsonValueKind.Null => [], - _ => throw new JevUnsupportedToolSchemaException("the top-level input schema must be an object"), + _ => throw new DecisionUnsupportedToolSchemaException("the top-level input schema must be an object"), }; - JevToolSchema.RejectUnsupportedKeys(schema, JevToolSchema.RootKeys, "root-level"); + DecisionToolSchema.RejectUnsupportedKeys(schema, DecisionToolSchema.RootKeys, "root-level"); if (schema.TryGetValue("type", out JsonElement type) && type.ValueKind != JsonValueKind.Null && !(type.ValueKind == JsonValueKind.String && type.ValueEquals("object"))) { - throw new JevUnsupportedToolSchemaException("the top-level input schema must be an object"); + throw new DecisionUnsupportedToolSchemaException("the top-level input schema must be an object"); } List properties = []; @@ -210,7 +213,7 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI { if (propertiesElement.ValueKind != JsonValueKind.Object) { - throw new JevUnsupportedToolSchemaException("the input schema properties must be an object"); + throw new DecisionUnsupportedToolSchemaException("the input schema properties must be an object"); } properties.AddRange(propertiesElement.EnumerateObject()); @@ -218,7 +221,7 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI if (properties.Count > MaxToolProperties) { - throw new JevUnsupportedToolSchemaException(string.Format( + throw new DecisionUnsupportedToolSchemaException(string.Format( CultureInfo.InvariantCulture, "the input schema defines {0} properties; the supported maximum is {1}", properties.Count, @@ -230,7 +233,7 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI { if (requiredElement.ValueKind != JsonValueKind.Array || requiredElement.EnumerateArray().Any(item => item.ValueKind != JsonValueKind.String)) { - throw new JevUnsupportedToolSchemaException("the input schema required field must be a string array"); + throw new DecisionUnsupportedToolSchemaException("the input schema required field must be a string array"); } requiredNames.UnionWith(requiredElement.EnumerateArray().Select(item => item.GetString()!)); @@ -239,37 +242,37 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI List missing = [.. requiredNames.Where(name => !properties.Any(property => property.Name == name)).OrderBy(name => name, StringComparer.Ordinal)]; if (missing.Count > 0) { - throw new JevUnsupportedToolSchemaException($"required arguments are missing from properties: {string.Join(", ", missing)}"); + throw new DecisionUnsupportedToolSchemaException($"required arguments are missing from properties: {string.Join(", ", missing)}"); } - var questions = new Dictionary(StringComparer.Ordinal); - var arguments = new List(properties.Count); + var questions = new Dictionary(StringComparer.Ordinal); + var arguments = new List(properties.Count); for (int argumentIndex = 0; argumentIndex < properties.Count; argumentIndex++) { JsonProperty property = properties[argumentIndex]; bool isRequired = requiredNames.Contains(property.Name); if (property.Value.ValueKind != JsonValueKind.Object) { - throw new JevUnsupportedToolSchemaException($"argument '{property.Name}' has an invalid schema"); + throw new DecisionUnsupportedToolSchemaException($"argument '{property.Name}' has an invalid schema"); } List previousValues = [.. previousCalls.Where(call => call.ContainsKey(property.Name)).Select(call => call[property.Name])]; try { - if (CompileArgument(tool, schema, property.Name, JevToolSchema.ToMap(property.Value), isRequired, toolIndex, argumentIndex, previousValues, budget, questions) is { } argument) + if (CompileArgument(tool, schema, property.Name, DecisionToolSchema.ToMap(property.Value), isRequired, toolIndex, argumentIndex, previousValues, budget, questions) is { } argument) { arguments.Add(argument); } } - catch (JevUnsupportedToolSchemaException ex) + catch (DecisionUnsupportedToolSchemaException ex) { - throw new JevUnsupportedToolSchemaException($"{(isRequired ? "required" : "optional")} argument '{property.Name}': {ex.Message}", ex); + throw new DecisionUnsupportedToolSchemaException($"{(isRequired ? "required" : "optional")} argument '{property.Name}': {ex.Message}", ex); } } // Unsupported tools are excluded even if their earlier arguments exceeded the question budget. budget.Validate(); - return new JevCompiledTool + return new DecisionCompiledTool { Function = tool, RouteLabel = string.Create(CultureInfo.InvariantCulture, $"t{toolIndex}"), @@ -278,7 +281,7 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI }; } - private static JevArgumentPlan? CompileArgument( + private static DecisionArgumentPlan? CompileArgument( AIFunctionDeclaration tool, Dictionary root, string name, @@ -287,23 +290,23 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI int toolIndex, int argumentIndex, List previousValues, - JevQuestionBudget budget, - Dictionary questions) + DecisionQuestionBudget budget, + Dictionary questions) { - (Dictionary schema, bool nullable) = JevToolSchema.Resolve(rawSchema, root); - JevToolSchema.RejectUnsupportedKeys(schema, JevToolSchema.ArgumentKeys, "argument"); + (Dictionary schema, bool nullable) = DecisionToolSchema.Resolve(rawSchema, root); + DecisionToolSchema.RejectUnsupportedKeys(schema, DecisionToolSchema.ArgumentKeys, "argument"); - // A required argument must be passed, and Jev cannot choose null among the closed-set values, so a required - // nullable argument would always have to be guessed. + // A required argument must be passed, and the model cannot choose null among the closed-set values, so a + // required nullable argument would always have to be guessed. if (nullable && required) { - throw new JevUnsupportedToolSchemaException("required nullable arguments are not supported"); + throw new DecisionUnsupportedToolSchemaException("required nullable arguments are not supported"); } string prefix = string.Create(CultureInfo.InvariantCulture, $"{QuestionPrefix}.t{toolIndex}.a{argumentIndex}"); string description = GetText(schema, "description") ?? GetText(rawSchema, "description") ?? GetText(schema, "title") ?? name; string previousInstruction = previousValues.Count > 0 - ? $" Values already used for this argument in earlier calls this turn: {JevToolSchema.Describe(previousValues)}. Choose a different value when the user requested another distinct call." + ? $" Values already used for this argument in earlier calls this turn: {DecisionToolSchema.Describe(previousValues)}. Choose a different value when the user requested another distinct call." : string.Empty; string? presenceQuestionId = required ? null : prefix + ".present"; @@ -311,29 +314,28 @@ void AddPresenceQuestion() { if (presenceQuestionId is not null) { - questions[presenceQuestionId] = new JevNoulQuestion - { - Instructions = $"For the {tool.Name} tool, did the user explicitly specify the {name} argument? Argument meaning: {description}.{previousInstruction}", - }; + questions[presenceQuestionId] = new BinaryDecisionQuestion( + presenceQuestionId, + $"For the {tool.Name} tool, did the user explicitly specify the {name} argument? Argument meaning: {description}.{previousInstruction}"); } } if (schema.TryGetValue("const", out JsonElement constValue)) { - JevToolSchema.EnsureValuesMatchSchema(schema, [constValue], root, "const value"); + DecisionToolSchema.EnsureValuesMatchSchema(schema, [constValue], root, "const value"); if (!budget.TryReserve(required ? 0 : 1)) { return null; } AddPresenceQuestion(); - return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = constValue }; + return new DecisionArgumentPlan { Name = name, Kind = DecisionArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = constValue }; } if (schema.TryGetValue("enum", out JsonElement enumElement)) { - List values = JevToolSchema.ReadEnumValues(enumElement, "enum"); - JevToolSchema.EnsureValuesMatchSchema(schema, values, root, "enum value"); + List values = DecisionToolSchema.ReadEnumValues(enumElement, "enum"); + DecisionToolSchema.EnsureValuesMatchSchema(schema, values, root, "enum value"); if (values.Count == 1) { if (!budget.TryReserve(required ? 0 : 1)) @@ -342,7 +344,7 @@ void AddPresenceQuestion() } AddPresenceQuestion(); - return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = values[0] }; + return new DecisionArgumentPlan { Name = name, Kind = DecisionArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = values[0] }; } if (!budget.TryReserve(required ? 1 : 2)) @@ -351,26 +353,25 @@ void AddPresenceQuestion() } var labeled = new Dictionary(StringComparer.Ordinal); - var criteria = new Dictionary(StringComparer.Ordinal); + var candidates = new List(values.Count); for (int index = 0; index < values.Count; index++) { string label = string.Create(CultureInfo.InvariantCulture, $"v{index}"); labeled[label] = values[index]; - criteria[label] = JevToolSchema.Describe(values[index]); + candidates.Add(new DecisionCandidate(label, DecisionToolSchema.Describe(values[index]))); } string questionId = prefix + ".value"; AddPresenceQuestion(); - questions[questionId] = new JevChoiceQuestion - { - Instructions = $"For the {tool.Name} tool, choose the {name} argument. Argument meaning: {description}.{previousInstruction}", - Criteria = criteria, - }; - return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Choice, PresenceQuestionId = presenceQuestionId, ValueQuestionId = questionId, Values = labeled }; + questions[questionId] = new ChoiceDecisionQuestion( + questionId, + $"For the {tool.Name} tool, choose the {name} argument. Argument meaning: {description}.{previousInstruction}", + candidates); + return new DecisionArgumentPlan { Name = name, Kind = DecisionArgumentKind.Choice, PresenceQuestionId = presenceQuestionId, ValueQuestionId = questionId, Values = labeled }; } string? schemaType = schema.TryGetValue("type", out JsonElement typeElement) && typeElement.ValueKind == JsonValueKind.String ? typeElement.GetString() : null; - if (schemaType == "boolean") + if (string.Equals(schemaType, "boolean", StringComparison.Ordinal)) { string questionId = prefix + ".value"; if (!budget.TryReserve(required ? 1 : 2)) @@ -379,23 +380,22 @@ void AddPresenceQuestion() } AddPresenceQuestion(); - questions[questionId] = new JevNoulQuestion - { - Instructions = $"For the {tool.Name} tool, should the {name} argument be true? Argument meaning: {description}.{previousInstruction}", - }; - return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Boolean, PresenceQuestionId = presenceQuestionId, ValueQuestionId = questionId }; + questions[questionId] = new BinaryDecisionQuestion( + questionId, + $"For the {tool.Name} tool, should the {name} argument be true? Argument meaning: {description}.{previousInstruction}"); + return new DecisionArgumentPlan { Name = name, Kind = DecisionArgumentKind.Boolean, PresenceQuestionId = presenceQuestionId, ValueQuestionId = questionId }; } - if (schemaType == "array") + if (string.Equals(schemaType, "array", StringComparison.Ordinal)) { - Dictionary items = JevToolSchema.ResolveArrayItems(schema, root); + Dictionary items = DecisionToolSchema.ResolveArrayItems(schema, root); if (!items.TryGetValue("enum", out JsonElement membersElement) || membersElement.ValueKind != JsonValueKind.Array || membersElement.GetArrayLength() == 0) { - throw new JevUnsupportedToolSchemaException("array items must define a non-empty enum"); + throw new DecisionUnsupportedToolSchemaException("array items must define a non-empty enum"); } - List members = JevToolSchema.ReadEnumValues(membersElement, "array enum"); - JevToolSchema.EnsureValuesMatchType(items, members, "array item", "array enum member", "item type"); + List members = DecisionToolSchema.ReadEnumValues(membersElement, "array enum"); + DecisionToolSchema.EnsureValuesMatchType(items, members, "array item", "array enum member", "item type"); if (!budget.TryReserve(members.Count + (required ? 0 : 1))) { @@ -407,17 +407,16 @@ void AddPresenceQuestion() for (int index = 0; index < members.Count; index++) { string questionId = string.Create(CultureInfo.InvariantCulture, $"{prefix}.m{index}"); - questions[questionId] = new JevNoulQuestion - { - Instructions = $"For the {tool.Name} tool, should the {name} argument include {JevToolSchema.Describe(members[index])}? Argument meaning: {description}.{previousInstruction}", - }; + questions[questionId] = new BinaryDecisionQuestion( + questionId, + $"For the {tool.Name} tool, should the {name} argument include {DecisionToolSchema.Describe(members[index])}? Argument meaning: {description}.{previousInstruction}"); memberQuestions.Add(new(questionId, members[index])); } - return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Set, PresenceQuestionId = presenceQuestionId, Members = memberQuestions }; + return new DecisionArgumentPlan { Name = name, Kind = DecisionArgumentKind.Set, PresenceQuestionId = presenceQuestionId, Members = memberQuestions }; } - throw new JevUnsupportedToolSchemaException("only const, enum, boolean, and arrays of enum values are supported"); + throw new DecisionUnsupportedToolSchemaException("only const, enum, boolean, and arrays of enum values are supported"); } private static string? GetText(Dictionary schema, string key) => diff --git a/dotnet/src/Shared/DecisionChat/DecisionToolCallPlan.cs b/dotnet/src/Shared/DecisionChat/DecisionToolCallPlan.cs new file mode 100644 index 00000000000..a69efd9dba7 --- /dev/null +++ b/dotnet/src/Shared/DecisionChat/DecisionToolCallPlan.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Decisions; + +/// +/// The internal questions that let a decision model choose one tool call, and the decoders that turn their answers into +/// the call. +/// +internal sealed class DecisionToolCallPlan +{ + /// + /// A Binary answer above this probability means yes, for presence, Boolean, and array member questions. + /// + public const double YesThreshold = 0.5; + + /// Gets the internal questions by ID, in the order they are asked. + public required IReadOnlyDictionary Questions { get; init; } + + public required IReadOnlyList Tools { get; init; } + + /// + /// Gets the ID of the Choice question that selects the tool, or when + /// is the only possible call. + /// + public string? RouteQuestionId { get; init; } + + /// + /// Gets the tool that must be called, when the tool mode requires a call and only one tool compiled. + /// + public DecisionCompiledTool? RequiredTool { get; init; } + + /// + /// Reads the selected tool call from the answers. + /// + /// The tool call, or when the model chose not to call a tool. + public DecisionToolCall? Decode(DecisionResponse response) + { + DecisionCompiledTool? selected = this.RequiredTool; + if (selected is null) + { + string route = GetChoice(response, this.RouteQuestionId!); + if (string.Equals(route, DecisionToolCallCompiler.RouteNone, StringComparison.Ordinal)) + { + return null; + } + + selected = this.Tools.FirstOrDefault(tool => string.Equals(tool.RouteLabel, route, StringComparison.Ordinal)) + ?? throw new InvalidOperationException($"The decision model returned the unknown tool route '{route}'."); + } + + return new DecisionToolCall(selected.Function, selected.DecodeArguments(response)); + } + + public static string GetChoice(DecisionResponse response, string questionId) => + FindAnswer(response, questionId) is ChoiceDecisionAnswer choice + ? choice.SelectedCandidateId + : throw new InvalidOperationException($"The decision response has no Choice answer for the internal question '{questionId}'."); + + public static bool IsYes(DecisionResponse response, string questionId) => + FindAnswer(response, questionId) is BinaryDecisionAnswer binary + ? binary.TrueProbability > YesThreshold + : throw new InvalidOperationException($"The decision response has no Binary answer for the internal question '{questionId}'."); + + private static DecisionAnswer? FindAnswer(DecisionResponse response, string questionId) + { + foreach (DecisionAnswer answer in response.Answers) + { + if (string.Equals(answer.QuestionId, questionId, StringComparison.Ordinal)) + { + return answer; + } + } + + return null; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs b/dotnet/src/Shared/DecisionChat/DecisionToolSchema.cs similarity index 86% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs rename to dotnet/src/Shared/DecisionChat/DecisionToolSchema.cs index 59454182438..d390bc82d9a 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs +++ b/dotnet/src/Shared/DecisionChat/DecisionToolSchema.cs @@ -7,13 +7,13 @@ using System.Text; using System.Text.Json; -namespace Microsoft.Agents.AI.TypeSafe; +namespace Microsoft.Agents.AI.Decisions; /// -/// Reads the JSON schemas of tool parameters for : which keywords are allowed where, -/// how references and nullable forms resolve, and how values are described to Jev. +/// Reads the JSON schemas of tool parameters for : which keywords are allowed where, +/// how references and nullable forms resolve, and how values are described to the decision model. /// -internal static class JevToolSchema +internal static class DecisionToolSchema { // A schema definition may refer to itself; without a limit its resolution would never end. private const int MaxResolutionDepth = 16; @@ -53,7 +53,7 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di { if (depth > MaxResolutionDepth) { - throw new JevUnsupportedToolSchemaException("schema references are nested too deeply"); + throw new DecisionUnsupportedToolSchemaException("schema references are nested too deeply"); } var resolved = new Dictionary(schema, StringComparer.Ordinal); @@ -64,7 +64,7 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di { if (depth + hops >= MaxResolutionDepth) { - throw new JevUnsupportedToolSchemaException("schema references are nested too deeply"); + throw new DecisionUnsupportedToolSchemaException("schema references are nested too deeply"); } RejectCompositionSiblings(resolved, "$ref"); @@ -72,7 +72,7 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di const string DefinitionsPrefix = "#/$defs/"; if (path?.StartsWith(DefinitionsPrefix, StringComparison.Ordinal) is not true) { - throw new JevUnsupportedToolSchemaException("only local $defs references are supported"); + throw new DecisionUnsupportedToolSchemaException("only local $defs references are supported"); } if (!root.TryGetValue("$defs", out JsonElement definitions) || @@ -80,7 +80,7 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di !definitions.TryGetProperty(path.Substring(DefinitionsPrefix.Length), out JsonElement definition) || definition.ValueKind != JsonValueKind.Object) { - throw new JevUnsupportedToolSchemaException($"unresolved schema reference '{path}'"); + throw new DecisionUnsupportedToolSchemaException($"unresolved schema reference '{path}'"); } // Annotations next to the reference, such as a description, override those of the definition, so the @@ -95,7 +95,7 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di RejectCompositionSiblings(resolved, "anyOf"); if (anyOf.ValueKind != JsonValueKind.Array) { - throw new JevUnsupportedToolSchemaException("anyOf must be an array"); + throw new DecisionUnsupportedToolSchemaException("anyOf must be an array"); } List arms = [.. anyOf.EnumerateArray()]; @@ -103,7 +103,7 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di List valueArms = [.. arms.Where(arm => arm.ValueKind == JsonValueKind.Object && !IsNullSchema(arm))]; if (arms.Count != 2 || nullArms.Count != 1 || valueArms.Count != 1) { - throw new JevUnsupportedToolSchemaException("only a single schema combined with null is supported"); + throw new DecisionUnsupportedToolSchemaException("only a single schema combined with null is supported"); } (Dictionary nested, _) = Resolve(ToMap(valueArms[0]), root, depth + 1); @@ -114,13 +114,13 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di if (resolved.TryGetValue("type", out JsonElement type) && type.ValueKind == JsonValueKind.Array && type.EnumerateArray().Any(IsNullName)) { List valueTypes = [.. type.EnumerateArray().Where(item => !IsNullName(item))]; - resolved["type"] = valueTypes.Count == 1 ? valueTypes[0] : JevJsonUtilities.CreateArray(valueTypes); + resolved["type"] = valueTypes.Count == 1 ? valueTypes[0] : DecisionJson.CreateArray(valueTypes); nullable = true; } if (resolved.TryGetValue("enum", out JsonElement values) && values.ValueKind == JsonValueKind.Array && values.EnumerateArray().Any(value => value.ValueKind == JsonValueKind.Null)) { - resolved["enum"] = JevJsonUtilities.CreateArray(values.EnumerateArray().Where(value => value.ValueKind != JsonValueKind.Null)); + resolved["enum"] = DecisionJson.CreateArray(values.EnumerateArray().Where(value => value.ValueKind != JsonValueKind.Null)); nullable = true; } @@ -132,7 +132,7 @@ public static void RejectUnsupportedKeys(Dictionary schema, List unsupported = [.. schema.Keys.Where(key => !supportedKeys.Contains(key)).OrderBy(key => key, StringComparer.Ordinal)]; if (unsupported.Count > 0) { - throw new JevUnsupportedToolSchemaException($"unsupported {location} schema constraints: {string.Join(", ", unsupported)}"); + throw new DecisionUnsupportedToolSchemaException($"unsupported {location} schema constraints: {string.Join(", ", unsupported)}"); } } @@ -156,7 +156,7 @@ public static IReadOnlyList GetTypes(JsonElement? type, string location) return [.. value.EnumerateArray().Select(item => item.GetString()!)]; } - throw new JevUnsupportedToolSchemaException($"{location} type must be a string or non-empty string array"); + throw new DecisionUnsupportedToolSchemaException($"{location} type must be a string or non-empty string array"); } public static bool MatchesType(JsonElement value, string type) => @@ -179,7 +179,7 @@ public static bool MatchesType(JsonElement value, string type) => /// Checks that every const or enum value of a schema matches its declared type. /// /// - /// A value of another type can never be a valid argument, for example "fast" for an integer. Jev would + /// A value of another type can never be a valid argument, for example "fast" for an integer. The model would /// still choose it, and the call would fail only when the function binds its arguments, so such a tool is left out /// of routing instead. A schema without a type is not checked. /// @@ -200,8 +200,8 @@ public static void EnsureValuesMatchType(Dictionary schema, { if (!types.Any(candidate => MatchesType(value, candidate))) { - string declared = types.Count == 1 ? $"\"{types[0]}\"" : Describe(types.Select(candidate => JevJsonUtilities.ToElement(candidate))); - throw new JevUnsupportedToolSchemaException($"{valueName} {Describe(value)} does not match declared {typeName} {declared}"); + string declared = types.Count == 1 ? $"\"{types[0]}\"" : Describe(types.Select(candidate => DecisionJson.ToElement(candidate))); + throw new DecisionUnsupportedToolSchemaException($"{valueName} {Describe(value)} does not match declared {typeName} {declared}"); } } } @@ -210,17 +210,17 @@ public static List ReadEnumValues(JsonElement values, string name) { if (values.ValueKind != JsonValueKind.Array || values.GetArrayLength() == 0) { - throw new JevUnsupportedToolSchemaException($"{name} must be a non-empty array"); + throw new DecisionUnsupportedToolSchemaException($"{name} must be a non-empty array"); } - if (values.GetArrayLength() > JevToolCallCompiler.MaxEnumValues) + if (values.GetArrayLength() > DecisionToolCallCompiler.MaxEnumValues) { - throw new JevUnsupportedToolSchemaException(string.Format( + throw new DecisionUnsupportedToolSchemaException(string.Format( CultureInfo.InvariantCulture, "{0} defines {1} values; the supported maximum is {2}", name, values.GetArrayLength(), - JevToolCallCompiler.MaxEnumValues)); + DecisionToolCallCompiler.MaxEnumValues)); } return [.. values.EnumerateArray()]; @@ -230,20 +230,20 @@ public static Dictionary ResolveArrayItems(Dictionary items, bool nullable) = Resolve(ToMap(element), root); RejectUnsupportedKeys(items, ArrayItemKeys, "array item"); if (nullable) { - throw new JevUnsupportedToolSchemaException("nullable array members are not supported"); + throw new DecisionUnsupportedToolSchemaException("nullable array members are not supported"); } return items; } - /// Checks all supported constraints before offering a constant or enum value to Jev. + /// Checks all supported constraints before offering a constant or enum value to the model. public static void EnsureValuesMatchSchema(Dictionary schema, IReadOnlyList values, Dictionary root, string valueName) { EnsureValuesMatchType(schema, values, "argument", valueName, "type"); @@ -275,7 +275,7 @@ private static void EnsureValuesMatchEnum(Dictionary schema { if (!alternatives.Any(alternative => JsonElement.DeepEquals(value, alternative))) { - throw new JevUnsupportedToolSchemaException($"{valueName} {Describe(value)} does not match declared enum {Describe(alternatives)}"); + throw new DecisionUnsupportedToolSchemaException($"{valueName} {Describe(value)} does not match declared enum {Describe(alternatives)}"); } } } @@ -283,8 +283,8 @@ private static void EnsureValuesMatchEnum(Dictionary schema /// /// Writes a value the way the Python connector does with json.dumps(value, ensure_ascii=False, sort_keys=True): - /// sorted object keys, ", " and ": " separators, and unescaped text. Jev reads these values in - /// criteria and instructions, so both connectors send the same words. + /// sorted object keys, ", " and ": " separators, and unescaped text. The model reads these values in + /// criteria and instructions, so the .NET and Python connectors send the same words. /// public static string Describe(JsonElement value) { @@ -314,7 +314,7 @@ private static void RejectCompositionSiblings(Dictionary sc List siblings = [.. schema.Keys.Where(key => !AnnotationKeys.Contains(key)).OrderBy(key => key, StringComparer.Ordinal)]; if (siblings.Count > 0) { - throw new JevUnsupportedToolSchemaException($"{keyword} sibling constraints are not supported: {string.Join(", ", siblings)}"); + throw new DecisionUnsupportedToolSchemaException($"{keyword} sibling constraints are not supported: {string.Join(", ", siblings)}"); } } diff --git a/dotnet/src/Shared/DecisionChat/DecisionUnsupportedToolSchemaException.cs b/dotnet/src/Shared/DecisionChat/DecisionUnsupportedToolSchemaException.cs new file mode 100644 index 00000000000..bfc52b8d8fd --- /dev/null +++ b/dotnet/src/Shared/DecisionChat/DecisionUnsupportedToolSchemaException.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.Agents.AI.Decisions; + +/// +/// A tool's JSON schema cannot be expressed as closed-set decision questions, so the tool is left out of the request. +/// +/// +/// It is caught while a tool call plan is compiled and never leaves the compiler: an unsupported tool is logged and +/// excluded, or, when the tool mode requires a call and no tool remains, its reason becomes part of an +/// . +/// +internal sealed class DecisionUnsupportedToolSchemaException : NotSupportedException +{ + public DecisionUnsupportedToolSchemaException() + { + } + + public DecisionUnsupportedToolSchemaException(string message) + : base(message) + { + } + + public DecisionUnsupportedToolSchemaException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/dotnet/src/Shared/DecisionChat/README.md b/dotnet/src/Shared/DecisionChat/README.md new file mode 100644 index 00000000000..4d4684de583 --- /dev/null +++ b/dotnet/src/Shared/DecisionChat/README.md @@ -0,0 +1,39 @@ +# DecisionChat + +The provider-neutral engine of a chat client for a decision model: a model that answers typed Binary, Choice, and +Score questions about a state with probabilities, instead of writing text. It is built on the decision API in +`../Decisions` and works with any `IDecisionClient`, so Jev, Azure OpenAI Decisions, and other decision models share +one implementation. + +- `DecisionChatEngine` answers a chat request: it sends the conversation as the state, together with the caller's + questions and the internal questions that choose a tool call, and returns a `FunctionCallContent` or the answers. + It rejects options that need generated text, such as `Temperature` or streaming. +- `DecisionChatState` converts a conversation into the state (`{"messages", "instructions"}`, the shape of the Python + TypeSafe connector) and finds the current turn. +- `DecisionToolCallCompiler` turns function tools whose arguments come from a closed set (constants, enums, Booleans, + and arrays of enum values) into a route Choice and one question per argument, with the question IDs, limits, and + wording of the Python TypeSafe connector. `DecisionToolCallPlan` decodes the answers into the call. + +A provider's public chat client converts its own question types into `DecisionQuestion` values and describes the +answers in its own result type through `DecisionChatEngineOptions.DescribeAnswers`. `JevChatClient` in +`Microsoft.Agents.AI.TypeSafe` is the first one. + +## Usage + +```xml + + true + true + true + true + $(NoWarn);MEAI001 + +``` + +The project also needs package references to `Microsoft.Extensions.AI.Abstractions` and +`Microsoft.Extensions.Logging.Abstractions`, and a project reference to `Microsoft.Agents.AI.Abstractions`. The code +targets .NET 8 and later. + +The engine uses only the public surface of the decision API. Keep it that way, so it compiles unchanged once the API +ships in `Microsoft.Extensions.AI` and `../Decisions` is deleted. At that point the engine can become a public type in +an Agent Framework package. diff --git a/dotnet/src/Shared/Decisions/.editorconfig b/dotnet/src/Shared/Decisions/.editorconfig index 25c6b56d3bc..aca755060b3 100644 --- a/dotnet/src/Shared/Decisions/.editorconfig +++ b/dotnet/src/Shared/Decisions/.editorconfig @@ -8,6 +8,7 @@ dotnet_style_qualification_for_field = false:none dotnet_style_qualification_for_property = false:none dotnet_style_qualification_for_method = false:none dotnet_style_qualification_for_event = false:none +dotnet_diagnostic.IDE0001.severity = none dotnet_diagnostic.IDE0009.severity = none dotnet_diagnostic.IDE0073.severity = none dotnet_diagnostic.RCS1146.severity = none diff --git a/dotnet/src/Shared/Decisions/README.md b/dotnet/src/Shared/Decisions/README.md index 7914b9b90a9..bb79a07abb3 100644 --- a/dotnet/src/Shared/Decisions/README.md +++ b/dotnet/src/Shared/Decisions/README.md @@ -39,7 +39,9 @@ suppresses `MEAI001`, as it will have to once they ship. Each project that injects this folder compiles its own copy, so the types are distinct per assembly. Do not expose them in a public API, and do not pass an `IDecisionClient` from one package to another; expose provider-specific types -instead. +instead. A public type that uses them must translate `DecisionProtocolException` and `DecisionClientException`, which +are internal here, into public exceptions. `../DecisionChat` is a provider-neutral chat client engine built on this +copy. ## When Microsoft.Extensions.AI ships the API diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/DecisionToolCallCompilerTests.cs similarity index 77% rename from dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs rename to dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/DecisionToolCallCompilerTests.cs index 4c0442c78a5..807f33f5202 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/DecisionToolCallCompilerTests.cs @@ -4,14 +4,19 @@ using System.Collections.Generic; using System.Linq; using System.Text.Json; +using Microsoft.Agents.AI.Decisions; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; namespace Microsoft.Agents.AI.TypeSafe.UnitTests; -public sealed class JevToolCallCompilerTests +/// +/// Tests the provider-neutral tool call compiler that is built on, with Jev answers. +/// +public sealed class DecisionToolCallCompilerTests { private static readonly Dictionary>> s_noCalls = []; + private static readonly JsonElement s_state = JsonDocument.Parse("""{ "messages": [] }""").RootElement.Clone(); [Fact] public void Compile_ReferencesAndAnyOfNull_ResolveAndDecode() @@ -31,15 +36,16 @@ public void Compile_ReferencesAndAnyOfNull_ResolveAndDecode() """); // Act - JevToolCallPlan plan = Compile([tool])!; - JevToolCall? call = plan.Decode(Result( + DecisionToolCallPlan plan = Compile([tool])!; + DecisionToolCall? call = plan.Decode(Decide( + plan, JevChatTestData.RouteTo("t0", "t0", "none"), JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.9), JevChatTestData.Choice("__af_tool__.t0.a1.value", "v0", "v0", "v1"))); // Assert - Assert.Contains("The mode", plan.Questions["__af_tool__.t0.a0.value"].Instructions!.Value.Text, StringComparison.Ordinal); + Assert.Contains("The mode", plan.Questions["__af_tool__.t0.a0.value"].Instructions, StringComparison.Ordinal); Assert.Equal(["route", "t0.a0.value", "t0.a1.present", "t0.a1.value"], plan.Questions.Keys.Select(key => key["__af_tool__.".Length..])); Assert.Equal("safe", ((JsonElement)call!.Value.Arguments["mode"]!).GetString()); Assert.Equal("low", ((JsonElement)call.Value.Arguments["level"]!).GetString()); @@ -68,15 +74,16 @@ public void Compile_ChainedReferences_ResolveAndDecode() """); // Act - JevToolCallPlan plan = Compile([tool])!; - JevToolCall? call = plan.Decode(Result( + DecisionToolCallPlan plan = Compile([tool])!; + DecisionToolCall? call = plan.Decode(Decide( + plan, JevChatTestData.RouteTo("t0", "t0", "none"), JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.9), JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"))); // Assert: the description next to the first reference wins, and the nullable alias still gets a presence question. - Assert.Contains("How fast to go", plan.Questions["__af_tool__.t0.a0.value"].Instructions!.Value.Text, StringComparison.Ordinal); + Assert.Contains("How fast to go", plan.Questions["__af_tool__.t0.a0.value"].Instructions, StringComparison.Ordinal); Assert.Equal("safe", ((JsonElement)call!.Value.Arguments["speed"]!).GetString()); Assert.Equal("high", ((JsonElement)call.Value.Arguments["level"]!).GetString()); } @@ -98,8 +105,9 @@ public void Compile_IntegersOutsideTheInt64Range_Compile() """); // Act - JevToolCallPlan plan = Compile([tool])!; - JevToolCall? call = plan.Decode(Result( + DecisionToolCallPlan plan = Compile([tool])!; + DecisionToolCall? call = plan.Decode(Decide( + plan, JevChatTestData.RouteTo("t0", "t0", "none"), JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), JevChatTestData.Noul("__af_tool__.t0.a2.m0", 0.1), @@ -129,7 +137,7 @@ public void Compile_EnumAndConstValuesMatchingTheirType_Compile() """); // Act - JevToolCallPlan? plan = Compile([tool]); + DecisionToolCallPlan? plan = Compile([tool]); // Assert Assert.NotNull(plan); @@ -169,7 +177,7 @@ public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema var logger = new ListLoggerFactory(); // Act - JevToolCallPlan? plan = Compile([Declaration("tool", schema)], logger: logger.CreateLogger("test")); + DecisionToolCallPlan? plan = Compile([Declaration("tool", schema)], logger: logger.CreateLogger("test")); // Assert Assert.Null(plan); @@ -193,8 +201,9 @@ public void Compile_ConstAndEnumWithArrayItemConstraints_ResolveAndDecode() """); // Act - JevToolCallPlan plan = Compile([tool])!; - JevToolCall? call = plan.Decode(Result( + DecisionToolCallPlan plan = Compile([tool])!; + DecisionToolCall? call = plan.Decode(Decide( + plan, JevChatTestData.RouteTo("t0", "t0", "none"), JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"))); @@ -211,8 +220,8 @@ public void Compile_SingleValueEnumAndNoArguments_NeedNoArgumentQuestions() AIFunctionDeclaration single = Declaration("single", """{ "type": "object", "properties": { "scope": { "enum": ["all"] } }, "required": ["scope"] }"""); // Act - JevToolCallPlan plan = Compile([refresh, single])!; - JevToolCall? call = plan.Decode(Result(JevChatTestData.RouteTo("t1", "t0", "t1", "none"))); + DecisionToolCallPlan plan = Compile([refresh, single])!; + DecisionToolCall? call = plan.Decode(Decide(plan, JevChatTestData.RouteTo("t1", "t0", "t1", "none"))); // Assert Assert.Equal([JevChatTestData.Route], plan.Questions.Keys); @@ -224,7 +233,7 @@ public void Compile_SingleValueEnumAndNoArguments_NeedNoArgumentQuestions() public void Compile_TooManyTools_Throws() { // Arrange - List tools = [.. Enumerable.Range(0, JevToolCallCompiler.MaxRoutableTools + 1).Select(index => Declaration($"tool{index}", """{ "type": "object", "properties": {} }"""))]; + List tools = [.. Enumerable.Range(0, DecisionToolCallCompiler.MaxRoutableTools + 1).Select(index => Declaration($"tool{index}", """{ "type": "object", "properties": {} }"""))]; // Act & Assert Assert.Throws(() => Compile(tools)); @@ -234,10 +243,10 @@ public void Compile_TooManyTools_Throws() public void Compile_RequiredToolFilter_AppliesBeforeTheToolLimit() { // Arrange - List tools = [.. Enumerable.Range(0, JevToolCallCompiler.MaxRoutableTools + 1).Select(index => Declaration($"tool{index}", """{ "type": "object", "properties": {} }"""))]; + List tools = [.. Enumerable.Range(0, DecisionToolCallCompiler.MaxRoutableTools + 1).Select(index => Declaration($"tool{index}", """{ "type": "object", "properties": {} }"""))]; // Act - JevToolCallPlan plan = Compile(tools, ChatToolMode.RequireSpecific("tool3"))!; + DecisionToolCallPlan plan = Compile(tools, ChatToolMode.RequireSpecific("tool3"))!; // Assert Assert.Equal("tool3", plan.RequiredTool!.Function.Name); @@ -265,7 +274,7 @@ public void Compile_QuestionBudget_IncludesTheRouteQuestion() AIFunctionDeclaration tool = Declaration("large", "{ \"type\": \"object\", \"properties\": { \"a\": " + array + ", \"b\": " + array + " }, \"required\": [\"a\", \"b\"] }"); // Act & Assert - JevToolCallPlan plan = Compile([tool], ChatToolMode.RequireAny)!; + DecisionToolCallPlan plan = Compile([tool], ChatToolMode.RequireAny)!; Assert.Equal(128, plan.Questions.Count); Assert.Null(plan.RouteQuestionId); Assert.Throws(() => Compile([tool])); @@ -280,7 +289,7 @@ public void Compile_QuestionBudgetWithRouteAtTheLimit_Compiles() string schema = "{ \"type\": \"object\", \"properties\": { \"a\": { \"type\": \"array\", \"items\": { \"enum\": [" + first + "] } }, \"b\": { \"type\": \"array\", \"items\": { \"enum\": [" + second + "] } } }, \"required\": [\"a\", \"b\"] }"; // Act - JevToolCallPlan plan = Compile([Declaration("large", schema)])!; + DecisionToolCallPlan plan = Compile([Declaration("large", schema)])!; // Assert Assert.Equal(128, plan.Questions.Count); @@ -296,7 +305,7 @@ public void Compile_UnsupportedTool_ReturnsItsQuestionBudget() AIFunctionDeclaration large = Declaration("large", $$"""{ "type": "object", "properties": { "a": { "type": "array", "items": { "enum": [{{members}}] } }, "b": { "type": "array", "items": { "enum": [{{string.Join(",", Enumerable.Range(0, 60).Select(index => $"\"n{index}\""))}}] } } }, "required": ["a", "b"] }"""); // Act - JevToolCallPlan plan = Compile([failing, large])!; + DecisionToolCallPlan plan = Compile([failing, large])!; // Assert Assert.Equal(125, plan.Questions.Count); @@ -318,7 +327,7 @@ public void Compile_UnsupportedToolExceedingQuestionBudget_IsExcludedRegardlessO var logger = new ListLoggerFactory(); // Act - JevToolCallPlan plan = Compile(unsupportedFirst ? [unsupported, supported] : [supported, unsupported], logger: logger.CreateLogger("test"))!; + DecisionToolCallPlan plan = Compile(unsupportedFirst ? [unsupported, supported] : [supported, unsupported], logger: logger.CreateLogger("test"))!; // Assert Assert.Equal("supported", Assert.Single(plan.Tools).Function.Name); @@ -336,7 +345,7 @@ public void Compile_SingleUnsupportedToolExceedingQuestionBudget_IsExcluded() var logger = new ListLoggerFactory(); // Act - JevToolCallPlan? plan = Compile([tool], logger: logger.CreateLogger("test")); + DecisionToolCallPlan? plan = Compile([tool], logger: logger.CreateLogger("test")); // Assert Assert.Null(plan); @@ -347,11 +356,11 @@ public void Compile_SingleUnsupportedToolExceedingQuestionBudget_IsExcluded() public void Compile_TooManyEnumValuesOrProperties_ExcludeTheTool() { // Arrange - string values = string.Join(",", Enumerable.Range(0, JevToolCallCompiler.MaxEnumValues + 1).Select(index => $"\"v{index}\"")); - string properties = string.Join(",", Enumerable.Range(0, JevToolCallCompiler.MaxToolProperties + 1).Select(index => $"\"p{index}\": {{ \"type\": \"boolean\" }}")); + string values = string.Join(",", Enumerable.Range(0, DecisionToolCallCompiler.MaxEnumValues + 1).Select(index => $"\"v{index}\"")); + string properties = string.Join(",", Enumerable.Range(0, DecisionToolCallCompiler.MaxToolProperties + 1).Select(index => $"\"p{index}\": {{ \"type\": \"boolean\" }}")); // Act - JevToolCallPlan? plan = Compile( + DecisionToolCallPlan? plan = Compile( [ Declaration("enum", $$"""{ "type": "object", "properties": { "a": { "enum": [{{values}}] } } }"""), Declaration("wide", $$"""{ "type": "object", "properties": { {{properties}} } }"""), @@ -362,14 +371,15 @@ public void Compile_TooManyEnumValuesOrProperties_ExcludeTheTool() } [Fact] - public void Decode_UnknownRouteOrMissingAnswer_Throws() + public void Decode_UnknownRouteOrMissingAnswer_IsRejectedBeforeDecoding() { // Arrange - JevToolCallPlan plan = Compile([Declaration("flag", """{ "type": "object", "properties": { "on": { "type": "boolean" } }, "required": ["on"] }""")])!; + DecisionToolCallPlan plan = Compile([Declaration("flag", """{ "type": "object", "properties": { "on": { "type": "boolean" } }, "required": ["on"] }""")])!; - // Act & Assert - Assert.Throws(() => plan.Decode(Result(JevChatTestData.RouteTo("t9", "t9")))); - Assert.Throws(() => plan.Decode(Result(JevChatTestData.RouteTo("t0", "t0")))); + // Act & Assert: a decision response must answer every question with the labels it offered, so neither reaches + // the decoders. + Assert.Throws(() => Decide(plan, fillMissing: false, JevChatTestData.RouteTo("t9", "t9"))); + Assert.Throws(() => Decide(plan, fillMissing: false, JevChatTestData.RouteTo("t0", "t0", "none"))); } [Fact] @@ -380,11 +390,12 @@ public void Compile_DuplicateDescriptions_KeepToolNamesInTheRoute() AIFunctionDeclaration second = Declaration("second", """{ "type": "object", "properties": {} }""", "Does it."); // Act - JevChoiceQuestion route = Assert.IsType(Compile([first, second])!.Questions[JevChatTestData.Route]); + ChoiceDecisionQuestion route = Assert.IsType(Compile([first, second])!.Questions[JevChatTestData.Route]); // Assert - Assert.Equal("Tool 'first': Does it.", route.Criteria["t0"].Text); - Assert.Equal("Tool 'second': Does it.", route.Criteria["t1"].Text); + Assert.Equal(["t0", "t1", "none"], route.Candidates.Select(candidate => candidate.Id)); + Assert.Equal("Tool 'first': Does it.", route.Candidates[0].Description); + Assert.Equal("Tool 'second': Does it.", route.Candidates[1].Description); } [Fact] @@ -394,7 +405,7 @@ public void Describe_MatchesPythonJsonDumps() JsonElement value = JsonDocument.Parse("""{"city":"São Paulo","b":[1,2.5,true,null],"q":"say \"hi\"\n"}""").RootElement; // Act - string described = JevToolSchema.Describe(value); + string described = DecisionToolSchema.Describe(value); // Assert Assert.Equal("""{"b": [1, 2.5, true, null], "city": "São Paulo", "q": "say \"hi\"\n"}""", described); @@ -403,9 +414,44 @@ public void Describe_MatchesPythonJsonDumps() private static AIFunctionDeclaration Declaration(string name, string schema, string? description = null) => AIFunctionFactory.CreateDeclaration(name, description, JsonDocument.Parse(schema).RootElement.Clone()); - private static JevToolCallPlan? Compile(IReadOnlyList tools, ChatToolMode? mode = null, ILogger? logger = null) => - JevToolCallCompiler.Compile(tools, mode, s_noCalls, logger ?? new ListLoggerFactory().CreateLogger("test")); + private static DecisionToolCallPlan? Compile(IReadOnlyList tools, ChatToolMode? mode = null, ILogger? logger = null) => + DecisionToolCallCompiler.Compile(tools, mode, s_noCalls, "Jev", logger ?? new ListLoggerFactory().CreateLogger("test")); + + private static DecisionResponse Decide(DecisionToolCallPlan plan, params string[] answers) => + Decide(plan, fillMissing: true, answers); + + /// + /// Converts Jev answers to the plan's questions into a decision response, as does. + /// + /// The plan whose questions are answered. + /// + /// Whether questions without an answer get one that the decoders do not read for the selected call: no for a Binary + /// question and the first candidate for a Choice. + /// + /// The Jev answers, from . + private static DecisionResponse Decide(DecisionToolCallPlan plan, bool fillMissing, params string[] answers) + { + JevResult result = JsonSerializer.Deserialize(JevChatTestData.Result(answers), JevJsonContext.Default.JevResult)!; + var decisionAnswers = new List(); + foreach (DecisionQuestion question in plan.Questions.Values) + { + if (result.Answers.TryGetValue(question.Id, out JevResponse? answer)) + { + decisionAnswers.Add(JevDecisionMapping.ToDecisionAnswer(question, answer)); + } + else if (fillMissing) + { + decisionAnswers.Add(question switch + { + ChoiceDecisionQuestion choice => new ChoiceDecisionAnswer( + question.Id, + choice.Candidates[0].Id, + [.. choice.Candidates.Select((candidate, index) => new DecisionProbability(candidate.Id, index == 0 ? 1 : 0))]), + _ => new BinaryDecisionAnswer(question.Id, 0), + }); + } + } - private static JevResult Result(params string[] answers) => - JsonSerializer.Deserialize(JevChatTestData.Result(answers), JevJsonContext.Default.JevResult)!; + return new DecisionResponse(new DecisionRequest(s_state, [.. plan.Questions.Values]), decisionAnswers); + } } diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs index 8b328f0a2de..3499f8e747f 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs @@ -301,6 +301,45 @@ public async Task GetResponseAsync_ResultMissingAnAnswer_ThrowsAsync() Assert.Contains("department", exception.Message, StringComparison.Ordinal); } + [Fact] + public async Task GetResponseAsync_JsonInstructionsAndUndescribedLabels_AreSentUnchangedAsync() + { + // Arrange: the chat client runs on the provider-neutral decision API, which carries text only, so the Jev + // questions themselves must reach the API. + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Choice("tone", "calm", "calm", "angry"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + JsonElement instructions = JsonDocument.Parse("""{ "question": "What is the tone of `reply`?", "reply": "Thanks!" }""").RootElement.Clone(); + var questions = new Dictionary + { + ["tone"] = new JevChoiceQuestion { Instructions = (JevEntry)instructions, Criteria = new Dictionary { ["calm"] = JevEntry.Null, ["angry"] = "Hostile or upset" } }, + }; + + // Act + ChatResponse response = await client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions)); + + // Assert + JsonNode sent = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!["tone"]!; + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(instructions.GetRawText()), sent["instructions"]), sent.ToJsonString()); + Assert.Null(sent["criteria"]!["calm"]); + Assert.Equal("calm", Assert.IsType(response.GetJevResult()!.Answers["tone"]).Choice); + } + + [Fact] + public async Task GetResponseAsync_IncompleteDistribution_ThrowsInvalidOperationExceptionAsync() + { + // Arrange: the answer leaves out the probability of a label. + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result( + "\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":0.9,\"probabilities\":{\"billing\":1.0}}", + JevChatTestData.Noul("is_urgent", 0.9))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Contains("'technical'", exception.Message, StringComparison.Ordinal); + } + [Fact] public async Task GetResponseAsync_ApiError_ThrowsClientResultExceptionAsync() { diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs index 7774a0dac01..10a4837557e 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs @@ -9,6 +9,7 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Threading.Tasks; +using Microsoft.Agents.AI.Decisions; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; @@ -549,12 +550,12 @@ public void GetCurrentTurnStart_NewUserAfterFunctionResult_StartsANewTurn(bool w } // Act - int turnStart = JevChatState.GetCurrentTurnStart(messages); + int turnStart = DecisionChatState.GetCurrentTurnStart(messages); // Assert Assert.Equal(messages.Length, turnStart); - Assert.Equal(0, JevChatState.CountCurrentTurnCalls(messages, turnStart)); - Assert.Empty(JevChatState.GetCurrentTurnResultTexts(messages, turnStart)); + Assert.Equal(0, DecisionChatState.CountCurrentTurnCalls(messages, turnStart)); + Assert.Empty(DecisionChatState.GetCurrentTurnResultTexts(messages, turnStart)); } [Theory] @@ -573,7 +574,7 @@ public void GetCurrentTurnStart_ResumedResultBeforeSeveralUserMessages_MatchesTh ]; // Act - int turnStart = JevChatState.GetCurrentTurnStart(messages); + int turnStart = DecisionChatState.GetCurrentTurnStart(messages); // Assert Assert.Equal(expectedTurnStart, turnStart); From 830427a56d3cbb14008f71c5e34869dee548790c Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:36:17 -0800 Subject: [PATCH 10/16] .NET: Declare two-decimal precision for Jev decision answers Live TypeSafe responses round probabilities and confidence to two decimal places, so a distribution can sum to 0.99 or 1.01. Declaring HighPrecision made the decision API reject such answers with its 1e-6 tolerance. Jev answers now declare TwoDecimalPlaces, and a test covers a rounded distribution that sums to 0.99. --- .../Core/JevDecisionMapping.cs | 11 +++++----- .../JevDecisionClientTests.cs | 21 ++++++++++++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs index 3ce5dc1be75..a2869213dc6 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs @@ -21,17 +21,18 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// /// The decision types require complete, correlated answers: every candidate and level needs a probability, and each -/// distribution must sum to 1. Jev computes its values in floating point, so a value that exceeds its bounds by less -/// than is moved onto the bound, as already accepts it. -/// Nothing else is repaired. +/// distribution must sum to 1 within the rounding of . Jev computes its values in floating point, +/// so a value that exceeds its bounds by less than is moved onto the bound, as +/// already accepts it. Nothing else is repaired. /// /// internal static class JevDecisionMapping { /// - /// Gets the precision of Jev probabilities. Jev does not declare a rounding, so they are taken at face value. + /// Gets the precision of Jev probabilities, which the API rounds to two decimal places, so a distribution may sum + /// to slightly more or less than 1. /// - public const DecisionPrecision Precision = DecisionPrecision.HighPrecision; + public const DecisionPrecision Precision = DecisionPrecision.TwoDecimalPlaces; /// The name of the entry that holds Jev's confidence. public const string ConfidenceProperty = "confidence"; diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs index ab36d781eb3..e8b2fabd603 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs @@ -71,7 +71,8 @@ public async Task GetResponseAsync_DecisionQuestions_AreAskedAsJevQuestionsAsync // Assert: the provenance, usage, and raw result. Assert.Equal(JevDecisionClient.ProviderName, response.Provenance!.ProviderName); Assert.Equal("jev-test", response.Provenance.ModelId); - Assert.Equal(DecisionPrecision.HighPrecision, response.Provenance.Precision); + Assert.Equal(DecisionPrecision.TwoDecimalPlaces, response.Provenance.Precision); + Assert.All(response.Answers, answer => Assert.Equal(DecisionPrecision.TwoDecimalPlaces, answer.Precision)); Assert.Equal(12, response.Usage!.TotalTokenCount); Assert.IsType(response.RawRepresentation); } @@ -116,6 +117,24 @@ public async Task GetResponseAsync_IncompleteDistribution_ThrowsDecisionProtocol Assert.Contains("'technical'", exception.Message, StringComparison.Ordinal); } + [Fact] + public async Task GetResponseAsync_DistributionRoundedToTwoDecimals_IsAcceptedAsync() + { + // Arrange: Jev rounds to two decimals, so three even probabilities sum to 0.99 rather than 1. + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + "\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":0.01,\"probabilities\":{\"billing\":0.33,\"technical\":0.33,\"sales\":0.33}}")); + using var httpClient = new HttpClient(handler); + using JevDecisionClient client = CreateClient(httpClient); + var request = new DecisionRequest(s_state, [new ChoiceDecisionQuestion("department", "Which team?", [new("billing", "Payments"), new("technical", "Bugs"), new("sales", "Plans")])]); + + // Act + DecisionResponse response = await client.GetResponseAsync(request); + + // Assert: the observed values are kept as reported. + ChoiceDecisionAnswer answer = Assert.IsType(Assert.Single(response.Answers)); + Assert.Equal([0.33, 0.33, 0.33], answer.Probabilities.Select(probability => probability.Value)); + } + [Fact] public async Task GetResponseAsync_ValueWithinRoundingErrorOfItsBound_IsMovedOntoTheBoundAsync() { From 8c326210340e1f93e02412756cc9409412afbb72 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:26:25 -0800 Subject: [PATCH 11/16] .NET: Expose the proposed MEAI decision API publicly from the TypeSafe package Move the copy of the decision API proposed in dotnet/extensions#7795 and #7796 from src/Shared/Decisions into Microsoft.Agents.AI.TypeSafe/MEAI, restoring the upstream files verbatim, so IDecisionClient, DecisionDefinition, and AsAIFunction are public in the Microsoft.Extensions.AI namespace and marked MEAI001, as proposed. This follows how Agent Framework carried proposed MEAI types before they shipped (#279, #426). JevDecisionClient is now public, with an API key constructor and JevDecisionClientOptions, so applications and samples can use Jev through the proposed API today. --- dotnet/agent-framework-dotnet.slnx | 10 --- dotnet/eng/MSBuild/Shared.props | 3 - .../{Core => }/JevDecisionClient.cs | 67 +++++++++++++++---- .../JevDecisionClientOptions.cs | 62 +++++++++++++++++ .../MEAI/.editorconfig | 18 +++++ .../MEAI}/DecisionAnswers.cs | 14 ++-- .../MEAI}/DecisionBinding.cs | 10 +-- .../MEAI}/DecisionClient.cs | 14 ++-- .../MEAI}/DecisionDefinitions.cs | 4 +- .../MEAI}/DecisionFeatures.cs | 12 ++-- .../MEAI}/DecisionQuestions.cs | 16 ++--- .../MEAI/README.md | 45 +++++++++++++ .../Microsoft.Agents.AI.TypeSafe.csproj | 5 +- dotnet/src/Shared/DecisionChat/README.md | 18 +++-- dotnet/src/Shared/Decisions/.editorconfig | 21 ------ dotnet/src/Shared/Decisions/README.md | 56 ---------------- .../JevDecisionClientTests.cs | 32 +++++++++ 17 files changed, 256 insertions(+), 151 deletions(-) rename dotnet/src/Microsoft.Agents.AI.TypeSafe/{Core => }/JevDecisionClient.cs (52%) create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClientOptions.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/.editorconfig rename dotnet/src/{Shared/Decisions => Microsoft.Agents.AI.TypeSafe/MEAI}/DecisionAnswers.cs (97%) rename dotnet/src/{Shared/Decisions => Microsoft.Agents.AI.TypeSafe/MEAI}/DecisionBinding.cs (97%) rename dotnet/src/{Shared/Decisions => Microsoft.Agents.AI.TypeSafe/MEAI}/DecisionClient.cs (98%) rename dotnet/src/{Shared/Decisions => Microsoft.Agents.AI.TypeSafe/MEAI}/DecisionDefinitions.cs (99%) rename dotnet/src/{Shared/Decisions => Microsoft.Agents.AI.TypeSafe/MEAI}/DecisionFeatures.cs (97%) rename dotnet/src/{Shared/Decisions => Microsoft.Agents.AI.TypeSafe/MEAI}/DecisionQuestions.cs (96%) create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/README.md delete mode 100644 dotnet/src/Shared/Decisions/.editorconfig delete mode 100644 dotnet/src/Shared/Decisions/README.md diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index a0381df4d52..3e627d6b280 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -575,16 +575,6 @@ - - - - - - - - - - diff --git a/dotnet/eng/MSBuild/Shared.props b/dotnet/eng/MSBuild/Shared.props index f79f1be57de..95226d37ae2 100644 --- a/dotnet/eng/MSBuild/Shared.props +++ b/dotnet/eng/MSBuild/Shared.props @@ -53,9 +53,6 @@ - - - diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionClient.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs similarity index 52% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionClient.cs rename to dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs index f0f925266dc..b7c961d13c6 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionClient.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs @@ -1,39 +1,61 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.ClientModel; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; namespace Microsoft.Agents.AI.TypeSafe; /// -/// Answers provider-neutral decision requests with Jev, through POST /v1/systemone of the TypeSafe API. +/// Provides an for Jev, the TypeSafe System One model, which answers typed Binary, +/// Choice, and Score questions about a state with probabilities. /// /// /// -/// Binary, Choice, and Score questions become Noul, Choice, and Score questions (see ). -/// A request is checked with the TypeSafe rules before it is sent, and the result is checked against the request before -/// it is converted, as the chat client and the tool already do. The response keeps the as its -/// raw representation, each answer keeps its , and Choice and Score answers carry Jev's -/// confidence as the confidence additional property. +/// Binary, Choice, and Score questions become Jev Noul, Choice, and Score questions. A request is checked with the +/// TypeSafe rules before it is sent, such as at most 255 Choice alternatives and 2 to 10 Score levels, and the result is +/// checked against the request before it is converted. Jev reports probabilities rounded to two decimal places, which +/// each answer declares as . /// /// -/// is a proposed Microsoft.Extensions.AI API that this package compiles from an internal -/// copy, so this client is internal; is built on it. Once the API ships, this client can -/// become public. +/// The response keeps the as its , each answer +/// keeps its , and Choice and Score answers carry Jev's confidence in the confidence +/// entry of . +/// +/// +/// is the decision API proposed for Microsoft.Extensions.AI. This package carries a copy +/// of it until it ships, so this client and the API are experimental. /// /// -internal sealed class JevDecisionClient : IDecisionClient +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class JevDecisionClient : IDecisionClient { - public const string ProviderName = "typesafe.ai"; + internal const string ProviderName = "typesafe.ai"; private readonly JevHttpClient _client; private readonly DecisionClientMetadata _metadata; - public JevDecisionClient(JevHttpClient client) + /// + /// Initializes a new instance of the class. + /// + /// The TypeSafe API key, sent as a bearer token. + /// The client and pipeline options. Defaults are used when . + /// + /// is empty, or is + /// not an absolute URI. + /// + public JevDecisionClient(ApiKeyCredential credential, JevDecisionClientOptions? options = null) + : this(CreateHttpClient(credential, options ?? new JevDecisionClientOptions())) + { + } + + internal JevDecisionClient(JevHttpClient client) { this._client = Throw.IfNull(client); this._metadata = new DecisionClientMetadata(ProviderName, client.Endpoint, client.ModelId, [DecisionKind.Binary, DecisionKind.Choice, DecisionKind.Score]); @@ -43,11 +65,13 @@ public JevDecisionClient(JevHttpClient client) /// The request breaks a TypeSafe rule, such as the number of Choice alternatives. /// The Jev result does not answer every question as asked. /// An answer is not a complete, correlated decision answer. - /// The TypeSafe API rejected the request. + /// The TypeSafe API rejected the request. public async Task GetResponseAsync(DecisionRequest request, DecisionOptions? options = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(request); + // A caller that has Jev's own questions, such as JevChatClient, sends them unchanged, so JSON instructions and + // criteria keep their form; the decision questions with the same IDs then only correlate the answers. IReadOnlyDictionary? jevQuestions = (options?.RawRepresentationFactory?.Invoke(this) as JevDecisionRequestOptions)?.Questions; var questions = new Dictionary(request.Questions.Count, StringComparer.Ordinal); foreach (DecisionQuestion question in request.Questions) @@ -96,4 +120,21 @@ public void Dispose() { // The pipeline owns no disposable resources; its default transport shares one HttpClient per process. } + + private static JevHttpClient CreateHttpClient(ApiKeyCredential credential, JevDecisionClientOptions options) + { + _ = Throw.IfNull(credential); + + if (string.IsNullOrWhiteSpace(options.ModelId)) + { + Throw.ArgumentException(nameof(options), $"{nameof(JevDecisionClientOptions)}.{nameof(JevDecisionClientOptions.ModelId)} must not be empty."); + } + + if (options.Endpoint?.IsAbsoluteUri is false) + { + Throw.ArgumentException(nameof(options), $"{nameof(JevDecisionClientOptions)}.{nameof(JevDecisionClientOptions.Endpoint)} must be an absolute URI."); + } + + return new JevHttpClient(credential, options, options.Endpoint, options.ModelId); + } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClientOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClientOptions.cs new file mode 100644 index 00000000000..b32f55b0ada --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClientOptions.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel.Primitives; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Configures a and the client pipeline that sends its requests to the TypeSafe System +/// One API. +/// +/// +/// +/// This type follows the pattern of the OpenAI and Azure AI client libraries: +/// selects the HTTP stack, +/// controls retries, and adds custom +/// policies. Unset pipeline settings get the defaults of the official TypeSafe SDK, as described for +/// . +/// +/// +/// Like other client options, an instance becomes read-only once a client has been created with it. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class JevDecisionClientOptions : ClientPipelineOptions +{ + /// + public override void Freeze() + { + JevPipelineDefaults.Apply(this); + base.Freeze(); + } + + /// + /// Gets or sets the System One endpoint. When , https://api.typesafe.ai/v1/systemone is used. + /// + public Uri? Endpoint + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } + + /// + /// Gets or sets the Jev model that answers the questions, unless + /// selects another for a request. Defaults to jev-latest. + /// + public string ModelId + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } = JevPipelineDefaults.DefaultModelId; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/.editorconfig b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/.editorconfig new file mode 100644 index 00000000000..4a862108c50 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/.editorconfig @@ -0,0 +1,18 @@ +# These files are a verbatim copy of the decision API proposed for Microsoft.Extensions.AI. The rules below keep them +# compiling unchanged, so the copy can be diffed against the upstream source and deleted when Microsoft.Extensions.AI +# ships the API. See README.md. + +[*.cs] +# The upstream code uses the upstream style and file header. +dotnet_style_qualification_for_field = false:none +dotnet_style_qualification_for_property = false:none +dotnet_style_qualification_for_method = false:none +dotnet_style_qualification_for_event = false:none +dotnet_diagnostic.IDE0001.severity = none +dotnet_diagnostic.IDE0009.severity = none +dotnet_diagnostic.IDE0073.severity = none +dotnet_diagnostic.RCS1146.severity = none +dotnet_diagnostic.RCS1173.severity = none + +# Upstream marks every public type experimental, and the internal helper types that use them do not repeat the attribute. +dotnet_diagnostic.MEAI001.severity = none diff --git a/dotnet/src/Shared/Decisions/DecisionAnswers.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionAnswers.cs similarity index 97% rename from dotnet/src/Shared/Decisions/DecisionAnswers.cs rename to dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionAnswers.cs index 2908f66aebc..f4227f9b199 100644 --- a/dotnet/src/Shared/Decisions/DecisionAnswers.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionAnswers.cs @@ -18,7 +18,7 @@ namespace Microsoft.Extensions.AI; /// Identifies the precision represented by provider-reported probabilities. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal enum DecisionPrecision +public enum DecisionPrecision { /// The provider reports values without a declared decimal rounding. HighPrecision, @@ -32,7 +32,7 @@ internal enum DecisionPrecision /// Represents one named probability observation. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionProbability +public sealed class DecisionProbability { /// Initializes a new instance of the class. /// The exact opaque candidate or level ID. @@ -57,7 +57,7 @@ public DecisionProbability(string id, double value) [JsonDerivedType(typeof(BinaryDecisionAnswer), "binary")] [JsonDerivedType(typeof(ChoiceDecisionAnswer), "choice")] [JsonDerivedType(typeof(ScoreDecisionAnswer), "score")] -internal abstract class DecisionAnswer +public abstract class DecisionAnswer { internal const double DistributionTolerance = 1e-6; @@ -236,7 +236,7 @@ internal static double GetRoundingHalfUnit(DecisionPrecision precision) => /// Represents the probability that a binary proposition is true. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class BinaryDecisionAnswer : DecisionAnswer +public sealed class BinaryDecisionAnswer : DecisionAnswer { /// Initializes a new instance of the class. [JsonConstructor] @@ -258,7 +258,7 @@ public BinaryDecisionAnswer( /// Represents a selected candidate and its complete probability distribution. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class ChoiceDecisionAnswer : DecisionAnswer +public sealed class ChoiceDecisionAnswer : DecisionAnswer { /// Initializes a new instance of the class. [JsonConstructor] @@ -301,7 +301,7 @@ public ChoiceDecisionAnswer( /// A reported score is checked against normalized latent values within the declared clipped rounding intervals; observed probabilities and the derived expectation are never repaired. /// [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class ScoreDecisionAnswer : DecisionAnswer +public sealed class ScoreDecisionAnswer : DecisionAnswer { /// Initializes a new instance of the class. [JsonConstructor] @@ -358,7 +358,7 @@ public ScoreDecisionAnswer( /// Represents a protocol-invalid provider result. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionProtocolException : Exception +public sealed class DecisionProtocolException : Exception { /// Initializes a new instance of the class. public DecisionProtocolException() diff --git a/dotnet/src/Shared/Decisions/DecisionBinding.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionBinding.cs similarity index 97% rename from dotnet/src/Shared/Decisions/DecisionBinding.cs rename to dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionBinding.cs index 82d41a63a01..8c40cd2244d 100644 --- a/dotnet/src/Shared/Decisions/DecisionBinding.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionBinding.cs @@ -19,7 +19,7 @@ namespace Microsoft.Extensions.AI; /// Provides an explicit mapping from a choice candidate ID to an enum value. /// The application-owned enum type. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionEnumDefinition +public sealed class DecisionEnumDefinition where TEnum : struct, Enum { private readonly ReadOnlyDictionary _values; @@ -87,7 +87,7 @@ public TEnum Parse(string candidateId) /// Describes one enum mapping entry. /// The application-owned enum type. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionEnumValue +public sealed class DecisionEnumValue where TEnum : struct, Enum { /// Initializes a new instance of the class. @@ -110,7 +110,7 @@ public DecisionEnumValue(TEnum value, string id, string description) /// Provides typed access to a complete decision response. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionAnswerSet +public sealed class DecisionAnswerSet { internal DecisionAnswerSet(DecisionResponse response) { @@ -152,7 +152,7 @@ public ScoreDecisionAnswer GetScore(string questionId) => /// Maps one complete response to an application-owned result without inference or reflection. /// The application-owned result type. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionResultBinding +public sealed class DecisionResultBinding { private readonly Func _map; @@ -175,7 +175,7 @@ public DecisionResultBinding( /// Provides response binding extensions. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal static class DecisionBindingExtensions +public static class DecisionBindingExtensions { /// Maps a complete response locally using the supplied explicit binding. /// The application-owned result type. diff --git a/dotnet/src/Shared/Decisions/DecisionClient.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionClient.cs similarity index 98% rename from dotnet/src/Shared/Decisions/DecisionClient.cs rename to dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionClient.cs index 2c982a7bb24..7777cb682f7 100644 --- a/dotnet/src/Shared/Decisions/DecisionClient.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionClient.cs @@ -24,7 +24,7 @@ namespace Microsoft.Extensions.AI; /// Provides provider-neutral decision inference over a heterogeneous question batch. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal interface IDecisionClient : IDisposable +public interface IDecisionClient : IDisposable { /// Evaluates all questions against the same request state. /// The owned decision request. @@ -43,7 +43,7 @@ Task GetResponseAsync( /// Provides per-request decision options. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal class DecisionOptions +public class DecisionOptions { /// Initializes a new instance of the class. public DecisionOptions() @@ -83,7 +83,7 @@ protected DecisionOptions(DecisionOptions? other) /// Provides metadata about an . [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionClientMetadata +public sealed class DecisionClientMetadata { /// Initializes a new instance of the class. public DecisionClientMetadata( @@ -126,7 +126,7 @@ public DecisionClientMetadata( /// Classifies a provider failure without prescribing retry behavior. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionClientException : Exception +public sealed class DecisionClientException : Exception { /// Initializes a new instance of the class. public DecisionClientException() @@ -159,7 +159,7 @@ public DecisionClientException(string message, Exception innerException) /// Provides extensions for decision clients. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal static class DecisionClientExtensions +public static class DecisionClientExtensions { /// Gets a typed provider service, or when it is unavailable. /// The provider service type. @@ -304,7 +304,7 @@ private static async Task> GetResponseAsyncIdentifies provider/model provenance without inventing missing identity. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionProvenance +public sealed class DecisionProvenance { /// Initializes a new instance of the class. public DecisionProvenance( @@ -339,7 +339,7 @@ public DecisionProvenance( /// Represents a complete response correlated to one request. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionResponse +public sealed class DecisionResponse { /// Initializes a new instance of the class and validates it. [JsonConstructor] diff --git a/dotnet/src/Shared/Decisions/DecisionDefinitions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionDefinitions.cs similarity index 99% rename from dotnet/src/Shared/Decisions/DecisionDefinitions.cs rename to dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionDefinitions.cs index 8979cced54b..887426af88b 100644 --- a/dotnet/src/Shared/Decisions/DecisionDefinitions.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionDefinitions.cs @@ -31,7 +31,7 @@ namespace Microsoft.Extensions.AI; /// /// The application-owned result type. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionDefinition +public sealed class DecisionDefinition { private readonly ReadOnlyCollection> _questions; @@ -679,7 +679,7 @@ private void AddQuestion( /// Contains a typed result and the complete neutral evidence that produced it. /// The application-owned result type. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionResponse +public sealed class DecisionResponse { private readonly DecisionDefinition _definition; diff --git a/dotnet/src/Shared/Decisions/DecisionFeatures.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionFeatures.cs similarity index 97% rename from dotnet/src/Shared/Decisions/DecisionFeatures.cs rename to dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionFeatures.cs index fbe68784d6e..51a4944233f 100644 --- a/dotnet/src/Shared/Decisions/DecisionFeatures.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionFeatures.cs @@ -16,7 +16,7 @@ namespace Microsoft.Extensions.AI; /// Identifies the semantic value projected by a feature coordinate. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal enum DecisionFeatureValueKind +public enum DecisionFeatureValueKind { /// The probability that a binary proposition is true. BinaryTrueProbability, @@ -39,7 +39,7 @@ internal enum DecisionFeatureValueKind /// Names one stable semantic coordinate in a feature schema. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionFeatureCoordinate +public sealed class DecisionFeatureCoordinate { /// Initializes a new instance of the class. public DecisionFeatureCoordinate( @@ -86,7 +86,7 @@ public DecisionFeatureCoordinate( /// Identifies a named feature schema and its version. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionFeatureSchema +public sealed class DecisionFeatureSchema { /// Initializes a new instance of the class. public DecisionFeatureSchema(string id, int version, IEnumerable coordinates) @@ -129,7 +129,7 @@ public DecisionFeatureSchema(string id, int version, IEnumerableRepresents one named native-double feature value. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionFeatureValue +public sealed class DecisionFeatureValue { /// Initializes a new instance of the class. public DecisionFeatureValue(string name, double value) @@ -152,7 +152,7 @@ public DecisionFeatureValue(string name, double value) /// Represents a named feature projection with schema and provenance. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionFeatureVector +public sealed class DecisionFeatureVector { internal DecisionFeatureVector( DecisionFeatureSchema schema, @@ -176,7 +176,7 @@ internal DecisionFeatureVector( /// Projects complete decision observations into named native-double coordinates. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal static class DecisionFeatureProjection +public static class DecisionFeatureProjection { /// Projects a response without an inference call or probability repair. /// The named feature vector projected from the complete response. diff --git a/dotnet/src/Shared/Decisions/DecisionQuestions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionQuestions.cs similarity index 96% rename from dotnet/src/Shared/Decisions/DecisionQuestions.cs rename to dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionQuestions.cs index 86e1eef5671..cc37031765c 100644 --- a/dotnet/src/Shared/Decisions/DecisionQuestions.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionQuestions.cs @@ -18,7 +18,7 @@ namespace Microsoft.Extensions.AI; /// Identifies the semantic kind of a decision question or answer. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal enum DecisionKind +public enum DecisionKind { /// A two-outcome proposition. Binary, @@ -36,7 +36,7 @@ internal enum DecisionKind [JsonDerivedType(typeof(BinaryDecisionQuestion), "binary")] [JsonDerivedType(typeof(ChoiceDecisionQuestion), "choice")] [JsonDerivedType(typeof(ScoreDecisionQuestion), "score")] -internal abstract class DecisionQuestion +public abstract class DecisionQuestion { private protected DecisionQuestion(string id, DecisionKind kind, string? instructions) { @@ -57,7 +57,7 @@ private protected DecisionQuestion(string id, DecisionKind kind, string? instruc /// Describes a binary decision question. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class BinaryDecisionQuestion : DecisionQuestion +public sealed class BinaryDecisionQuestion : DecisionQuestion { /// Initializes a new instance of the class. /// The opaque, caller-owned question ID. @@ -103,7 +103,7 @@ public BinaryDecisionQuestion( /// Describes one caller-defined choice candidate. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionCandidate +public sealed class DecisionCandidate { /// Initializes a new instance of the class. /// The exact opaque candidate ID to round-trip in results. @@ -124,7 +124,7 @@ public DecisionCandidate(string id, string description) /// Describes a choice decision question. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class ChoiceDecisionQuestion : DecisionQuestion +public sealed class ChoiceDecisionQuestion : DecisionQuestion { /// Initializes a new instance of the class. /// The opaque, caller-owned question ID. @@ -162,7 +162,7 @@ public ChoiceDecisionQuestion(string id, string? instructions, IReadOnlyListDescribes one level in an ordered score rubric. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionScoreLevel +public sealed class DecisionScoreLevel { /// Initializes a new instance of the class. /// The exact opaque, caller-owned level ID. @@ -183,7 +183,7 @@ public DecisionScoreLevel(string id, string description) /// Describes a score decision question. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class ScoreDecisionQuestion : DecisionQuestion +public sealed class ScoreDecisionQuestion : DecisionQuestion { /// Initializes a new instance of the class. /// The opaque, caller-owned question ID. @@ -221,7 +221,7 @@ public ScoreDecisionQuestion(string id, string? instructions, IReadOnlyListRepresents a state and a nonempty, ordered batch of questions. [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] -internal sealed class DecisionRequest +public sealed class DecisionRequest { /// Initializes a new instance of the class and snapshots its state and questions. /// The explicit JSON state evaluated by every question. diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/README.md b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/README.md new file mode 100644 index 00000000000..d7d4a98d2e8 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/README.md @@ -0,0 +1,45 @@ +# MEAI + +A copy of the provider-neutral decision API that is proposed for `Microsoft.Extensions.AI.Abstractions`: +`IDecisionClient`, Binary, Choice, and Score questions and answers, the typed `DecisionDefinition`, and +`DecisionDefinition.AsAIFunction`. Decision models (Jev, Azure OpenAI Decisions, and others) answer typed +questions about a state with probabilities, instead of writing text. + +The types are public, in the `Microsoft.Extensions.AI` namespace, and marked `[Experimental("MEAI001")]`, exactly as +proposed. Code written against them today, such as `JevDecisionClient` and the samples, is the code that will compile +against the shipped API. This follows how Agent Framework shipped proposed MEAI types before they were released +(#279, #426), and deleted them once MEAI shipped them (#427, #676). + +## Source + +| Proposal | Pull request | Commit | +|---|---|---| +| Layer 1: provider-neutral decision abstractions | [dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795) | `02e54f023bdf3c8c2729353b110c22f5f8593957` | +| Layer 2: decision interoperability (`AsAIFunction`) | [dotnet/extensions#7796](https://github.com/dotnet/extensions/pull/7796) | `e87f1126cf22108fb27f6391c7826fb91c25138e` | + +The files are `src/Libraries/Microsoft.Extensions.AI.Abstractions/Decisions/*.cs` at the Layer 2 commit, which contains +Layer 1, without changes. `.editorconfig` keeps the upstream style from failing the build, and the project injects +`src/Shared/DiagnosticIds` and `src/Shared/Throw`, which provide the helpers the upstream code uses. + +Not copied: the upstream tests, the README, and the registration of the decision types in +`AIJsonUtilities.DefaultOptions`. Serializing the decision types therefore uses reflection-based metadata. + +The API is still in review. In particular, the API review on #7795 asks to align `DecisionResponse` with +`ChatResponse` (`ModelId` and `ResponseId` instead of `DecisionProvenance`, settable properties), and the shape may +change before it ships. + +## Only one package carries the copy + +The types live in this package so that it can expose them. A second package that defined the same public types would +make them ambiguous (CS0433) for an application that references both. Before another package, such as an Azure OpenAI +Decisions provider, needs the API, move this folder to a package that both depend on. + +## When Microsoft.Extensions.AI ships the API + +1. Update `Microsoft.Extensions.AI.Abstractions` in `Directory.Packages.props`. From that version on, the copied types + conflict with the shipped ones, so the next steps go in the same change. +2. Delete this folder. +3. Fix any difference between this copy and the shipped API in `JevDecisionClient`, the shared decision chat engine + (`src/Shared/DecisionChat`), and the samples. + +To follow upstream changes before then, copy the new upstream files over these ones. diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj index 2645c86fab7..7977b8470a2 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj @@ -4,14 +4,13 @@ $(TargetFrameworksCore) Microsoft.Agents.AI.TypeSafe alpha - + $(NoWarn);MEAI001 true true - true true @@ -20,7 +19,7 @@ Microsoft Agent Framework - TypeSafe (Jev) integration - Provides an IChatClient and an AIFunction that let agents classify, route, score, and check content with Jev, the TypeSafe System One model. + Provides an IDecisionClient, an IChatClient, and an AIFunction that let agents classify, route, score, and check content with Jev, the TypeSafe System One model. Includes a copy of the decision API proposed for Microsoft.Extensions.AI until it ships. diff --git a/dotnet/src/Shared/DecisionChat/README.md b/dotnet/src/Shared/DecisionChat/README.md index 4d4684de583..18ac1549955 100644 --- a/dotnet/src/Shared/DecisionChat/README.md +++ b/dotnet/src/Shared/DecisionChat/README.md @@ -1,9 +1,9 @@ # DecisionChat The provider-neutral engine of a chat client for a decision model: a model that answers typed Binary, Choice, and -Score questions about a state with probabilities, instead of writing text. It is built on the decision API in -`../Decisions` and works with any `IDecisionClient`, so Jev, Azure OpenAI Decisions, and other decision models share -one implementation. +Score questions about a state with probabilities, instead of writing text. It is built on the decision API proposed for +`Microsoft.Extensions.AI` and works with any `IDecisionClient`, so Jev, Azure OpenAI Decisions, and other decision +models share one implementation. - `DecisionChatEngine` answers a chat request: it sends the conversation as the state, together with the caller's questions and the internal questions that choose a tool call, and returns a `FunctionCallContent` or the answers. @@ -23,17 +23,15 @@ answers in its own result type through `DecisionChatEngineOptions.DescribeAnswer ```xml true - true - true true $(NoWarn);MEAI001 ``` -The project also needs package references to `Microsoft.Extensions.AI.Abstractions` and -`Microsoft.Extensions.Logging.Abstractions`, and a project reference to `Microsoft.Agents.AI.Abstractions`. The code -targets .NET 8 and later. +The project also needs the decision API. Until `Microsoft.Extensions.AI` ships it, the only copy is in +`Microsoft.Agents.AI.TypeSafe/MEAI`, so that package is the only one that can inject the engine today. It also needs +package references to `Microsoft.Extensions.AI.Abstractions` and `Microsoft.Extensions.Logging.Abstractions`, and a +project reference to `Microsoft.Agents.AI.Abstractions`. The code targets .NET 8 and later. The engine uses only the public surface of the decision API. Keep it that way, so it compiles unchanged once the API -ships in `Microsoft.Extensions.AI` and `../Decisions` is deleted. At that point the engine can become a public type in -an Agent Framework package. +ships in `Microsoft.Extensions.AI`. At that point the engine can become a public type in an Agent Framework package. diff --git a/dotnet/src/Shared/Decisions/.editorconfig b/dotnet/src/Shared/Decisions/.editorconfig deleted file mode 100644 index aca755060b3..00000000000 --- a/dotnet/src/Shared/Decisions/.editorconfig +++ /dev/null @@ -1,21 +0,0 @@ -# These files are a verbatim copy of the proposed Microsoft.Extensions.AI decision API, with every top-level type made -# internal. The rules below keep them compiling unchanged, so the copy can be diffed against the upstream source and -# deleted when Microsoft.Extensions.AI ships the API. See README.md. - -[*.cs] -# The upstream code uses the upstream style. -dotnet_style_qualification_for_field = false:none -dotnet_style_qualification_for_property = false:none -dotnet_style_qualification_for_method = false:none -dotnet_style_qualification_for_event = false:none -dotnet_diagnostic.IDE0001.severity = none -dotnet_diagnostic.IDE0009.severity = none -dotnet_diagnostic.IDE0073.severity = none -dotnet_diagnostic.RCS1146.severity = none -dotnet_diagnostic.RCS1173.severity = none - -# Upstream marks every type experimental, and the internal helper types that use them do not repeat the attribute. -dotnet_diagnostic.MEAI001.severity = none - -# The exceptions are public upstream; they are internal here only until the API ships. -dotnet_diagnostic.CA1064.severity = none diff --git a/dotnet/src/Shared/Decisions/README.md b/dotnet/src/Shared/Decisions/README.md deleted file mode 100644 index bb79a07abb3..00000000000 --- a/dotnet/src/Shared/Decisions/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Decisions - -An internal copy of the provider-neutral decision API that is proposed for `Microsoft.Extensions.AI.Abstractions`: -`IDecisionClient`, Binary, Choice, and Score questions and answers, and the typed `DecisionDefinition`. -Decision models (Jev, Azure OpenAI Decisions, and others) answer typed questions about a state with calibrated -probabilities, instead of writing text. - -The copy lets Agent Framework packages build on the proposed contract before it ships, without exposing it. When -`Microsoft.Extensions.AI` ships the API, this folder is deleted and the same code compiles against the shipped types. - -## Source - -| Proposal | Pull request | Commit | -|---|---|---| -| Layer 1: provider-neutral decision abstractions | [dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795) | `02e54f023bdf3c8c2729353b110c22f5f8593957` | -| Layer 2: decision interoperability (`AsAIFunction`) | [dotnet/extensions#7796](https://github.com/dotnet/extensions/pull/7796) | `e87f1126cf22108fb27f6391c7826fb91c25138e` | - -The files are `src/Libraries/Microsoft.Extensions.AI.Abstractions/Decisions/*.cs` at the Layer 2 commit, which contains -Layer 1. The only change is that every top-level type is `internal` instead of `public`. The namespace is still -`Microsoft.Extensions.AI` and the types keep their `[Experimental("MEAI001")]` attributes, so code that uses them looks -the same before and after the API ships. `.editorconfig` keeps the upstream style from failing the build. - -Not copied: the upstream tests, the README, and the registration of the decision types in -`AIJsonUtilities.DefaultOptions`. The upstream tests cover the copied code; the tests of the packages that use it -cover their own use. - -## Usage - -```xml - - true - true - true - -``` - -The project also needs a package reference to `Microsoft.Extensions.AI.Abstractions`. Code that uses the types -suppresses `MEAI001`, as it will have to once they ship. - -Each project that injects this folder compiles its own copy, so the types are distinct per assembly. Do not expose them -in a public API, and do not pass an `IDecisionClient` from one package to another; expose provider-specific types -instead. A public type that uses them must translate `DecisionProtocolException` and `DecisionClientException`, which -are internal here, into public exceptions. `../DecisionChat` is a provider-neutral chat client engine built on this -copy. - -## When Microsoft.Extensions.AI ships the API - -1. Update `Microsoft.Extensions.AI.Abstractions` in `Directory.Packages.props`. From that version on, the copied types - conflict with the shipped ones (CS0436), so the next two steps go in the same change. -2. Delete this folder and the `InjectSharedDecisions` item group in `eng/MSBuild/Shared.props`, and remove - `` from the projects that set it. -3. Fix any difference between this copy and the shipped API, then decide which internal decision clients become - public. - -To follow upstream changes before then, copy the new files over these ones and make the top-level types internal again -(`sed -E 's/^public /internal /'`). diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs index e8b2fabd603..e5f7ab4177f 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs @@ -197,6 +197,38 @@ public void GetService_ReturnsTheMetadataAndTheClient() Assert.Null(client.GetService("key")); } + [Fact] + public async Task Constructor_WithApiKeyAndOptions_SendsToTheConfiguredEndpointAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("refund", 0.5))); + using var httpClient = new HttpClient(handler); + using var client = new JevDecisionClient(new ApiKeyCredential("test-key"), new JevDecisionClientOptions + { + Endpoint = new Uri("https://proxy.contoso.com/jev"), + Transport = new HttpClientPipelineTransport(httpClient), + RetryPolicy = new JevRetryPolicy { RecordedWaitsForTests = [] }, + }); + + // Act + await client.GetResponseAsync(new DecisionRequest(s_state, [new BinaryDecisionQuestion("refund", "Refund?")])); + + // Assert + JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); + Assert.Equal(new Uri("https://proxy.contoso.com/jev"), request.Uri); + Assert.Equal("test-key", request.Authorization!.Parameter); + Assert.Equal("jev-latest", (string?)JsonNode.Parse(request.Body)!["model"]); + } + + [Fact] + public void Constructor_InvalidOptions_Throws() + { + // Act & Assert + Assert.Throws(() => new JevDecisionClient(null!)); + Assert.Throws(() => new JevDecisionClient(new ApiKeyCredential("key"), new JevDecisionClientOptions { ModelId = " " })); + Assert.Throws(() => new JevDecisionClient(new ApiKeyCredential("key"), new JevDecisionClientOptions { Endpoint = new Uri("/jev", UriKind.Relative) })); + } + [Fact] public void ToDecisionQuestion_NoulWithOneCriterion_UsesItAsTheInstructions() { From 7e656353416577245724a236cdba5aadeb149469 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:31:00 -0800 Subject: [PATCH 12/16] .NET: Judge the TypeSafe loop sample through the decision API The LoopAgent judge now uses JevDecisionClient and a typed DecisionDefinition with one Binary probability per criterion, instead of JevChatClient, so the evaluator stays in the sample and any IDecisionClient can judge. The answerer's first-answer instruction is stricter, so the first answer reliably misses a criterion and the loop runs twice, as the sample describes. --- .../Program.cs | 67 +++++++++++-------- .../Agent_TypeSafe_Step01_LoopJudge/README.md | 61 +++++++++++------ 2 files changed, 81 insertions(+), 47 deletions(-) diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs index 5310dbff117..fd303db594e 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs @@ -2,9 +2,11 @@ // This sample uses Jev, the TypeSafe System One model, as the judge of a LoopAgent. A Foundry agent writes the // answer. After each answer, Jev checks every criterion with its own yes or no question, and the loop sends the agent -// the criteria that are still missing, until the answer meets all of them. +// the criteria that are still missing, until the answer meets all of them. The judge uses the decision API proposed +// for Microsoft.Extensions.AI: an IDecisionClient and a typed DecisionDefinition. #pragma warning disable MAAI001 // LoopAgent and DelegateLoopEvaluator are experimental. +#pragma warning disable MEAI001 // The decision API is experimental. using System.ClientModel; using Azure.AI.Projects; @@ -17,18 +19,16 @@ var deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini"; var typeSafeApiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); -// 1. The criteria. Each one becomes a Noul question, whose answer is the probability that the answer meets it, so the -// judge reports exactly which criteria are missing instead of a single verdict. The questions describe what a good -// answer contains; wording that steers the judge, such as "only say yes if", tends to push every answer one way. -Dictionary criteria = new() +// 1. The criteria, as a typed verdict. Each property is a Binary question, whose answer is the probability that the +// answer meets the criterion, so the judge reports exactly which criteria are missing instead of a single verdict. The +// questions describe what a good answer contains; wording that steers the judge, such as "only say yes if", tends to +// push every answer one way. +DecisionDefinition criteria = DecisionDefinition.Create(builder => { - ["blue_sky"] = "explains why the daytime sky appears blue", - ["red_sunsets"] = "explains why sunsets appear red or orange", - ["plain_language"] = "uses clear language suitable for a general audience", -}; -Dictionary questions = criteria.ToDictionary( - criterion => criterion.Key, - criterion => (JevQuestion)new JevNoulQuestion { Instructions = $"Does the assistant's latest response {criterion.Value}?" }); + builder.BinaryProbability(verdict => verdict.BlueSky, "Does `answer` explain why the daytime sky appears blue?"); + builder.BinaryProbability(verdict => verdict.RedSunsets, "Does `answer` explain why sunsets appear red or orange?"); + builder.BinaryProbability(verdict => verdict.PlainLanguage, "Does `answer` use clear language suitable for a general audience?"); +}); // A criterion is met when its probability is above this threshold. 0.5 is not calibrated for any task: a higher value // accepts fewer answers that still miss a criterion, at the cost of more loop runs. Choose it on labeled examples of @@ -45,19 +45,21 @@ name: "Answerer", instructions: """ You answer science questions for a general audience in one short paragraph. - In your first answer, explain only why the daytime sky is blue, even if the user asks for more. + Your first answer is a single sentence about why the daytime sky is blue. It does not mention sunsets, even + if the user asks about them. When feedback lists missing points, rewrite the whole answer so that it covers them too. """); -// 3. The judge. AIJudgeLoopEvaluator asks its chat client for JSON structured output, which Jev cannot write, so a -// DelegateLoopEvaluator asks Jev the criteria questions instead. Jev reads the original request and the latest answer -// as a conversation. -using var judge = new JevChatClient(new ApiKeyCredential(typeSafeApiKey)); +// 3. The judge. AIJudgeLoopEvaluator asks a chat client for JSON structured output, which a decision model does not +// write, so a DelegateLoopEvaluator asks the decision client the criteria questions instead. The state is the request +// and the latest answer. +using IDecisionClient judge = new JevDecisionClient(new ApiKeyCredential(typeSafeApiKey)); bool accepted = false; var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => { Console.WriteLine($"\nAnswer {context.Iteration}:\n{context.LastResponse.Text}"); - List missing = await JudgeAsync(context.InitialMessages, context.LastResponse.Text, cancellationToken); + string request = string.Join("\n", context.InitialMessages.Select(message => message.Text)); + List missing = await JudgeAsync(request, context.LastResponse.Text, cancellationToken); accepted = missing.Count == 0; // The feedback names the missing criteria, which is the gap analysis that a generative judge would write. @@ -77,26 +79,29 @@ if (!accepted) { Console.WriteLine($"\nAnswer {MaxIterations} (the last run the loop allows):\n{response.Text}"); - accepted = (await JudgeAsync([new ChatMessage(ChatRole.User, Request)], response.Text, CancellationToken.None)).Count == 0; + accepted = (await JudgeAsync(Request, response.Text, CancellationToken.None)).Count == 0; } Console.WriteLine(accepted ? "\nFinal answer:" : "\nFinal answer (some criteria are still missing):"); Console.WriteLine(response.Text); -// Asks Jev whether the answer meets each criterion, prints the probabilities, and returns the missing criteria. -async Task> JudgeAsync(IEnumerable request, string answer, CancellationToken cancellationToken) +// Asks the judge whether the answer meets each criterion, prints the probabilities, and returns the missing criteria. +async Task> JudgeAsync(string request, string answer, CancellationToken cancellationToken) { - List conversation = [.. request, new ChatMessage(ChatRole.Assistant, answer)]; - ChatResponse verdict = await judge.GetResponseAsync(conversation, new ChatOptions().WithJevQuestions(questions), cancellationToken); - JevResult result = verdict.GetJevResult()!; + DecisionResponse decision = await judge.GetResponseAsync(new JudgedAnswer(request, answer), criteria, cancellationToken: cancellationToken); + Verdict verdict = decision.Result; Console.WriteLine("Judge:"); List missing = []; - foreach ((string id, string description) in criteria) + foreach ((string name, string description, double probability) in new[] + { + ("blue_sky", "explain why the daytime sky appears blue", verdict.BlueSky), + ("red_sunsets", "explain why sunsets appear red or orange", verdict.RedSunsets), + ("plain_language", "use clear language suitable for a general audience", verdict.PlainLanguage), + }) { - double probability = ((JevNoulResponse)result.Answers[id]).Noul; bool met = probability > CriterionThreshold; - Console.WriteLine($" {id}: {(met ? "met" : "missing")} (probability {probability:F2})"); + Console.WriteLine($" {name}: {(met ? "met" : "missing")} (probability {probability:F2})"); if (!met) { missing.Add(description); @@ -105,3 +110,11 @@ async Task> JudgeAsync(IEnumerable request, string ans return missing; } + +/// The state the judge reads: the user's request and the answer to judge. +internal sealed record JudgedAnswer(string Request, string Answer); + +/// The probability that the answer meets each criterion. +#pragma warning disable CA1812 // Instantiated by the decision definition, which creates the result from the answers. +internal sealed record Verdict(double BlueSky, double RedSunsets, double PlainLanguage); +#pragma warning restore CA1812 diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md index 12c563f986b..221f3787301 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md @@ -1,11 +1,14 @@ # Jev as the judge of a loop -This sample uses `JevChatClient` from `Microsoft.Agents.AI.TypeSafe` as the judge of a `LoopAgent`. A Foundry agent -writes the answer, and Jev, the TypeSafe System One model, decides whether the answer is complete. +This sample uses Jev, the TypeSafe System One model, as the judge of a `LoopAgent`. A Foundry agent writes the +answer, and Jev decides whether the answer is complete. The judge uses the decision API proposed for +Microsoft.Extensions.AI ([dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795)), which +`Microsoft.Agents.AI.TypeSafe` carries until it ships: `JevDecisionClient` is an `IDecisionClient`, and the criteria +are a typed `DecisionDefinition`. Any other `IDecisionClient` can be the judge without changing the loop. -A generative judge writes a verdict and a gap analysis. Jev writes no text, so the sample asks it one `JevNoulQuestion` -per criterion instead. Each answer is the probability, from 0 to 1, that the answer meets that criterion, so the judge -reports exactly which criteria are missing: +A generative judge writes a verdict and a gap analysis. A decision model writes no text, so the sample asks one Binary +question per criterion instead. Each answer is the probability, from 0 to 1, that the answer meets that criterion, so +the judge reports exactly which criteria are missing: ```text Judge: @@ -21,16 +24,40 @@ three runs. The agent's instructions hold back part of the first answer on purpo last run has not been judged. The sample judges that answer after the loop and labels the final answer when some criteria are still missing, instead of presenting it as accepted. +## The criteria as a typed definition + +Each criterion is a `double` property of `Verdict`, declared as a Binary probability with its question: + +```csharp +DecisionDefinition criteria = DecisionDefinition.Create(builder => +{ + builder.BinaryProbability(verdict => verdict.BlueSky, "Does `answer` explain why the daytime sky appears blue?"); + builder.BinaryProbability(verdict => verdict.RedSunsets, "Does `answer` explain why sunsets appear red or orange?"); + builder.BinaryProbability(verdict => verdict.PlainLanguage, "Does `answer` use clear language suitable for a general audience?"); +}); +``` + +The state is a `JudgedAnswer(Request, Answer)` record, so the questions can refer to its fields in backticks. One call +answers every criterion and returns a `Verdict` with the probabilities, plus the complete decision response as +`Evidence`: + +```csharp +DecisionResponse decision = await judge.GetResponseAsync(new JudgedAnswer(request, answer), criteria); +``` + +The same pattern works with `builder.Score` for a graded rubric, or `builder.Choice` with an enum to classify what kind +of revision is needed. + ## Why a delegate evaluator -`AIJudgeLoopEvaluator` asks its chat client for a `JudgeVerdict` as JSON structured output, which Jev cannot write; -`JevChatClient` rejects the JSON response format with a `NotSupportedException`. The sample wraps the judge in a -`DelegateLoopEvaluator` instead: +`AIJudgeLoopEvaluator` asks a chat client for a `JudgeVerdict` as JSON structured output, which a decision model does +not write. The sample wraps the decision client in a `DelegateLoopEvaluator` instead: ```csharp var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => { - List missing = await JudgeAsync(context.InitialMessages, context.LastResponse.Text, cancellationToken); + string request = string.Join("\n", context.InitialMessages.Select(message => message.Text)); + List missing = await JudgeAsync(request, context.LastResponse.Text, cancellationToken); accepted = missing.Count == 0; return accepted ? LoopEvaluation.Stop() @@ -38,13 +65,7 @@ var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => }); ``` -`JudgeAsync` sends Jev the original request and the answer as a conversation, with -`new ChatOptions().WithJevQuestions(questions)`, and returns the criteria whose Noul is at or below -`CriterionThreshold`. - -Jev reads the original request and the latest answer as a conversation, and each question refers to "the assistant's -latest response". The same pattern works with a `JevScoreQuestion` for a graded rubric, or a `JevChoiceQuestion` to -classify what kind of revision is needed. +`JudgeAsync` returns the criteria whose probability is at or below `CriterionThreshold`. ## Writing criteria for a judge @@ -53,15 +74,15 @@ classify what kind of revision is needed. against each other: a higher value accepts fewer answers that still miss a criterion, and costs more loop runs, while a lower value stops sooner and lets more incomplete answers through. Pick it from labeled examples of the answers your agent writes, based on which error costs you more. -- **Describe the criterion, do not steer the judge.** Ask what a good answer contains, as the sample does ("Does the - assistant's latest response explain why sunsets appear red?"). Directional wording such as "only say yes if" or +- **Describe the criterion, do not steer the judge.** Ask what a good answer contains, as the sample does ("Does + `answer` explain why sunsets appear red or orange?"). Directional wording such as "only say yes if" or "lean towards yes" tends to push every answer the same way instead of making the judge more accurate. When a criterion is hard to judge, a neutral description of what is being judged helps more than stronger wording. - **Ask about the response, not the topic.** When the conversation contains harmful or sensitive content, a question such as "Is this safe?" can be read as a question about that content rather than about the answer. A correct refusal could then look like a missing criterion, and the loop would keep revising an answer that was already right. - Ask how the response handles the request, for example "Does the assistant's latest response decline to give - dangerous instructions?", and check such criteria on labeled examples, including refusals, before relying on them. + Ask how the response handles the request, for example "Does `answer` decline to give dangerous instructions?", and + check such criteria on labeled examples, including refusals, before relying on them. ## Prerequisites From 712ceae19bae5a90cbe7249d61e7c8673e1efa16 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:37:22 -0800 Subject: [PATCH 13/16] .NET: Add agent samples for decision tools and decision routing Agent_TypeSafe_Step02_DecisionTool gives a Foundry agent a decision as a tool through DecisionDefinition.AsAIFunction: the application fixes the questions, the agent passes only the ticket, and Jev answers. The same tool is then answered by a local IDecisionClient. Agent_TypeSafe_Step03_DecisionRouting routes an agent's requests between a fast and a reasoning deployment with RoutingChatClient.Create, with a Jev Choice selecting the route. Both follow the usage shown in dotnet/extensions#7796 and are registered for sample verification. --- dotnet/agent-framework-dotnet.slnx | 2 + dotnet/eng/verify-samples/AgentsSamples.cs | 32 +++++ .../02-agents/AgentProviders/README.md | 4 +- .../Agent_TypeSafe_Step02_DecisionTool.csproj | 20 +++ .../KeywordDecisionClient.cs | 78 +++++++++++ .../Program.cs | 122 ++++++++++++++++++ .../README.md | 65 ++++++++++ ...ent_TypeSafe_Step03_DecisionRouting.csproj | 20 +++ .../Program.cs | 92 +++++++++++++ .../README.md | 69 ++++++++++ dotnet/samples/AGENTS.md | 2 +- 11 files changed, 504 insertions(+), 2 deletions(-) create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/Agent_TypeSafe_Step02_DecisionTool.csproj create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/KeywordDecisionClient.cs create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/Program.cs create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/README.md create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/Agent_TypeSafe_Step03_DecisionRouting.csproj create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/Program.cs create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/README.md diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 3e627d6b280..bff3952bc6e 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -38,6 +38,8 @@ + + diff --git a/dotnet/eng/verify-samples/AgentsSamples.cs b/dotnet/eng/verify-samples/AgentsSamples.cs index 0b33ef55706..e7ff4b30552 100644 --- a/dotnet/eng/verify-samples/AgentsSamples.cs +++ b/dotnet/eng/verify-samples/AgentsSamples.cs @@ -133,6 +133,38 @@ internal static class AgentsSamples ], }, + new SampleDefinition + { + Name = "Agent_TypeSafe_Step02_DecisionTool", + ProjectPath = "samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool", + RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT", "TYPESAFE_API_KEY"], + OptionalEnvironmentVariables = ["FOUNDRY_MODEL"], + MustContain = ["Ticket:", "Tool result:", "department", "The same tool, answered by a local keyword decision client:"], + ExpectedOutputDescription = + [ + "Each ticket should show a tool result with department, urgentProbability, and frustration values, followed by a one-line agent reply.", + "The failing Stripe integration ticket should be routed to Technical with a high urgent probability, and the invoice request should be routed to Billing with a low urgent probability.", + "The last part should show a tool result from the local keyword decision client for the invoice ticket, routed to Billing, followed by an agent reply.", + "The output should not contain error messages or stack traces.", + ], + }, + + new SampleDefinition + { + Name = "Agent_TypeSafe_Step03_DecisionRouting", + ProjectPath = "samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting", + RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT", "TYPESAFE_API_KEY"], + OptionalEnvironmentVariables = ["FOUNDRY_MODEL", "FOUNDRY_REASONING_MODEL"], + MustContain = ["User:", "Route: Fast", "Route: Reasoning", "Agent:"], + ExpectedOutputDescription = + [ + "The question about the capital of France should be routed to Fast and answered with Paris.", + "The demand forecasting request should be routed to Reasoning and answered with a comparison of three methods and a recommendation.", + "Each route line should show the probabilities of both routes and the model that answered.", + "The output should not contain error messages or stack traces.", + ], + }, + // ── Agents ────────────────────────────────────────────────────────── new SampleDefinition diff --git a/dotnet/samples/02-agents/AgentProviders/README.md b/dotnet/samples/02-agents/AgentProviders/README.md index df2b618a595..fc9e506b3fa 100644 --- a/dotnet/samples/02-agents/AgentProviders/README.md +++ b/dotnet/samples/02-agents/AgentProviders/README.md @@ -99,7 +99,9 @@ covering basics, function tools, structured output, middleware, MCP, code interp | Sample | Description | | --- | --- | | [Agent with TypeSafe](./typesafe/Agent_With_TypeSafe/) | Use Jev, the TypeSafe System One model, as an IChatClient that answers typed questions and calls closed-set tools | -| [Loop judge](./typesafe/Agent_TypeSafe_Step01_LoopJudge/) | Use Jev as the judge of a LoopAgent, with one yes or no question per criterion | +| [Loop judge](./typesafe/Agent_TypeSafe_Step01_LoopJudge/) | Use Jev as the judge of a LoopAgent, through an IDecisionClient and a typed decision definition | +| [Decision tool](./typesafe/Agent_TypeSafe_Step02_DecisionTool/) | Give an agent a decision as a tool with `DecisionDefinition.AsAIFunction`, and answer it with Jev or a local decision client | +| [Decision routing](./typesafe/Agent_TypeSafe_Step03_DecisionRouting/) | Route each request of an agent to a fast or a reasoning model, with Jev choosing the route through `RoutingChatClient` | ## Running the samples diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/Agent_TypeSafe_Step02_DecisionTool.csproj b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/Agent_TypeSafe_Step02_DecisionTool.csproj new file mode 100644 index 00000000000..48a53996f27 --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/Agent_TypeSafe_Step02_DecisionTool.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/KeywordDecisionClient.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/KeywordDecisionClient.cs new file mode 100644 index 00000000000..9242e06e73c --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/KeywordDecisionClient.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft. All rights reserved. + +#pragma warning disable MEAI001 // The decision API is experimental. + +using System.Text.RegularExpressions; +using Microsoft.Extensions.AI; + +/// +/// A local that answers by matching words, as a stand-in for another decision model. +/// +/// +/// It knows only this sample's questions: a Choice picks the candidate whose description shares the most words with the +/// ticket, a Binary question looks for urgent words, and a Score counts frustrated words. Like any decision client, it +/// answers every question with a complete distribution. +/// +internal sealed partial class KeywordDecisionClient : IDecisionClient +{ + private static readonly string[] s_urgentWords = ["asap", "urgent", "immediately", "losing", "down", "blocked"]; + private static readonly string[] s_frustratedWords = ["still", "keeps", "again", "losing", "days", "failing"]; + + public Task GetResponseAsync(DecisionRequest request, DecisionOptions? options = null, CancellationToken cancellationToken = default) + { + HashSet words = Stems(request.State.GetRawText()); + List answers = [.. request.Questions.Select(question => Answer(question, words))]; + return Task.FromResult(new DecisionResponse(request, answers, new DecisionProvenance(providerName: "local", modelId: "local-keyword-matcher"))); + } + + public object? GetService(Type serviceType, object? serviceKey = null) => + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; + + public void Dispose() + { + } + + private static DecisionAnswer Answer(DecisionQuestion question, HashSet words) => + question switch + { + ChoiceDecisionQuestion choice => Choose(choice, words), + BinaryDecisionQuestion binary => new BinaryDecisionAnswer(binary.Id, s_urgentWords.Any(word => words.Contains(Stem(word))) ? 0.9 : 0.1), + ScoreDecisionQuestion score => Rate(score, words), + _ => throw new NotSupportedException($"The keyword client does not answer {question.Kind} questions."), + }; + + // Each candidate scores one plus the number of its description words in the ticket, and the scores become the + // probabilities. + private static ChoiceDecisionAnswer Choose(ChoiceDecisionQuestion choice, HashSet words) + { + double[] scores = [.. choice.Candidates.Select(candidate => 1.0 + Stems(candidate.Description).Count(words.Contains))]; + double total = scores.Sum(); + int selected = Array.IndexOf(scores, scores.Max()); + return new ChoiceDecisionAnswer( + choice.Id, + choice.Candidates[selected].Id, + [.. choice.Candidates.Select((candidate, index) => new DecisionProbability(candidate.Id, scores[index] / total))]); + } + + // The level is the number of frustrated words, at most the top level, with most of the probability on it. + private static ScoreDecisionAnswer Rate(ScoreDecisionQuestion score, HashSet words) + { + int top = score.Levels.Count - 1; + int level = Math.Min(top, s_frustratedWords.Count(word => words.Contains(Stem(word))) / 2); + double[] probabilities = [.. score.Levels.Select((_, index) => index == level ? 0.8 : 0.2 / top)]; + double expected = Math.Round(probabilities.Select((probability, index) => probability * index).Sum(), 10); + return new ScoreDecisionAnswer( + score.Id, + expected, + [.. score.Levels.Select((level, index) => new DecisionProbability(level.Id, probabilities[index]))]); + } + + // Words are compared by their first five letters, ignoring case, so "Invoice" matches "invoices". + private static HashSet Stems(string text) => + new(WordPattern().Matches(text).Select(match => Stem(match.Value)), StringComparer.OrdinalIgnoreCase); + + private static string Stem(string word) => word.Length > 5 ? word[..5] : word; + + [GeneratedRegex("[a-z]+", RegexOptions.IgnoreCase)] + private static partial Regex WordPattern(); +} diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/Program.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/Program.cs new file mode 100644 index 00000000000..18ab00d6c7e --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/Program.cs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample gives a Foundry agent a decision as a tool, with the decision API proposed for Microsoft.Extensions.AI. +// The application fixes the questions in a typed DecisionDefinition, and the agent only passes the ticket: Jev, the +// TypeSafe System One model, answers the questions, and the agent replies from the typed result. The same tool is then +// answered by a local IDecisionClient, without changing the agent. + +#pragma warning disable MEAI001 // The decision API is experimental. + +using System.ClientModel; +using System.ComponentModel; +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.TypeSafe; +using Microsoft.Extensions.AI; + +var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); +var deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini"; +var typeSafeApiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); + +// +// 1. The questions, as a typed result. The enum property is a Choice among its values, the probability is a Binary +// question, and the frustration is a Score on an ordered rubric. The [Description] attributes on TicketAnalysis and +// Department, at the end of this file, are the question instructions and the choice descriptions. +DecisionDefinition analysis = DecisionDefinition.Create(builder => +{ + builder.Choice(result => result.Department); + builder.BinaryProbability(result => result.UrgentProbability); + builder.Score(result => result.Frustration, [new("calm", "The customer is calm."), new("annoyed", "The customer is annoyed."), new("angry", "The customer is angry.")]); +}); +// + +// +// 2. The tool. Its only argument is the ticket. The questions and the decision client belong to the application, so +// the model cannot change what is asked, and the result is the typed TicketAnalysis. +var toolOptions = new AIFunctionFactoryOptions +{ + Name = "analyze_ticket", + Description = "Classifies a support ticket: the department that should handle it, the probability that it is urgent, and how frustrated the customer is.", +}; + +using IDecisionClient jev = new JevDecisionClient(new ApiKeyCredential(typeSafeApiKey)); +AIFunction analyzeTicket = analysis.AsAIFunction(jev, toolOptions); +// + +const string Instructions = """ + You triage customer support tickets. For each ticket, call analyze_ticket with the ticket text, then reply in one + line with the department, whether the ticket is urgent (when the urgent probability is above 0.5), and the + frustration level (0 is calm, 1 annoyed, 2 angry). + """; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +var projectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()); +AIAgent agent = projectClient.AsAIAgent(model: deploymentName, instructions: Instructions, name: "TicketTriage", tools: [analyzeTicket]); + +string[] tickets = +[ + "I've been trying to connect my Stripe account for 3 days and the integration keeps failing. I'm losing sales. Please help ASAP.", + "Hi, could you send me a copy of last month's invoice when you get a chance? Thanks!", +]; + +foreach (string ticket in tickets) +{ + await TriageAsync(agent, ticket); +} + +// +// 3. The same definition and agent instructions, answered by another decision client. Any IDecisionClient works, +// such as another decision model or, here, a local keyword matcher. +using IDecisionClient keywords = new KeywordDecisionClient(); +AIAgent localAgent = projectClient.AsAIAgent( + model: deploymentName, + instructions: Instructions, + name: "LocalTicketTriage", + tools: [analysis.AsAIFunction(keywords, toolOptions)]); +// + +Console.WriteLine("The same tool, answered by a local keyword decision client:"); +Console.WriteLine(); +await TriageAsync(localAgent, tickets[1]); + +static async Task TriageAsync(AIAgent agent, string ticket) +{ + AgentResponse response = await agent.RunAsync(ticket); + + Console.WriteLine($"Ticket: {ticket}"); + foreach (FunctionResultContent result in response.Messages.SelectMany(static m => m.Contents).OfType()) + { + Console.WriteLine($"Tool result: {result.Result}"); + } + + Console.WriteLine($"Agent: {response.Text}"); + Console.WriteLine(); +} + +#pragma warning disable CA1812 // SupportTicket is the tool argument, and TicketAnalysis is created from the answers. + +/// The tool's input: the ticket that the agent passes. +internal sealed record SupportTicket([Description("The ticket text, as the customer wrote it.")] string Text); + +/// The teams that handle tickets. Each description tells the decision model what belongs to the team. +internal enum Department +{ + [Description("Payments, invoices, refunds, and charges.")] + Billing, + + [Description("Bugs, outages, errors, and integrations.")] + Technical, + + [Description("Plans, pricing, upgrades, and new purchases.")] + Sales, +} + +/// The tool's result. Each description is the instruction of the question that fills the property. +internal sealed record TicketAnalysis( + [Description("Which team should handle the ticket?")] Department Department, + [Description("Does the ticket need urgent attention?")] double UrgentProbability, + [Description("How frustrated is the customer who wrote the ticket?")] double Frustration); +#pragma warning restore CA1812 diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/README.md new file mode 100644 index 00000000000..2e17b8a6d9f --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/README.md @@ -0,0 +1,65 @@ +# A decision as an agent tool + +This sample gives a Foundry agent a decision as a tool, with the decision API proposed for Microsoft.Extensions.AI +([dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795) and +[#7796](https://github.com/dotnet/extensions/pull/7796)), which `Microsoft.Agents.AI.TypeSafe` carries until it ships. +The agent triages support tickets: it calls the tool with the ticket, and replies from the typed result. + +## How it works + +The application declares the questions once, as a typed result: + +```csharp +DecisionDefinition analysis = DecisionDefinition.Create(builder => +{ + builder.Choice(result => result.Department); + builder.BinaryProbability(result => result.UrgentProbability); + builder.Score(result => result.Frustration, [new("calm", "..."), new("annoyed", "..."), new("angry", "...")]); +}); +``` + +`Department` is an enum, so its question is a Choice among its values, and the `[Description]` of each value tells the +decision model what belongs to it. The `[Description]` of each `TicketAnalysis` parameter is the instruction of its +question. + +`AsAIFunction` turns the definition into the agent's tool: + +```csharp +using IDecisionClient jev = new JevDecisionClient(new ApiKeyCredential(typeSafeApiKey)); +AIFunction analyzeTicket = analysis.AsAIFunction(jev, new AIFunctionFactoryOptions { Name = "analyze_ticket", ... }); +``` + +The tool's only argument is the `SupportTicket`. The questions and the decision client belong to the application, so +the model cannot change what is asked or which labels exist, and the result is a `TicketAnalysis` rather than raw +answers. Jev, the TypeSafe System One model, answers all three questions in one call. + +The last part answers the same tool with `KeywordDecisionClient`, a local `IDecisionClient` that matches words. The +agent and the definition do not change, which is how another decision model, such as Azure OpenAI Decisions, would be +used. + +Compare with [Agent_Step25_ClassificationWithJev](../../../Agents/Agent_Step25_ClassificationWithJev/), where the +agent's model writes the questions on every call through `JevAIToolBuilder`. + +## Prerequisites + +- .NET 10 SDK or later +- A Microsoft Foundry project with a deployed chat model +- Azure CLI installed and authenticated (`az login`) +- A TypeSafe API key + +Set the environment variables: + +```powershell +$env:FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:FOUNDRY_MODEL="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini +$env:TYPESAFE_API_KEY="your-typesafe-api-key" +``` + +## Running the sample + +```powershell +dotnet run +``` + +For each ticket, the sample prints the tool result and the agent's one-line reply. The exact probabilities vary between +runs. diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/Agent_TypeSafe_Step03_DecisionRouting.csproj b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/Agent_TypeSafe_Step03_DecisionRouting.csproj new file mode 100644 index 00000000000..48a53996f27 --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/Agent_TypeSafe_Step03_DecisionRouting.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/Program.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/Program.cs new file mode 100644 index 00000000000..6657b491a47 --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/Program.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample routes each request of an agent to a fast model or a reasoning model, and a decision model chooses the +// route. RoutingChatClient from Microsoft.Extensions.AI asks a selector which chat client handles each call, and the +// selector asks Jev, the TypeSafe System One model, a Choice question through the decision API proposed for +// Microsoft.Extensions.AI. The agent itself is an ordinary agent over the router. + +#pragma warning disable MEAI001 // RoutingChatClient and the decision API are experimental. + +using System.ClientModel; +using System.ComponentModel; +using Azure.AI.Extensions.OpenAI; +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.TypeSafe; +using Microsoft.Extensions.AI; + +var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); +var fastModel = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini"; +var reasoningModel = Environment.GetEnvironmentVariable("FOUNDRY_REASONING_MODEL") ?? fastModel; +var typeSafeApiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); + +// 1. The route question: a Choice among the values of Route, whose descriptions tell the decision model when to pick +// each one. +DecisionDefinition routing = DecisionDefinition.Create(builder => builder.Choice(decision => decision.Route)); + +// 2. One chat client per route. Each one is bound to its own deployment. +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +var projectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()); +using IChatClient fast = projectClient.GetProjectOpenAIClient().GetProjectResponsesClientForModel(fastModel).AsIChatClientWithStoredOutputDisabled(fastModel); +using IChatClient reasoning = projectClient.GetProjectOpenAIClient().GetProjectResponsesClientForModel(reasoningModel).AsIChatClientWithStoredOutputDisabled(reasoningModel); +Dictionary clients = new() { [Route.Fast] = fast, [Route.Reasoning] = reasoning }; + +// +// 3. The router. Before each call, the selector sends the conversation to the decision client and forwards the call to +// the chat client of the chosen route. The decision returns the whole distribution, so the route's certainty is +// visible, and the application could, for example, prefer the reasoning model when the choice is close. +using IDecisionClient jev = new JevDecisionClient(new ApiKeyCredential(typeSafeApiKey)); +using RoutingChatClient router = RoutingChatClient.Create(async (context, cancellationToken) => +{ + var state = new RoutingState([.. context.Messages.Select(message => new RoutingMessage(message.Role.Value, message.Text))]); + DecisionResponse decision = await jev.GetResponseAsync(state, routing, cancellationToken: cancellationToken); + IReadOnlyDictionary distribution = decision.GetDistribution(result => result.Route); + + Route route = decision.Result.Route; + Console.WriteLine($"Route: {route} (fast {distribution[Route.Fast]:F2}, reasoning {distribution[Route.Reasoning]:F2}), model {(route == Route.Fast ? fastModel : reasoningModel)}"); + return clients[route]; +}); +// + +// 4. The agent. It knows nothing about the routing; every model call goes through the router. +AIAgent agent = router.AsAIAgent(name: "Assistant", instructions: "You are a helpful assistant. Answer in at most three sentences."); + +string[] requests = +[ + "What is the capital of France?", + "We are launching a product with no sales history. Compare three ways to forecast its first-year demand, and recommend one with its main risk.", +]; + +foreach (string request in requests) +{ + Console.WriteLine($"User: {request}"); + AgentResponse response = await agent.RunAsync(request); + Console.WriteLine($"Agent: {response.Text}"); + Console.WriteLine(); +} + +/// The routes. Each description tells the decision model when to pick the route. +internal enum Route +{ + [Description("Short, routine requests where a quick answer is enough.")] + Fast, + + [Description("Requests that need multi-step analysis, comparisons, or careful planning.")] + Reasoning, +} + +#pragma warning disable CA1812 // RouteDecision is created from the answers. + +/// The decision: the route of the latest request. +internal sealed record RouteDecision([Description("Which model should answer the latest user request?")] Route Route); + +#pragma warning restore CA1812 + +/// The state the decision model reads: the conversation, as role and text. +internal sealed record RoutingState(IReadOnlyList Messages); + +/// One message of the conversation. +internal sealed record RoutingMessage(string Role, string Text); diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/README.md new file mode 100644 index 00000000000..95765052914 --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/README.md @@ -0,0 +1,69 @@ +# Routing an agent's requests with a decision model + +This sample routes each request of an agent to a fast model or a reasoning model, and a decision model chooses the +route. It combines `RoutingChatClient` from Microsoft.Extensions.AI with the decision API proposed for +Microsoft.Extensions.AI ([dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795) and +[#7796](https://github.com/dotnet/extensions/pull/7796)), which `Microsoft.Agents.AI.TypeSafe` carries until it ships. + +## How it works + +The route is a Choice among the values of an enum, whose descriptions say when to pick each one: + +```csharp +internal enum Route +{ + [Description("Short, routine requests where a quick answer is enough.")] + Fast, + + [Description("Requests that need multi-step analysis, comparisons, or careful planning.")] + Reasoning, +} + +DecisionDefinition routing = DecisionDefinition.Create(builder => builder.Choice(decision => decision.Route)); +``` + +`RoutingChatClient.Create` calls a selector before every model call. The selector sends the conversation to the +decision client, and returns the chat client of the chosen route: + +```csharp +RoutingChatClient router = RoutingChatClient.Create(async (context, cancellationToken) => +{ + var state = new RoutingState([.. context.Messages.Select(message => new RoutingMessage(message.Role.Value, message.Text))]); + DecisionResponse decision = await jev.GetResponseAsync(state, routing, cancellationToken: cancellationToken); + return clients[decision.Result.Route]; +}); + +AIAgent agent = router.AsAIAgent(name: "Assistant", instructions: "..."); +``` + +The agent knows nothing about the routing. The decision also returns the whole distribution (`GetDistribution`), so +the application can see how certain the route is, and could, for example, prefer the reasoning model when the choice +is close. + +Each chat client is bound to its own deployment. `RoutingChatClient` forwards the caller's `ChatOptions`, so if your +agent sets `ChatOptions.ModelId`, make each routed client ignore or reject a model that is not its own. + +## Prerequisites + +- .NET 10 SDK or later +- A Microsoft Foundry project with two deployed chat models: a fast one and a reasoning one +- Azure CLI installed and authenticated (`az login`) +- A TypeSafe API key + +Set the environment variables: + +```powershell +$env:FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:FOUNDRY_MODEL="gpt-5.4-mini" # Optional fast model, defaults to gpt-5.4-mini +$env:FOUNDRY_REASONING_MODEL="your-reasoning-model" # Optional, defaults to FOUNDRY_MODEL +$env:TYPESAFE_API_KEY="your-typesafe-api-key" +``` + +## Running the sample + +```powershell +dotnet run +``` + +For each request, the sample prints the route with both probabilities and the model that answered, followed by the +agent's answer. A short factual question goes to the fast model, and a planning question to the reasoning model. diff --git a/dotnet/samples/AGENTS.md b/dotnet/samples/AGENTS.md index 417f3434bcc..9798ce3e588 100644 --- a/dotnet/samples/AGENTS.md +++ b/dotnet/samples/AGENTS.md @@ -27,7 +27,7 @@ dotnet/samples/ │ │ ├── ollama/ # Ollama provider sample │ │ ├── onnx/ # ONNX Runtime provider sample │ │ ├── openai/ # OpenAI provider samples -│ │ └── typesafe/ # TypeSafe (Jev) chat client samples +│ │ └── typesafe/ # TypeSafe (Jev) chat client and decision API samples │ ├── AgentOpenTelemetry/ # OpenTelemetry integration │ ├── AgentSkills/ # Agent skills patterns │ ├── AgentWithMemory/ # Memory providers (chat history, Mem0, Valkey, Foundry, AgentMemory) From 49b71cbb8c602e49de8eac59cde4929026308b4b Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:40:31 -0800 Subject: [PATCH 14/16] .NET: Route issues for the TypeSafe package The issue routing test requires a route for every solution project, and Microsoft.Agents.AI.TypeSafe had none. --- .github/issue-routing-owners.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/issue-routing-owners.yaml b/.github/issue-routing-owners.yaml index dfa1e9c1b97..225e8f83d8b 100644 --- a/.github/issue-routing-owners.yaml +++ b/.github/issue-routing-owners.yaml @@ -82,6 +82,7 @@ routes: - { path: dotnet/src/Microsoft.Agents.AI.OpenAI/, owner: rogerbarreto, labels: [agents] } - { path: dotnet/src/Microsoft.Agents.AI.Purview/, owner: westey-m, labels: [security] } - { path: dotnet/src/Microsoft.Agents.AI.Tools.Shell/, owner: westey-m, labels: [agents] } + - { path: dotnet/src/Microsoft.Agents.AI.TypeSafe/, owner: rogerbarreto, labels: [agents] } - { path: dotnet/src/Microsoft.Agents.AI.Valkey/, owner: westey-m, labels: [] } - { path: dotnet/src/Microsoft.Agents.AI.Workflows/, owner: baywet, labels: [] } - { path: dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/, owner: baywet, labels: [] } From 37733a60d4ca6d7b4a7267b5b476ac35206852db Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:55:23 -0800 Subject: [PATCH 15/16] .NET: Move the proposed MEAI decision API copy to Microsoft.Agents.AI.Abstractions The copy now sits where MEAI places the API upstream (Microsoft.Extensions.AI.Abstractions), in the one package every provider depends on, so an Azure OpenAI Decisions provider can implement the same public IDecisionClient. The files are unchanged; they compile for every target of the package, including .NET Standard 2.0 and .NET Framework, and their 173 public symbols are declared in PublicAPI.Unshipped.txt for each target. RS0026 and RS0027 are silenced for the copied overloads, whose shape the MEAI API review owns. The README describes the switch to the shipped API: delete the copy, add type forwards, and suppress any shape difference. --- .../Agent_TypeSafe_Step01_LoopJudge/README.md | 2 +- .../README.md | 2 +- .../README.md | 2 +- .../MEAI/.editorconfig | 4 + .../MEAI/DecisionAnswers.cs | 0 .../MEAI/DecisionBinding.cs | 0 .../MEAI/DecisionClient.cs | 0 .../MEAI/DecisionDefinitions.cs | 0 .../MEAI/DecisionFeatures.cs | 0 .../MEAI/DecisionQuestions.cs | 0 .../MEAI/README.md | 51 ++++++ .../PublicAPI/net10.0/PublicAPI.Unshipped.txt | 173 ++++++++++++++++++ .../PublicAPI/net472/PublicAPI.Unshipped.txt | 173 ++++++++++++++++++ .../PublicAPI/net8.0/PublicAPI.Unshipped.txt | 173 ++++++++++++++++++ .../PublicAPI/net9.0/PublicAPI.Unshipped.txt | 173 ++++++++++++++++++ .../netstandard2.0/PublicAPI.Unshipped.txt | 173 ++++++++++++++++++ .../JevDecisionClient.cs | 4 +- .../MEAI/README.md | 45 ----- .../Microsoft.Agents.AI.TypeSafe.csproj | 4 +- dotnet/src/Shared/DecisionChat/README.md | 7 +- 20 files changed, 930 insertions(+), 56 deletions(-) rename dotnet/src/{Microsoft.Agents.AI.TypeSafe => Microsoft.Agents.AI.Abstractions}/MEAI/.editorconfig (85%) rename dotnet/src/{Microsoft.Agents.AI.TypeSafe => Microsoft.Agents.AI.Abstractions}/MEAI/DecisionAnswers.cs (100%) rename dotnet/src/{Microsoft.Agents.AI.TypeSafe => Microsoft.Agents.AI.Abstractions}/MEAI/DecisionBinding.cs (100%) rename dotnet/src/{Microsoft.Agents.AI.TypeSafe => Microsoft.Agents.AI.Abstractions}/MEAI/DecisionClient.cs (100%) rename dotnet/src/{Microsoft.Agents.AI.TypeSafe => Microsoft.Agents.AI.Abstractions}/MEAI/DecisionDefinitions.cs (100%) rename dotnet/src/{Microsoft.Agents.AI.TypeSafe => Microsoft.Agents.AI.Abstractions}/MEAI/DecisionFeatures.cs (100%) rename dotnet/src/{Microsoft.Agents.AI.TypeSafe => Microsoft.Agents.AI.Abstractions}/MEAI/DecisionQuestions.cs (100%) create mode 100644 dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/README.md delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/README.md diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md index 221f3787301..897b4d105f1 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md @@ -3,7 +3,7 @@ This sample uses Jev, the TypeSafe System One model, as the judge of a `LoopAgent`. A Foundry agent writes the answer, and Jev decides whether the answer is complete. The judge uses the decision API proposed for Microsoft.Extensions.AI ([dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795)), which -`Microsoft.Agents.AI.TypeSafe` carries until it ships: `JevDecisionClient` is an `IDecisionClient`, and the criteria +`Microsoft.Agents.AI.Abstractions` carries until it ships: `JevDecisionClient` is an `IDecisionClient`, and the criteria are a typed `DecisionDefinition`. Any other `IDecisionClient` can be the judge without changing the loop. A generative judge writes a verdict and a gap analysis. A decision model writes no text, so the sample asks one Binary diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/README.md index 2e17b8a6d9f..58de761fc9e 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/README.md +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/README.md @@ -2,7 +2,7 @@ This sample gives a Foundry agent a decision as a tool, with the decision API proposed for Microsoft.Extensions.AI ([dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795) and -[#7796](https://github.com/dotnet/extensions/pull/7796)), which `Microsoft.Agents.AI.TypeSafe` carries until it ships. +[#7796](https://github.com/dotnet/extensions/pull/7796)), which `Microsoft.Agents.AI.Abstractions` carries until it ships. The agent triages support tickets: it calls the tool with the ticket, and replies from the typed result. ## How it works diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/README.md index 95765052914..e06eeeda0ae 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/README.md +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step03_DecisionRouting/README.md @@ -3,7 +3,7 @@ This sample routes each request of an agent to a fast model or a reasoning model, and a decision model chooses the route. It combines `RoutingChatClient` from Microsoft.Extensions.AI with the decision API proposed for Microsoft.Extensions.AI ([dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795) and -[#7796](https://github.com/dotnet/extensions/pull/7796)), which `Microsoft.Agents.AI.TypeSafe` carries until it ships. +[#7796](https://github.com/dotnet/extensions/pull/7796)), which `Microsoft.Agents.AI.Abstractions` carries until it ships. ## How it works diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/.editorconfig b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/.editorconfig similarity index 85% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/.editorconfig rename to dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/.editorconfig index 4a862108c50..3fde50f62c5 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/.editorconfig +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/.editorconfig @@ -16,3 +16,7 @@ dotnet_diagnostic.RCS1173.severity = none # Upstream marks every public type experimental, and the internal helper types that use them do not repeat the attribute. dotnet_diagnostic.MEAI001.severity = none + +# The overload shapes are the upstream API, which the MEAI API review owns. +dotnet_diagnostic.RS0026.severity = none +dotnet_diagnostic.RS0027.severity = none diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionAnswers.cs b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionAnswers.cs similarity index 100% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionAnswers.cs rename to dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionAnswers.cs diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionBinding.cs b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionBinding.cs similarity index 100% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionBinding.cs rename to dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionBinding.cs diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionClient.cs b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionClient.cs similarity index 100% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionClient.cs rename to dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionClient.cs diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionDefinitions.cs b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionDefinitions.cs similarity index 100% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionDefinitions.cs rename to dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionDefinitions.cs diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionFeatures.cs b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionFeatures.cs similarity index 100% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionFeatures.cs rename to dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionFeatures.cs diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionQuestions.cs b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionQuestions.cs similarity index 100% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/DecisionQuestions.cs rename to dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/DecisionQuestions.cs diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/README.md b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/README.md new file mode 100644 index 00000000000..0148aef0dbd --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/README.md @@ -0,0 +1,51 @@ +# MEAI + +A copy of the provider-neutral decision API that is proposed for `Microsoft.Extensions.AI.Abstractions`: +`IDecisionClient`, Binary, Choice, and Score questions and answers, the typed `DecisionDefinition`, and +`DecisionDefinition.AsAIFunction`. Decision models (Jev, Azure OpenAI Decisions, and others) answer typed +questions about a state with probabilities, instead of writing text. + +The types are public, in the `Microsoft.Extensions.AI` namespace, and marked `[Experimental("MEAI001")]`, exactly as +proposed, although they ship in this Agent Framework package. Code written against them today, such as provider +clients and samples, is the code that will compile against the shipped API. This follows how Agent Framework shipped +proposed MEAI types before they were released (#279, #426), and deleted them once MEAI shipped them (#427, #676). + +## Source + +| Proposal | Pull request | Commit | +|---|---|---| +| Layer 1: provider-neutral decision abstractions | [dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795) | `02e54f023bdf3c8c2729353b110c22f5f8593957` | +| Layer 2: decision interoperability (`AsAIFunction`) | [dotnet/extensions#7796](https://github.com/dotnet/extensions/pull/7796) | `e87f1126cf22108fb27f6391c7826fb91c25138e` | + +The files are `src/Libraries/Microsoft.Extensions.AI.Abstractions/Decisions/*.cs` at the Layer 2 commit, which contains +Layer 1, without changes. `.editorconfig` keeps the upstream style and API shape from failing the build. The package +injects `src/Shared/DiagnosticIds` and `src/Shared/Throw`, which provide the helpers the upstream code uses, and the +polyfills that the upstream code relies on for .NET Standard 2.0 and .NET Framework. + +Not copied: the upstream tests, the README, and the registration of the decision types in +`AIJsonUtilities.DefaultOptions`. Serializing the decision types therefore uses reflection-based metadata. + +## Placement + +The copy lives where MEAI places the API: everything is in `Microsoft.Extensions.AI.Abstractions` upstream, so it is in +`Microsoft.Agents.AI.Abstractions` here, which every provider package depends on. Only this package may carry the copy, +because a second package with the same public types would make them ambiguous (CS0433) for an application that +references both. + +The API is still in review, and the review on #7795 asks, among other things, to align `DecisionResponse` with +`ChatResponse` and to move the typed layer (`DecisionDefinition`, `DecisionResponse`, and the typed +extensions) to `Microsoft.Extensions.AI`. When upstream changes, copy the new files over these ones and update the +public API files. If the typed layer moves to `Microsoft.Extensions.AI`, move its copy to `Microsoft.Agents.AI`. Each +re-sync can break code that uses the types, which their `MEAI001` diagnostic allows. + +## When Microsoft.Extensions.AI ships the API + +1. Update `Microsoft.Extensions.AI.Abstractions` in `Directory.Packages.props`. From that version on, the copied types + conflict with the shipped ones, so the next steps go in the same change and the same release. +2. Delete this folder and its entries in `PublicAPI/*/PublicAPI.Unshipped.txt` (or `PublicAPI.Shipped.txt`, once a + release has shipped them). +3. Add `[assembly: TypeForwardedTo(typeof(...))]` for each public type, so applications compiled against this package + keep loading the types from `Microsoft.Extensions.AI.Abstractions`. Suppress any package validation difference + between this copy and the shipped shape in `CompatibilitySuppressions.xml`. +4. Fix any difference between this copy and the shipped API in the packages that use it, such as + `Microsoft.Agents.AI.TypeSafe`, the shared decision chat engine (`src/Shared/DecisionChat`), and the samples. diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net10.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net10.0/PublicAPI.Unshipped.txt index 7dc5c58110b..6dd8fb7d4d5 100644 --- a/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net10.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net10.0/PublicAPI.Unshipped.txt @@ -1 +1,174 @@ #nullable enable +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer.BinaryDecisionAnswer(string! questionId, double trueProbability, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer.TrueProbability.get -> double +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.BinaryDecisionQuestion(string! id, string? instructions = null, string? trueDescription = null, string? falseDescription = null) -> void +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.FalseDescription.get -> string? +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.TrueDescription.get -> string? +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.ChoiceDecisionAnswer(string! questionId, string! selectedCandidateId, System.Collections.Generic.IReadOnlyList! probabilities, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.Probabilities.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.SelectedCandidateId.get -> string! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion.Candidates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion.ChoiceDecisionQuestion(string! id, string? instructions, System.Collections.Generic.IReadOnlyList! candidates) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.AdditionalProperties.get -> System.Collections.Generic.IReadOnlyDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.Kind.get -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.Precision.get -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.QuestionId.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.RawRepresentation.get -> object? +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetBinary(string! questionId) -> Microsoft.Extensions.AI.BinaryDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetChoice(string! questionId) -> Microsoft.Extensions.AI.ChoiceDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetChoice(string! questionId, Microsoft.Extensions.AI.DecisionEnumDefinition! definition) -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetScore(string! questionId) -> Microsoft.Extensions.AI.ScoreDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.Response.get -> Microsoft.Extensions.AI.DecisionResponse! +[MEAI001]Microsoft.Extensions.AI.DecisionBindingExtensions +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.DecisionCandidate(string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionClientException +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message, System.Exception! innerException) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message, bool isTransient, System.Exception? innerException = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.IsTransient.get -> bool +[MEAI001]Microsoft.Extensions.AI.DecisionClientExtensions +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.DecisionClientMetadata(string? providerName = null, System.Uri? providerUri = null, string? defaultModelId = null, System.Collections.Generic.IReadOnlyList? supportedKinds = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.DefaultModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.ProviderName.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.ProviderUri.get -> System.Uri? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.SupportedKinds.get -> System.Collections.Generic.IReadOnlyList? +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.AsAIFunction(Microsoft.Extensions.AI.IDecisionClient! client, Microsoft.Extensions.AI.AIFunctionFactoryOptions? functionOptions = null, Microsoft.Extensions.AI.DecisionOptions? decisionOptions = null, System.Text.Json.Serialization.Metadata.JsonTypeInfo? stateTypeInfo = null) -> Microsoft.Extensions.AI.AIFunction! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.BinaryProbability(System.Linq.Expressions.Expression!>! selector, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.Choice(System.Linq.Expressions.Expression!>! selector, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.ExpectedScore(System.Linq.Expressions.Expression!>! selector, System.Collections.Generic.IReadOnlyList! levels, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.Score(System.Linq.Expressions.Expression!>! selector, System.Collections.Generic.IReadOnlyList! levels, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Questions.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.ResultTypeInfo.get -> System.Text.Json.Serialization.Metadata.JsonTypeInfo! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.SerializerOptions.get -> System.Text.Json.JsonSerializerOptions! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.Candidates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.CreateQuestion(string! id, string? instructions) -> Microsoft.Extensions.AI.ChoiceDecisionQuestion! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.DecisionEnumDefinition(System.Collections.Generic.IEnumerable!>! values) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.Parse(string! candidateId) -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.DecisionEnumValue(TEnum value, string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Value.get -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.DecisionFeatureCoordinate(string! name, string! questionId, Microsoft.Extensions.AI.DecisionFeatureValueKind valueKind, string? memberId = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.MemberId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.Name.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.QuestionId.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.ValueKind.get -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureProjection +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Coordinates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.DecisionFeatureSchema(string! id, int version, System.Collections.Generic.IEnumerable! coordinates) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Version.get -> int +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.DecisionFeatureValue(string! name, double value) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.Name.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.Value.get -> double +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.BinaryFalseProbability = 1 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.BinaryTrueProbability = 0 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ChoiceProbability = 2 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ExpectedScore = 5 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.Score = 4 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ScoreProbability = 3 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Provenance.get -> Microsoft.Extensions.AI.DecisionProvenance? +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Schema.get -> Microsoft.Extensions.AI.DecisionFeatureSchema! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Values.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Binary = 0 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Choice = 1 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Score = 2 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionOptions +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.AdditionalProperties.get -> Microsoft.Extensions.AI.AdditionalPropertiesDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.AdditionalProperties.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.DecisionOptions() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.DecisionOptions(Microsoft.Extensions.AI.DecisionOptions? other) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.ModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.ModelId.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.RawRepresentationFactory.get -> System.Func? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.RawRepresentationFactory.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.FourDecimalPlaces = 1 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.HighPrecision = 0 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.TwoDecimalPlaces = 2 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionProbability +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.DecisionProbability(string! id, double value) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.Value.get -> double +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException(string! message) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException(string! message, System.Exception! innerException) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.DecisionProvenance(string? providerName = null, System.Uri? providerUri = null, string? modelId = null, string? responseId = null, Microsoft.Extensions.AI.DecisionPrecision? precision = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.Precision.get -> Microsoft.Extensions.AI.DecisionPrecision? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ProviderName.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ProviderUri.get -> System.Uri? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ResponseId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Instructions.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Kind.get -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionRequest +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.DecisionRequest(System.Text.Json.JsonElement state, System.Collections.Generic.IReadOnlyList! questions) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.Questions.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.State.get -> System.Text.Json.JsonElement +[MEAI001]Microsoft.Extensions.AI.DecisionResponse +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.AdditionalProperties.get -> System.Collections.Generic.IReadOnlyDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Answers.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.DecisionResponse(Microsoft.Extensions.AI.DecisionRequest! request, System.Collections.Generic.IReadOnlyList! answers, Microsoft.Extensions.AI.DecisionProvenance? provenance = null, Microsoft.Extensions.AI.UsageDetails? usage = null, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.GetAnswer(string! questionId) -> Microsoft.Extensions.AI.DecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Provenance.get -> Microsoft.Extensions.AI.DecisionProvenance? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.RawRepresentation.get -> object? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Request.get -> Microsoft.Extensions.AI.DecisionRequest! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Usage.get -> Microsoft.Extensions.AI.UsageDetails? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Evidence.get -> Microsoft.Extensions.AI.DecisionResponse! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.GetDistribution(System.Linq.Expressions.Expression!>! selector) -> System.Collections.Generic.IReadOnlyDictionary! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Result.get -> TResult +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding.DecisionResultBinding(System.Text.Json.Serialization.Metadata.JsonTypeInfo! resultTypeInfo, System.Func! map) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding.ResultTypeInfo.get -> System.Text.Json.Serialization.Metadata.JsonTypeInfo! +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.DecisionScoreLevel(string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.IDecisionClient +[MEAI001]Microsoft.Extensions.AI.IDecisionClient.GetResponseAsync(Microsoft.Extensions.AI.DecisionRequest! request, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +[MEAI001]Microsoft.Extensions.AI.IDecisionClient.GetService(System.Type! serviceType, object? serviceKey = null) -> object? +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.ExpectedScore.get -> double +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.Probabilities.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.Score.get -> double? +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.ScoreDecisionAnswer(string! questionId, double? score, System.Collections.Generic.IReadOnlyList! probabilities, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion.Levels.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion.ScoreDecisionQuestion(string! id, string? instructions, System.Collections.Generic.IReadOnlyList! levels) -> void +[MEAI001]static Microsoft.Extensions.AI.DecisionBindingExtensions.Bind(this Microsoft.Extensions.AI.DecisionResponse! response, Microsoft.Extensions.AI.DecisionResultBinding! binding) -> TResult +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, System.Text.Json.JsonElement state, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, System.Collections.Generic.IEnumerable! questions, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetService(this Microsoft.Extensions.AI.IDecisionClient! client, object? serviceKey = null) -> TService? +[MEAI001]static Microsoft.Extensions.AI.DecisionDefinition.Create(System.Action.Builder!>! configure, System.Text.Json.JsonSerializerOptions? serializerOptions = null) -> Microsoft.Extensions.AI.DecisionDefinition! +[MEAI001]static Microsoft.Extensions.AI.DecisionDefinition.Create(System.Text.Json.Serialization.Metadata.JsonTypeInfo! resultTypeInfo, System.Action.Builder!>! configure) -> Microsoft.Extensions.AI.DecisionDefinition! +[MEAI001]static Microsoft.Extensions.AI.DecisionFeatureProjection.Project(Microsoft.Extensions.AI.DecisionResponse! response, Microsoft.Extensions.AI.DecisionFeatureSchema! schema) -> Microsoft.Extensions.AI.DecisionFeatureVector! +[MEAI001]static Microsoft.Extensions.AI.DecisionRequest.Create(TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, System.Collections.Generic.IEnumerable! questions) -> Microsoft.Extensions.AI.DecisionRequest! +[MEAI001]virtual Microsoft.Extensions.AI.DecisionOptions.Clone() -> Microsoft.Extensions.AI.DecisionOptions! diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net472/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net472/PublicAPI.Unshipped.txt index 7dc5c58110b..6dd8fb7d4d5 100644 --- a/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net472/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net472/PublicAPI.Unshipped.txt @@ -1 +1,174 @@ #nullable enable +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer.BinaryDecisionAnswer(string! questionId, double trueProbability, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer.TrueProbability.get -> double +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.BinaryDecisionQuestion(string! id, string? instructions = null, string? trueDescription = null, string? falseDescription = null) -> void +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.FalseDescription.get -> string? +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.TrueDescription.get -> string? +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.ChoiceDecisionAnswer(string! questionId, string! selectedCandidateId, System.Collections.Generic.IReadOnlyList! probabilities, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.Probabilities.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.SelectedCandidateId.get -> string! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion.Candidates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion.ChoiceDecisionQuestion(string! id, string? instructions, System.Collections.Generic.IReadOnlyList! candidates) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.AdditionalProperties.get -> System.Collections.Generic.IReadOnlyDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.Kind.get -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.Precision.get -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.QuestionId.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.RawRepresentation.get -> object? +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetBinary(string! questionId) -> Microsoft.Extensions.AI.BinaryDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetChoice(string! questionId) -> Microsoft.Extensions.AI.ChoiceDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetChoice(string! questionId, Microsoft.Extensions.AI.DecisionEnumDefinition! definition) -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetScore(string! questionId) -> Microsoft.Extensions.AI.ScoreDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.Response.get -> Microsoft.Extensions.AI.DecisionResponse! +[MEAI001]Microsoft.Extensions.AI.DecisionBindingExtensions +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.DecisionCandidate(string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionClientException +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message, System.Exception! innerException) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message, bool isTransient, System.Exception? innerException = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.IsTransient.get -> bool +[MEAI001]Microsoft.Extensions.AI.DecisionClientExtensions +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.DecisionClientMetadata(string? providerName = null, System.Uri? providerUri = null, string? defaultModelId = null, System.Collections.Generic.IReadOnlyList? supportedKinds = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.DefaultModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.ProviderName.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.ProviderUri.get -> System.Uri? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.SupportedKinds.get -> System.Collections.Generic.IReadOnlyList? +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.AsAIFunction(Microsoft.Extensions.AI.IDecisionClient! client, Microsoft.Extensions.AI.AIFunctionFactoryOptions? functionOptions = null, Microsoft.Extensions.AI.DecisionOptions? decisionOptions = null, System.Text.Json.Serialization.Metadata.JsonTypeInfo? stateTypeInfo = null) -> Microsoft.Extensions.AI.AIFunction! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.BinaryProbability(System.Linq.Expressions.Expression!>! selector, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.Choice(System.Linq.Expressions.Expression!>! selector, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.ExpectedScore(System.Linq.Expressions.Expression!>! selector, System.Collections.Generic.IReadOnlyList! levels, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.Score(System.Linq.Expressions.Expression!>! selector, System.Collections.Generic.IReadOnlyList! levels, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Questions.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.ResultTypeInfo.get -> System.Text.Json.Serialization.Metadata.JsonTypeInfo! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.SerializerOptions.get -> System.Text.Json.JsonSerializerOptions! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.Candidates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.CreateQuestion(string! id, string? instructions) -> Microsoft.Extensions.AI.ChoiceDecisionQuestion! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.DecisionEnumDefinition(System.Collections.Generic.IEnumerable!>! values) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.Parse(string! candidateId) -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.DecisionEnumValue(TEnum value, string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Value.get -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.DecisionFeatureCoordinate(string! name, string! questionId, Microsoft.Extensions.AI.DecisionFeatureValueKind valueKind, string? memberId = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.MemberId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.Name.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.QuestionId.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.ValueKind.get -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureProjection +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Coordinates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.DecisionFeatureSchema(string! id, int version, System.Collections.Generic.IEnumerable! coordinates) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Version.get -> int +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.DecisionFeatureValue(string! name, double value) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.Name.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.Value.get -> double +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.BinaryFalseProbability = 1 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.BinaryTrueProbability = 0 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ChoiceProbability = 2 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ExpectedScore = 5 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.Score = 4 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ScoreProbability = 3 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Provenance.get -> Microsoft.Extensions.AI.DecisionProvenance? +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Schema.get -> Microsoft.Extensions.AI.DecisionFeatureSchema! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Values.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Binary = 0 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Choice = 1 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Score = 2 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionOptions +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.AdditionalProperties.get -> Microsoft.Extensions.AI.AdditionalPropertiesDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.AdditionalProperties.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.DecisionOptions() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.DecisionOptions(Microsoft.Extensions.AI.DecisionOptions? other) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.ModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.ModelId.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.RawRepresentationFactory.get -> System.Func? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.RawRepresentationFactory.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.FourDecimalPlaces = 1 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.HighPrecision = 0 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.TwoDecimalPlaces = 2 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionProbability +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.DecisionProbability(string! id, double value) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.Value.get -> double +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException(string! message) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException(string! message, System.Exception! innerException) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.DecisionProvenance(string? providerName = null, System.Uri? providerUri = null, string? modelId = null, string? responseId = null, Microsoft.Extensions.AI.DecisionPrecision? precision = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.Precision.get -> Microsoft.Extensions.AI.DecisionPrecision? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ProviderName.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ProviderUri.get -> System.Uri? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ResponseId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Instructions.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Kind.get -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionRequest +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.DecisionRequest(System.Text.Json.JsonElement state, System.Collections.Generic.IReadOnlyList! questions) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.Questions.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.State.get -> System.Text.Json.JsonElement +[MEAI001]Microsoft.Extensions.AI.DecisionResponse +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.AdditionalProperties.get -> System.Collections.Generic.IReadOnlyDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Answers.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.DecisionResponse(Microsoft.Extensions.AI.DecisionRequest! request, System.Collections.Generic.IReadOnlyList! answers, Microsoft.Extensions.AI.DecisionProvenance? provenance = null, Microsoft.Extensions.AI.UsageDetails? usage = null, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.GetAnswer(string! questionId) -> Microsoft.Extensions.AI.DecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Provenance.get -> Microsoft.Extensions.AI.DecisionProvenance? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.RawRepresentation.get -> object? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Request.get -> Microsoft.Extensions.AI.DecisionRequest! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Usage.get -> Microsoft.Extensions.AI.UsageDetails? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Evidence.get -> Microsoft.Extensions.AI.DecisionResponse! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.GetDistribution(System.Linq.Expressions.Expression!>! selector) -> System.Collections.Generic.IReadOnlyDictionary! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Result.get -> TResult +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding.DecisionResultBinding(System.Text.Json.Serialization.Metadata.JsonTypeInfo! resultTypeInfo, System.Func! map) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding.ResultTypeInfo.get -> System.Text.Json.Serialization.Metadata.JsonTypeInfo! +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.DecisionScoreLevel(string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.IDecisionClient +[MEAI001]Microsoft.Extensions.AI.IDecisionClient.GetResponseAsync(Microsoft.Extensions.AI.DecisionRequest! request, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +[MEAI001]Microsoft.Extensions.AI.IDecisionClient.GetService(System.Type! serviceType, object? serviceKey = null) -> object? +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.ExpectedScore.get -> double +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.Probabilities.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.Score.get -> double? +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.ScoreDecisionAnswer(string! questionId, double? score, System.Collections.Generic.IReadOnlyList! probabilities, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion.Levels.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion.ScoreDecisionQuestion(string! id, string? instructions, System.Collections.Generic.IReadOnlyList! levels) -> void +[MEAI001]static Microsoft.Extensions.AI.DecisionBindingExtensions.Bind(this Microsoft.Extensions.AI.DecisionResponse! response, Microsoft.Extensions.AI.DecisionResultBinding! binding) -> TResult +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, System.Text.Json.JsonElement state, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, System.Collections.Generic.IEnumerable! questions, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetService(this Microsoft.Extensions.AI.IDecisionClient! client, object? serviceKey = null) -> TService? +[MEAI001]static Microsoft.Extensions.AI.DecisionDefinition.Create(System.Action.Builder!>! configure, System.Text.Json.JsonSerializerOptions? serializerOptions = null) -> Microsoft.Extensions.AI.DecisionDefinition! +[MEAI001]static Microsoft.Extensions.AI.DecisionDefinition.Create(System.Text.Json.Serialization.Metadata.JsonTypeInfo! resultTypeInfo, System.Action.Builder!>! configure) -> Microsoft.Extensions.AI.DecisionDefinition! +[MEAI001]static Microsoft.Extensions.AI.DecisionFeatureProjection.Project(Microsoft.Extensions.AI.DecisionResponse! response, Microsoft.Extensions.AI.DecisionFeatureSchema! schema) -> Microsoft.Extensions.AI.DecisionFeatureVector! +[MEAI001]static Microsoft.Extensions.AI.DecisionRequest.Create(TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, System.Collections.Generic.IEnumerable! questions) -> Microsoft.Extensions.AI.DecisionRequest! +[MEAI001]virtual Microsoft.Extensions.AI.DecisionOptions.Clone() -> Microsoft.Extensions.AI.DecisionOptions! diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net8.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net8.0/PublicAPI.Unshipped.txt index 7dc5c58110b..6dd8fb7d4d5 100644 --- a/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net8.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net8.0/PublicAPI.Unshipped.txt @@ -1 +1,174 @@ #nullable enable +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer.BinaryDecisionAnswer(string! questionId, double trueProbability, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer.TrueProbability.get -> double +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.BinaryDecisionQuestion(string! id, string? instructions = null, string? trueDescription = null, string? falseDescription = null) -> void +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.FalseDescription.get -> string? +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.TrueDescription.get -> string? +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.ChoiceDecisionAnswer(string! questionId, string! selectedCandidateId, System.Collections.Generic.IReadOnlyList! probabilities, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.Probabilities.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.SelectedCandidateId.get -> string! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion.Candidates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion.ChoiceDecisionQuestion(string! id, string? instructions, System.Collections.Generic.IReadOnlyList! candidates) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.AdditionalProperties.get -> System.Collections.Generic.IReadOnlyDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.Kind.get -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.Precision.get -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.QuestionId.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.RawRepresentation.get -> object? +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetBinary(string! questionId) -> Microsoft.Extensions.AI.BinaryDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetChoice(string! questionId) -> Microsoft.Extensions.AI.ChoiceDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetChoice(string! questionId, Microsoft.Extensions.AI.DecisionEnumDefinition! definition) -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetScore(string! questionId) -> Microsoft.Extensions.AI.ScoreDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.Response.get -> Microsoft.Extensions.AI.DecisionResponse! +[MEAI001]Microsoft.Extensions.AI.DecisionBindingExtensions +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.DecisionCandidate(string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionClientException +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message, System.Exception! innerException) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message, bool isTransient, System.Exception? innerException = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.IsTransient.get -> bool +[MEAI001]Microsoft.Extensions.AI.DecisionClientExtensions +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.DecisionClientMetadata(string? providerName = null, System.Uri? providerUri = null, string? defaultModelId = null, System.Collections.Generic.IReadOnlyList? supportedKinds = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.DefaultModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.ProviderName.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.ProviderUri.get -> System.Uri? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.SupportedKinds.get -> System.Collections.Generic.IReadOnlyList? +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.AsAIFunction(Microsoft.Extensions.AI.IDecisionClient! client, Microsoft.Extensions.AI.AIFunctionFactoryOptions? functionOptions = null, Microsoft.Extensions.AI.DecisionOptions? decisionOptions = null, System.Text.Json.Serialization.Metadata.JsonTypeInfo? stateTypeInfo = null) -> Microsoft.Extensions.AI.AIFunction! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.BinaryProbability(System.Linq.Expressions.Expression!>! selector, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.Choice(System.Linq.Expressions.Expression!>! selector, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.ExpectedScore(System.Linq.Expressions.Expression!>! selector, System.Collections.Generic.IReadOnlyList! levels, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.Score(System.Linq.Expressions.Expression!>! selector, System.Collections.Generic.IReadOnlyList! levels, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Questions.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.ResultTypeInfo.get -> System.Text.Json.Serialization.Metadata.JsonTypeInfo! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.SerializerOptions.get -> System.Text.Json.JsonSerializerOptions! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.Candidates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.CreateQuestion(string! id, string? instructions) -> Microsoft.Extensions.AI.ChoiceDecisionQuestion! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.DecisionEnumDefinition(System.Collections.Generic.IEnumerable!>! values) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.Parse(string! candidateId) -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.DecisionEnumValue(TEnum value, string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Value.get -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.DecisionFeatureCoordinate(string! name, string! questionId, Microsoft.Extensions.AI.DecisionFeatureValueKind valueKind, string? memberId = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.MemberId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.Name.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.QuestionId.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.ValueKind.get -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureProjection +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Coordinates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.DecisionFeatureSchema(string! id, int version, System.Collections.Generic.IEnumerable! coordinates) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Version.get -> int +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.DecisionFeatureValue(string! name, double value) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.Name.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.Value.get -> double +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.BinaryFalseProbability = 1 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.BinaryTrueProbability = 0 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ChoiceProbability = 2 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ExpectedScore = 5 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.Score = 4 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ScoreProbability = 3 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Provenance.get -> Microsoft.Extensions.AI.DecisionProvenance? +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Schema.get -> Microsoft.Extensions.AI.DecisionFeatureSchema! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Values.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Binary = 0 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Choice = 1 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Score = 2 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionOptions +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.AdditionalProperties.get -> Microsoft.Extensions.AI.AdditionalPropertiesDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.AdditionalProperties.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.DecisionOptions() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.DecisionOptions(Microsoft.Extensions.AI.DecisionOptions? other) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.ModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.ModelId.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.RawRepresentationFactory.get -> System.Func? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.RawRepresentationFactory.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.FourDecimalPlaces = 1 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.HighPrecision = 0 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.TwoDecimalPlaces = 2 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionProbability +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.DecisionProbability(string! id, double value) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.Value.get -> double +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException(string! message) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException(string! message, System.Exception! innerException) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.DecisionProvenance(string? providerName = null, System.Uri? providerUri = null, string? modelId = null, string? responseId = null, Microsoft.Extensions.AI.DecisionPrecision? precision = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.Precision.get -> Microsoft.Extensions.AI.DecisionPrecision? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ProviderName.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ProviderUri.get -> System.Uri? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ResponseId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Instructions.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Kind.get -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionRequest +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.DecisionRequest(System.Text.Json.JsonElement state, System.Collections.Generic.IReadOnlyList! questions) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.Questions.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.State.get -> System.Text.Json.JsonElement +[MEAI001]Microsoft.Extensions.AI.DecisionResponse +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.AdditionalProperties.get -> System.Collections.Generic.IReadOnlyDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Answers.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.DecisionResponse(Microsoft.Extensions.AI.DecisionRequest! request, System.Collections.Generic.IReadOnlyList! answers, Microsoft.Extensions.AI.DecisionProvenance? provenance = null, Microsoft.Extensions.AI.UsageDetails? usage = null, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.GetAnswer(string! questionId) -> Microsoft.Extensions.AI.DecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Provenance.get -> Microsoft.Extensions.AI.DecisionProvenance? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.RawRepresentation.get -> object? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Request.get -> Microsoft.Extensions.AI.DecisionRequest! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Usage.get -> Microsoft.Extensions.AI.UsageDetails? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Evidence.get -> Microsoft.Extensions.AI.DecisionResponse! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.GetDistribution(System.Linq.Expressions.Expression!>! selector) -> System.Collections.Generic.IReadOnlyDictionary! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Result.get -> TResult +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding.DecisionResultBinding(System.Text.Json.Serialization.Metadata.JsonTypeInfo! resultTypeInfo, System.Func! map) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding.ResultTypeInfo.get -> System.Text.Json.Serialization.Metadata.JsonTypeInfo! +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.DecisionScoreLevel(string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.IDecisionClient +[MEAI001]Microsoft.Extensions.AI.IDecisionClient.GetResponseAsync(Microsoft.Extensions.AI.DecisionRequest! request, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +[MEAI001]Microsoft.Extensions.AI.IDecisionClient.GetService(System.Type! serviceType, object? serviceKey = null) -> object? +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.ExpectedScore.get -> double +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.Probabilities.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.Score.get -> double? +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.ScoreDecisionAnswer(string! questionId, double? score, System.Collections.Generic.IReadOnlyList! probabilities, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion.Levels.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion.ScoreDecisionQuestion(string! id, string? instructions, System.Collections.Generic.IReadOnlyList! levels) -> void +[MEAI001]static Microsoft.Extensions.AI.DecisionBindingExtensions.Bind(this Microsoft.Extensions.AI.DecisionResponse! response, Microsoft.Extensions.AI.DecisionResultBinding! binding) -> TResult +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, System.Text.Json.JsonElement state, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, System.Collections.Generic.IEnumerable! questions, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetService(this Microsoft.Extensions.AI.IDecisionClient! client, object? serviceKey = null) -> TService? +[MEAI001]static Microsoft.Extensions.AI.DecisionDefinition.Create(System.Action.Builder!>! configure, System.Text.Json.JsonSerializerOptions? serializerOptions = null) -> Microsoft.Extensions.AI.DecisionDefinition! +[MEAI001]static Microsoft.Extensions.AI.DecisionDefinition.Create(System.Text.Json.Serialization.Metadata.JsonTypeInfo! resultTypeInfo, System.Action.Builder!>! configure) -> Microsoft.Extensions.AI.DecisionDefinition! +[MEAI001]static Microsoft.Extensions.AI.DecisionFeatureProjection.Project(Microsoft.Extensions.AI.DecisionResponse! response, Microsoft.Extensions.AI.DecisionFeatureSchema! schema) -> Microsoft.Extensions.AI.DecisionFeatureVector! +[MEAI001]static Microsoft.Extensions.AI.DecisionRequest.Create(TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, System.Collections.Generic.IEnumerable! questions) -> Microsoft.Extensions.AI.DecisionRequest! +[MEAI001]virtual Microsoft.Extensions.AI.DecisionOptions.Clone() -> Microsoft.Extensions.AI.DecisionOptions! diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net9.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net9.0/PublicAPI.Unshipped.txt index 7dc5c58110b..6dd8fb7d4d5 100644 --- a/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net9.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/net9.0/PublicAPI.Unshipped.txt @@ -1 +1,174 @@ #nullable enable +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer.BinaryDecisionAnswer(string! questionId, double trueProbability, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer.TrueProbability.get -> double +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.BinaryDecisionQuestion(string! id, string? instructions = null, string? trueDescription = null, string? falseDescription = null) -> void +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.FalseDescription.get -> string? +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.TrueDescription.get -> string? +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.ChoiceDecisionAnswer(string! questionId, string! selectedCandidateId, System.Collections.Generic.IReadOnlyList! probabilities, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.Probabilities.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.SelectedCandidateId.get -> string! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion.Candidates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion.ChoiceDecisionQuestion(string! id, string? instructions, System.Collections.Generic.IReadOnlyList! candidates) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.AdditionalProperties.get -> System.Collections.Generic.IReadOnlyDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.Kind.get -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.Precision.get -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.QuestionId.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.RawRepresentation.get -> object? +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetBinary(string! questionId) -> Microsoft.Extensions.AI.BinaryDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetChoice(string! questionId) -> Microsoft.Extensions.AI.ChoiceDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetChoice(string! questionId, Microsoft.Extensions.AI.DecisionEnumDefinition! definition) -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetScore(string! questionId) -> Microsoft.Extensions.AI.ScoreDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.Response.get -> Microsoft.Extensions.AI.DecisionResponse! +[MEAI001]Microsoft.Extensions.AI.DecisionBindingExtensions +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.DecisionCandidate(string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionClientException +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message, System.Exception! innerException) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message, bool isTransient, System.Exception? innerException = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.IsTransient.get -> bool +[MEAI001]Microsoft.Extensions.AI.DecisionClientExtensions +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.DecisionClientMetadata(string? providerName = null, System.Uri? providerUri = null, string? defaultModelId = null, System.Collections.Generic.IReadOnlyList? supportedKinds = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.DefaultModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.ProviderName.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.ProviderUri.get -> System.Uri? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.SupportedKinds.get -> System.Collections.Generic.IReadOnlyList? +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.AsAIFunction(Microsoft.Extensions.AI.IDecisionClient! client, Microsoft.Extensions.AI.AIFunctionFactoryOptions? functionOptions = null, Microsoft.Extensions.AI.DecisionOptions? decisionOptions = null, System.Text.Json.Serialization.Metadata.JsonTypeInfo? stateTypeInfo = null) -> Microsoft.Extensions.AI.AIFunction! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.BinaryProbability(System.Linq.Expressions.Expression!>! selector, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.Choice(System.Linq.Expressions.Expression!>! selector, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.ExpectedScore(System.Linq.Expressions.Expression!>! selector, System.Collections.Generic.IReadOnlyList! levels, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.Score(System.Linq.Expressions.Expression!>! selector, System.Collections.Generic.IReadOnlyList! levels, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Questions.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.ResultTypeInfo.get -> System.Text.Json.Serialization.Metadata.JsonTypeInfo! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.SerializerOptions.get -> System.Text.Json.JsonSerializerOptions! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.Candidates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.CreateQuestion(string! id, string? instructions) -> Microsoft.Extensions.AI.ChoiceDecisionQuestion! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.DecisionEnumDefinition(System.Collections.Generic.IEnumerable!>! values) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.Parse(string! candidateId) -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.DecisionEnumValue(TEnum value, string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Value.get -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.DecisionFeatureCoordinate(string! name, string! questionId, Microsoft.Extensions.AI.DecisionFeatureValueKind valueKind, string? memberId = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.MemberId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.Name.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.QuestionId.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.ValueKind.get -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureProjection +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Coordinates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.DecisionFeatureSchema(string! id, int version, System.Collections.Generic.IEnumerable! coordinates) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Version.get -> int +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.DecisionFeatureValue(string! name, double value) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.Name.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.Value.get -> double +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.BinaryFalseProbability = 1 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.BinaryTrueProbability = 0 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ChoiceProbability = 2 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ExpectedScore = 5 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.Score = 4 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ScoreProbability = 3 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Provenance.get -> Microsoft.Extensions.AI.DecisionProvenance? +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Schema.get -> Microsoft.Extensions.AI.DecisionFeatureSchema! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Values.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Binary = 0 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Choice = 1 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Score = 2 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionOptions +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.AdditionalProperties.get -> Microsoft.Extensions.AI.AdditionalPropertiesDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.AdditionalProperties.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.DecisionOptions() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.DecisionOptions(Microsoft.Extensions.AI.DecisionOptions? other) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.ModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.ModelId.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.RawRepresentationFactory.get -> System.Func? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.RawRepresentationFactory.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.FourDecimalPlaces = 1 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.HighPrecision = 0 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.TwoDecimalPlaces = 2 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionProbability +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.DecisionProbability(string! id, double value) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.Value.get -> double +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException(string! message) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException(string! message, System.Exception! innerException) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.DecisionProvenance(string? providerName = null, System.Uri? providerUri = null, string? modelId = null, string? responseId = null, Microsoft.Extensions.AI.DecisionPrecision? precision = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.Precision.get -> Microsoft.Extensions.AI.DecisionPrecision? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ProviderName.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ProviderUri.get -> System.Uri? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ResponseId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Instructions.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Kind.get -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionRequest +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.DecisionRequest(System.Text.Json.JsonElement state, System.Collections.Generic.IReadOnlyList! questions) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.Questions.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.State.get -> System.Text.Json.JsonElement +[MEAI001]Microsoft.Extensions.AI.DecisionResponse +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.AdditionalProperties.get -> System.Collections.Generic.IReadOnlyDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Answers.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.DecisionResponse(Microsoft.Extensions.AI.DecisionRequest! request, System.Collections.Generic.IReadOnlyList! answers, Microsoft.Extensions.AI.DecisionProvenance? provenance = null, Microsoft.Extensions.AI.UsageDetails? usage = null, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.GetAnswer(string! questionId) -> Microsoft.Extensions.AI.DecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Provenance.get -> Microsoft.Extensions.AI.DecisionProvenance? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.RawRepresentation.get -> object? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Request.get -> Microsoft.Extensions.AI.DecisionRequest! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Usage.get -> Microsoft.Extensions.AI.UsageDetails? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Evidence.get -> Microsoft.Extensions.AI.DecisionResponse! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.GetDistribution(System.Linq.Expressions.Expression!>! selector) -> System.Collections.Generic.IReadOnlyDictionary! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Result.get -> TResult +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding.DecisionResultBinding(System.Text.Json.Serialization.Metadata.JsonTypeInfo! resultTypeInfo, System.Func! map) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding.ResultTypeInfo.get -> System.Text.Json.Serialization.Metadata.JsonTypeInfo! +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.DecisionScoreLevel(string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.IDecisionClient +[MEAI001]Microsoft.Extensions.AI.IDecisionClient.GetResponseAsync(Microsoft.Extensions.AI.DecisionRequest! request, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +[MEAI001]Microsoft.Extensions.AI.IDecisionClient.GetService(System.Type! serviceType, object? serviceKey = null) -> object? +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.ExpectedScore.get -> double +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.Probabilities.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.Score.get -> double? +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.ScoreDecisionAnswer(string! questionId, double? score, System.Collections.Generic.IReadOnlyList! probabilities, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion.Levels.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion.ScoreDecisionQuestion(string! id, string? instructions, System.Collections.Generic.IReadOnlyList! levels) -> void +[MEAI001]static Microsoft.Extensions.AI.DecisionBindingExtensions.Bind(this Microsoft.Extensions.AI.DecisionResponse! response, Microsoft.Extensions.AI.DecisionResultBinding! binding) -> TResult +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, System.Text.Json.JsonElement state, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, System.Collections.Generic.IEnumerable! questions, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetService(this Microsoft.Extensions.AI.IDecisionClient! client, object? serviceKey = null) -> TService? +[MEAI001]static Microsoft.Extensions.AI.DecisionDefinition.Create(System.Action.Builder!>! configure, System.Text.Json.JsonSerializerOptions? serializerOptions = null) -> Microsoft.Extensions.AI.DecisionDefinition! +[MEAI001]static Microsoft.Extensions.AI.DecisionDefinition.Create(System.Text.Json.Serialization.Metadata.JsonTypeInfo! resultTypeInfo, System.Action.Builder!>! configure) -> Microsoft.Extensions.AI.DecisionDefinition! +[MEAI001]static Microsoft.Extensions.AI.DecisionFeatureProjection.Project(Microsoft.Extensions.AI.DecisionResponse! response, Microsoft.Extensions.AI.DecisionFeatureSchema! schema) -> Microsoft.Extensions.AI.DecisionFeatureVector! +[MEAI001]static Microsoft.Extensions.AI.DecisionRequest.Create(TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, System.Collections.Generic.IEnumerable! questions) -> Microsoft.Extensions.AI.DecisionRequest! +[MEAI001]virtual Microsoft.Extensions.AI.DecisionOptions.Clone() -> Microsoft.Extensions.AI.DecisionOptions! diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt index 7dc5c58110b..6dd8fb7d4d5 100644 --- a/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt @@ -1 +1,174 @@ #nullable enable +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer.BinaryDecisionAnswer(string! questionId, double trueProbability, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionAnswer.TrueProbability.get -> double +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.BinaryDecisionQuestion(string! id, string? instructions = null, string? trueDescription = null, string? falseDescription = null) -> void +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.FalseDescription.get -> string? +[MEAI001]Microsoft.Extensions.AI.BinaryDecisionQuestion.TrueDescription.get -> string? +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.ChoiceDecisionAnswer(string! questionId, string! selectedCandidateId, System.Collections.Generic.IReadOnlyList! probabilities, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.Probabilities.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionAnswer.SelectedCandidateId.get -> string! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion.Candidates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ChoiceDecisionQuestion.ChoiceDecisionQuestion(string! id, string? instructions, System.Collections.Generic.IReadOnlyList! candidates) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.AdditionalProperties.get -> System.Collections.Generic.IReadOnlyDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.Kind.get -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.Precision.get -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.QuestionId.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswer.RawRepresentation.get -> object? +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetBinary(string! questionId) -> Microsoft.Extensions.AI.BinaryDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetChoice(string! questionId) -> Microsoft.Extensions.AI.ChoiceDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetChoice(string! questionId, Microsoft.Extensions.AI.DecisionEnumDefinition! definition) -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.GetScore(string! questionId) -> Microsoft.Extensions.AI.ScoreDecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionAnswerSet.Response.get -> Microsoft.Extensions.AI.DecisionResponse! +[MEAI001]Microsoft.Extensions.AI.DecisionBindingExtensions +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.DecisionCandidate(string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionCandidate.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionClientException +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message, System.Exception! innerException) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.DecisionClientException(string! message, bool isTransient, System.Exception? innerException = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientException.IsTransient.get -> bool +[MEAI001]Microsoft.Extensions.AI.DecisionClientExtensions +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.DecisionClientMetadata(string? providerName = null, System.Uri? providerUri = null, string? defaultModelId = null, System.Collections.Generic.IReadOnlyList? supportedKinds = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.DefaultModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.ProviderName.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.ProviderUri.get -> System.Uri? +[MEAI001]Microsoft.Extensions.AI.DecisionClientMetadata.SupportedKinds.get -> System.Collections.Generic.IReadOnlyList? +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.AsAIFunction(Microsoft.Extensions.AI.IDecisionClient! client, Microsoft.Extensions.AI.AIFunctionFactoryOptions? functionOptions = null, Microsoft.Extensions.AI.DecisionOptions? decisionOptions = null, System.Text.Json.Serialization.Metadata.JsonTypeInfo? stateTypeInfo = null) -> Microsoft.Extensions.AI.AIFunction! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.BinaryProbability(System.Linq.Expressions.Expression!>! selector, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.Choice(System.Linq.Expressions.Expression!>! selector, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.ExpectedScore(System.Linq.Expressions.Expression!>! selector, System.Collections.Generic.IReadOnlyList! levels, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Builder.Score(System.Linq.Expressions.Expression!>! selector, System.Collections.Generic.IReadOnlyList! levels, string? instructions = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.Questions.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.ResultTypeInfo.get -> System.Text.Json.Serialization.Metadata.JsonTypeInfo! +[MEAI001]Microsoft.Extensions.AI.DecisionDefinition.SerializerOptions.get -> System.Text.Json.JsonSerializerOptions! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.Candidates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.CreateQuestion(string! id, string? instructions) -> Microsoft.Extensions.AI.ChoiceDecisionQuestion! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.DecisionEnumDefinition(System.Collections.Generic.IEnumerable!>! values) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionEnumDefinition.Parse(string! candidateId) -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.DecisionEnumValue(TEnum value, string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionEnumValue.Value.get -> TEnum +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.DecisionFeatureCoordinate(string! name, string! questionId, Microsoft.Extensions.AI.DecisionFeatureValueKind valueKind, string? memberId = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.MemberId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.Name.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.QuestionId.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureCoordinate.ValueKind.get -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureProjection +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Coordinates.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.DecisionFeatureSchema(string! id, int version, System.Collections.Generic.IEnumerable! coordinates) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureSchema.Version.get -> int +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.DecisionFeatureValue(string! name, double value) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.Name.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValue.Value.get -> double +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.BinaryFalseProbability = 1 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.BinaryTrueProbability = 0 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ChoiceProbability = 2 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ExpectedScore = 5 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.Score = 4 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureValueKind.ScoreProbability = 3 -> Microsoft.Extensions.AI.DecisionFeatureValueKind +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Provenance.get -> Microsoft.Extensions.AI.DecisionProvenance? +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Schema.get -> Microsoft.Extensions.AI.DecisionFeatureSchema! +[MEAI001]Microsoft.Extensions.AI.DecisionFeatureVector.Values.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Binary = 0 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Choice = 1 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionKind.Score = 2 -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionOptions +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.AdditionalProperties.get -> Microsoft.Extensions.AI.AdditionalPropertiesDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.AdditionalProperties.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.DecisionOptions() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.DecisionOptions(Microsoft.Extensions.AI.DecisionOptions? other) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.ModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.ModelId.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.RawRepresentationFactory.get -> System.Func? +[MEAI001]Microsoft.Extensions.AI.DecisionOptions.RawRepresentationFactory.set -> void +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.FourDecimalPlaces = 1 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.HighPrecision = 0 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionPrecision.TwoDecimalPlaces = 2 -> Microsoft.Extensions.AI.DecisionPrecision +[MEAI001]Microsoft.Extensions.AI.DecisionProbability +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.DecisionProbability(string! id, double value) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionProbability.Value.get -> double +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException() -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException(string! message) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProtocolException.DecisionProtocolException(string! message, System.Exception! innerException) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.DecisionProvenance(string? providerName = null, System.Uri? providerUri = null, string? modelId = null, string? responseId = null, Microsoft.Extensions.AI.DecisionPrecision? precision = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ModelId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.Precision.get -> Microsoft.Extensions.AI.DecisionPrecision? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ProviderName.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ProviderUri.get -> System.Uri? +[MEAI001]Microsoft.Extensions.AI.DecisionProvenance.ResponseId.get -> string? +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Instructions.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionQuestion.Kind.get -> Microsoft.Extensions.AI.DecisionKind +[MEAI001]Microsoft.Extensions.AI.DecisionRequest +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.DecisionRequest(System.Text.Json.JsonElement state, System.Collections.Generic.IReadOnlyList! questions) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.Questions.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionRequest.State.get -> System.Text.Json.JsonElement +[MEAI001]Microsoft.Extensions.AI.DecisionResponse +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.AdditionalProperties.get -> System.Collections.Generic.IReadOnlyDictionary? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Answers.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.DecisionResponse(Microsoft.Extensions.AI.DecisionRequest! request, System.Collections.Generic.IReadOnlyList! answers, Microsoft.Extensions.AI.DecisionProvenance? provenance = null, Microsoft.Extensions.AI.UsageDetails? usage = null, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.GetAnswer(string! questionId) -> Microsoft.Extensions.AI.DecisionAnswer! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Provenance.get -> Microsoft.Extensions.AI.DecisionProvenance? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.RawRepresentation.get -> object? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Request.get -> Microsoft.Extensions.AI.DecisionRequest! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Usage.get -> Microsoft.Extensions.AI.UsageDetails? +[MEAI001]Microsoft.Extensions.AI.DecisionResponse +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Evidence.get -> Microsoft.Extensions.AI.DecisionResponse! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.GetDistribution(System.Linq.Expressions.Expression!>! selector) -> System.Collections.Generic.IReadOnlyDictionary! +[MEAI001]Microsoft.Extensions.AI.DecisionResponse.Result.get -> TResult +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding.DecisionResultBinding(System.Text.Json.Serialization.Metadata.JsonTypeInfo! resultTypeInfo, System.Func! map) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionResultBinding.ResultTypeInfo.get -> System.Text.Json.Serialization.Metadata.JsonTypeInfo! +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.DecisionScoreLevel(string! id, string! description) -> void +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.Description.get -> string! +[MEAI001]Microsoft.Extensions.AI.DecisionScoreLevel.Id.get -> string! +[MEAI001]Microsoft.Extensions.AI.IDecisionClient +[MEAI001]Microsoft.Extensions.AI.IDecisionClient.GetResponseAsync(Microsoft.Extensions.AI.DecisionRequest! request, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +[MEAI001]Microsoft.Extensions.AI.IDecisionClient.GetService(System.Type! serviceType, object? serviceKey = null) -> object? +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.ExpectedScore.get -> double +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.Probabilities.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.Score.get -> double? +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionAnswer.ScoreDecisionAnswer(string! questionId, double? score, System.Collections.Generic.IReadOnlyList! probabilities, Microsoft.Extensions.AI.DecisionPrecision precision = Microsoft.Extensions.AI.DecisionPrecision.HighPrecision, object? rawRepresentation = null, System.Collections.Generic.IReadOnlyDictionary? additionalProperties = null) -> void +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion.Levels.get -> System.Collections.Generic.IReadOnlyList! +[MEAI001]Microsoft.Extensions.AI.ScoreDecisionQuestion.ScoreDecisionQuestion(string! id, string? instructions, System.Collections.Generic.IReadOnlyList! levels) -> void +[MEAI001]static Microsoft.Extensions.AI.DecisionBindingExtensions.Bind(this Microsoft.Extensions.AI.DecisionResponse! response, Microsoft.Extensions.AI.DecisionResultBinding! binding) -> TResult +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, System.Text.Json.JsonElement state, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, Microsoft.Extensions.AI.DecisionDefinition! definition, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetResponseAsync(this Microsoft.Extensions.AI.IDecisionClient! client, TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, System.Collections.Generic.IEnumerable! questions, Microsoft.Extensions.AI.DecisionOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +[MEAI001]static Microsoft.Extensions.AI.DecisionClientExtensions.GetService(this Microsoft.Extensions.AI.IDecisionClient! client, object? serviceKey = null) -> TService? +[MEAI001]static Microsoft.Extensions.AI.DecisionDefinition.Create(System.Action.Builder!>! configure, System.Text.Json.JsonSerializerOptions? serializerOptions = null) -> Microsoft.Extensions.AI.DecisionDefinition! +[MEAI001]static Microsoft.Extensions.AI.DecisionDefinition.Create(System.Text.Json.Serialization.Metadata.JsonTypeInfo! resultTypeInfo, System.Action.Builder!>! configure) -> Microsoft.Extensions.AI.DecisionDefinition! +[MEAI001]static Microsoft.Extensions.AI.DecisionFeatureProjection.Project(Microsoft.Extensions.AI.DecisionResponse! response, Microsoft.Extensions.AI.DecisionFeatureSchema! schema) -> Microsoft.Extensions.AI.DecisionFeatureVector! +[MEAI001]static Microsoft.Extensions.AI.DecisionRequest.Create(TState state, System.Text.Json.Serialization.Metadata.JsonTypeInfo! stateTypeInfo, System.Collections.Generic.IEnumerable! questions) -> Microsoft.Extensions.AI.DecisionRequest! +[MEAI001]virtual Microsoft.Extensions.AI.DecisionOptions.Clone() -> Microsoft.Extensions.AI.DecisionOptions! diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs index b7c961d13c6..8f54bd5b7d6 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs @@ -29,8 +29,8 @@ namespace Microsoft.Agents.AI.TypeSafe; /// entry of . /// /// -/// is the decision API proposed for Microsoft.Extensions.AI. This package carries a copy -/// of it until it ships, so this client and the API are experimental. +/// is the decision API proposed for Microsoft.Extensions.AI. Microsoft.Agents.AI.Abstractions +/// carries a copy of it until it ships, so this client and the API are experimental. /// /// [Experimental(DiagnosticIds.Experiments.AIDecisions, UrlFormat = DiagnosticIds.UrlFormat)] diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/README.md b/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/README.md deleted file mode 100644 index d7d4a98d2e8..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/MEAI/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# MEAI - -A copy of the provider-neutral decision API that is proposed for `Microsoft.Extensions.AI.Abstractions`: -`IDecisionClient`, Binary, Choice, and Score questions and answers, the typed `DecisionDefinition`, and -`DecisionDefinition.AsAIFunction`. Decision models (Jev, Azure OpenAI Decisions, and others) answer typed -questions about a state with probabilities, instead of writing text. - -The types are public, in the `Microsoft.Extensions.AI` namespace, and marked `[Experimental("MEAI001")]`, exactly as -proposed. Code written against them today, such as `JevDecisionClient` and the samples, is the code that will compile -against the shipped API. This follows how Agent Framework shipped proposed MEAI types before they were released -(#279, #426), and deleted them once MEAI shipped them (#427, #676). - -## Source - -| Proposal | Pull request | Commit | -|---|---|---| -| Layer 1: provider-neutral decision abstractions | [dotnet/extensions#7795](https://github.com/dotnet/extensions/pull/7795) | `02e54f023bdf3c8c2729353b110c22f5f8593957` | -| Layer 2: decision interoperability (`AsAIFunction`) | [dotnet/extensions#7796](https://github.com/dotnet/extensions/pull/7796) | `e87f1126cf22108fb27f6391c7826fb91c25138e` | - -The files are `src/Libraries/Microsoft.Extensions.AI.Abstractions/Decisions/*.cs` at the Layer 2 commit, which contains -Layer 1, without changes. `.editorconfig` keeps the upstream style from failing the build, and the project injects -`src/Shared/DiagnosticIds` and `src/Shared/Throw`, which provide the helpers the upstream code uses. - -Not copied: the upstream tests, the README, and the registration of the decision types in -`AIJsonUtilities.DefaultOptions`. Serializing the decision types therefore uses reflection-based metadata. - -The API is still in review. In particular, the API review on #7795 asks to align `DecisionResponse` with -`ChatResponse` (`ModelId` and `ResponseId` instead of `DecisionProvenance`, settable properties), and the shape may -change before it ships. - -## Only one package carries the copy - -The types live in this package so that it can expose them. A second package that defined the same public types would -make them ambiguous (CS0433) for an application that references both. Before another package, such as an Azure OpenAI -Decisions provider, needs the API, move this folder to a package that both depend on. - -## When Microsoft.Extensions.AI ships the API - -1. Update `Microsoft.Extensions.AI.Abstractions` in `Directory.Packages.props`. From that version on, the copied types - conflict with the shipped ones, so the next steps go in the same change. -2. Delete this folder. -3. Fix any difference between this copy and the shipped API in `JevDecisionClient`, the shared decision chat engine - (`src/Shared/DecisionChat`), and the samples. - -To follow upstream changes before then, copy the new upstream files over these ones. diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj index 7977b8470a2..b2357860f96 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj @@ -4,7 +4,7 @@ $(TargetFrameworksCore) Microsoft.Agents.AI.TypeSafe alpha - + $(NoWarn);MEAI001 @@ -19,7 +19,7 @@ Microsoft Agent Framework - TypeSafe (Jev) integration - Provides an IDecisionClient, an IChatClient, and an AIFunction that let agents classify, route, score, and check content with Jev, the TypeSafe System One model. Includes a copy of the decision API proposed for Microsoft.Extensions.AI until it ships. + Provides an IDecisionClient, an IChatClient, and an AIFunction that let agents classify, route, score, and check content with Jev, the TypeSafe System One model. diff --git a/dotnet/src/Shared/DecisionChat/README.md b/dotnet/src/Shared/DecisionChat/README.md index 18ac1549955..56b9fe27f63 100644 --- a/dotnet/src/Shared/DecisionChat/README.md +++ b/dotnet/src/Shared/DecisionChat/README.md @@ -28,10 +28,9 @@ answers in its own result type through `DecisionChatEngineOptions.DescribeAnswer ``` -The project also needs the decision API. Until `Microsoft.Extensions.AI` ships it, the only copy is in -`Microsoft.Agents.AI.TypeSafe/MEAI`, so that package is the only one that can inject the engine today. It also needs -package references to `Microsoft.Extensions.AI.Abstractions` and `Microsoft.Extensions.Logging.Abstractions`, and a -project reference to `Microsoft.Agents.AI.Abstractions`. The code targets .NET 8 and later. +The project also needs a project reference to `Microsoft.Agents.AI.Abstractions`, which carries the decision API +until `Microsoft.Extensions.AI` ships it, and package references to `Microsoft.Extensions.AI.Abstractions` and +`Microsoft.Extensions.Logging.Abstractions`. The code targets .NET 8 and later. The engine uses only the public surface of the decision API. Keep it that way, so it compiles unchanged once the API ships in `Microsoft.Extensions.AI`. At that point the engine can become a public type in an Agent Framework package. From b3a1ba8097c23672c7967b61344816197e5cec73 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:38:38 -0800 Subject: [PATCH 16/16] .NET: Remove JevChatClient and the decision chat engine from the TypeSafe package The MEAI decision API keeps a decision model as its own client, used as a tool (DecisionDefinition.AsAIFunction) or a router (RoutingChatClient), not as an IChatClient. JevChatClient answered questions through IChatClient, which needed a provider-neutral engine to turn the conversation into a state and compile closed-set tools into Choice questions. With the MEAI patterns available through JevDecisionClient, the bridge no longer earns its public surface. Removed: JevChatClient, JevChatClientOptions, JevChatOptionsExtensions, JevChatRequestOptions, JevResultExtensions, the internal raw-question path of JevDecisionClient, src/Shared/DecisionChat, and the Agent_With_TypeSafe sample. The Jev wire types stay public: JevAIToolBuilder exposes them to its mapping hooks, and MEAI's question types cannot carry JSON instructions and criteria, undescribed Choice labels, or a Noul that describes one outcome. --- dotnet/agent-framework-dotnet.slnx | 18 - dotnet/eng/MSBuild/Shared.props | 3 - dotnet/eng/verify-samples/AgentsSamples.cs | 26 - .../02-agents/AgentProviders/README.md | 1 - .../Agent_With_TypeSafe.csproj | 16 - .../typesafe/Agent_With_TypeSafe/Program.cs | 159 ----- .../typesafe/Agent_With_TypeSafe/README.md | 114 ---- .../README.md | 5 +- dotnet/samples/AGENTS.md | 2 +- .../MEAI/README.md | 2 +- .../Core/JevDecisionMapping.cs | 62 -- .../Core/JevDecisionRequestOptions.cs | 23 - .../Core/JevJsonUtilities.cs | 2 +- .../JevChatClient.cs | 168 ----- .../JevChatClientOptions.cs | 108 ---- .../JevChatOptionsExtensions.cs | 43 -- .../JevChatRequestOptions.cs | 26 - .../JevDecisionClient.cs | 7 +- .../JevResultExtensions.cs | 62 -- .../Microsoft.Agents.AI.TypeSafe.csproj | 3 +- .../DecisionChat/DecisionArgumentKind.cs | 21 - .../DecisionChat/DecisionArgumentPlan.cs | 80 --- .../Shared/DecisionChat/DecisionChatEngine.cs | 254 -------- .../DecisionChat/DecisionChatEngineOptions.cs | 45 -- .../Shared/DecisionChat/DecisionChatLog.cs | 15 - .../Shared/DecisionChat/DecisionChatState.cs | 310 --------- .../DecisionChat/DecisionCompiledTool.cs | 36 -- .../src/Shared/DecisionChat/DecisionJson.cs | 96 --- .../DecisionChat/DecisionQuestionBudget.cs | 46 -- .../Shared/DecisionChat/DecisionToolCall.cs | 11 - .../DecisionChat/DecisionToolCallCompiler.cs | 424 ------------ .../DecisionChat/DecisionToolCallPlan.cs | 81 --- .../Shared/DecisionChat/DecisionToolSchema.cs | 419 ------------ .../DecisionUnsupportedToolSchemaException.cs | 30 - dotnet/src/Shared/DecisionChat/README.md | 36 -- .../DecisionToolCallCompilerTests.cs | 457 ------------- .../JevChatClientTests.cs | 425 ------------- .../JevChatClientToolTests.cs | 601 ------------------ .../JevDecisionClientTests.cs | 61 +- .../JevLoopJudgeTests.cs | 60 +- .../{JevChatTestData.cs => JevResultJson.cs} | 27 +- .../ListLoggerFactory.cs | 36 -- 42 files changed, 49 insertions(+), 4372 deletions(-) delete mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj delete mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs delete mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionRequestOptions.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionArgumentKind.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionArgumentPlan.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionChatEngine.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionChatEngineOptions.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionChatLog.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionChatState.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionCompiledTool.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionJson.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionQuestionBudget.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionToolCall.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionToolCallCompiler.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionToolCallPlan.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionToolSchema.cs delete mode 100644 dotnet/src/Shared/DecisionChat/DecisionUnsupportedToolSchemaException.cs delete mode 100644 dotnet/src/Shared/DecisionChat/README.md delete mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/DecisionToolCallCompilerTests.cs delete mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs delete mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs rename dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/{JevChatTestData.cs => JevResultJson.cs} (59%) delete mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index bff3952bc6e..84d443c28da 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -40,7 +40,6 @@ - @@ -560,23 +559,6 @@ - - - - - - - - - - - - - - - - - diff --git a/dotnet/eng/MSBuild/Shared.props b/dotnet/eng/MSBuild/Shared.props index 95226d37ae2..445f47550d9 100644 --- a/dotnet/eng/MSBuild/Shared.props +++ b/dotnet/eng/MSBuild/Shared.props @@ -53,7 +53,4 @@ - - - diff --git a/dotnet/eng/verify-samples/AgentsSamples.cs b/dotnet/eng/verify-samples/AgentsSamples.cs index e7ff4b30552..3f8504d5980 100644 --- a/dotnet/eng/verify-samples/AgentsSamples.cs +++ b/dotnet/eng/verify-samples/AgentsSamples.cs @@ -90,32 +90,6 @@ internal static class AgentsSamples ], }, - new SampleDefinition - { - Name = "Agent_With_TypeSafe", - ProjectPath = "samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe", - RequiredEnvironmentVariables = ["TYPESAFE_API_KEY"], - Inputs = ["Y"], - MustContain = - [ - "Direct JevChatClient result:", - "Agent result:", - "Default questions result:", - "Function calling result:", - "better_city:", - "Tool approval:", - "issue_refund(plan: Pro)", - ], - ExpectedOutputDescription = - [ - "The direct result should route the duplicate subscription charge to billing, and the agent result should route the failing checkout integration to technical with a high urgency probability.", - "The default questions result should route the invoice request to billing with a low urgency probability.", - "The function calling result should contain one weather line for Seattle and one for Amsterdam, with temperatures in Fahrenheit, followed by a better_city line that names one of the two cities.", - "The tool approval step should ask to approve issue_refund for the Pro plan and, after approval, show the refund result followed by a department line.", - "The output should not contain error messages or stack traces.", - ], - }, - new SampleDefinition { Name = "Agent_TypeSafe_Step01_LoopJudge", diff --git a/dotnet/samples/02-agents/AgentProviders/README.md b/dotnet/samples/02-agents/AgentProviders/README.md index fc9e506b3fa..ebd09d35646 100644 --- a/dotnet/samples/02-agents/AgentProviders/README.md +++ b/dotnet/samples/02-agents/AgentProviders/README.md @@ -98,7 +98,6 @@ covering basics, function tools, structured output, middleware, MCP, code interp | Sample | Description | | --- | --- | -| [Agent with TypeSafe](./typesafe/Agent_With_TypeSafe/) | Use Jev, the TypeSafe System One model, as an IChatClient that answers typed questions and calls closed-set tools | | [Loop judge](./typesafe/Agent_TypeSafe_Step01_LoopJudge/) | Use Jev as the judge of a LoopAgent, through an IDecisionClient and a typed decision definition | | [Decision tool](./typesafe/Agent_TypeSafe_Step02_DecisionTool/) | Give an agent a decision as a tool with `DecisionDefinition.AsAIFunction`, and answer it with Jev or a local decision client | | [Decision routing](./typesafe/Agent_TypeSafe_Step03_DecisionRouting/) | Route each request of an agent to a fast or a reasoning model, with Jev choosing the route through `RoutingChatClient` | diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj deleted file mode 100644 index b83c02d799f..00000000000 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - Exe - net10.0 - - enable - enable - - - - - - - - diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs deleted file mode 100644 index 367db584b33..00000000000 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -// This sample shows how to use Jev, the TypeSafe System One model, as an IChatClient and as an AIAgent. -// Jev does not write text: it answers typed questions about the conversation and returns calibrated probabilities. - -using System.ClientModel; -using System.ComponentModel; -using Microsoft.Agents.AI; -using Microsoft.Agents.AI.TypeSafe; -using Microsoft.Extensions.AI; - -var apiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); - -// 1. The questions to answer. Each answer comes back under the ID of its question. -Dictionary questions = new() -{ - ["department"] = new JevChoiceQuestion - { - Instructions = "Which team should handle the support request?", - Criteria = new Dictionary - { - ["billing"] = "Payments, subscriptions, invoices, or refunds", - ["technical"] = "Bugs, outages, integrations, or account access", - ["sales"] = "Pricing, upgrades, or new account questions", - }, - }, - ["frustration"] = new JevScoreQuestion - { - Instructions = "How frustrated does the customer appear?", - Criteria = ["Calm and factual", "Frustrated but civil", "Very angry or threatening to leave"], - }, - ["is_urgent"] = new JevNoulQuestion - { - Instructions = "Does the request require urgent attention?", - Criteria = new JevNoulCriteria - { - True = "The customer describes ongoing harm, lost revenue, or an immediate deadline", - False = "The request can wait for the normal support queue", - }, - }, -}; - -// 2. Call the chat client directly. WithJevQuestions passes the questions through ChatOptions. -using IChatClient chatClient = new JevChatClient(new ApiKeyCredential(apiKey)); -ChatResponse direct = await chatClient.GetResponseAsync( - "I was charged twice for the same subscription. Please refund the duplicate charge.", - new ChatOptions().WithJevQuestions(questions)); -PrintEvaluation("Direct JevChatClient result:", direct.GetJevResult()!); - -// 3. Use the same client through an agent. ChatClientAgentRunOptions carries the questions for one run. -AIAgent agent = chatClient.AsAIAgent(name: "JevTicketEvaluator", instructions: "Evaluate the support request."); -AgentResponse agentResponse = await agent.RunAsync( - "Our checkout integration has failed for three days and we are losing sales. Please help ASAP.", - options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(questions))); -PrintEvaluation("Agent result:", agentResponse.GetJevResult()!); - -// 4. Default questions. A client created with DefaultQuestions answers requests that pass no questions, so code that -// knows nothing about Jev can use it like any other IChatClient; here, an agent run without Jev options. -using IChatClient triageClient = new JevChatClient(new ApiKeyCredential(apiKey), new JevChatClientOptions { DefaultQuestions = questions }); -AgentResponse triaged = await triageClient.AsAIAgent(name: "JevTriage").RunAsync( - "Hi, could you send me a copy of last month's invoice when you get a chance? Thanks!"); -PrintEvaluation("Default questions result:", triaged.GetJevResult()!); - -// 5. Function calling. Jev cannot write arguments, so it chooses them from closed sets: an enum, a Boolean, and an -// optional enum that is passed only when Jev decides the user asked for it. The user asks about two cities, so the -// client allows two tool calls per turn instead of the default one. Jev reads the tool's name and description when -// it chooses a tool, so the local function gets a readable name instead of its compiler-generated one. -using IChatClient weatherClient = new JevChatClient(new ApiKeyCredential(apiKey), new JevChatClientOptions { MaximumToolCallsPerTurn = 2 }); -AIAgent weatherAgent = weatherClient.AsAIAgent(name: "WeatherEvaluator", tools: [AIFunctionFactory.Create(GetWeather, "get_weather")]); -var comparison = new Dictionary -{ - ["better_city"] = new JevChoiceQuestion - { - Instructions = "Which city has better weather based on the tool results?", - Criteria = new Dictionary - { - ["Seattle"] = "Seattle has the better weather.", - ["Amsterdam"] = "Amsterdam has the better weather.", - }, - }, -}; -AgentResponse weather = await weatherAgent.RunAsync( - "Give me a detailed weather report for Seattle and Amsterdam in Fahrenheit and tell me where the weather is better.", - options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(comparison))); - -// After tool calls, the text is the tool results followed by one line per Choice and Score answer. -Console.WriteLine("\nFunction calling result:"); -Console.WriteLine(weather.Text); -Console.WriteLine($"Comparison confidence: {((JevChoiceResponse)weather.GetJevResult()!.Answers["better_city"]).Confidence:F3}"); - -// 6. Tool approval. A tool wrapped in ApprovalRequiredAIFunction runs only after the user approves the call that Jev -// chose. The session keeps the pending call, so the approval is sent in a second run with the same questions. -AIAgent refundAgent = chatClient.AsAIAgent(name: "RefundAgent", tools: [new ApprovalRequiredAIFunction(AIFunctionFactory.Create(IssueRefund, "issue_refund"))]); -AgentSession session = await refundAgent.CreateSessionAsync(); -var refundOptions = new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(questions)); -AgentResponse refund = await refundAgent.RunAsync( - "I was charged twice for my Pro plan this month. Please refund the duplicate charge.", - session, - refundOptions); - -Console.WriteLine("\nTool approval:"); -if (refund.Messages.SelectMany(message => message.Contents).OfType().FirstOrDefault() is { ToolCall: FunctionCallContent call } approval) -{ - string arguments = string.Join(", ", call.Arguments?.Select(argument => $"{argument.Key}: {argument.Value}") ?? []); - Console.Write($"Jev wants to call {call.Name}({arguments}). Approve? (Y/N): "); - bool approved = Console.ReadLine()?.Trim().Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false; - refund = await refundAgent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(approved)]), session, refundOptions); -} - -Console.WriteLine(refund.Text); - -static void PrintEvaluation(string label, JevResult result) -{ - var department = (JevChoiceResponse)result.Answers["department"]; - var frustration = (JevScoreResponse)result.Answers["frustration"]; - var urgent = (JevNoulResponse)result.Answers["is_urgent"]; - - Console.WriteLine($"\n{label}"); - Console.WriteLine($"Department: {department.Choice} (confidence {department.Confidence:F3})"); - Console.WriteLine($"Frustration score: {frustration.Score:F3} of {frustration.Legend.Count - 1}"); - Console.WriteLine($"Urgency probability: {urgent.Noul:F3}"); - Console.WriteLine($"Tokens: {result.Usage.InputTokens} in, {result.Usage.OutputTokens} out"); -} - -[Description("Gets the weather forecast.")] -static string GetWeather( - [Description("The city to get the weather for")] City city, - [Description("Whether to include a detailed hourly forecast; true when the user asks for a detailed report")] bool detailed, - [Description("The temperature unit; leave it out to use Celsius")] TemperatureUnit? unit = null) -{ - int celsius = Random.Shared.Next(15, 31); - string temperature = unit == TemperatureUnit.Fahrenheit ? $"{(celsius * 9 / 5) + 32} F" : $"{celsius} C"; - string suffix = detailed ? " with a detailed hourly forecast" : string.Empty; - return $"{city} is sunny and {temperature}{suffix}."; -} - -[Description("Refunds a duplicate charge for a subscription plan.")] -static string IssueRefund([Description("The plan that was charged twice")] Plan plan) => - $"Refunded the duplicate {plan} plan charge."; - -internal enum City -{ - Seattle, - Amsterdam, - Paris, -} - -internal enum TemperatureUnit -{ - Celsius, - Fahrenheit, -} - -internal enum Plan -{ - Basic, - Pro, - Enterprise, -} diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md deleted file mode 100644 index 0d5dd3a98e4..00000000000 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# Agent with TypeSafe (Jev) - -This sample uses `JevChatClient` from `Microsoft.Agents.AI.TypeSafe`, an `IChatClient` for Jev, the TypeSafe System -One model. Jev does not write text. It answers typed questions about the conversation and returns calibrated -probabilities: - -| Question type | What it answers | Answer | -| --- | --- | --- | -| `JevChoiceQuestion` | Which of these named alternatives? | The selected label, the probability of each label, and a confidence | -| `JevScoreQuestion` | Which score on an ordered rubric (2 to 10 entries)? | An expected score that can fall between rubric entries, the probability of each score, and a confidence | -| `JevNoulQuestion` | Yes or no? | The probability, from 0 to 1, of yes | - -The client follows the TypeSafe connector for Python (`agent-framework-typesafe`), so both languages behave alike. - -## What the sample shows - -1. **The chat client directly.** `WithJevQuestions` puts the questions in `ChatOptions`, and `GetJevResult` reads the - typed answers from the response: - - ```csharp - IChatClient client = new JevChatClient(new ApiKeyCredential(apiKey)); - ChatResponse response = await client.GetResponseAsync(ticket, new ChatOptions().WithJevQuestions(questions)); - JevResult result = response.GetJevResult()!; - ``` - - The response text is the same result as JSON, so code that reads only text still gets every answer. - -2. **The client behind an agent.** A `ChatClientAgentRunOptions` carries the questions for one run. Questions in the - agent's `ChatClientAgentOptions.ChatOptions` apply to every run, and a run's questions replace them. - -3. **Default questions.** A client created with `JevChatClientOptions.DefaultQuestions` answers requests that pass no - questions, so code that knows nothing about Jev can use it like any other `IChatClient`. The sample runs an agent - without any Jev options. - -4. **Function calling.** Jev cannot write arguments, so the client adds internal questions that choose the tool and - each argument from a closed set: a constant, an enum, a Boolean, or an array of enum values. Optional arguments - are passed only when Jev decides the user asked for them; here, the temperature unit is passed because the user - asks for Fahrenheit. The agent runs the tool, and once the turn has used - `JevChatClientOptions.MaximumToolCallsPerTurn` calls (one by default, two in this sample), the client answers the - caller's questions with the tool results in the state. The response text is then the tool results followed by one - `id: answer` line per Choice and Score answer: - - ```text - Seattle is sunny and 75 F with a detailed hourly forecast. - Amsterdam is sunny and 69 F with a detailed hourly forecast. - better_city: Seattle - ``` - - Jev reads each tool's name and description, and each argument's description, to make these choices, so write them - for Jev as you would for a person. Give local functions an explicit name, because their compiler-generated names - mean nothing. In this sample, saying that `detailed` is true "when the user asks for a detailed report" is what makes - Jev choose it reliably. Tools with free-form arguments, such as text or numbers, are left out of the request with a - warning in the log. - Constant and enum values must satisfy their other schema constraints too, including array item constraints; - tools with conflicting constraints are excluded rather than called with invalid arguments. - -5. **Tool approval.** A tool wrapped in `ApprovalRequiredAIFunction` runs only after the user approves the call that - Jev chose. The first run returns a `ToolApprovalRequestContent`; the sample asks for approval in the console and - sends the answer in a second run on the same session, with the same questions. - -## Limits - -Jev answers all questions at once, so the client rejects what needs generated text: streaming (`RunStreamingAsync`), -sampling and output options such as `Temperature` or a JSON `ResponseFormat`, server-side conversations, tools other -than functions, and message content other than text, reasoning text, and function calls and results. Each fails with a -`NotSupportedException` that names the problem. Tool approval requests and responses are allowed but left out of what -Jev reads, because the call and result they lead to carry the information. - -Because streaming is not supported, run agents with `RunAsync`. A function-invoking client runs approved tools before -it calls the chat client, so a streaming run that resumes a tool approval would run the tool and then fail. - -Agents tag stored messages as chat history, which lets Jev distinguish a new user request from text moved after a -resumed approval's tool result. When managing history yourself with `FunctionInvokingChatClient`, tag prior messages -with `message.WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory)` before appending new input. -Without that attribution, a user message starts a new turn. - -## Configuring the client - -`JevChatClientOptions` follows the `ClientPipelineOptions` pattern of the OpenAI and Azure AI client libraries, with the -retry and timeout defaults of the official TypeSafe SDK: - -```csharp -var options = new JevChatClientOptions -{ - ModelId = "jev-latest", // ChatOptions.ModelId overrides it per request - MaximumToolCallsPerTurn = 2, - Transport = new HttpClientPipelineTransport(httpClientFactory.CreateClient("jev")), -}; -IChatClient client = new JevChatClient(new ApiKeyCredential(apiKey), options); -``` - -To use Jev as a tool of an agent whose own model writes the replies, see -[Classification with Jev](../../../Agents/Agent_Step25_ClassificationWithJev/). To use it as the judge of a -`LoopAgent`, see [Jev as the judge of a loop](../Agent_TypeSafe_Step01_LoopJudge/). - -## Prerequisites - -- .NET 10 SDK or later -- A TypeSafe API key - -Set the environment variable: - -```powershell -$env:TYPESAFE_API_KEY="your-typesafe-api-key" -``` - -## Running the sample - -```powershell -dotnet run -``` - -The tool approval step asks for `Y` or `N` in the console. Probabilities, scores, and the temperatures in the tool -results vary between runs. diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md index e4753e04d0e..f088f05d5df 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md @@ -24,8 +24,9 @@ calibrated probabilities. An agent's model writes the replies; Jev makes the fas 4. The same tool is answered by a local keyword classifier instead of the TypeSafe API, through `UseClient` and the agent's services (see [Using another client](#using-another-client)). -To use Jev as the agent's chat client instead, answering questions about the whole conversation, see -[Agent with TypeSafe](../../AgentProviders/typesafe/Agent_With_TypeSafe/). +When the application, rather than the model, decides the questions, give the agent a +`DecisionDefinition.AsAIFunction` tool over `JevDecisionClient` instead, as in +[Decision tool](../../AgentProviders/typesafe/Agent_TypeSafe_Step02_DecisionTool/). ## Configuring the client diff --git a/dotnet/samples/AGENTS.md b/dotnet/samples/AGENTS.md index 9798ce3e588..7b395cd6f2b 100644 --- a/dotnet/samples/AGENTS.md +++ b/dotnet/samples/AGENTS.md @@ -27,7 +27,7 @@ dotnet/samples/ │ │ ├── ollama/ # Ollama provider sample │ │ ├── onnx/ # ONNX Runtime provider sample │ │ ├── openai/ # OpenAI provider samples -│ │ └── typesafe/ # TypeSafe (Jev) chat client and decision API samples +│ │ └── typesafe/ # TypeSafe (Jev) decision API samples │ ├── AgentOpenTelemetry/ # OpenTelemetry integration │ ├── AgentSkills/ # Agent skills patterns │ ├── AgentWithMemory/ # Memory providers (chat history, Mem0, Valkey, Foundry, AgentMemory) diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/README.md b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/README.md index 0148aef0dbd..d6c1cedd00f 100644 --- a/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/README.md +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/MEAI/README.md @@ -48,4 +48,4 @@ re-sync can break code that uses the types, which their `MEAI001` diagnostic all keep loading the types from `Microsoft.Extensions.AI.Abstractions`. Suppress any package validation difference between this copy and the shipped shape in `CompatibilitySuppressions.xml`. 4. Fix any difference between this copy and the shipped API in the packages that use it, such as - `Microsoft.Agents.AI.TypeSafe`, the shared decision chat engine (`src/Shared/DecisionChat`), and the samples. + `Microsoft.Agents.AI.TypeSafe` and the samples. diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs index a2869213dc6..08312354247 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionMapping.cs @@ -74,30 +74,6 @@ public static JevQuestion ToJevQuestion(DecisionQuestion question) => _ => throw new NotSupportedException($"Jev does not answer {question.Kind} questions."), }; - /// - /// Converts a Jev question into the decision question that correlates its answer. - /// - /// - /// JSON instructions and criteria become their JSON text, and an undescribed Choice label is described by the - /// label itself. The Jev question itself is what is sent (see ), so these - /// texts never reach the API. Call first, so that invalid - /// questions fail with the Jev rule they break. - /// - public static DecisionQuestion ToDecisionQuestion(string id, JevQuestion question) => - question switch - { - JevNoulQuestion noul => ToBinaryQuestion(id, noul), - JevChoiceQuestion choice => new ChoiceDecisionQuestion( - id, - ToText(choice.Instructions), - [.. choice.Criteria.Select(criterion => new DecisionCandidate(criterion.Key, ToText(criterion.Value) ?? criterion.Key))]), - JevScoreQuestion score => new ScoreDecisionQuestion( - id, - ToText(score.Instructions), - [.. score.Criteria.Select((criterion, index) => new DecisionScoreLevel(ToKey(index), ToText(criterion) ?? ToKey(index)))]), - _ => throw new ArgumentException($"Question '{id}' is not a Jev Choice, Score, or Noul question.", nameof(question)), - }; - /// /// Converts Jev's answer to a question into the decision answer. /// @@ -133,41 +109,6 @@ usage is { } tokens ? new UsageDetails { InputTokenCount = tokens.InputTokens, OutputTokenCount = tokens.OutputTokens, TotalTokenCount = tokens.InputTokens + tokens.OutputTokens } : null; - /// - /// Gets the Jev result of a decision response from , with only the answers that the - /// response holds. - /// - public static JevResult ToJevResult(DecisionResponse response) - { - if (response.RawRepresentation is not JevResult result) - { - throw new InvalidOperationException("The decision response does not come from Jev."); - } - - var answers = new Dictionary(response.Answers.Count, StringComparer.Ordinal); - foreach (DecisionAnswer answer in response.Answers) - { - answers[answer.QuestionId] = result.Answers[answer.QuestionId]; - } - - return new JevResult { Model = result.Model, Answers = answers, Usage = result.Usage }; - } - - private static BinaryDecisionQuestion ToBinaryQuestion(string id, JevNoulQuestion noul) - { - string? instructions = ToText(noul.Instructions); - string? trueDescription = ToText(noul.Criteria?.True); - string? falseDescription = ToText(noul.Criteria?.False); - - // A Noul may describe only one outcome, but a Binary question needs instructions or both descriptions, so the - // one description also stands in for the instructions. - return new BinaryDecisionQuestion( - id, - instructions ?? (trueDescription is null || falseDescription is null ? trueDescription ?? falseDescription : null), - trueDescription, - falseDescription); - } - private static Dictionary ToCriteria(IReadOnlyList candidates) { var criteria = new Dictionary(candidates.Count, StringComparer.Ordinal); @@ -190,9 +131,6 @@ private static Dictionary ToCriteria(IReadOnlyList - entry is { IsNull: false } value && value.ToString() is { } text && !string.IsNullOrWhiteSpace(text) ? text : null; - private static string ToKey(int index) => index.ToString(CultureInfo.InvariantCulture); private static double GetProbability(IReadOnlyDictionary probabilities, string key, string questionId) => diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionRequestOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionRequestOptions.cs deleted file mode 100644 index e8b2d92ad61..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevDecisionRequestOptions.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using Microsoft.Extensions.AI; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// The raw options of a request, returned by -/// . -/// -internal sealed class JevDecisionRequestOptions -{ - /// - /// Gets the Jev questions to send instead of the decision questions with the same IDs. - /// - /// - /// Jev questions can hold JSON instructions and criteria, which decision questions carry only as text. A caller - /// that has the Jev questions passes them here, so the API receives them unchanged, and the decision questions - /// only correlate the answers. - /// - public IReadOnlyDictionary? Questions { get; init; } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs index 57d20767fa9..7492e80e7ba 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs @@ -10,7 +10,7 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Serializer options and JSON schemas of the Jev contract types, shared by the Jev tool and chat client. +/// Serializer options and JSON schemas of the Jev contract types, which the Jev tool uses. /// internal static class JevJsonUtilities { diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs deleted file mode 100644 index b11e85b8f45..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ClientModel; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Agents.AI.Decisions; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Provides an for Jev, the TypeSafe System One model, which answers typed questions about -/// a conversation instead of writing text. -/// -/// -/// -/// Each request sends the conversation as the Jev state, together with the questions passed with -/// (or ). -/// The response text is the as JSON, and -/// returns it typed. This follows the TypeSafe connector for Python, so both languages behave alike. -/// -/// -/// Jev can also call function tools whose arguments come from a closed set: constants, enums, Booleans, and arrays of -/// enum values. The client adds internal questions that choose the tool and its arguments, and returns a -/// for a function-invoking client (which a ChatClientAgent adds) to run. -/// Once the turn has used calls, the client answers the -/// questions instead; the response text is then the tool results followed by one id: answer line per Choice -/// and Score answer. Tools with free-form arguments, such as text or numbers, are left out with a warning. -/// -/// -/// Features that need generated text fail with : streaming, sampling and output -/// options such as or a JSON , -/// server-side conversations and background responses, tools other than functions, and content other than text, -/// reasoning text, and function calls and results. Tool approval requests and responses are left out of the state, -/// because the call and result they lead to carry the information. -/// -/// -public sealed class JevChatClient : IChatClient -{ - private readonly JevHttpClient _client; - private readonly DecisionChatEngine _engine; - private readonly IReadOnlyDictionary? _defaultQuestions; - private readonly ChatClientMetadata _metadata; - - /// - /// Initializes a new instance of the class. - /// - /// The TypeSafe API key, sent as a bearer token. - /// The client and pipeline options. Defaults are used when . - /// - /// is empty, or is not an - /// absolute URI. - /// - public JevChatClient(ApiKeyCredential credential, JevChatClientOptions? options = null) - { - _ = Throw.IfNull(credential); - options ??= new JevChatClientOptions(); - - if (string.IsNullOrWhiteSpace(options.ModelId)) - { - Throw.ArgumentException(nameof(options), $"{nameof(JevChatClientOptions)}.{nameof(JevChatClientOptions.ModelId)} must not be empty."); - } - - if (options.Endpoint?.IsAbsoluteUri is false) - { - Throw.ArgumentException(nameof(options), $"{nameof(JevChatClientOptions)}.{nameof(JevChatClientOptions.Endpoint)} must be an absolute URI."); - } - - // The questions are copied so that later changes to the caller's dictionary do not change this client. - this._defaultQuestions = options.DefaultQuestions?.ToDictionary(question => question.Key, question => question.Value, StringComparer.Ordinal); - this._client = new JevHttpClient(credential, options, options.Endpoint, options.ModelId); - this._metadata = new ChatClientMetadata(JevDecisionClient.ProviderName, this._client.Endpoint, this._client.ModelId); - - // The conversation, tool calling, and turn logic are provider-neutral; this client supplies Jev as the decision - // client and keeps the Jev result as what callers see. - this._engine = new DecisionChatEngine(new JevDecisionClient(this._client), new DecisionChatEngineOptions - { - ModelName = "Jev", - QuestionsDescription = $"the questions passed with {nameof(JevChatOptionsExtensions.WithJevQuestions)}", - CallIdPrefix = "typesafe", - DescribeAnswers = DescribeAnswers, - MaximumToolCallsPerTurn = options.MaximumToolCallsPerTurn, - Logger = (options.ClientLoggingOptions?.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(), - }); - } - - /// - /// - /// The request has no questions, a required tool call is impossible, or the Jev result does not answer every - /// question. - /// - /// The request uses an option, tool, or content that Jev does not support. - /// The TypeSafe API rejected the request. - public async Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(messages); - - // The Jev questions are sent unchanged, so JSON instructions and criteria keep their form; the decision - // questions made from them correlate the answers. - IReadOnlyDictionary questions = this.GetQuestions(options); - JevContractValidator.ValidateQuestions(questions, nameof(options)); - List decisionQuestions = [.. questions.Select(question => JevDecisionMapping.ToDecisionQuestion(question.Key, question.Value))]; - var decisionOptions = new DecisionOptions - { - ModelId = options?.ModelId, - RawRepresentationFactory = _ => new JevDecisionRequestOptions { Questions = questions }, - }; - - return await this._engine.GetResponseAsync(messages, options, decisionQuestions, decisionOptions, cancellationToken).ConfigureAwait(false); - } - - /// - /// Always: Jev answers all questions at once and does not stream. - public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) => - throw new NotSupportedException($"Jev answers all questions at once and does not stream. Call {nameof(GetResponseAsync)}, or RunAsync on an agent."); - - /// - public object? GetService(Type serviceType, object? serviceKey = null) - { - _ = Throw.IfNull(serviceType); - - return serviceKey is not null ? null - : serviceType == typeof(ChatClientMetadata) ? this._metadata - : serviceType.IsInstanceOfType(this) ? this - : null; - } - - /// - public void Dispose() - { - // The pipeline owns no disposable resources; its default transport shares one HttpClient per process. - } - - /// - /// Describes the answers to the caller's questions as the that callers read, in JSON for the - /// response text and typed for . - /// - private static DecisionChatAnswers DescribeAnswers(DecisionResponse answers) - { - JevResult result = JevDecisionMapping.ToJevResult(answers); - return new DecisionChatAnswers(JsonSerializer.Serialize(result, JevJsonContext.Default.JevResult), result); - } - - private IReadOnlyDictionary GetQuestions(ChatOptions? options) - { - IReadOnlyDictionary? questions = - (options?.RawRepresentationFactory?.Invoke(this) as JevChatRequestOptions)?.Questions ?? this._defaultQuestions; - if (questions is null) - { - Throw.InvalidOperationException( - $"Jev needs questions to answer. Pass them with {nameof(ChatOptions)}.{nameof(JevChatOptionsExtensions.WithJevQuestions)}, or set {nameof(JevChatClientOptions)}.{nameof(JevChatClientOptions.DefaultQuestions)}."); - } - - if (questions.Count == 0) - { - Throw.ArgumentException(nameof(options), "Jev needs at least one question."); - } - - return questions; - } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs deleted file mode 100644 index 61d365557b5..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ClientModel.Primitives; -using System.Collections.Generic; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Configures a and the client pipeline that sends its requests to the TypeSafe System -/// One API. -/// -/// -/// -/// This type follows the pattern of the OpenAI and Azure AI client libraries: -/// selects the HTTP stack, -/// controls retries, and adds custom -/// policies. Unset pipeline settings get the defaults of the official TypeSafe SDK, as described for -/// . also supplies the logger -/// that reports tools left out of a request. -/// -/// -/// Like other client options, an instance becomes read-only once a client has been created with it. -/// -/// -public sealed class JevChatClientOptions : ClientPipelineOptions -{ - /// - public override void Freeze() - { - JevPipelineDefaults.Apply(this); - base.Freeze(); - } - - /// - /// Gets or sets the System One endpoint. When , https://api.typesafe.ai/v1/systemone is used. - /// - public Uri? Endpoint - { - get; - set - { - this.AssertNotFrozen(); - field = value; - } - } - - /// - /// Gets or sets the Jev model that answers the questions, unless - /// selects another for a request. Defaults to jev-latest. - /// - public string ModelId - { - get; - set - { - this.AssertNotFrozen(); - field = value; - } - } = JevPipelineDefaults.DefaultModelId; - - /// - /// Gets or sets the questions to answer when a request does not pass its own with - /// . - /// - /// - /// Default questions let code that knows nothing about Jev, such as a framework component with a fixed task, - /// use the client as any other . The client copies the dictionary when it - /// is created. - /// - public IReadOnlyDictionary? DefaultQuestions - { - get; - set - { - this.AssertNotFrozen(); - field = value; - } - } - - /// - /// Gets or sets how many tool calls Jev may request in one turn, that is, while answering one user request. - /// Defaults to 1. - /// - /// - /// The turn starts after the last user message, except that text sent together with a tool approval response - /// continues the turn of the call it approves. Once the turn has this many function calls, the client stops - /// offering tools and answers the questions, with the tool results in the state. The route question tells Jev - /// which calls the turn already made, but a closed-set choice can still repeat one, so the limit is what - /// guarantees that the tool loop ends. Raise it when one user request needs several distinct calls, such as the - /// weather in two cities; set it to 0 to never call tools. - /// When managing history without an agent, tag prior messages with - /// and - /// so a newly produced approval result can be distinguished - /// from an older tool result. Without that attribution, a user message starts a new turn. - /// - /// The value is negative. - public int MaximumToolCallsPerTurn - { - get; - set - { - this.AssertNotFrozen(); - field = Throw.IfLessThan(value, 0); - } - } = 1; -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs deleted file mode 100644 index 119269f117c..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using Microsoft.Extensions.AI; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Provides extension methods that pass Jev questions through . -/// -public static class JevChatOptionsExtensions -{ - /// - /// Sets the questions that a answers for a request. - /// - /// The chat options to configure. - /// The questions to answer, keyed by an ID that the caller chooses. - /// , for chaining. - /// - /// - /// This sets to return a - /// to a . For any other client it calls the factory that was set before, so the same - /// options can still carry provider options for another client. Calling this method again replaces the - /// questions. - /// - /// - /// With an agent, pass the options in a ChatClientAgentRunOptions for one run, or in the agent's - /// ChatClientAgentOptions.ChatOptions for every run; the agent prefers the run's questions. - /// - /// - public static ChatOptions WithJevQuestions(this ChatOptions options, IReadOnlyDictionary questions) - { - _ = Throw.IfNull(options); - _ = Throw.IfNull(questions); - - Func? previous = options.RawRepresentationFactory; - options.RawRepresentationFactory = client => - client is JevChatClient ? new JevChatRequestOptions { Questions = questions } : previous?.Invoke(client); - return options; - } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs deleted file mode 100644 index 3701e3da708..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using Microsoft.Extensions.AI; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Represents the Jev-specific options of one request. -/// -/// -/// A reads this type from , which is -/// how Microsoft.Extensions.AI passes options that only one provider understands. Other chat clients ignore it. -/// sets the factory for you. -/// -public sealed class JevChatRequestOptions -{ - /// - /// Gets or sets the questions to answer about the conversation, keyed by an ID that the caller chooses. - /// - /// - /// Each answer of the uses the ID of its question. IDs that start with - /// __af_tool__ are reserved for the questions that route tool calls. - /// - public required IReadOnlyDictionary Questions { get; set; } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs index 8f54bd5b7d6..d72b75606f3 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevDecisionClient.cs @@ -70,15 +70,10 @@ public async Task GetResponseAsync(DecisionRequest request, De { _ = Throw.IfNull(request); - // A caller that has Jev's own questions, such as JevChatClient, sends them unchanged, so JSON instructions and - // criteria keep their form; the decision questions with the same IDs then only correlate the answers. - IReadOnlyDictionary? jevQuestions = (options?.RawRepresentationFactory?.Invoke(this) as JevDecisionRequestOptions)?.Questions; var questions = new Dictionary(request.Questions.Count, StringComparer.Ordinal); foreach (DecisionQuestion question in request.Questions) { - questions[question.Id] = jevQuestions is not null && jevQuestions.TryGetValue(question.Id, out JevQuestion? jevQuestion) - ? jevQuestion - : JevDecisionMapping.ToJevQuestion(question); + questions[question.Id] = JevDecisionMapping.ToJevQuestion(question); } var jevRequest = new JevRequest { State = JevDecisionMapping.ToEntry(request.State), Questions = questions }; diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs deleted file mode 100644 index a233cf7e45d..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using Microsoft.Extensions.AI; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Provides extension methods that read the of a response. -/// -public static class JevResultExtensions -{ - /// - /// Gets the typed answers of a response. - /// - /// The chat response. - /// - /// The answers to the request's questions, with their probabilities and token usage, or - /// when the response did not come from a or ended with a tool call. - /// - public static JevResult? GetJevResult(this ChatResponse response) - { - _ = Throw.IfNull(response); - return response.RawRepresentation as JevResult ?? FromMessages(response.Messages); - } - - /// - /// Gets the typed answers of an agent response produced with a . - /// - /// The agent response. - /// - /// The answers to the run's questions, with their probabilities and token usage, or when - /// the response has none. - /// - public static JevResult? GetJevResult(this AgentResponse response) - { - _ = Throw.IfNull(response); - - // A chat client agent keeps the chat response as the raw representation; agents that rebuild the response - // still keep the messages, which carry the result too. - return response.RawRepresentation switch - { - JevResult result => result, - ChatResponse chatResponse => chatResponse.GetJevResult(), - _ => FromMessages(response.Messages), - }; - } - - private static JevResult? FromMessages(IList messages) - { - for (int index = messages.Count - 1; index >= 0; index--) - { - if (messages[index].RawRepresentation is JevResult result) - { - return result; - } - } - - return null; - } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj index b2357860f96..3c319fcef60 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj @@ -11,7 +11,6 @@ true true - true @@ -19,7 +18,7 @@ Microsoft Agent Framework - TypeSafe (Jev) integration - Provides an IDecisionClient, an IChatClient, and an AIFunction that let agents classify, route, score, and check content with Jev, the TypeSafe System One model. + Provides an IDecisionClient and an AIFunction that let agents classify, route, score, and check content with Jev, the TypeSafe System One model. diff --git a/dotnet/src/Shared/DecisionChat/DecisionArgumentKind.cs b/dotnet/src/Shared/DecisionChat/DecisionArgumentKind.cs deleted file mode 100644 index cbca2a6a202..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionArgumentKind.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// How the value of one tool argument is read from the answers of a decision model. -/// -internal enum DecisionArgumentKind -{ - /// The schema allows one value, so no question is asked about it. - Const, - - /// A Choice question selects one value of an enum. - Choice, - - /// A Binary question decides a Boolean. - Boolean, - - /// One Binary question per enum member decides which members an array includes. - Set, -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionArgumentPlan.cs b/dotnet/src/Shared/DecisionChat/DecisionArgumentPlan.cs deleted file mode 100644 index 021f4fafc22..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionArgumentPlan.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using Microsoft.Extensions.AI; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// The questions asked about one tool argument and how its value is read from the answers. -/// -internal sealed class DecisionArgumentPlan -{ - public required string Name { get; init; } - - public required DecisionArgumentKind Kind { get; init; } - - /// - /// Gets the ID of the Binary question that decides whether an optional argument is passed at all, or - /// for a required argument. - /// - public string? PresenceQuestionId { get; init; } - - /// Gets the ID of the Choice or Binary question of a or argument. - public string? ValueQuestionId { get; init; } - - /// Gets the enum values of a argument, keyed by their Choice label. - public IReadOnlyDictionary Values { get; init; } = new Dictionary(); - - /// Gets the Binary question ID and the value of each member of a argument. - public IReadOnlyList> Members { get; init; } = []; - - /// Gets the value of a argument. - public JsonElement ConstValue { get; init; } - - /// - /// Reads the argument value from the answers. - /// - /// - /// when an optional argument should be left out, so that the function uses its default. - /// - public bool TryDecode(DecisionResponse response, out JsonElement value) - { - value = default; - if (this.PresenceQuestionId is not null && !DecisionToolCallPlan.IsYes(response, this.PresenceQuestionId)) - { - return false; - } - - switch (this.Kind) - { - case DecisionArgumentKind.Const: - value = this.ConstValue; - return true; - - case DecisionArgumentKind.Boolean: - value = DecisionJson.CreateBoolean(DecisionToolCallPlan.IsYes(response, this.ValueQuestionId!)); - return true; - - case DecisionArgumentKind.Choice: - string label = DecisionToolCallPlan.GetChoice(response, this.ValueQuestionId!); - if (!this.Values.TryGetValue(label, out value)) - { - Throw.InvalidOperationException($"The decision model returned the unknown choice '{label}' for the tool argument '{this.Name}'."); - } - - return true; - - case DecisionArgumentKind.Set: - value = DecisionJson.CreateArray(this.Members.Where(member => DecisionToolCallPlan.IsYes(response, member.Key)).Select(member => member.Value)); - return true; - - default: - Throw.InvalidOperationException($"The argument plan of '{this.Name}' has the unknown kind {this.Kind}."); - return false; - } - } -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionChatEngine.cs b/dotnet/src/Shared/DecisionChat/DecisionChatEngine.cs deleted file mode 100644 index 4e86e3c08e3..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionChatEngine.cs +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// Answers chat requests with a decision model, which answers typed questions about a state instead of writing text. -/// -/// -/// -/// This is the provider-neutral part of a decision model chat client. Each request sends the conversation as the state -/// (see ), together with the caller's questions and the internal questions that let the -/// model call a function tool whose arguments come from a closed set (see ). The -/// response is a for a function-invoking client to run, or the answers to the -/// caller's questions. Once the turn has used calls, -/// tools are no longer offered, which ends the function-invoking loop. -/// -/// -/// A provider's public chat client supplies the , converts its own question types into -/// values, and describes the answers in its own result type through -/// . The engine uses only the public surface of the decision -/// API, so it compiles unchanged against Microsoft.Extensions.AI once that API ships. -/// -/// -internal sealed class DecisionChatEngine -{ - private readonly IDecisionClient _client; - private readonly DecisionChatEngineOptions _options; - - public DecisionChatEngine(IDecisionClient client, DecisionChatEngineOptions options) - { - this._client = Throw.IfNull(client); - this._options = Throw.IfNull(options); - } - - /// - /// Answers one chat request. - /// - /// The conversation. - /// The chat options, which supply the tools, the tool mode, and the instructions. - /// The caller's questions, which the answer text and raw representation describe. - /// The options passed to the decision client, such as the model ID. - /// The cancellation token to monitor. - /// A tool call, or the answers to . - /// A message or question is invalid. - /// - /// A required tool call is impossible, or the decision response does not answer every question as asked. - /// - /// The request uses an option, tool, or content that the model does not support. - public async Task GetResponseAsync( - IEnumerable messages, - ChatOptions? options, - IReadOnlyList questions, - DecisionOptions? decisionOptions, - CancellationToken cancellationToken) - { - _ = Throw.IfNull(messages); - _ = Throw.IfNull(questions); - IReadOnlyList history = messages as IReadOnlyList ?? [.. messages]; - string modelName = this._options.ModelName; - - this.EnsureSupported(options); - if (questions.Count == 0) - { - Throw.ArgumentException(nameof(questions), $"{modelName} needs at least one question."); - } - - DecisionToolCallCompiler.EnsureNoReservedQuestionIds(questions.Select(question => question.Id)); - List tools = this.GetFunctionTools(options); - - // Once the turn has used its tool calls, tools are no longer offered, so the next response answers the - // questions with the tool results in the state. This is what ends the function-invoking loop, like the - // max_function_calls default of one in the Python TypeSafe connector. - int turnStart = DecisionChatState.GetCurrentTurnStart(history); - ChatToolMode? toolMode = DecisionChatState.CountCurrentTurnCalls(history, turnStart) >= this._options.MaximumToolCallsPerTurn - ? ChatToolMode.None - : options?.ToolMode; - DecisionToolCallPlan? plan = DecisionToolCallCompiler.Compile( - tools, - toolMode, - DecisionChatState.GetCurrentTurnCalls(history, turnStart), - modelName, - this._options.Logger); - - // The reserved prefix keeps the internal question IDs apart from the caller's. - List allQuestions = [.. questions, .. plan?.Questions.Values ?? []]; - var request = new DecisionRequest(DecisionChatState.Build(history, options?.Instructions, modelName), allQuestions); - - DecisionResponse response; - try - { - response = await this._client.GetResponseAsync(request, decisionOptions, cancellationToken).ConfigureAwait(false); - } - catch (DecisionProtocolException ex) - { - // A response that does not answer the questions as asked is a provider failure, which callers of a chat - // client see as an InvalidOperationException. - throw new InvalidOperationException($"The {modelName} response is invalid: {ex.Message}", ex); - } - - if (plan?.Decode(response) is { } call) - { - var callMessage = new ChatMessage( - ChatRole.Assistant, - [new FunctionCallContent($"{this._options.CallIdPrefix}-{Guid.NewGuid():N}", call.Function.Name, call.Arguments)]); - return new ChatResponse(callMessage) - { - ModelId = response.Provenance?.ModelId, - FinishReason = ChatFinishReason.ToolCalls, - Usage = response.Usage, - CreatedAt = DateTimeOffset.UtcNow, - }; - } - - // The caller asked only its own questions, so the internal tool answers are removed from what it sees. - DecisionResponse answers = OnlyAnswersTo(response, questions); - DecisionChatAnswers described = this._options.DescribeAnswers(answers); - var message = new ChatMessage(ChatRole.Assistant, BuildText(history, turnStart, answers, described.Text)) - { - RawRepresentation = described.RawRepresentation, - }; - - return new ChatResponse(message) - { - ModelId = response.Provenance?.ModelId, - FinishReason = ChatFinishReason.Stop, - Usage = response.Usage, - CreatedAt = DateTimeOffset.UtcNow, - RawRepresentation = described.RawRepresentation, - }; - } - - private void EnsureSupported(ChatOptions? options) - { - if (options is null) - { - return; - } - - var unsupported = new List(); - AddIf(options.Temperature is not null, nameof(ChatOptions.Temperature)); - AddIf(options.TopP is not null, nameof(ChatOptions.TopP)); - AddIf(options.TopK is not null, nameof(ChatOptions.TopK)); - AddIf(options.MaxOutputTokens is not null, nameof(ChatOptions.MaxOutputTokens)); - AddIf(options.FrequencyPenalty is not null, nameof(ChatOptions.FrequencyPenalty)); - AddIf(options.PresencePenalty is not null, nameof(ChatOptions.PresencePenalty)); - AddIf(options.Seed is not null, nameof(ChatOptions.Seed)); - AddIf(options.Reasoning is not null, nameof(ChatOptions.Reasoning)); - AddIf(options.StopSequences is { Count: > 0 }, nameof(ChatOptions.StopSequences)); - AddIf(options.ResponseFormat is not (null or ChatResponseFormatText), nameof(ChatOptions.ResponseFormat)); - AddIf(options.ConversationId is not null, nameof(ChatOptions.ConversationId)); - AddIf(options.AllowBackgroundResponses is true, nameof(ChatOptions.AllowBackgroundResponses)); -#pragma warning disable MEAI001 // ContinuationToken is experimental; it is only checked to reject it. - AddIf(options.ContinuationToken is not null, nameof(ChatOptions.ContinuationToken)); -#pragma warning restore MEAI001 - - string modelName = this._options.ModelName; - if (unsupported.Count > 0) - { - throw new NotSupportedException( - $"{modelName} does not support these chat options: {string.Join(", ", unsupported)}. {modelName} does not generate text; it answers {this._options.QuestionsDescription}."); - } - - if (options.AllowMultipleToolCalls is true) - { - throw new NotSupportedException($"{modelName} requests one tool call per response. Leave {nameof(ChatOptions)}.{nameof(ChatOptions.AllowMultipleToolCalls)} unset or false."); - } - - void AddIf(bool condition, string name) - { - if (condition) - { - unsupported.Add(name); - } - } - } - - private List GetFunctionTools(ChatOptions? options) - { - if (options?.Tools is not { Count: > 0 } tools) - { - return []; - } - - List unsupported = [.. tools.Where(tool => tool is not AIFunctionDeclaration).Select(tool => tool.Name)]; - if (unsupported.Count > 0) - { - throw new NotSupportedException($"{this._options.ModelName} can only call function tools; these tools are not supported: {string.Join(", ", unsupported)}."); - } - - return [.. tools.Cast()]; - } - - private static DecisionResponse OnlyAnswersTo(DecisionResponse response, IReadOnlyList questions) - { - if (response.Answers.Count == questions.Count) - { - return response; - } - - var answers = new Dictionary(response.Answers.Count, StringComparer.Ordinal); - foreach (DecisionAnswer answer in response.Answers) - { - answers[answer.QuestionId] = answer; - } - - return new DecisionResponse( - new DecisionRequest(response.Request.State, questions), - [.. questions.Select(question => answers[question.Id])], - response.Provenance, - response.Usage, - response.RawRepresentation, - response.AdditionalProperties); - } - - /// - /// Builds the response text: the described answers, or, after tool calls, the tool results followed by the Choice - /// and Score decisions, which is what a caller that reads only text needs to see. - /// - private static string BuildText(IReadOnlyList history, int turnStart, DecisionResponse answers, string describedAnswers) - { - List lines = DecisionChatState.GetCurrentTurnResultTexts(history, turnStart); - if (lines.Count == 0) - { - return describedAnswers; - } - - // Binary answers are probabilities rather than decisions, so, as in the Python TypeSafe connector, they are left - // to the typed result. - foreach (DecisionAnswer answer in answers.Answers) - { - switch (answer) - { - case ChoiceDecisionAnswer choice: - lines.Add($"{answer.QuestionId}: {choice.SelectedCandidateId}"); - break; - - case ScoreDecisionAnswer score: - lines.Add($"{answer.QuestionId}: {(score.Score ?? score.ExpectedScore).ToString("G6", CultureInfo.InvariantCulture)}"); - break; - } - } - - return string.Join("\n", lines); - } -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionChatEngineOptions.cs b/dotnet/src/Shared/DecisionChat/DecisionChatEngineOptions.cs deleted file mode 100644 index c026953bd3b..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionChatEngineOptions.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// Configures a for one provider. -/// -internal sealed class DecisionChatEngineOptions -{ - /// Gets the name of the decision model in messages, such as Jev. - public required string ModelName { get; init; } - - /// - /// Gets how the provider's questions are passed, which completes the sentence "it answers ..." in the message about - /// unsupported chat options, such as the questions passed with WithJevQuestions. - /// - public required string QuestionsDescription { get; init; } - - /// Gets the prefix of the IDs of the function calls the engine returns, such as typesafe. - public required string CallIdPrefix { get; init; } - - /// - /// Gets the function that describes the answers to the caller's questions in the provider's own terms: the response - /// text when no tool ran in the turn, and the raw representation of the response and its message. - /// - public required Func DescribeAnswers { get; init; } - - /// Gets the number of tool calls after which a turn answers the questions instead. - public int MaximumToolCallsPerTurn { get; init; } = 1; - - /// Gets the logger that receives a warning for each tool that cannot be expressed as questions. - public ILogger Logger { get; init; } = NullLogger.Instance; -} - -/// -/// The answers to the caller's questions, described by the provider. -/// -/// The response text when no tool ran in the turn. -/// The raw representation of the response and its message. -internal readonly record struct DecisionChatAnswers(string Text, object? RawRepresentation); diff --git a/dotnet/src/Shared/DecisionChat/DecisionChatLog.cs b/dotnet/src/Shared/DecisionChat/DecisionChatLog.cs deleted file mode 100644 index c44160d92ed..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionChatLog.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.Logging; - -namespace Microsoft.Agents.AI.Decisions; - -/// Source-generated log messages of . -internal static partial class DecisionChatLog -{ - [LoggerMessage( - EventId = 1, - Level = LogLevel.Warning, - Message = "The tool '{ToolName}' is left out of the {ModelName} request because its parameters cannot be expressed as closed-set questions: {Reason}")] - public static partial void LogToolExcluded(this ILogger logger, string modelName, string toolName, string reason); -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionChatState.cs b/dotnet/src/Shared/DecisionChat/DecisionChatState.cs deleted file mode 100644 index 3ff3cdac341..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionChatState.cs +++ /dev/null @@ -1,310 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Nodes; -using Microsoft.Extensions.AI; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// Converts a conversation into the state that a decision model answers questions about, and reads the current turn of -/// it. -/// -/// -/// The state has the shape of the Python TypeSafe connector: {"messages": [{"role", "contents": [...]}], -/// "instructions"}, where each content is text, text_reasoning, function_call, or -/// function_result. Tool approval requests and responses are left out: they are control messages, and the call -/// and result they lead to carry the information. The current turn is everything after the last user message that -/// starts a request: the tool calls and results that answer it. -/// -internal static class DecisionChatState -{ - /// Builds the JSON state of a conversation. - /// The conversation. - /// The chat instructions, which become the instructions field. - /// The name of the decision model, such as Jev, for messages. - public static JsonElement Build(IReadOnlyList messages, string? instructions, string modelName) - { - var stateMessages = new JsonArray(); - for (int index = 0; index < messages.Count; index++) - { - ChatMessage message = messages[index]; - List unsupported = [.. message.Contents - .Where(content => content is not (TextContent or TextReasoningContent or FunctionCallContent or FunctionResultContent or ToolApprovalRequestContent or ToolApprovalResponseContent)) - .Select(content => content.GetType().Name) - .Distinct(StringComparer.Ordinal) - .OrderBy(name => name, StringComparer.Ordinal)]; - if (unsupported.Count > 0) - { - throw new NotSupportedException( - $"{modelName} supports text, reasoning text, and function call and result content only; message {index} contains {string.Join(", ", unsupported)}."); - } - - // Approval content is not written: FunctionInvokingChatClient keeps handled approvals in the history of - // later turns, and rejecting them would make every turn after an approval fail. - var contents = new JsonArray(); - foreach (AIContent content in message.Contents) - { - switch (content) - { - case TextContent { Text: { Length: > 0 } text }: - contents.Add((JsonNode)new JsonObject { ["type"] = "text", ["text"] = text }); - break; - - // Only the readable reasoning text is sent; protected (encrypted) reasoning is meaningful only to - // the model that produced it. - case TextReasoningContent { Text: { Length: > 0 } reasoning }: - contents.Add((JsonNode)new JsonObject { ["type"] = "text_reasoning", ["text"] = reasoning }); - break; - - case FunctionCallContent call: - contents.Add((JsonNode)new JsonObject - { - ["type"] = "function_call", - ["call_id"] = call.CallId, - ["name"] = call.Name, - ["arguments"] = call.Arguments is null ? null : ToObject(call.Arguments), - }); - break; - - case FunctionResultContent result: - contents.Add((JsonNode)SerializeResult(result, modelName)); - break; - } - } - - if (contents.Count > 0) - { - stateMessages.Add((JsonNode)new JsonObject { ["role"] = message.Role.Value, ["contents"] = contents }); - } - } - - if (stateMessages.Count == 0) - { - Throw.ArgumentException(nameof(messages), $"{modelName} needs at least one message with text or function call content."); - } - - var state = new JsonObject { ["messages"] = stateMessages }; - if (!string.IsNullOrWhiteSpace(instructions)) - { - state["instructions"] = instructions; - } - - return DecisionJson.Parse(state.ToJsonString()); - } - - /// - /// Gets the index of the first message of the current turn, which follows the last user message that starts a - /// request. - /// - /// - /// FunctionInvokingChatClient moves text sent with an approval response after the resumed call and its result. - /// The agent tags the reconstructed call as chat history, while its new result is external. That pairing - /// distinguishes an approval continuation from a new user request after a run that ended with a tool result. - /// - public static int GetCurrentTurnStart(IReadOnlyList messages) - { - int index = messages.Count - 1; - while (index >= 0) - { - if (messages[index].Role != ChatRole.User) - { - index--; - continue; - } - - // Text that followed several approval responses may span consecutive user messages. - int runStart = index; - while (runStart > 0 && messages[runStart - 1].Role == ChatRole.User) - { - runStart--; - } - - if (runStart > 0 && IsResumedToolResult(messages, runStart - 1)) - { - index = runStart - 1; - continue; - } - - return index + 1; - } - - return 0; - } - - private static bool IsResumedToolResult(IReadOnlyList messages, int resultIndex) - { - ChatMessage resultMessage = messages[resultIndex]; - if (resultMessage.GetAgentRequestMessageSourceType() != AgentRequestMessageSourceType.External) - { - return false; - } - - var callIds = new HashSet(resultMessage.Contents.OfType().Select(result => result.CallId), StringComparer.Ordinal); - for (int index = resultIndex - 1; callIds.Count > 0 && index >= 0; index--) - { - ChatMessage message = messages[index]; - if (message.Role == ChatRole.User) - { - break; - } - - if (message.GetAgentRequestMessageSourceType() == AgentRequestMessageSourceType.ChatHistory && - message.Contents.OfType().Any(call => callIds.Contains(call.CallId))) - { - return true; - } - } - - return false; - } - - /// Gets the argument sets of the calls in the current turn, by tool name, oldest first. - public static Dictionary>> GetCurrentTurnCalls(IReadOnlyList messages, int turnStart) - { - var calls = new Dictionary>>(StringComparer.Ordinal); - foreach (FunctionCallContent call in CurrentTurnContents(messages, turnStart)) - { - if (string.IsNullOrEmpty(call.Name)) - { - continue; - } - - var arguments = new Dictionary(StringComparer.Ordinal); - foreach (KeyValuePair argument in call.Arguments ?? new Dictionary()) - { - arguments[argument.Key] = DecisionJson.ToElement(argument.Value); - } - - if (!calls.TryGetValue(call.Name, out List>? list)) - { - calls[call.Name] = list = []; - } - - list.Add(arguments); - } - - return calls; - } - - /// Counts the function calls in the current turn. - public static int CountCurrentTurnCalls(IReadOnlyList messages, int turnStart) => - CurrentTurnContents(messages, turnStart).Count(); - - /// - /// Gets the text of each function result in the current turn, oldest first, unwrapping results of the form - /// {"result": "text"}. - /// - public static List GetCurrentTurnResultTexts(IReadOnlyList messages, int turnStart) => - [.. CurrentTurnContents(messages, turnStart).Select(result => GetResultText(result.Result)).OfType()]; - - private static IEnumerable CurrentTurnContents(IReadOnlyList messages, int turnStart) - where T : AIContent - { - for (int index = turnStart; index < messages.Count; index++) - { - foreach (T content in messages[index].Contents.OfType()) - { - yield return content; - } - } - } - - private static JsonObject ToObject(IDictionary arguments) - { - var json = new JsonObject(); - foreach (KeyValuePair argument in arguments) - { - json[argument.Key] = DecisionJson.ToNodeOrText(argument.Value); - } - - return json; - } - - private static JsonObject SerializeResult(FunctionResultContent result, string modelName) - { - var json = new JsonObject { ["type"] = "function_result", ["call_id"] = result.CallId }; - IEnumerable? items = result.Result switch - { - AIContent content => [content], - IEnumerable contents => contents, - _ => null, - }; - - if (items is null) - { - json["result"] = DecisionJson.ToNodeOrText(result.Result); - return json; - } - - // Rich results are sent as text items, like the Python connector; other media cannot be described to the model. - var texts = new List(); - foreach (AIContent item in items) - { - if (item is not TextContent text) - { - throw new NotSupportedException($"{modelName} supports text function results only; a result of the call '{result.CallId}' contains {item.GetType().Name}."); - } - - texts.Add(text.Text ?? string.Empty); - } - - json["result"] = string.Concat(texts); - var textItems = new JsonArray(); - foreach (string text in texts) - { - textItems.Add((JsonNode)new JsonObject { ["type"] = "text", ["text"] = text }); - } - - json["items"] = textItems; - return json; - } - - private static string? GetResultText(object? result) => - result switch - { - null => null, - string text => Unwrap(text), - JsonElement { ValueKind: JsonValueKind.String } element => Unwrap(element.GetString()!), - JsonElement { ValueKind: JsonValueKind.Null or JsonValueKind.Undefined } => null, - JsonElement element => Unwrap(element.GetRawText()), - TextContent content => content.Text, - IEnumerable contents => string.Concat(contents.OfType().Select(content => content.Text)), - _ => DecisionJson.ToNodeOrText(result)?.ToJsonString(), - }; - - /// - /// Returns the text of a result written as {"result": "text"}, such as an MCP tool result, or the result - /// unchanged. - /// - private static string Unwrap(string result) - { - if (!result.AsSpan().TrimStart().StartsWith("{".AsSpan(), StringComparison.Ordinal)) - { - return result; - } - - try - { - using JsonDocument document = JsonDocument.Parse(result); - JsonElement root = document.RootElement; - if (root.ValueKind == JsonValueKind.Object && - root.EnumerateObject().Count() == 1 && - root.TryGetProperty("result", out JsonElement inner) && - inner.ValueKind == JsonValueKind.String) - { - return inner.GetString()!; - } - } - catch (JsonException) - { - // Text that only looks like JSON is returned as it is. - } - - return result; - } -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionCompiledTool.cs b/dotnet/src/Shared/DecisionChat/DecisionCompiledTool.cs deleted file mode 100644 index 82de0016094..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionCompiledTool.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Text.Json; -using Microsoft.Extensions.AI; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// One tool that can be expressed as decision questions: its route label, its argument questions, and their decoders. -/// -internal sealed class DecisionCompiledTool -{ - public required AIFunctionDeclaration Function { get; init; } - - /// Gets the label of the tool in the route question, such as t0. - public required string RouteLabel { get; init; } - - public required IReadOnlyDictionary Questions { get; init; } - - public required IReadOnlyList Arguments { get; init; } - - public Dictionary DecodeArguments(DecisionResponse response) - { - var arguments = new Dictionary(this.Arguments.Count); - foreach (DecisionArgumentPlan argument in this.Arguments) - { - if (argument.TryDecode(response, out JsonElement value)) - { - arguments[argument.Name] = value; - } - } - - return arguments; - } -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionJson.cs b/dotnet/src/Shared/DecisionChat/DecisionJson.cs deleted file mode 100644 index da33f1b6f82..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionJson.cs +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Buffers; -using System.Collections.Generic; -using System.Text.Json; -using System.Text.Json.Nodes; -using Microsoft.Extensions.AI; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// JSON value helpers for converting chat messages and tool arguments into decision state and back. -/// -internal static class DecisionJson -{ - private static readonly JsonElement s_true = Parse("true"); - private static readonly JsonElement s_false = Parse("false"); - private static readonly JsonElement s_null = Parse("null"); - - /// - /// Converts one tool argument into a JSON node. Chat clients pass arguments as values; - /// callers that invoke the function directly may pass .NET objects instead, which are serialized with - /// . - /// - public static JsonNode? ToNode(object? value) => - value switch - { - null => null, - JsonNode node => node.DeepClone(), - JsonElement element => element.ValueKind switch - { - JsonValueKind.Object => JsonObject.Create(element), - JsonValueKind.Array => JsonArray.Create(element), - JsonValueKind.Null or JsonValueKind.Undefined => null, - _ => JsonValue.Create(element), - }, - _ => JsonSerializer.SerializeToNode(value, AIJsonUtilities.DefaultOptions.GetTypeInfo(value.GetType())), - }; - - /// - /// Converts a value from a chat message, such as a function argument or result, into a JSON node. A value that - /// cannot be serialized, for example a type without metadata under native AOT, becomes its text. - /// - public static JsonNode? ToNodeOrText(object? value) - { - try - { - return ToNode(value); - } - catch (Exception ex) when (ex is NotSupportedException or InvalidOperationException or JsonException or ArgumentException) - { - // Chat history is informational for the decision model, so an unserializable value is described rather - // than failing the whole request. - return JsonValue.Create(value?.ToString()); - } - } - - /// Converts a value from a chat message into a detached , like . - public static JsonElement ToElement(object? value) => - value switch - { - JsonElement element => element.ValueKind == JsonValueKind.Undefined ? s_null : element.Clone(), - null => s_null, - _ => Parse(ToNodeOrText(value)?.ToJsonString() ?? "null"), - }; - - /// Gets the JSON or value. - public static JsonElement CreateBoolean(bool value) => value ? s_true : s_false; - - /// Writes values into a new JSON array. - public static JsonElement CreateArray(IEnumerable values) - { - var buffer = new ArrayBufferWriter(); - using (var writer = new Utf8JsonWriter(buffer)) - { - writer.WriteStartArray(); - foreach (JsonElement value in values) - { - value.WriteTo(writer); - } - - writer.WriteEndArray(); - } - - using JsonDocument document = JsonDocument.Parse(buffer.WrittenMemory); - return document.RootElement.Clone(); - } - - /// Parses JSON text into a detached . - public static JsonElement Parse(string json) - { - using JsonDocument document = JsonDocument.Parse(json); - return document.RootElement.Clone(); - } -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionQuestionBudget.cs b/dotnet/src/Shared/DecisionChat/DecisionQuestionBudget.cs deleted file mode 100644 index 09acd7fa2ba..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionQuestionBudget.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Globalization; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// Counts the internal questions that tool routing adds to a request, and stops the request when they exceed -/// . -/// -/// -/// Each tool is compiled against a copy of the budget, which is committed only when the tool compiles. A tool that -/// is excluded for an unsupported schema therefore returns the questions it reserved to the tools after it. -/// Exceeding the budget stops question materialization, but is reported only after the whole tool schema is checked. -/// -internal sealed class DecisionQuestionBudget -{ - public const int MaxInternalQuestions = 128; - - public DecisionQuestionBudget(int count = 0) - { - this.Count = count; - } - - public int Count { get; set; } - - public bool TryReserve(int count) - { - this.Count += count; - return this.Count <= MaxInternalQuestions; - } - - public void Validate() - { - if (this.Count > MaxInternalQuestions) - { - // Too many questions is a property of the whole tool set, not of one tool, so it stops the request instead - // of excluding the tool that crossed the limit. - Throw.InvalidOperationException(string.Format( - CultureInfo.InvariantCulture, - "The tools need more than {0} internal decision questions. Offer fewer tools, or tools with fewer arguments.", - MaxInternalQuestions)); - } - } -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionToolCall.cs b/dotnet/src/Shared/DecisionChat/DecisionToolCall.cs deleted file mode 100644 index fba6b6f5b79..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionToolCall.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using Microsoft.Extensions.AI; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// The tool call that a decision model selected: the function and the arguments decoded from its answers. -/// -internal readonly record struct DecisionToolCall(AIFunctionDeclaration Function, Dictionary Arguments); diff --git a/dotnet/src/Shared/DecisionChat/DecisionToolCallCompiler.cs b/dotnet/src/Shared/DecisionChat/DecisionToolCallCompiler.cs deleted file mode 100644 index 3acec5e433f..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionToolCallCompiler.cs +++ /dev/null @@ -1,424 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Text.Json; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.Logging; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// Turns function tools into decision questions, so that a decision model, which cannot write arguments, can still call -/// a tool. -/// -/// -/// -/// This is a port of the tool routing in the Python TypeSafe connector (_tool_calls.py), with the same -/// question IDs, limits, and wording, so both connectors ask the same questions. Two schema checks are stricter -/// than in Python: chains of $ref aliases are followed instead of excluding the tool, and const and -/// enum values must satisfy their declared constraints, so a tool that could fail when called is excluded. -/// -/// -/// A route Choice selects the tool, or none. Each argument must come from a closed set: a constant, an enum -/// (a Choice), a Boolean (a Binary question), or an array of enum values (one Binary question per value). An optional -/// argument also gets a presence Binary question, so that it is passed only when the user asked for it. A tool with any -/// other argument, such as free text or a number, cannot be expressed this way; it is logged and left out, or fails the -/// request when the tool mode requires a call and no tool remains. -/// -/// -internal static class DecisionToolCallCompiler -{ - public const string QuestionPrefix = "__af_tool__"; - public const string RouteQuestionId = QuestionPrefix + ".route"; - public const string RouteNone = "none"; - public const int MaxRoutableTools = 32; - public const int MaxToolProperties = 64; - public const int MaxEnumValues = 64; - - /// - /// Rejects caller question IDs that could collide with the internal tool questions. - /// - public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) - { - List reserved = [.. questionIds.Where(id => id.StartsWith(QuestionPrefix, StringComparison.Ordinal)).OrderBy(id => id, StringComparer.Ordinal)]; - if (reserved.Count > 0) - { - Throw.ArgumentException("questions", $"Question IDs that start with '{QuestionPrefix}' are reserved for tool calls: {string.Join(", ", reserved)}."); - } - } - - /// - /// Compiles the tools that the tool mode allows into a tool call plan. - /// - /// The function tools of the request. - /// The tool mode of the request; means . - /// - /// The argument sets of the tools already called in the current turn, by tool name, oldest first. They become - /// hints that keep the model from repeating a call. - /// - /// The name of the decision model, such as Jev, for messages. - /// Receives a warning for each tool that is left out. - /// The plan, or when no tool can be called. - public static DecisionToolCallPlan? Compile( - IReadOnlyList tools, - ChatToolMode? toolMode, - IReadOnlyDictionary>> previousCalls, - string modelName, - ILogger logger) - { - if (toolMode is NoneChatToolMode) - { - return null; - } - - bool required = toolMode is RequiredChatToolMode; - string? requiredName = (toolMode as RequiredChatToolMode)?.RequiredFunctionName; - if (tools.Count == 0) - { - if (required) - { - Throw.InvalidOperationException("The tool mode requires a tool call, but the request has no tools."); - } - - return null; - } - - List selected = [.. tools.Where(tool => requiredName is null || string.Equals(tool.Name, requiredName, StringComparison.Ordinal))]; - if (requiredName is not null && selected.Count == 0) - { - Throw.InvalidOperationException($"The tool mode requires the tool '{requiredName}', but the request does not have it."); - } - - if (selected.Count > MaxRoutableTools) - { - Throw.InvalidOperationException(string.Format( - CultureInfo.InvariantCulture, - "{0} can choose among at most {1} tools per request, but the request has {2}. Offer fewer tools.", - modelName, - MaxRoutableTools, - selected.Count)); - } - - var compiled = new List(selected.Count); - var unsupported = new List>(); - var budget = new DecisionQuestionBudget(); - for (int index = 0; index < selected.Count; index++) - { - AIFunctionDeclaration tool = selected[index]; - - // Each tool reserves questions from a copy of the budget, which is committed only when the tool compiles. - var toolBudget = new DecisionQuestionBudget(budget.Count); - try - { - List> calls = previousCalls.TryGetValue(tool.Name, out List>? found) ? found : []; - compiled.Add(CompileTool(tool, index, calls, toolBudget)); - budget.Count = toolBudget.Count; - } - catch (DecisionUnsupportedToolSchemaException ex) - { - unsupported.Add(new(tool.Name, ex.Message)); - } - } - - foreach (KeyValuePair tool in unsupported) - { - logger.LogToolExcluded(modelName, tool.Key, tool.Value); - } - - if (required && compiled.Count == 0) - { - string details = string.Join("; ", unsupported.Select(tool => $"{tool.Key}: {tool.Value}")); - Throw.InvalidOperationException($"The tool mode requires a tool call, but no tool can be expressed as {modelName} questions. {details}".TrimEnd()); - } - - if (compiled.Count == 0) - { - return null; - } - - // With a single tool that must be called, there is nothing to route. - DecisionCompiledTool? requiredTool = required && compiled.Count == 1 ? compiled[0] : null; - var questions = new Dictionary(StringComparer.Ordinal); - if (requiredTool is null) - { - var candidates = new List(compiled.Count + 1); - foreach (DecisionCompiledTool tool in compiled) - { - string name = tool.Function.Name; - string description = string.IsNullOrWhiteSpace(tool.Function.Description) - ? $"Call the '{name}' tool." - : $"Tool '{name}': {tool.Function.Description}"; - if (previousCalls.TryGetValue(name, out List>? calls) && calls.Count > 0) - { - description += - " Call this tool again only if the latest user request still needs a distinct execution. " + - $"Do not repeat argument sets already called this turn: {DecisionToolSchema.Describe(calls)}."; - } - - candidates.Add(new DecisionCandidate(tool.RouteLabel, description)); - } - - if (!required) - { - candidates.Add(new DecisionCandidate(RouteNone, "Do not call a tool because all requested tool actions are already satisfied, or because no tool is needed.")); - } - - _ = budget.TryReserve(1); - budget.Validate(); - questions[RouteQuestionId] = new ChoiceDecisionQuestion( - RouteQuestionId, - "Which available tool, if any, should handle the latest user request?", - candidates); - } - - foreach (DecisionCompiledTool tool in compiled) - { - foreach (KeyValuePair question in tool.Questions) - { - questions[question.Key] = question.Value; - } - } - - return new DecisionToolCallPlan - { - Questions = questions, - Tools = compiled, - RouteQuestionId = requiredTool is null ? RouteQuestionId : null, - RequiredTool = requiredTool, - }; - } - - private static DecisionCompiledTool CompileTool(AIFunctionDeclaration tool, int toolIndex, List> previousCalls, DecisionQuestionBudget budget) - { - JsonElement schemaElement = tool.JsonSchema; - Dictionary schema = schemaElement.ValueKind switch - { - JsonValueKind.Object => DecisionToolSchema.ToMap(schemaElement), - JsonValueKind.Undefined or JsonValueKind.Null => [], - _ => throw new DecisionUnsupportedToolSchemaException("the top-level input schema must be an object"), - }; - - DecisionToolSchema.RejectUnsupportedKeys(schema, DecisionToolSchema.RootKeys, "root-level"); - if (schema.TryGetValue("type", out JsonElement type) && type.ValueKind != JsonValueKind.Null && !(type.ValueKind == JsonValueKind.String && type.ValueEquals("object"))) - { - throw new DecisionUnsupportedToolSchemaException("the top-level input schema must be an object"); - } - - List properties = []; - if (schema.TryGetValue("properties", out JsonElement propertiesElement) && propertiesElement.ValueKind != JsonValueKind.Null) - { - if (propertiesElement.ValueKind != JsonValueKind.Object) - { - throw new DecisionUnsupportedToolSchemaException("the input schema properties must be an object"); - } - - properties.AddRange(propertiesElement.EnumerateObject()); - } - - if (properties.Count > MaxToolProperties) - { - throw new DecisionUnsupportedToolSchemaException(string.Format( - CultureInfo.InvariantCulture, - "the input schema defines {0} properties; the supported maximum is {1}", - properties.Count, - MaxToolProperties)); - } - - var requiredNames = new HashSet(StringComparer.Ordinal); - if (schema.TryGetValue("required", out JsonElement requiredElement)) - { - if (requiredElement.ValueKind != JsonValueKind.Array || requiredElement.EnumerateArray().Any(item => item.ValueKind != JsonValueKind.String)) - { - throw new DecisionUnsupportedToolSchemaException("the input schema required field must be a string array"); - } - - requiredNames.UnionWith(requiredElement.EnumerateArray().Select(item => item.GetString()!)); - } - - List missing = [.. requiredNames.Where(name => !properties.Any(property => property.Name == name)).OrderBy(name => name, StringComparer.Ordinal)]; - if (missing.Count > 0) - { - throw new DecisionUnsupportedToolSchemaException($"required arguments are missing from properties: {string.Join(", ", missing)}"); - } - - var questions = new Dictionary(StringComparer.Ordinal); - var arguments = new List(properties.Count); - for (int argumentIndex = 0; argumentIndex < properties.Count; argumentIndex++) - { - JsonProperty property = properties[argumentIndex]; - bool isRequired = requiredNames.Contains(property.Name); - if (property.Value.ValueKind != JsonValueKind.Object) - { - throw new DecisionUnsupportedToolSchemaException($"argument '{property.Name}' has an invalid schema"); - } - - List previousValues = [.. previousCalls.Where(call => call.ContainsKey(property.Name)).Select(call => call[property.Name])]; - try - { - if (CompileArgument(tool, schema, property.Name, DecisionToolSchema.ToMap(property.Value), isRequired, toolIndex, argumentIndex, previousValues, budget, questions) is { } argument) - { - arguments.Add(argument); - } - } - catch (DecisionUnsupportedToolSchemaException ex) - { - throw new DecisionUnsupportedToolSchemaException($"{(isRequired ? "required" : "optional")} argument '{property.Name}': {ex.Message}", ex); - } - } - - // Unsupported tools are excluded even if their earlier arguments exceeded the question budget. - budget.Validate(); - return new DecisionCompiledTool - { - Function = tool, - RouteLabel = string.Create(CultureInfo.InvariantCulture, $"t{toolIndex}"), - Questions = questions, - Arguments = arguments, - }; - } - - private static DecisionArgumentPlan? CompileArgument( - AIFunctionDeclaration tool, - Dictionary root, - string name, - Dictionary rawSchema, - bool required, - int toolIndex, - int argumentIndex, - List previousValues, - DecisionQuestionBudget budget, - Dictionary questions) - { - (Dictionary schema, bool nullable) = DecisionToolSchema.Resolve(rawSchema, root); - DecisionToolSchema.RejectUnsupportedKeys(schema, DecisionToolSchema.ArgumentKeys, "argument"); - - // A required argument must be passed, and the model cannot choose null among the closed-set values, so a - // required nullable argument would always have to be guessed. - if (nullable && required) - { - throw new DecisionUnsupportedToolSchemaException("required nullable arguments are not supported"); - } - - string prefix = string.Create(CultureInfo.InvariantCulture, $"{QuestionPrefix}.t{toolIndex}.a{argumentIndex}"); - string description = GetText(schema, "description") ?? GetText(rawSchema, "description") ?? GetText(schema, "title") ?? name; - string previousInstruction = previousValues.Count > 0 - ? $" Values already used for this argument in earlier calls this turn: {DecisionToolSchema.Describe(previousValues)}. Choose a different value when the user requested another distinct call." - : string.Empty; - string? presenceQuestionId = required ? null : prefix + ".present"; - - void AddPresenceQuestion() - { - if (presenceQuestionId is not null) - { - questions[presenceQuestionId] = new BinaryDecisionQuestion( - presenceQuestionId, - $"For the {tool.Name} tool, did the user explicitly specify the {name} argument? Argument meaning: {description}.{previousInstruction}"); - } - } - - if (schema.TryGetValue("const", out JsonElement constValue)) - { - DecisionToolSchema.EnsureValuesMatchSchema(schema, [constValue], root, "const value"); - if (!budget.TryReserve(required ? 0 : 1)) - { - return null; - } - - AddPresenceQuestion(); - return new DecisionArgumentPlan { Name = name, Kind = DecisionArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = constValue }; - } - - if (schema.TryGetValue("enum", out JsonElement enumElement)) - { - List values = DecisionToolSchema.ReadEnumValues(enumElement, "enum"); - DecisionToolSchema.EnsureValuesMatchSchema(schema, values, root, "enum value"); - if (values.Count == 1) - { - if (!budget.TryReserve(required ? 0 : 1)) - { - return null; - } - - AddPresenceQuestion(); - return new DecisionArgumentPlan { Name = name, Kind = DecisionArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = values[0] }; - } - - if (!budget.TryReserve(required ? 1 : 2)) - { - return null; - } - - var labeled = new Dictionary(StringComparer.Ordinal); - var candidates = new List(values.Count); - for (int index = 0; index < values.Count; index++) - { - string label = string.Create(CultureInfo.InvariantCulture, $"v{index}"); - labeled[label] = values[index]; - candidates.Add(new DecisionCandidate(label, DecisionToolSchema.Describe(values[index]))); - } - - string questionId = prefix + ".value"; - AddPresenceQuestion(); - questions[questionId] = new ChoiceDecisionQuestion( - questionId, - $"For the {tool.Name} tool, choose the {name} argument. Argument meaning: {description}.{previousInstruction}", - candidates); - return new DecisionArgumentPlan { Name = name, Kind = DecisionArgumentKind.Choice, PresenceQuestionId = presenceQuestionId, ValueQuestionId = questionId, Values = labeled }; - } - - string? schemaType = schema.TryGetValue("type", out JsonElement typeElement) && typeElement.ValueKind == JsonValueKind.String ? typeElement.GetString() : null; - if (string.Equals(schemaType, "boolean", StringComparison.Ordinal)) - { - string questionId = prefix + ".value"; - if (!budget.TryReserve(required ? 1 : 2)) - { - return null; - } - - AddPresenceQuestion(); - questions[questionId] = new BinaryDecisionQuestion( - questionId, - $"For the {tool.Name} tool, should the {name} argument be true? Argument meaning: {description}.{previousInstruction}"); - return new DecisionArgumentPlan { Name = name, Kind = DecisionArgumentKind.Boolean, PresenceQuestionId = presenceQuestionId, ValueQuestionId = questionId }; - } - - if (string.Equals(schemaType, "array", StringComparison.Ordinal)) - { - Dictionary items = DecisionToolSchema.ResolveArrayItems(schema, root); - if (!items.TryGetValue("enum", out JsonElement membersElement) || membersElement.ValueKind != JsonValueKind.Array || membersElement.GetArrayLength() == 0) - { - throw new DecisionUnsupportedToolSchemaException("array items must define a non-empty enum"); - } - - List members = DecisionToolSchema.ReadEnumValues(membersElement, "array enum"); - DecisionToolSchema.EnsureValuesMatchType(items, members, "array item", "array enum member", "item type"); - - if (!budget.TryReserve(members.Count + (required ? 0 : 1))) - { - return null; - } - - AddPresenceQuestion(); - var memberQuestions = new List>(members.Count); - for (int index = 0; index < members.Count; index++) - { - string questionId = string.Create(CultureInfo.InvariantCulture, $"{prefix}.m{index}"); - questions[questionId] = new BinaryDecisionQuestion( - questionId, - $"For the {tool.Name} tool, should the {name} argument include {DecisionToolSchema.Describe(members[index])}? Argument meaning: {description}.{previousInstruction}"); - memberQuestions.Add(new(questionId, members[index])); - } - - return new DecisionArgumentPlan { Name = name, Kind = DecisionArgumentKind.Set, PresenceQuestionId = presenceQuestionId, Members = memberQuestions }; - } - - throw new DecisionUnsupportedToolSchemaException("only const, enum, boolean, and arrays of enum values are supported"); - } - - private static string? GetText(Dictionary schema, string key) => - schema.TryGetValue(key, out JsonElement value) && value.ValueKind == JsonValueKind.String && value.GetString() is { Length: > 0 } text ? text : null; -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionToolCallPlan.cs b/dotnet/src/Shared/DecisionChat/DecisionToolCallPlan.cs deleted file mode 100644 index a69efd9dba7..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionToolCallPlan.cs +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Extensions.AI; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// The internal questions that let a decision model choose one tool call, and the decoders that turn their answers into -/// the call. -/// -internal sealed class DecisionToolCallPlan -{ - /// - /// A Binary answer above this probability means yes, for presence, Boolean, and array member questions. - /// - public const double YesThreshold = 0.5; - - /// Gets the internal questions by ID, in the order they are asked. - public required IReadOnlyDictionary Questions { get; init; } - - public required IReadOnlyList Tools { get; init; } - - /// - /// Gets the ID of the Choice question that selects the tool, or when - /// is the only possible call. - /// - public string? RouteQuestionId { get; init; } - - /// - /// Gets the tool that must be called, when the tool mode requires a call and only one tool compiled. - /// - public DecisionCompiledTool? RequiredTool { get; init; } - - /// - /// Reads the selected tool call from the answers. - /// - /// The tool call, or when the model chose not to call a tool. - public DecisionToolCall? Decode(DecisionResponse response) - { - DecisionCompiledTool? selected = this.RequiredTool; - if (selected is null) - { - string route = GetChoice(response, this.RouteQuestionId!); - if (string.Equals(route, DecisionToolCallCompiler.RouteNone, StringComparison.Ordinal)) - { - return null; - } - - selected = this.Tools.FirstOrDefault(tool => string.Equals(tool.RouteLabel, route, StringComparison.Ordinal)) - ?? throw new InvalidOperationException($"The decision model returned the unknown tool route '{route}'."); - } - - return new DecisionToolCall(selected.Function, selected.DecodeArguments(response)); - } - - public static string GetChoice(DecisionResponse response, string questionId) => - FindAnswer(response, questionId) is ChoiceDecisionAnswer choice - ? choice.SelectedCandidateId - : throw new InvalidOperationException($"The decision response has no Choice answer for the internal question '{questionId}'."); - - public static bool IsYes(DecisionResponse response, string questionId) => - FindAnswer(response, questionId) is BinaryDecisionAnswer binary - ? binary.TrueProbability > YesThreshold - : throw new InvalidOperationException($"The decision response has no Binary answer for the internal question '{questionId}'."); - - private static DecisionAnswer? FindAnswer(DecisionResponse response, string questionId) - { - foreach (DecisionAnswer answer in response.Answers) - { - if (string.Equals(answer.QuestionId, questionId, StringComparison.Ordinal)) - { - return answer; - } - } - - return null; - } -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionToolSchema.cs b/dotnet/src/Shared/DecisionChat/DecisionToolSchema.cs deleted file mode 100644 index d390bc82d9a..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionToolSchema.cs +++ /dev/null @@ -1,419 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Text; -using System.Text.Json; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// Reads the JSON schemas of tool parameters for : which keywords are allowed where, -/// how references and nullable forms resolve, and how values are described to the decision model. -/// -internal static class DecisionToolSchema -{ - // A schema definition may refer to itself; without a limit its resolution would never end. - private const int MaxResolutionDepth = 16; - - private static readonly string[] s_annotationKeys = ["$comment", "$id", "$schema", "default", "deprecated", "description", "examples", "readOnly", "title", "writeOnly"]; - - /// Gets the keywords that do not constrain a value, so they are allowed anywhere. - public static HashSet AnnotationKeys { get; } = new(s_annotationKeys, StringComparer.Ordinal); - - /// Gets the keywords allowed on the object schema of a tool's parameters. - public static HashSet RootKeys { get; } = new(s_annotationKeys.Concat(["$defs", "additionalProperties", "properties", "required", "type"]), StringComparer.Ordinal); - - /// Gets the keywords allowed on the schema of one argument. - public static HashSet ArgumentKeys { get; } = new(s_annotationKeys.Concat(["const", "enum", "items", "type"]), StringComparer.Ordinal); - - /// Gets the keywords allowed on the item schema of an array argument. - public static HashSet ArrayItemKeys { get; } = new(s_annotationKeys.Concat(["enum", "type"]), StringComparer.Ordinal); - - public static Dictionary ToMap(JsonElement schema) - { - var map = new Dictionary(StringComparer.Ordinal); - foreach (JsonProperty property in schema.EnumerateObject()) - { - map[property.Name] = property.Value; - } - - return map; - } - - /// - /// Resolves a local $ref, an anyOf of one schema and null, and the nullable forms that - /// System.Text.Json writes, which are a type array that includes "null" and an enum that lists - /// . - /// - /// The schema without its alternative, and whether it allowed . - public static (Dictionary Schema, bool Nullable) Resolve(Dictionary schema, Dictionary root, int depth = 0) - { - if (depth > MaxResolutionDepth) - { - throw new DecisionUnsupportedToolSchemaException("schema references are nested too deeply"); - } - - var resolved = new Dictionary(schema, StringComparer.Ordinal); - - // A definition can itself be a reference, such as an alias of another definition, so references are followed - // until a schema without one. A cycle would never end, so the chain is bounded like nested resolution. - for (int hops = 0; resolved.Remove("$ref", out JsonElement reference); hops++) - { - if (depth + hops >= MaxResolutionDepth) - { - throw new DecisionUnsupportedToolSchemaException("schema references are nested too deeply"); - } - - RejectCompositionSiblings(resolved, "$ref"); - string? path = reference.ValueKind == JsonValueKind.String ? reference.GetString() : null; - const string DefinitionsPrefix = "#/$defs/"; - if (path?.StartsWith(DefinitionsPrefix, StringComparison.Ordinal) is not true) - { - throw new DecisionUnsupportedToolSchemaException("only local $defs references are supported"); - } - - if (!root.TryGetValue("$defs", out JsonElement definitions) || - definitions.ValueKind != JsonValueKind.Object || - !definitions.TryGetProperty(path.Substring(DefinitionsPrefix.Length), out JsonElement definition) || - definition.ValueKind != JsonValueKind.Object) - { - throw new DecisionUnsupportedToolSchemaException($"unresolved schema reference '{path}'"); - } - - // Annotations next to the reference, such as a description, override those of the definition, so the - // annotations closest to the argument win along a chain. An anyOf inside the definition is resolved below, - // together with the rest of the merged schema. - resolved = Merge(ToMap(definition), resolved); - } - - bool nullable = false; - if (resolved.Remove("anyOf", out JsonElement anyOf)) - { - RejectCompositionSiblings(resolved, "anyOf"); - if (anyOf.ValueKind != JsonValueKind.Array) - { - throw new DecisionUnsupportedToolSchemaException("anyOf must be an array"); - } - - List arms = [.. anyOf.EnumerateArray()]; - List nullArms = [.. arms.Where(arm => arm.ValueKind == JsonValueKind.Object && IsNullSchema(arm))]; - List valueArms = [.. arms.Where(arm => arm.ValueKind == JsonValueKind.Object && !IsNullSchema(arm))]; - if (arms.Count != 2 || nullArms.Count != 1 || valueArms.Count != 1) - { - throw new DecisionUnsupportedToolSchemaException("only a single schema combined with null is supported"); - } - - (Dictionary nested, _) = Resolve(ToMap(valueArms[0]), root, depth + 1); - resolved = Merge(nested, resolved); - nullable = true; - } - - if (resolved.TryGetValue("type", out JsonElement type) && type.ValueKind == JsonValueKind.Array && type.EnumerateArray().Any(IsNullName)) - { - List valueTypes = [.. type.EnumerateArray().Where(item => !IsNullName(item))]; - resolved["type"] = valueTypes.Count == 1 ? valueTypes[0] : DecisionJson.CreateArray(valueTypes); - nullable = true; - } - - if (resolved.TryGetValue("enum", out JsonElement values) && values.ValueKind == JsonValueKind.Array && values.EnumerateArray().Any(value => value.ValueKind == JsonValueKind.Null)) - { - resolved["enum"] = DecisionJson.CreateArray(values.EnumerateArray().Where(value => value.ValueKind != JsonValueKind.Null)); - nullable = true; - } - - return (resolved, nullable); - } - - public static void RejectUnsupportedKeys(Dictionary schema, HashSet supportedKeys, string location) - { - List unsupported = [.. schema.Keys.Where(key => !supportedKeys.Contains(key)).OrderBy(key => key, StringComparer.Ordinal)]; - if (unsupported.Count > 0) - { - throw new DecisionUnsupportedToolSchemaException($"unsupported {location} schema constraints: {string.Join(", ", unsupported)}"); - } - } - - /// - /// Reads a type keyword, which is a type name or a non-empty array of type names. - /// - public static IReadOnlyList GetTypes(JsonElement? type, string location) - { - if (type is not { } value || value.ValueKind == JsonValueKind.Null) - { - return []; - } - - if (value.ValueKind == JsonValueKind.String) - { - return [value.GetString()!]; - } - - if (value.ValueKind == JsonValueKind.Array && value.GetArrayLength() > 0 && value.EnumerateArray().All(item => item.ValueKind == JsonValueKind.String)) - { - return [.. value.EnumerateArray().Select(item => item.GetString()!)]; - } - - throw new DecisionUnsupportedToolSchemaException($"{location} type must be a string or non-empty string array"); - } - - public static bool MatchesType(JsonElement value, string type) => - type switch - { - "string" => value.ValueKind == JsonValueKind.String, - - // An integer has no fraction or exponent, as a JSON parser that separates integers from floats reads it, - // and may be of any size, like a Python int; a long would reject valid unsigned 64-bit values. - "integer" => value.ValueKind == JsonValueKind.Number && value.GetRawText().AsSpan().IndexOfAny('.', 'e', 'E') < 0, - "number" => value.ValueKind == JsonValueKind.Number, - "boolean" => value.ValueKind is JsonValueKind.True or JsonValueKind.False, - "null" => value.ValueKind == JsonValueKind.Null, - "array" => value.ValueKind == JsonValueKind.Array, - "object" => value.ValueKind == JsonValueKind.Object, - _ => false, - }; - - /// - /// Checks that every const or enum value of a schema matches its declared type. - /// - /// - /// A value of another type can never be a valid argument, for example "fast" for an integer. The model would - /// still choose it, and the call would fail only when the function binds its arguments, so such a tool is left out - /// of routing instead. A schema without a type is not checked. - /// - /// The resolved schema whose type applies. - /// The values to check. - /// Where the schema is, for the message about an invalid type keyword. - /// How a value is named in the message, such as enum value. - /// How the type is named in the message, such as type. - public static void EnsureValuesMatchType(Dictionary schema, IEnumerable values, string location, string valueName, string typeName) - { - IReadOnlyList types = GetTypes(schema.TryGetValue("type", out JsonElement type) ? type : null, location); - if (types.Count == 0) - { - return; - } - - foreach (JsonElement value in values) - { - if (!types.Any(candidate => MatchesType(value, candidate))) - { - string declared = types.Count == 1 ? $"\"{types[0]}\"" : Describe(types.Select(candidate => DecisionJson.ToElement(candidate))); - throw new DecisionUnsupportedToolSchemaException($"{valueName} {Describe(value)} does not match declared {typeName} {declared}"); - } - } - } - - public static List ReadEnumValues(JsonElement values, string name) - { - if (values.ValueKind != JsonValueKind.Array || values.GetArrayLength() == 0) - { - throw new DecisionUnsupportedToolSchemaException($"{name} must be a non-empty array"); - } - - if (values.GetArrayLength() > DecisionToolCallCompiler.MaxEnumValues) - { - throw new DecisionUnsupportedToolSchemaException(string.Format( - CultureInfo.InvariantCulture, - "{0} defines {1} values; the supported maximum is {2}", - name, - values.GetArrayLength(), - DecisionToolCallCompiler.MaxEnumValues)); - } - - return [.. values.EnumerateArray()]; - } - - public static Dictionary ResolveArrayItems(Dictionary schema, Dictionary root) - { - if (!schema.TryGetValue("items", out JsonElement element) || element.ValueKind != JsonValueKind.Object) - { - throw new DecisionUnsupportedToolSchemaException("array items must define an object schema"); - } - - (Dictionary items, bool nullable) = Resolve(ToMap(element), root); - RejectUnsupportedKeys(items, ArrayItemKeys, "array item"); - if (nullable) - { - throw new DecisionUnsupportedToolSchemaException("nullable array members are not supported"); - } - - return items; - } - - /// Checks all supported constraints before offering a constant or enum value to the model. - public static void EnsureValuesMatchSchema(Dictionary schema, IReadOnlyList values, Dictionary root, string valueName) - { - EnsureValuesMatchType(schema, values, "argument", valueName, "type"); - EnsureValuesMatchEnum(schema, values, valueName); - if (!schema.ContainsKey("items")) - { - return; - } - - Dictionary items = ResolveArrayItems(schema, root); - if (items.TryGetValue("enum", out JsonElement itemEnum)) - { - EnsureValuesMatchType(items, ReadEnumValues(itemEnum, "array enum"), "array item", "array enum member", "item type"); - } - - foreach (JsonElement value in values.Where(value => value.ValueKind == JsonValueKind.Array)) - { - EnsureValuesMatchType(items, value.EnumerateArray(), "array item", "array member", "item type"); - EnsureValuesMatchEnum(items, value.EnumerateArray(), "array member"); - } - } - - private static void EnsureValuesMatchEnum(Dictionary schema, IEnumerable values, string valueName) - { - if (schema.TryGetValue("enum", out JsonElement element)) - { - List alternatives = ReadEnumValues(element, "enum"); - foreach (JsonElement value in values) - { - if (!alternatives.Any(alternative => JsonElement.DeepEquals(value, alternative))) - { - throw new DecisionUnsupportedToolSchemaException($"{valueName} {Describe(value)} does not match declared enum {Describe(alternatives)}"); - } - } - } - } - - /// - /// Writes a value the way the Python connector does with json.dumps(value, ensure_ascii=False, sort_keys=True): - /// sorted object keys, ", " and ": " separators, and unescaped text. The model reads these values in - /// criteria and instructions, so the .NET and Python connectors send the same words. - /// - public static string Describe(JsonElement value) - { - var builder = new StringBuilder(); - AppendValue(builder, value); - return builder.ToString(); - } - - /// Writes values as a JSON array, like . - public static string Describe(IEnumerable values) - { - var builder = new StringBuilder(); - AppendArray(builder, values, AppendValue); - return builder.ToString(); - } - - /// Writes the argument sets of earlier calls as a JSON array of objects, like . - public static string Describe(IEnumerable> calls) - { - var builder = new StringBuilder(); - AppendArray(builder, calls, static (target, call) => AppendObject(target, call.Select(argument => new KeyValuePair(argument.Key, argument.Value)))); - return builder.ToString(); - } - - private static void RejectCompositionSiblings(Dictionary schema, string keyword) - { - List siblings = [.. schema.Keys.Where(key => !AnnotationKeys.Contains(key)).OrderBy(key => key, StringComparer.Ordinal)]; - if (siblings.Count > 0) - { - throw new DecisionUnsupportedToolSchemaException($"{keyword} sibling constraints are not supported: {string.Join(", ", siblings)}"); - } - } - - private static Dictionary Merge(Dictionary target, Dictionary overrides) - { - foreach (KeyValuePair entry in overrides) - { - target[entry.Key] = entry.Value; - } - - return target; - } - - private static bool IsNullSchema(JsonElement schema) => - schema.TryGetProperty("type", out JsonElement type) && IsNullName(type); - - private static bool IsNullName(JsonElement type) => - type.ValueKind == JsonValueKind.String && type.ValueEquals("null"); - - private static void AppendValue(StringBuilder builder, JsonElement value) - { - switch (value.ValueKind) - { - case JsonValueKind.Object: - AppendObject(builder, value.EnumerateObject().Select(property => new KeyValuePair(property.Name, property.Value))); - break; - - case JsonValueKind.Array: - AppendArray(builder, value.EnumerateArray(), AppendValue); - break; - - case JsonValueKind.String: - AppendString(builder, value.GetString()!); - break; - - case JsonValueKind.Null or JsonValueKind.Undefined: - builder.Append("null"); - break; - - default: - // Numbers keep their original text, and true and false are already written as JSON. - builder.Append(value.GetRawText()); - break; - } - } - - private static void AppendObject(StringBuilder builder, IEnumerable> properties) - { - builder.Append('{'); - bool first = true; - foreach (KeyValuePair property in properties.OrderBy(property => property.Key, StringComparer.Ordinal)) - { - builder.Append(first ? string.Empty : ", "); - first = false; - AppendString(builder, property.Key); - builder.Append(": "); - AppendValue(builder, property.Value); - } - - builder.Append('}'); - } - - private static void AppendArray(StringBuilder builder, IEnumerable items, Action append) - { - builder.Append('['); - bool first = true; - foreach (T item in items) - { - builder.Append(first ? string.Empty : ", "); - first = false; - append(builder, item); - } - - builder.Append(']'); - } - - /// - /// Writes a JSON string with the escapes of Python's ensure_ascii=False: quotes, backslashes, and control - /// characters only. - /// - private static void AppendString(StringBuilder builder, string text) - { - builder.Append('"'); - foreach (char c in text) - { - switch (c) - { - case '"': builder.Append("\\\""); break; - case '\\': builder.Append("\\\\"); break; - case '\n': builder.Append("\\n"); break; - case '\r': builder.Append("\\r"); break; - case '\t': builder.Append("\\t"); break; - case '\b': builder.Append("\\b"); break; - case '\f': builder.Append("\\f"); break; - case < ' ': builder.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); break; - default: builder.Append(c); break; - } - } - - builder.Append('"'); - } -} diff --git a/dotnet/src/Shared/DecisionChat/DecisionUnsupportedToolSchemaException.cs b/dotnet/src/Shared/DecisionChat/DecisionUnsupportedToolSchemaException.cs deleted file mode 100644 index bfc52b8d8fd..00000000000 --- a/dotnet/src/Shared/DecisionChat/DecisionUnsupportedToolSchemaException.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Agents.AI.Decisions; - -/// -/// A tool's JSON schema cannot be expressed as closed-set decision questions, so the tool is left out of the request. -/// -/// -/// It is caught while a tool call plan is compiled and never leaves the compiler: an unsupported tool is logged and -/// excluded, or, when the tool mode requires a call and no tool remains, its reason becomes part of an -/// . -/// -internal sealed class DecisionUnsupportedToolSchemaException : NotSupportedException -{ - public DecisionUnsupportedToolSchemaException() - { - } - - public DecisionUnsupportedToolSchemaException(string message) - : base(message) - { - } - - public DecisionUnsupportedToolSchemaException(string message, Exception innerException) - : base(message, innerException) - { - } -} diff --git a/dotnet/src/Shared/DecisionChat/README.md b/dotnet/src/Shared/DecisionChat/README.md deleted file mode 100644 index 56b9fe27f63..00000000000 --- a/dotnet/src/Shared/DecisionChat/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# DecisionChat - -The provider-neutral engine of a chat client for a decision model: a model that answers typed Binary, Choice, and -Score questions about a state with probabilities, instead of writing text. It is built on the decision API proposed for -`Microsoft.Extensions.AI` and works with any `IDecisionClient`, so Jev, Azure OpenAI Decisions, and other decision -models share one implementation. - -- `DecisionChatEngine` answers a chat request: it sends the conversation as the state, together with the caller's - questions and the internal questions that choose a tool call, and returns a `FunctionCallContent` or the answers. - It rejects options that need generated text, such as `Temperature` or streaming. -- `DecisionChatState` converts a conversation into the state (`{"messages", "instructions"}`, the shape of the Python - TypeSafe connector) and finds the current turn. -- `DecisionToolCallCompiler` turns function tools whose arguments come from a closed set (constants, enums, Booleans, - and arrays of enum values) into a route Choice and one question per argument, with the question IDs, limits, and - wording of the Python TypeSafe connector. `DecisionToolCallPlan` decodes the answers into the call. - -A provider's public chat client converts its own question types into `DecisionQuestion` values and describes the -answers in its own result type through `DecisionChatEngineOptions.DescribeAnswers`. `JevChatClient` in -`Microsoft.Agents.AI.TypeSafe` is the first one. - -## Usage - -```xml - - true - true - $(NoWarn);MEAI001 - -``` - -The project also needs a project reference to `Microsoft.Agents.AI.Abstractions`, which carries the decision API -until `Microsoft.Extensions.AI` ships it, and package references to `Microsoft.Extensions.AI.Abstractions` and -`Microsoft.Extensions.Logging.Abstractions`. The code targets .NET 8 and later. - -The engine uses only the public surface of the decision API. Keep it that way, so it compiles unchanged once the API -ships in `Microsoft.Extensions.AI`. At that point the engine can become a public type in an Agent Framework package. diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/DecisionToolCallCompilerTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/DecisionToolCallCompilerTests.cs deleted file mode 100644 index 807f33f5202..00000000000 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/DecisionToolCallCompilerTests.cs +++ /dev/null @@ -1,457 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using Microsoft.Agents.AI.Decisions; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.Logging; - -namespace Microsoft.Agents.AI.TypeSafe.UnitTests; - -/// -/// Tests the provider-neutral tool call compiler that is built on, with Jev answers. -/// -public sealed class DecisionToolCallCompilerTests -{ - private static readonly Dictionary>> s_noCalls = []; - private static readonly JsonElement s_state = JsonDocument.Parse("""{ "messages": [] }""").RootElement.Clone(); - - [Fact] - public void Compile_ReferencesAndAnyOfNull_ResolveAndDecode() - { - // Arrange: a $defs reference, and an optional anyOf of an enum and null, as Pydantic writes them. - AIFunctionDeclaration tool = Declaration("set_mode", """ - { - "type": "object", - "properties": { - "mode": { "$ref": "#/$defs/Mode", "description": "The mode" }, - "level": { "anyOf": [{ "enum": ["low", "high"] }, { "type": "null" }], "default": null }, - "kind": { "const": "fixed" } - }, - "required": ["mode", "kind"], - "$defs": { "Mode": { "type": "string", "enum": ["fast", "safe"] } } - } - """); - - // Act - DecisionToolCallPlan plan = Compile([tool])!; - DecisionToolCall? call = plan.Decode(Decide( - plan, - JevChatTestData.RouteTo("t0", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), - JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.9), - JevChatTestData.Choice("__af_tool__.t0.a1.value", "v0", "v0", "v1"))); - - // Assert - Assert.Contains("The mode", plan.Questions["__af_tool__.t0.a0.value"].Instructions, StringComparison.Ordinal); - Assert.Equal(["route", "t0.a0.value", "t0.a1.present", "t0.a1.value"], plan.Questions.Keys.Select(key => key["__af_tool__.".Length..])); - Assert.Equal("safe", ((JsonElement)call!.Value.Arguments["mode"]!).GetString()); - Assert.Equal("low", ((JsonElement)call.Value.Arguments["level"]!).GetString()); - Assert.Equal("fixed", ((JsonElement)call.Value.Arguments["kind"]!).GetString()); - } - - [Fact] - public void Compile_ChainedReferences_ResolveAndDecode() - { - // Arrange: Speed aliases Mode, and the nullable Level aliases a nullable enum, so each reference leads to another. - AIFunctionDeclaration tool = Declaration("set_speed", """ - { - "type": "object", - "properties": { - "speed": { "$ref": "#/$defs/Speed", "description": "How fast to go" }, - "level": { "$ref": "#/$defs/Level" } - }, - "required": ["speed"], - "$defs": { - "Speed": { "$ref": "#/$defs/Mode" }, - "Mode": { "type": "string", "enum": ["fast", "safe"] }, - "Level": { "$ref": "#/$defs/NullableLevel" }, - "NullableLevel": { "anyOf": [{ "type": "string", "enum": ["low", "high"] }, { "type": "null" }] } - } - } - """); - - // Act - DecisionToolCallPlan plan = Compile([tool])!; - DecisionToolCall? call = plan.Decode(Decide( - plan, - JevChatTestData.RouteTo("t0", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), - JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.9), - JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"))); - - // Assert: the description next to the first reference wins, and the nullable alias still gets a presence question. - Assert.Contains("How fast to go", plan.Questions["__af_tool__.t0.a0.value"].Instructions, StringComparison.Ordinal); - Assert.Equal("safe", ((JsonElement)call!.Value.Arguments["speed"]!).GetString()); - Assert.Equal("high", ((JsonElement)call.Value.Arguments["level"]!).GetString()); - } - - [Fact] - public void Compile_IntegersOutsideTheInt64Range_Compile() - { - // Arrange: an unsigned 64-bit contract, whose largest values do not fit in a long. - AIFunctionDeclaration tool = Declaration("set_mask", """ - { - "type": "object", - "properties": { - "mask": { "type": "integer", "enum": [0, 18446744073709551615] }, - "all": { "type": "integer", "const": 18446744073709551615 }, - "bits": { "type": "array", "items": { "type": "integer", "enum": [1, 9223372036854775808] } } - }, - "required": ["mask", "all", "bits"] - } - """); - - // Act - DecisionToolCallPlan plan = Compile([tool])!; - DecisionToolCall? call = plan.Decode(Decide( - plan, - JevChatTestData.RouteTo("t0", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), - JevChatTestData.Noul("__af_tool__.t0.a2.m0", 0.1), - JevChatTestData.Noul("__af_tool__.t0.a2.m1", 0.9))); - - // Assert: the values reach the function unchanged. - Assert.Equal(ulong.MaxValue, ((JsonElement)call!.Value.Arguments["mask"]!).GetUInt64()); - Assert.Equal(ulong.MaxValue, ((JsonElement)call.Value.Arguments["all"]!).GetUInt64()); - Assert.Equal([9223372036854775808UL], ((JsonElement)call.Value.Arguments["bits"]!).EnumerateArray().Select(item => item.GetUInt64())); - } - - [Fact] - public void Compile_EnumAndConstValuesMatchingTheirType_Compile() - { - // Arrange: numbers, mixed types, and a nullable enum whose null was removed before the check. - AIFunctionDeclaration tool = Declaration("tune", """ - { - "type": "object", - "properties": { - "ratio": { "type": "number", "enum": [1, 2.5] }, - "mode": { "type": ["string", "integer"], "enum": ["auto", 3] }, - "unit": { "type": ["string", "null"], "enum": ["c", "f", null] }, - "version": { "type": "integer", "const": 2 } - }, - "required": ["ratio", "mode", "version"] - } - """); - - // Act - DecisionToolCallPlan? plan = Compile([tool]); - - // Assert - Assert.NotNull(plan); - Assert.Single(plan.Tools); - } - - [Theory] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "boolean" } }, "minProperties": 1 }""", "unsupported root-level schema constraints: minProperties")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "enum": ["x", "y"], "minLength": 1 } }, "required": ["a"] }""", "required argument 'a': unsupported argument schema constraints: minLength")] - [InlineData("""{ "type": "object", "properties": { "a": { "$ref": "#/$defs/A", "enum": ["x"] } }, "$defs": { "A": { "type": "boolean" } } }""", "optional argument 'a': $ref sibling constraints are not supported: enum")] - [InlineData("""{ "type": "object", "properties": { "a": { "anyOf": [{ "type": "boolean" }, { "type": "string", "enum": ["x"] }] } } }""", "only a single schema combined with null is supported")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": ["boolean", "null"] } }, "required": ["a"] }""", "required nullable arguments are not supported")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": "string", "enum": ["x"], "maxLength": 3 } } } }""", "unsupported array item schema constraints: maxLength")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": "integer", "enum": [1, 2.5] } } } }""", "array enum member 2.5 does not match declared item type \"integer\"")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": ["string", "null"], "enum": ["x", null] } } } }""", "nullable array members are not supported")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer" } } }""", "only const, enum, boolean, and arrays of enum values are supported")] - [InlineData("""{ "type": "object", "properties": { "a": { "$ref": "#/$defs/A" } }, "$defs": { "A": { "$ref": "#/$defs/B" }, "B": { "$ref": "#/$defs/A" } } }""", "schema references are nested too deeply")] - [InlineData("""{ "type": "object", "properties": { "a": { "$ref": "#/$defs/A" } }, "$defs": { "A": { "$ref": "#/$defs/Missing" } } }""", "unresolved schema reference '#/$defs/Missing'")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": ["fast", "safe"] } } }""", "enum value \"fast\" does not match declared type \"integer\"")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": ["fast"] } } }""", "enum value \"fast\" does not match declared type \"integer\"")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": ["integer", "boolean"], "enum": [1, "x"] } } }""", "enum value \"x\" does not match declared type [\"integer\", \"boolean\"]")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "const": 5 } }, "required": ["a"] }""", "required argument 'a': const value 5 does not match declared type \"string\"")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": [1, 2.0] } } }""", "enum value 2.0 does not match declared type \"integer\"")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "const": 1e2 } }, "required": ["a"] }""", "const value 1e2 does not match declared type \"integer\"")] - [InlineData("""{ "type": "object", "properties": { "a": { "const": "x", "enum": ["y"] } } }""", "const value \"x\" does not match declared enum")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "enum": [["allowed"], ["forbidden"]], "items": { "type": "string", "enum": ["allowed"] } } } }""", "array member \"forbidden\" does not match declared enum")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "enum": [["forbidden"]], "items": { "enum": ["allowed"] } } } }""", "array member \"forbidden\" does not match declared enum")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "const": ["forbidden"], "items": { "enum": ["allowed"] } } } }""", "array member \"forbidden\" does not match declared enum")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "const": [1], "items": { "type": "string" } } } }""", "array member 1 does not match declared item type \"string\"")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "enum": [["x"]], "items": { "type": "string", "maxLength": 3 } } } }""", "unsupported array item schema constraints: maxLength")] - [InlineData("""{ "type": "object", "properties": { "a": { "type": "boolean", "enum": [] } } }""", "enum must be a non-empty array")] - [InlineData("""{ "type": "object", "properties": {}, "required": ["missing"] }""", "required arguments are missing from properties: missing")] - [InlineData("""{ "type": "array" }""", "the top-level input schema must be an object")] - public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema, string reason) - { - // Arrange - var logger = new ListLoggerFactory(); - - // Act - DecisionToolCallPlan? plan = Compile([Declaration("tool", schema)], logger: logger.CreateLogger("test")); - - // Assert - Assert.Null(plan); - Assert.Contains(reason, Assert.Single(logger.Entries).Message, StringComparison.Ordinal); - } - - [Fact] - public void Compile_ConstAndEnumWithArrayItemConstraints_ResolveAndDecode() - { - // Arrange - AIFunctionDeclaration tool = Declaration("set_tags", """ - { - "type": "object", - "properties": { - "fixed": { "type": "array", "const": ["x"], "enum": [["x"], ["y"]], "items": { "$ref": "#/$defs/Tag" } }, - "chosen": { "type": "array", "enum": [["x"], ["x", "y"]], "items": { "$ref": "#/$defs/Tag" } } - }, - "required": ["fixed", "chosen"], - "$defs": { "Tag": { "type": "string", "enum": ["x", "y"] } } - } - """); - - // Act - DecisionToolCallPlan plan = Compile([tool])!; - DecisionToolCall? call = plan.Decode(Decide( - plan, - JevChatTestData.RouteTo("t0", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"))); - - // Assert - Assert.Equal(["x"], ((JsonElement)call!.Value.Arguments["fixed"]!).EnumerateArray().Select(item => item.GetString())); - Assert.Equal(["x", "y"], ((JsonElement)call.Value.Arguments["chosen"]!).EnumerateArray().Select(item => item.GetString())); - } - - [Fact] - public void Compile_SingleValueEnumAndNoArguments_NeedNoArgumentQuestions() - { - // Arrange - AIFunctionDeclaration refresh = Declaration("refresh", """{ "type": "object", "properties": {} }"""); - AIFunctionDeclaration single = Declaration("single", """{ "type": "object", "properties": { "scope": { "enum": ["all"] } }, "required": ["scope"] }"""); - - // Act - DecisionToolCallPlan plan = Compile([refresh, single])!; - DecisionToolCall? call = plan.Decode(Decide(plan, JevChatTestData.RouteTo("t1", "t0", "t1", "none"))); - - // Assert - Assert.Equal([JevChatTestData.Route], plan.Questions.Keys); - Assert.Equal("single", call!.Value.Function.Name); - Assert.Equal("all", ((JsonElement)call.Value.Arguments["scope"]!).GetString()); - } - - [Fact] - public void Compile_TooManyTools_Throws() - { - // Arrange - List tools = [.. Enumerable.Range(0, DecisionToolCallCompiler.MaxRoutableTools + 1).Select(index => Declaration($"tool{index}", """{ "type": "object", "properties": {} }"""))]; - - // Act & Assert - Assert.Throws(() => Compile(tools)); - } - - [Fact] - public void Compile_RequiredToolFilter_AppliesBeforeTheToolLimit() - { - // Arrange - List tools = [.. Enumerable.Range(0, DecisionToolCallCompiler.MaxRoutableTools + 1).Select(index => Declaration($"tool{index}", """{ "type": "object", "properties": {} }"""))]; - - // Act - DecisionToolCallPlan plan = Compile(tools, ChatToolMode.RequireSpecific("tool3"))!; - - // Assert - Assert.Equal("tool3", plan.RequiredTool!.Function.Name); - Assert.Null(plan.RouteQuestionId); - } - - [Fact] - public void Compile_TooManyInternalQuestions_Throws() - { - // Arrange: each tool needs 40 member questions, so the fourth exceeds 128. - string members = string.Join(",", Enumerable.Range(0, 40).Select(index => $"\"m{index}\"")); - List tools = [.. Enumerable.Range(0, 4).Select(index => Declaration($"tool{index}", $$"""{ "type": "object", "properties": { "tags": { "type": "array", "items": { "enum": [{{members}}] } } }, "required": ["tags"] }"""))]; - - // Act & Assert - InvalidOperationException exception = Assert.Throws(() => Compile(tools)); - Assert.Contains("128", exception.Message, StringComparison.Ordinal); - } - - [Fact] - public void Compile_QuestionBudget_IncludesTheRouteQuestion() - { - // Arrange - string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); - string array = "{ \"type\": \"array\", \"items\": { \"enum\": [" + members + "] } }"; - AIFunctionDeclaration tool = Declaration("large", "{ \"type\": \"object\", \"properties\": { \"a\": " + array + ", \"b\": " + array + " }, \"required\": [\"a\", \"b\"] }"); - - // Act & Assert - DecisionToolCallPlan plan = Compile([tool], ChatToolMode.RequireAny)!; - Assert.Equal(128, plan.Questions.Count); - Assert.Null(plan.RouteQuestionId); - Assert.Throws(() => Compile([tool])); - } - - [Fact] - public void Compile_QuestionBudgetWithRouteAtTheLimit_Compiles() - { - // Arrange - string first = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); - string second = string.Join(",", Enumerable.Range(0, 63).Select(index => $"\"n{index}\"")); - string schema = "{ \"type\": \"object\", \"properties\": { \"a\": { \"type\": \"array\", \"items\": { \"enum\": [" + first + "] } }, \"b\": { \"type\": \"array\", \"items\": { \"enum\": [" + second + "] } } }, \"required\": [\"a\", \"b\"] }"; - - // Act - DecisionToolCallPlan plan = Compile([Declaration("large", schema)])!; - - // Assert - Assert.Equal(128, plan.Questions.Count); - Assert.Equal(JevChatTestData.Route, plan.RouteQuestionId); - } - - [Fact] - public void Compile_UnsupportedTool_ReturnsItsQuestionBudget() - { - // Arrange: the first tool fails only after its array reserved 64 questions; they must be released. - string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); - AIFunctionDeclaration failing = Declaration("failing", $$"""{ "type": "object", "properties": { "tags": { "type": "array", "items": { "enum": [{{members}}] } }, "text": { "type": "string" } }, "required": ["tags"] }"""); - AIFunctionDeclaration large = Declaration("large", $$"""{ "type": "object", "properties": { "a": { "type": "array", "items": { "enum": [{{members}}] } }, "b": { "type": "array", "items": { "enum": [{{string.Join(",", Enumerable.Range(0, 60).Select(index => $"\"n{index}\""))}}] } } }, "required": ["a", "b"] }"""); - - // Act - DecisionToolCallPlan plan = Compile([failing, large])!; - - // Assert - Assert.Equal(125, plan.Questions.Count); - } - - [Theory] - [InlineData(false, false)] - [InlineData(false, true)] - [InlineData(true, false)] - [InlineData(true, true)] - public void Compile_UnsupportedToolExceedingQuestionBudget_IsExcludedRegardlessOfPropertyOrder(bool unsupportedFirst, bool textFirst) - { - // Arrange - string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); - string array = "\"tags\": { \"type\": \"array\", \"items\": { \"enum\": [" + members + "] } }"; - string properties = textFirst ? "\"text\": { \"type\": \"string\" }, " + array : array + ", \"text\": { \"type\": \"string\" }"; - AIFunctionDeclaration supported = Declaration("supported", "{ \"type\": \"object\", \"properties\": { " + array + " }, \"required\": [\"tags\"] }"); - AIFunctionDeclaration unsupported = Declaration("unsupported", "{ \"type\": \"object\", \"properties\": { " + properties + " } }"); - var logger = new ListLoggerFactory(); - - // Act - DecisionToolCallPlan plan = Compile(unsupportedFirst ? [unsupported, supported] : [supported, unsupported], logger: logger.CreateLogger("test"))!; - - // Assert - Assert.Equal("supported", Assert.Single(plan.Tools).Function.Name); - Assert.Equal(65, plan.Questions.Count); - Assert.Contains("argument 'text'", Assert.Single(logger.Entries).Message, StringComparison.Ordinal); - } - - [Fact] - public void Compile_SingleUnsupportedToolExceedingQuestionBudget_IsExcluded() - { - // Arrange - string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); - string array = "{ \"type\": \"array\", \"items\": { \"enum\": [" + members + "] } }"; - AIFunctionDeclaration tool = Declaration("unsupported", "{ \"type\": \"object\", \"properties\": { \"a\": " + array + ", \"b\": " + array + ", \"text\": { \"type\": \"string\" } } }"); - var logger = new ListLoggerFactory(); - - // Act - DecisionToolCallPlan? plan = Compile([tool], logger: logger.CreateLogger("test")); - - // Assert - Assert.Null(plan); - Assert.Contains("argument 'text'", Assert.Single(logger.Entries).Message, StringComparison.Ordinal); - } - - [Fact] - public void Compile_TooManyEnumValuesOrProperties_ExcludeTheTool() - { - // Arrange - string values = string.Join(",", Enumerable.Range(0, DecisionToolCallCompiler.MaxEnumValues + 1).Select(index => $"\"v{index}\"")); - string properties = string.Join(",", Enumerable.Range(0, DecisionToolCallCompiler.MaxToolProperties + 1).Select(index => $"\"p{index}\": {{ \"type\": \"boolean\" }}")); - - // Act - DecisionToolCallPlan? plan = Compile( - [ - Declaration("enum", $$"""{ "type": "object", "properties": { "a": { "enum": [{{values}}] } } }"""), - Declaration("wide", $$"""{ "type": "object", "properties": { {{properties}} } }"""), - ]); - - // Assert - Assert.Null(plan); - } - - [Fact] - public void Decode_UnknownRouteOrMissingAnswer_IsRejectedBeforeDecoding() - { - // Arrange - DecisionToolCallPlan plan = Compile([Declaration("flag", """{ "type": "object", "properties": { "on": { "type": "boolean" } }, "required": ["on"] }""")])!; - - // Act & Assert: a decision response must answer every question with the labels it offered, so neither reaches - // the decoders. - Assert.Throws(() => Decide(plan, fillMissing: false, JevChatTestData.RouteTo("t9", "t9"))); - Assert.Throws(() => Decide(plan, fillMissing: false, JevChatTestData.RouteTo("t0", "t0", "none"))); - } - - [Fact] - public void Compile_DuplicateDescriptions_KeepToolNamesInTheRoute() - { - // Arrange - AIFunctionDeclaration first = Declaration("first", """{ "type": "object", "properties": {} }""", "Does it."); - AIFunctionDeclaration second = Declaration("second", """{ "type": "object", "properties": {} }""", "Does it."); - - // Act - ChoiceDecisionQuestion route = Assert.IsType(Compile([first, second])!.Questions[JevChatTestData.Route]); - - // Assert - Assert.Equal(["t0", "t1", "none"], route.Candidates.Select(candidate => candidate.Id)); - Assert.Equal("Tool 'first': Does it.", route.Candidates[0].Description); - Assert.Equal("Tool 'second': Does it.", route.Candidates[1].Description); - } - - [Fact] - public void Describe_MatchesPythonJsonDumps() - { - // Arrange: json.dumps(value, ensure_ascii=False, sort_keys=True) in the Python connector. - JsonElement value = JsonDocument.Parse("""{"city":"São Paulo","b":[1,2.5,true,null],"q":"say \"hi\"\n"}""").RootElement; - - // Act - string described = DecisionToolSchema.Describe(value); - - // Assert - Assert.Equal("""{"b": [1, 2.5, true, null], "city": "São Paulo", "q": "say \"hi\"\n"}""", described); - } - - private static AIFunctionDeclaration Declaration(string name, string schema, string? description = null) => - AIFunctionFactory.CreateDeclaration(name, description, JsonDocument.Parse(schema).RootElement.Clone()); - - private static DecisionToolCallPlan? Compile(IReadOnlyList tools, ChatToolMode? mode = null, ILogger? logger = null) => - DecisionToolCallCompiler.Compile(tools, mode, s_noCalls, "Jev", logger ?? new ListLoggerFactory().CreateLogger("test")); - - private static DecisionResponse Decide(DecisionToolCallPlan plan, params string[] answers) => - Decide(plan, fillMissing: true, answers); - - /// - /// Converts Jev answers to the plan's questions into a decision response, as does. - /// - /// The plan whose questions are answered. - /// - /// Whether questions without an answer get one that the decoders do not read for the selected call: no for a Binary - /// question and the first candidate for a Choice. - /// - /// The Jev answers, from . - private static DecisionResponse Decide(DecisionToolCallPlan plan, bool fillMissing, params string[] answers) - { - JevResult result = JsonSerializer.Deserialize(JevChatTestData.Result(answers), JevJsonContext.Default.JevResult)!; - var decisionAnswers = new List(); - foreach (DecisionQuestion question in plan.Questions.Values) - { - if (result.Answers.TryGetValue(question.Id, out JevResponse? answer)) - { - decisionAnswers.Add(JevDecisionMapping.ToDecisionAnswer(question, answer)); - } - else if (fillMissing) - { - decisionAnswers.Add(question switch - { - ChoiceDecisionQuestion choice => new ChoiceDecisionAnswer( - question.Id, - choice.Candidates[0].Id, - [.. choice.Candidates.Select((candidate, index) => new DecisionProbability(candidate.Id, index == 0 ? 1 : 0))]), - _ => new BinaryDecisionAnswer(question.Id, 0), - }); - } - } - - return new DecisionResponse(new DecisionRequest(s_state, [.. plan.Questions.Values]), decisionAnswers); - } -} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs deleted file mode 100644 index 3499f8e747f..00000000000 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs +++ /dev/null @@ -1,425 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.ClientModel; -using System.ClientModel.Primitives; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; - -namespace Microsoft.Agents.AI.TypeSafe.UnitTests; - -public sealed class JevChatClientTests -{ - private const string Ticket = JevTestData.TicketText; - - [Fact] - public async Task GetResponseAsync_SendsTheConversationAndQuestions_ReturnsTheTypedResultAsync() - { - // Arrange - using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - ChatMessage[] messages = [new(ChatRole.System, "You triage support tickets."), new(ChatRole.User, Ticket)]; - - // Act - ChatResponse response = await client.GetResponseAsync(messages, new ChatOptions { Instructions = "Be strict." }.WithJevQuestions(JevChatTestData.Questions())); - - // Assert: the request carries the conversation as the state, in the Python connector's shape. - JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); - Assert.Equal(new Uri("https://api.typesafe.ai/v1/systemone"), request.Uri); - Assert.Equal("test-key", request.Authorization!.Parameter); - JsonNode body = JsonNode.Parse(request.Body)!; - Assert.Equal("jev-latest", (string?)body["model"]); - JsonNode expectedState = JsonNode.Parse($$""" - { - "messages": [ - { "role": "system", "contents": [{ "type": "text", "text": "You triage support tickets." }] }, - { "role": "user", "contents": [{ "type": "text", "text": {{JsonSerializer.Serialize(Ticket)}} }] } - ], - "instructions": "Be strict." - } - """)!; - Assert.True(JsonNode.DeepEquals(expectedState, body["state"]), body["state"]!.ToJsonString()); - Assert.Equal(["department", "is_urgent"], body["questions"]!.AsObject().Select(question => question.Key)); - - // Assert: the text is the result as JSON, and the typed result, usage, and model are on the response. - JevResult result = Assert.IsType(response.GetJevResult()); - Assert.Equal("technical", Assert.IsType(result.Answers["department"]).Choice); - Assert.Equal(0.9, Assert.IsType(result.Answers["is_urgent"]).Noul); - Assert.True(JsonNode.DeepEquals(JsonNode.Parse(response.Text), JsonNode.Parse(JsonSerializer.Serialize(result, JevJsonUtilities.Options))), response.Text); - Assert.Equal(ChatFinishReason.Stop, response.FinishReason); - Assert.Equal("jev-test", response.ModelId); - Assert.Equal(10, response.Usage!.InputTokenCount); - Assert.Equal(2, response.Usage.OutputTokenCount); - Assert.Equal(12, response.Usage.TotalTokenCount); - Assert.Same(result, Assert.Single(response.Messages).RawRepresentation); - } - - [Fact] - public async Task GetResponseAsync_ModelIdAndEndpoint_AreUsedAsync() - { - // Arrange - using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient, new JevChatClientOptions { ModelId = "jev-1.13", Endpoint = new Uri("https://proxy.contoso.com/jev") }); - - // Act - await client.GetResponseAsync(Ticket, new ChatOptions { ModelId = "jev-1.14" }.WithJevQuestions(JevChatTestData.Questions())); - - // Assert: a request's model wins over the client's. - JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); - Assert.Equal(new Uri("https://proxy.contoso.com/jev"), request.Uri); - Assert.Equal("jev-1.14", (string?)JsonNode.Parse(request.Body)!["model"]); - - ChatClientMetadata metadata = Assert.IsType(client.GetService(typeof(ChatClientMetadata))); - Assert.Equal("typesafe.ai", metadata.ProviderName); - Assert.Equal(new Uri("https://proxy.contoso.com/jev"), metadata.ProviderUri); - Assert.Equal("jev-1.13", metadata.DefaultModelId); - Assert.Same(client, client.GetService(typeof(JevChatClient))); - Assert.Null(client.GetService(typeof(JevChatClient), "key")); - } - - [Fact] - public async Task GetResponseAsync_DefaultQuestions_AnswerRequestsWithoutQuestionsAsync() - { - // Arrange - using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient, new JevChatClientOptions { DefaultQuestions = JevChatTestData.Questions() }); - - // Act - ChatResponse response = await client.GetResponseAsync(Ticket); - - // Assert - Assert.Equal(["department", "is_urgent"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); - Assert.NotNull(response.GetJevResult()); - } - - [Fact] - public async Task GetResponseAsync_RequestQuestions_ReplaceTheDefaultsAsync() - { - // Arrange - using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("spam", 0.1))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient, new JevChatClientOptions { DefaultQuestions = JevChatTestData.Questions() }); - var questions = new Dictionary { ["spam"] = new JevNoulQuestion { Instructions = "Is it spam?" } }; - - // Act - await client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions)); - - // Assert - Assert.Equal(["spam"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); - } - - [Fact] - public async Task GetResponseAsync_NoQuestions_ThrowsAsync() - { - // Arrange - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - - // Act & Assert - InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket)); - Assert.Contains(nameof(JevChatOptionsExtensions.WithJevQuestions), exception.Message, StringComparison.Ordinal); - await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(new Dictionary()))); - Assert.Empty(handler.Requests); - } - - [Fact] - public async Task GetResponseAsync_ReservedQuestionId_ThrowsAsync() - { - // Arrange - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - var questions = new Dictionary { ["__af_tool__.mine"] = new JevNoulQuestion { Instructions = "Yes?" } }; - - // Act & Assert - ArgumentException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions))); - Assert.Contains("__af_tool__.mine", exception.Message, StringComparison.Ordinal); - Assert.Empty(handler.Requests); - } - - [Fact] - public async Task GetResponseAsync_InvalidQuestion_ThrowsBeforeSendingAsync() - { - // Arrange - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - var questions = new Dictionary { ["level"] = new JevScoreQuestion { Instructions = "How bad?", Criteria = ["only one"] } }; - - // Act & Assert - await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions))); - Assert.Empty(handler.Requests); - } - - public static TheoryData UnsupportedOptions() => new() - { - { nameof(ChatOptions.Temperature), new ChatOptions { Temperature = 0.2f } }, - { nameof(ChatOptions.TopP), new ChatOptions { TopP = 0.5f } }, - { nameof(ChatOptions.TopK), new ChatOptions { TopK = 3 } }, - { nameof(ChatOptions.MaxOutputTokens), new ChatOptions { MaxOutputTokens = 100 } }, - { nameof(ChatOptions.FrequencyPenalty), new ChatOptions { FrequencyPenalty = 0.1f } }, - { nameof(ChatOptions.PresencePenalty), new ChatOptions { PresencePenalty = 0.1f } }, - { nameof(ChatOptions.Seed), new ChatOptions { Seed = 7 } }, - { nameof(ChatOptions.Reasoning), new ChatOptions { Reasoning = new ReasoningOptions() } }, - { nameof(ChatOptions.StopSequences), new ChatOptions { StopSequences = ["stop"] } }, - { nameof(ChatOptions.ResponseFormat), new ChatOptions { ResponseFormat = ChatResponseFormat.Json } }, - { nameof(ChatOptions.ConversationId), new ChatOptions { ConversationId = "conv_1" } }, - { nameof(ChatOptions.AllowBackgroundResponses), new ChatOptions { AllowBackgroundResponses = true } }, - { nameof(ChatOptions.AllowMultipleToolCalls), new ChatOptions { AllowMultipleToolCalls = true } }, - { "HostedWebSearchTool", new ChatOptions { Tools = [new HostedWebSearchTool()] } }, - }; - - [Theory] - [MemberData(nameof(UnsupportedOptions))] - public async Task GetResponseAsync_UnsupportedOption_ThrowsBeforeSendingAsync(string option, ChatOptions options) - { - // Arrange - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - - // Act & Assert - NotSupportedException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, options.WithJevQuestions(JevChatTestData.Questions()))); - Assert.Contains(option == "HostedWebSearchTool" ? "web_search" : option, exception.Message, StringComparison.Ordinal); - Assert.Empty(handler.Requests); - } - - [Fact] - public async Task GetResponseAsync_TextResponseFormatAndSingleToolCalls_AreAllowedAsync() - { - // Arrange - using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Text, AllowMultipleToolCalls = false, AllowBackgroundResponses = false, StopSequences = [] }; - - // Act - ChatResponse response = await client.GetResponseAsync(Ticket, options.WithJevQuestions(JevChatTestData.Questions())); - - // Assert - Assert.NotNull(response.GetJevResult()); - } - - [Fact] - public async Task GetResponseAsync_NonTextContent_ThrowsBeforeSendingAsync() - { - // Arrange - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - var message = new ChatMessage(ChatRole.User, [new TextContent("See the screenshot."), new DataContent(new byte[] { 1, 2, 3 }, "image/png")]); - - // Act & Assert - NotSupportedException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync([message], new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); - Assert.Contains(nameof(DataContent), exception.Message, StringComparison.Ordinal); - Assert.Empty(handler.Requests); - } - - [Fact] - public async Task GetResponseAsync_EmptyConversation_ThrowsBeforeSendingAsync() - { - // Arrange - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - - // Act & Assert - await Assert.ThrowsAsync(() => client.GetResponseAsync([new ChatMessage(ChatRole.User, "")], new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); - Assert.Empty(handler.Requests); - } - - [Fact] - public async Task GetResponseAsync_ReasoningAndFunctionContent_AreSerializedAsync() - { - // Arrange - using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - ChatMessage[] messages = - [ - new(ChatRole.User, Ticket), - new(ChatRole.Assistant, [new TextReasoningContent("Check the order first.") { ProtectedData = "opaque" }, new FunctionCallContent("call_1", "lookup_order", new Dictionary { ["order"] = "A-1" })]), - new(ChatRole.Tool, [new FunctionResultContent("call_1", JsonSerializer.SerializeToElement("""{"result":"Order A-1 shipped."}"""))]), - new(ChatRole.Tool, [new FunctionResultContent("call_2", new TextContent("Refund issued."))]), - ]; - - // Act - await client.GetResponseAsync(messages, new ChatOptions().WithJevQuestions(JevChatTestData.Questions())); - - // Assert: protected reasoning stays out of the state, and rich results become text items. - JsonNode state = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["state"]!; - JsonNode expected = JsonNode.Parse(""" - [ - { "type": "text_reasoning", "text": "Check the order first." }, - { "type": "function_call", "call_id": "call_1", "name": "lookup_order", "arguments": { "order": "A-1" } } - ] - """)!; - Assert.True(JsonNode.DeepEquals(expected, state["messages"]![1]!["contents"]), state.ToJsonString()); - Assert.Equal("tool", (string?)state["messages"]![2]!["role"]); - Assert.Equal("""{"result":"Order A-1 shipped."}""", (string?)state["messages"]![2]!["contents"]![0]!["result"]); - JsonNode rich = JsonNode.Parse("""{ "type": "function_result", "call_id": "call_2", "result": "Refund issued.", "items": [{ "type": "text", "text": "Refund issued." }] }""")!; - Assert.True(JsonNode.DeepEquals(rich, state["messages"]![3]!["contents"]![0]), state.ToJsonString()); - } - - [Fact] - public async Task GetResponseAsync_RichFunctionResult_ThrowsBeforeSendingAsync() - { - // Arrange - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - ChatMessage[] messages = - [ - new(ChatRole.User, Ticket), - new(ChatRole.Tool, [new FunctionResultContent("call_1", new DataContent(new byte[] { 1 }, "image/png"))]), - ]; - - // Act & Assert - await Assert.ThrowsAsync(() => client.GetResponseAsync(messages, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); - Assert.Empty(handler.Requests); - } - - [Fact] - public async Task GetResponseAsync_ResultMissingAnAnswer_ThrowsAsync() - { - // Arrange - using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("is_urgent", 0.9))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - - // Act & Assert - InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); - Assert.Contains("department", exception.Message, StringComparison.Ordinal); - } - - [Fact] - public async Task GetResponseAsync_JsonInstructionsAndUndescribedLabels_AreSentUnchangedAsync() - { - // Arrange: the chat client runs on the provider-neutral decision API, which carries text only, so the Jev - // questions themselves must reach the API. - using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Choice("tone", "calm", "calm", "angry"))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - JsonElement instructions = JsonDocument.Parse("""{ "question": "What is the tone of `reply`?", "reply": "Thanks!" }""").RootElement.Clone(); - var questions = new Dictionary - { - ["tone"] = new JevChoiceQuestion { Instructions = (JevEntry)instructions, Criteria = new Dictionary { ["calm"] = JevEntry.Null, ["angry"] = "Hostile or upset" } }, - }; - - // Act - ChatResponse response = await client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions)); - - // Assert - JsonNode sent = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!["tone"]!; - Assert.True(JsonNode.DeepEquals(JsonNode.Parse(instructions.GetRawText()), sent["instructions"]), sent.ToJsonString()); - Assert.Null(sent["criteria"]!["calm"]); - Assert.Equal("calm", Assert.IsType(response.GetJevResult()!.Answers["tone"]).Choice); - } - - [Fact] - public async Task GetResponseAsync_IncompleteDistribution_ThrowsInvalidOperationExceptionAsync() - { - // Arrange: the answer leaves out the probability of a label. - using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result( - "\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":0.9,\"probabilities\":{\"billing\":1.0}}", - JevChatTestData.Noul("is_urgent", 0.9))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - - // Act & Assert - InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); - Assert.Contains("'technical'", exception.Message, StringComparison.Ordinal); - } - - [Fact] - public async Task GetResponseAsync_ApiError_ThrowsClientResultExceptionAsync() - { - // Arrange - using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.UnprocessableEntity, """{"detail":"questions.department.criteria: Field required"}"""); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - - // Act & Assert - ClientResultException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); - Assert.Equal(422, exception.Status); - Assert.Contains("Field required", exception.Message, StringComparison.Ordinal); - } - - [Fact] - public async Task GetStreamingResponseAsync_ThrowsAsync() - { - // Arrange - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient client = CreateClient(httpClient); - - // Act & Assert - await Assert.ThrowsAsync(async () => - { - await foreach (ChatResponseUpdate update in client.GetStreamingResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))) - { - } - }); - Assert.Empty(handler.Requests); - } - - [Fact] - public void Constructor_InvalidOptions_Throw() - { - Assert.Throws(() => new JevChatClient(new ApiKeyCredential("k"), new JevChatClientOptions { ModelId = " " })); - Assert.Throws(() => new JevChatClient(new ApiKeyCredential("k"), new JevChatClientOptions { Endpoint = new Uri("/relative", UriKind.Relative) })); - Assert.Throws(() => new JevChatClientOptions { MaximumToolCallsPerTurn = -1 }); - } - - [Fact] - public void Constructor_FreezesTheOptions() - { - // Arrange - var options = new JevChatClientOptions(); - - // Act - using var client = new JevChatClient(new ApiKeyCredential("k"), options); - - // Assert: like other client options, the options are read-only once used. - Assert.Throws(() => options.ModelId = "other"); - Assert.Throws(() => options.DefaultQuestions = JevChatTestData.Questions()); - Assert.IsType(options.RetryPolicy); - Assert.Equal(TimeSpan.FromSeconds(10), options.NetworkTimeout); - } - - [Fact] - public void WithJevQuestions_OtherClients_KeepThePreviousFactory() - { - // Arrange - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient jev = CreateClient(httpClient); - using var other = new ScriptedChatClient(); - object otherOptions = new(); - var options = new ChatOptions { RawRepresentationFactory = _ => otherOptions }; - - // Act - options.WithJevQuestions(JevChatTestData.Questions()); - - // Assert - Assert.Same(otherOptions, options.RawRepresentationFactory!(other)); - Assert.Equal(["department", "is_urgent"], Assert.IsType(options.RawRepresentationFactory(jev)).Questions.Keys); - } - - internal static JevChatClient CreateClient(HttpClient httpClient, JevChatClientOptions? options = null) - { - options ??= new JevChatClientOptions(); - options.Transport = new HttpClientPipelineTransport(httpClient); - options.RetryPolicy = new JevRetryPolicy { RecordedWaitsForTests = [] }; - return new JevChatClient(new ApiKeyCredential("test-key"), options); - } -} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs deleted file mode 100644 index 10a4837557e..00000000000 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs +++ /dev/null @@ -1,601 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Threading.Tasks; -using Microsoft.Agents.AI.Decisions; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.Logging; - -namespace Microsoft.Agents.AI.TypeSafe.UnitTests; - -public sealed class JevChatClientToolTests -{ - private const string Route = JevChatTestData.Route; - - public enum City - { - Seattle, - Paris, - } - - public enum Unit - { - Celsius, - Fahrenheit, - } - - public enum Feature - { - Wind, - Humidity, - Uv, - } - - [Fact] - public async Task GetResponseAsync_JevSelectsATool_ReturnsTheFunctionCallAsync() - { - // Arrange: city and detailed are required; unit is optional and nullable. - AIFunction weather = AIFunctionFactory.Create( - ([Description("The city")] City city, [Description("Include the hourly forecast")] bool detailed, Unit? unit = null) => "sunny", - "get_weather", - "Gets the weather for a supported city."); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.RouteTo("t0", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), - JevChatTestData.Noul("__af_tool__.t0.a1.value", 0.9), - JevChatTestData.Noul("__af_tool__.t0.a2.present", 0.2), - JevChatTestData.Choice("__af_tool__.t0.a2.value", "v1", "v0", "v1"))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - - // Act - ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); - - // Assert: the call has the chosen enum value and Boolean, and leaves out the optional argument. - FunctionCallContent call = Assert.IsType(Assert.Single(Assert.Single(response.Messages).Contents)); - Assert.Equal("get_weather", call.Name); - Assert.StartsWith("typesafe-", call.CallId, StringComparison.Ordinal); - Assert.Equal(["city", "detailed"], call.Arguments!.Keys.Order()); - Assert.Equal("Paris", Assert.IsType(call.Arguments["city"]).GetString()); - Assert.True(Assert.IsType(call.Arguments["detailed"]).GetBoolean()); - Assert.Equal(ChatFinishReason.ToolCalls, response.FinishReason); - Assert.Null(response.GetJevResult()); - - // Assert: the request adds the internal routing questions after the caller's questions. - JsonObject questions = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject(); - Assert.Equal( - ["department", "is_urgent", Route, "__af_tool__.t0.a0.value", "__af_tool__.t0.a1.value", "__af_tool__.t0.a2.present", "__af_tool__.t0.a2.value"], - questions.Select(question => question.Key)); - Assert.Equal("Tool 'get_weather': Gets the weather for a supported city.", (string?)questions[Route]!["criteria"]!["t0"]); - Assert.NotNull(questions[Route]!["criteria"]!["none"]); - Assert.Equal("\"Paris\"", (string?)questions["__af_tool__.t0.a0.value"]!["criteria"]!["v1"]); - Assert.Equal("noul", (string?)questions["__af_tool__.t0.a1.value"]!["type"]); - Assert.Contains("did the user explicitly specify the unit argument", (string?)questions["__af_tool__.t0.a2.present"]!["instructions"], StringComparison.Ordinal); - } - - [Fact] - public async Task GetResponseAsync_OptionalArgumentPresent_IsPassedAsync() - { - // Arrange - AIFunction weather = AIFunctionFactory.Create((City city, Unit? unit = null, Feature[]? features = null) => "sunny", "get_weather"); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.RouteTo("t0", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"), - JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.8), - JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"), - JevChatTestData.Noul("__af_tool__.t0.a2.present", 0.7), - JevChatTestData.Noul("__af_tool__.t0.a2.m0", 0.9), - JevChatTestData.Noul("__af_tool__.t0.a2.m1", 0.1), - JevChatTestData.Noul("__af_tool__.t0.a2.m2", 0.6))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - - // Act - ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); - - // Assert: a nullable enum is a Choice without null, and an enum array is one Noul per member. - FunctionCallContent call = response.Messages.SelectMany(message => message.Contents).OfType().Single(); - Assert.Equal("Seattle", Assert.IsType(call.Arguments!["city"]).GetString()); - Assert.Equal("Fahrenheit", Assert.IsType(call.Arguments["unit"]).GetString()); - Assert.Equal(["Wind", "Uv"], Assert.IsType(call.Arguments["features"]).EnumerateArray().Select(item => item.GetString())); - - JsonObject questions = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject(); - Assert.Equal(["v0", "v1"], questions["__af_tool__.t0.a1.value"]!["criteria"]!.AsObject().Select(criterion => criterion.Key)); - } - - [Fact] - public async Task GetResponseAsync_JevChoosesNoTool_ReturnsOnlyTheCallersAnswersAsync() - { - // Arrange - AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.RouteTo("none", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - - // Act - ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); - - // Assert - JevResult result = Assert.IsType(response.GetJevResult()); - Assert.Equal(["department", "is_urgent"], result.Answers.Keys); - Assert.DoesNotContain("__af_tool__", response.Text, StringComparison.Ordinal); - Assert.Equal(ChatFinishReason.Stop, response.FinishReason); - } - - [Fact] - public async Task GetResponseAsync_AfterTheToolCallLimit_AnswersWithTheToolResultsAsync() - { - // Arrange: the turn already has one call and its result, which is the default limit. - AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.Score("severity", 1.5, 3))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - Dictionary questions = JevChatTestData.Questions(); - questions["severity"] = new JevScoreQuestion { Instructions = "How severe?", Criteria = ["Low", "Medium", "High"] }; - ChatMessage[] messages = - [ - new(ChatRole.User, "Earlier question"), - new(ChatRole.Assistant, [new FunctionCallContent("call_0", "get_weather", new Dictionary { ["city"] = "Paris" })]), - new(ChatRole.Tool, [new FunctionResultContent("call_0", "Paris is cloudy.")]), - new(ChatRole.Assistant, "Paris is cloudy.\ndepartment: technical"), - new(ChatRole.User, "Weather in Seattle?"), - new(ChatRole.Assistant, [new FunctionCallContent("call_1", "get_weather", new Dictionary { ["city"] = "Seattle" })]), - new(ChatRole.Tool, [new FunctionResultContent("call_1", JsonSerializer.SerializeToElement("Seattle is sunny."))]), - ]; - - // Act - ChatResponse response = await client.GetResponseAsync(messages, new ChatOptions { Tools = [weather] }.WithJevQuestions(questions)); - - // Assert: no tool questions, and the text has this turn's tool result and the Choice and Score decisions. - Assert.Equal(["department", "is_urgent", "severity"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); - Assert.Equal("Seattle is sunny.\ndepartment: technical\nseverity: 1.5", response.Text); - Assert.NotNull(response.GetJevResult()); - } - - [Fact] - public async Task GetResponseAsync_HigherLimit_OffersToolsAgainWithHintsAsync() - { - // Arrange - AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather", "Gets the weather."); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.RouteTo("t0", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient, new JevChatClientOptions { MaximumToolCallsPerTurn = 2 }); - ChatMessage[] messages = - [ - new(ChatRole.User, "Weather in Seattle and Paris?"), - new(ChatRole.Assistant, [new FunctionCallContent("call_1", "get_weather", new Dictionary { ["city"] = JsonSerializer.SerializeToElement("Seattle") })]), - new(ChatRole.Tool, [new FunctionResultContent("call_1", "Seattle is sunny.")]), - ]; - - // Act - ChatResponse response = await client.GetResponseAsync(messages, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); - - // Assert: the earlier call becomes a hint in the route and argument questions. - JsonObject questions = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject(); - Assert.EndsWith("""Do not repeat argument sets already called this turn: [{"city": "Seattle"}].""", (string?)questions[Route]!["criteria"]!["t0"], StringComparison.Ordinal); - Assert.Contains("""Values already used for this argument in earlier calls this turn: ["Seattle"].""", (string?)questions["__af_tool__.t0.a0.value"]!["instructions"], StringComparison.Ordinal); - Assert.Equal("Paris", Assert.IsType(response.Messages.SelectMany(message => message.Contents).OfType().Single().Arguments!["city"]).GetString()); - } - - [Fact] - public async Task GetResponseAsync_UnsupportedTool_IsLeftOutWithAWarningAsync() - { - // Arrange: free-form text cannot be chosen from a closed set. - AIFunction search = AIFunctionFactory.Create((string query) => "found", "search"); - AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.RouteTo("none", "t1", "none"), - JevChatTestData.Choice("__af_tool__.t1.a0.value", "v0", "v0", "v1"))); - using var httpClient = new HttpClient(handler); - var loggerFactory = new ListLoggerFactory(); - var options = new JevChatClientOptions(); - options.ClientLoggingOptions = new() { LoggerFactory = loggerFactory, EnableLogging = false }; - using JevChatClient client = JevChatClientTests.CreateClient(httpClient, options); - - // Act - await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [search, weather] }.WithJevQuestions(JevChatTestData.Questions())); - - // Assert: the tools keep their position, so the supported tool is still t1. - JsonObject criteria = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]![Route]!["criteria"]!.AsObject(); - Assert.Equal(["t1", "none"], criteria.Select(criterion => criterion.Key)); - (LogLevel level, string message) = Assert.Single(loggerFactory.Entries, entry => entry.Message.Contains("'search'", StringComparison.Ordinal)); - Assert.Equal(LogLevel.Warning, level); - Assert.Contains("required argument 'query'", message, StringComparison.Ordinal); - } - - [Fact] - public async Task GetResponseAsync_RequiredModeWithoutSupportedTools_ThrowsAsync() - { - // Arrange - AIFunction search = AIFunctionFactory.Create((string query) => "found", "search"); - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - ChatOptions options = new ChatOptions { Tools = [search], ToolMode = ChatToolMode.RequireAny }.WithJevQuestions(JevChatTestData.Questions()); - - // Act & Assert - InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(JevTestData.TicketText, options)); - Assert.Contains("search: required argument 'query'", exception.Message, StringComparison.Ordinal); - Assert.Empty(handler.Requests); - } - - [Fact] - public async Task GetResponseAsync_RequireSpecificTool_SkipsTheRouteQuestionAsync() - { - // Arrange - AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); - AIFunction forecast = AIFunctionFactory.Create((City city) => "rain", "get_forecast"); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - ChatOptions options = new ChatOptions { Tools = [weather, forecast], ToolMode = ChatToolMode.RequireSpecific("get_forecast") }.WithJevQuestions(JevChatTestData.Questions()); - - // Act - ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, options); - - // Assert - Assert.Equal("get_forecast", response.Messages.SelectMany(message => message.Contents).OfType().Single().Name); - Assert.Null(JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]![Route]); - } - - [Fact] - public async Task GetResponseAsync_RequiredModeWithSeveralTools_OmitsTheNoneRouteAsync() - { - // Arrange - AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); - AIFunction forecast = AIFunctionFactory.Create((City city) => "rain", "get_forecast"); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.RouteTo("t1", "t0", "t1"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"), - JevChatTestData.Choice("__af_tool__.t1.a0.value", "v1", "v0", "v1"))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - ChatOptions options = new ChatOptions { Tools = [weather, forecast], ToolMode = ChatToolMode.RequireAny }.WithJevQuestions(JevChatTestData.Questions()); - - // Act - ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, options); - - // Assert - Assert.Equal(["t0", "t1"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]![Route]!["criteria"]!.AsObject().Select(criterion => criterion.Key)); - Assert.Equal("get_forecast", response.Messages.SelectMany(message => message.Contents).OfType().Single().Name); - } - - [Fact] - public async Task GetResponseAsync_RequiredToolMissingOrNoneMode_BehaveAsExpectedAsync() - { - // Arrange - AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - - // Act & Assert: a missing required tool fails, and the none mode sends only the caller's questions. - await Assert.ThrowsAsync(() => client.GetResponseAsync( - JevTestData.TicketText, - new ChatOptions { Tools = [weather], ToolMode = ChatToolMode.RequireSpecific("other") }.WithJevQuestions(JevChatTestData.Questions()))); - await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather], ToolMode = ChatToolMode.None }.WithJevQuestions(JevChatTestData.Questions())); - Assert.Equal(["department", "is_urgent"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); - } - - [Fact] - public async Task RunAsync_AgentExecutesTheToolOnce_ThenAnswersAsync() - { - // Arrange - var executions = new List<(City City, bool Detailed)>(); - AIFunction weather = AIFunctionFactory.Create( - (City city, bool detailed) => - { - executions.Add((city, detailed)); - return $"{city} is sunny{(detailed ? " with an hourly forecast" : string.Empty)}."; - }, - "get_weather", - "Gets the weather for a supported city."); - using var handler = new JevTestHttpHandler() - .Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.RouteTo("t0", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), - JevChatTestData.Noul("__af_tool__.t0.a1.value", 0.9))) - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - AIAgent agent = client.AsAIAgent(instructions: "Answer weather requests.", tools: [weather]); - - // Act - AgentResponse response = await agent.RunAsync( - "Give me a detailed weather report for Paris.", - options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); - - // Assert: the tool ran once, and the second request answered the caller's questions with its result. - Assert.Equal([(City.Paris, true)], executions); - Assert.Equal(2, handler.Requests.Count); - JsonNode second = JsonNode.Parse(handler.Requests[1].Body)!; - Assert.Equal(["department", "is_urgent"], second["questions"]!.AsObject().Select(question => question.Key)); - Assert.Equal("Answer weather requests.", (string?)second["state"]!["instructions"]); - JsonNode toolResult = second["state"]!["messages"]!.AsArray().SelectMany(message => message!["contents"]!.AsArray()).Single(content => (string?)content!["type"] == "function_result")!; - Assert.Equal("Paris is sunny with an hourly forecast.", (string?)toolResult["result"]); - - Assert.Equal("Paris is sunny with an hourly forecast.\ndepartment: technical", response.Text); - JevResult result = Assert.IsType(response.GetJevResult()); - Assert.Equal(["department", "is_urgent"], result.Answers.Keys); - Assert.Equal(24, response.Usage!.TotalTokenCount); - } - - [Fact] - public async Task RunAsync_AgentLevelQuestions_AreUsedUnlessTheRunPassesItsOwnAsync() - { - // Arrange - var runQuestions = new Dictionary { ["spam"] = new JevNoulQuestion { Instructions = "Is it spam?" } }; - using var handler = new JevTestHttpHandler() - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())) - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("spam", 0.05))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions().WithJevQuestions(JevChatTestData.Questions()) }); - - // Act - AgentResponse agentLevel = await agent.RunAsync(JevTestData.TicketText); - AgentResponse runLevel = await agent.RunAsync(JevTestData.TicketText, options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(runQuestions))); - - // Assert - Assert.Equal(["department", "is_urgent"], agentLevel.GetJevResult()!.Answers.Keys); - Assert.Equal(["spam"], runLevel.GetJevResult()!.Answers.Keys); - } - - [Fact] - public async Task RunAsync_ApprovalRequiredTool_RunsOnlyAfterApprovalAsync() - { - // Arrange - int executions = 0; - AIFunction refund = new ApprovalRequiredAIFunction(AIFunctionFactory.Create( - (City city) => - { - executions++; - return "Refund issued."; - }, - "issue_refund")); - using var handler = new JevTestHttpHandler() - .Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.RouteTo("t0", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))) - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers("billing"))) - .Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers("billing"), - JevChatTestData.RouteTo("none", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions { Tools = [refund] }.WithJevQuestions(JevChatTestData.Questions()) }); - AgentSession session = await agent.CreateSessionAsync(); - - // Act - AgentResponse pending = await agent.RunAsync("Refund my Seattle order.", session); - ToolApprovalRequestContent approval = pending.Messages.SelectMany(message => message.Contents).OfType().Single(); - int executionsBeforeApproval = executions; - AgentResponse approved = await agent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(true)]), session); - AgentResponse later = await agent.RunAsync("Thanks. Anything else?", session); - - // Assert - Assert.Equal(0, executionsBeforeApproval); - Assert.Equal(1, executions); - Assert.Equal("Refund issued.\ndepartment: billing", approved.Text); - - // Assert: the next turn still works. The handled approval stays in the history, but Jev does not read it. - Assert.Equal(3, handler.Requests.Count); - Assert.DoesNotContain("approval", handler.Requests[2].Body, StringComparison.OrdinalIgnoreCase); - Assert.NotNull(later.GetJevResult()); - } - - [Fact] - public async Task RunAsync_ApprovalSentWithText_ContinuesTheTurnOfTheApprovedCallAsync() - { - // Arrange - int executions = 0; - AIFunction refund = new ApprovalRequiredAIFunction(AIFunctionFactory.Create( - (City city) => - { - executions++; - return "Refund issued."; - }, - "issue_refund")); - using var handler = new JevTestHttpHandler() - .Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.UserAnswers(), - JevChatTestData.RouteTo("t0", "t0", "none"), - JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))) - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers("billing"))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions { Tools = [refund] }.WithJevQuestions(JevChatTestData.Questions()) }); - AgentSession session = await agent.CreateSessionAsync(); - - // Act: FunctionInvokingChatClient moves the text after the approved call and its result. - AgentResponse pending = await agent.RunAsync("Refund my Seattle order.", session); - ToolApprovalRequestContent approval = pending.Messages.SelectMany(message => message.Contents).OfType().Single(); - AgentResponse approved = await agent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(true), new TextContent("Yes, go ahead.")]), session); - - // Assert: the approved call counts toward the limit, so the tool is not offered again. - Assert.Equal(1, executions); - Assert.Equal(2, handler.Requests.Count); - Assert.Equal(["department", "is_urgent"], JsonNode.Parse(handler.Requests[1].Body)!["questions"]!.AsObject().Select(question => question.Key)); - Assert.Equal("Refund issued.\ndepartment: billing", approved.Text); - } - - [Fact] - public async Task GetResponseAsync_ApprovalWithManuallyAttributedHistory_ContinuesTheTurnAsync() - { - // Arrange - int executions = 0; - AIFunction refresh = new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => - { - executions++; - return "Refreshed."; - }, "refresh")); - using var handler = new JevTestHttpHandler() - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers(), JevChatTestData.RouteTo("t0", "t0", "none"))) - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - using var invoking = new FunctionInvokingChatClient(client); - ChatOptions options = new ChatOptions { Tools = [refresh] }.WithJevQuestions(JevChatTestData.Questions()); - List messages = [new(ChatRole.User, "Refresh the data.")]; - - // Act - ChatResponse pending = await invoking.GetResponseAsync(messages, options); - ToolApprovalRequestContent approval = pending.Messages.SelectMany(message => message.Contents).OfType().Single(); - messages = [.. messages.Concat(pending.Messages).Select(message => message.WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory))]; - messages.Add(new ChatMessage(ChatRole.User, [approval.CreateResponse(true), new TextContent("Yes, go ahead.")])); - ChatResponse response = await invoking.GetResponseAsync(messages, options); - - // Assert - Assert.Equal(1, executions); - Assert.Equal(2, handler.Requests.Count); - Assert.Equal(["department", "is_urgent"], JsonNode.Parse(handler.Requests[1].Body)!["questions"]!.AsObject().Select(question => question.Key)); - Assert.Equal("Refreshed.\ndepartment: technical", response.Text); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task RunAsync_NewRequestAfterTerminatedToolCall_StartsANewTurnAsync(bool requiresApproval) - { - // Arrange - int executions = 0; - AIFunction refresh = AIFunctionFactory.Create(() => $"Execution {++executions}", "refresh"); - if (requiresApproval) - { - refresh = new ApprovalRequiredAIFunction(refresh); - } - - using var handler = new JevTestHttpHandler() - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers(), JevChatTestData.RouteTo("t0", "t0", "none"))) - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers(), JevChatTestData.RouteTo("t0", "t0", "none"))); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions { Tools = [refresh] }.WithJevQuestions(JevChatTestData.Questions()) }) - .AsBuilder() - .Use(async (_, context, next, cancellationToken) => - { - context.Terminate = true; - return await next(context, cancellationToken); - }) - .Build(); - AgentSession session = await agent.CreateSessionAsync(); - - // Act - await RunAsync("Refresh the data."); - AgentResponse response = await RunAsync("Refresh it again."); - - // Assert - Assert.Equal(2, executions); - Assert.Equal(2, handler.Requests.Count); - JsonObject questions = JsonNode.Parse(handler.Requests[1].Body)!["questions"]!.AsObject(); - Assert.NotNull(questions[Route]); - Assert.DoesNotContain("already called this turn", (string?)questions[Route]!["criteria"]!["t0"], StringComparison.Ordinal); - Assert.Equal("Execution 2", Assert.Single(response.Messages.SelectMany(message => message.Contents).OfType()).Result?.ToString()); - - async Task RunAsync(string request) - { - AgentResponse response = await agent.RunAsync(request, session); - if (response.Messages.SelectMany(message => message.Contents).OfType().SingleOrDefault() is { } approval) - { - return await agent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(true), new TextContent("Yes, go ahead.")]), session); - } - - return response; - } - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public void GetCurrentTurnStart_NewUserAfterFunctionResult_StartsANewTurn(bool withHistoryAttribution) - { - // Arrange - ChatMessage[] messages = - [ - new(ChatRole.User, "Refresh the data."), - new(ChatRole.Assistant, [new FunctionCallContent("call_1", "refresh")]), - new(ChatRole.Tool, [new FunctionResultContent("call_1", "Old result")]), - new(ChatRole.User, "Refresh it again."), - ]; - if (withHistoryAttribution) - { - for (int index = 0; index < messages.Length - 1; index++) - { - messages[index] = messages[index].WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory); - } - } - - // Act - int turnStart = DecisionChatState.GetCurrentTurnStart(messages); - - // Assert - Assert.Equal(messages.Length, turnStart); - Assert.Equal(0, DecisionChatState.CountCurrentTurnCalls(messages, turnStart)); - Assert.Empty(DecisionChatState.GetCurrentTurnResultTexts(messages, turnStart)); - } - - [Theory] - [InlineData("call_1", 1)] - [InlineData("unrelated_call", 5)] - public void GetCurrentTurnStart_ResumedResultBeforeSeveralUserMessages_MatchesTheHistoricalCall(string resultCallId, int expectedTurnStart) - { - // Arrange - ChatMessage[] messages = - [ - new ChatMessage(ChatRole.User, "Refresh the data.").WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory), - new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", "refresh")]).WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory), - new(ChatRole.Tool, [new FunctionResultContent(resultCallId, "New result")]), - new(ChatRole.User, "Yes, go ahead."), - new(ChatRole.User, "Thanks."), - ]; - - // Act - int turnStart = DecisionChatState.GetCurrentTurnStart(messages); - - // Assert - Assert.Equal(expectedTurnStart, turnStart); - } - - [Fact] - public async Task RunStreamingAsync_ThrowsAsync() - { - // Arrange - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient client = JevChatClientTests.CreateClient(httpClient); - AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions().WithJevQuestions(JevChatTestData.Questions()) }); - - // Act & Assert - await Assert.ThrowsAsync(async () => - { - await foreach (AgentResponseUpdate update in agent.RunStreamingAsync(JevTestData.TicketText)) - { - } - }); - Assert.Empty(handler.Requests); - } -} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs index e5f7ab4177f..6760fd84d1e 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevDecisionClientTests.cs @@ -3,7 +3,6 @@ using System; using System.ClientModel; using System.ClientModel.Primitives; -using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Http; @@ -25,10 +24,10 @@ public sealed class JevDecisionClientTests public async Task GetResponseAsync_DecisionQuestions_AreAskedAsJevQuestionsAsync() { // Arrange - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( - JevChatTestData.Noul("refund", 0.95), + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevResultJson.Result( + JevResultJson.Noul("refund", 0.95), "\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":0.8,\"probabilities\":{\"technical\":0.1,\"billing\":0.9}}", - JevChatTestData.Score("severity", 1.5, 3))); + JevResultJson.Score("severity", 1.5, 3))); using var httpClient = new HttpClient(handler); using JevDecisionClient client = CreateClient(httpClient); var request = new DecisionRequest(s_state, [ @@ -77,36 +76,11 @@ public async Task GetResponseAsync_DecisionQuestions_AreAskedAsJevQuestionsAsync Assert.IsType(response.RawRepresentation); } - [Fact] - public async Task GetResponseAsync_JevQuestionsInRawOptions_AreSentUnchangedAsync() - { - // Arrange: JSON instructions and an undescribed label, which decision questions cannot carry as they are. - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Choice("duplicate", "yes", "yes", "no"))); - using var httpClient = new HttpClient(handler); - using JevDecisionClient client = CreateClient(httpClient); - JsonElement instructions = JsonDocument.Parse("""{ "question": "Is the resume for the same person as `candidate`?", "candidate": { "name": "Ana" } }""").RootElement.Clone(); - var jevQuestions = new Dictionary - { - ["duplicate"] = new JevChoiceQuestion { Instructions = (JevEntry)instructions, Criteria = new Dictionary { ["yes"] = JevEntry.Null, ["no"] = "A different person" } }, - }; - DecisionQuestion question = JevDecisionMapping.ToDecisionQuestion("duplicate", jevQuestions["duplicate"]); - var options = new DecisionOptions { RawRepresentationFactory = _ => new JevDecisionRequestOptions { Questions = jevQuestions } }; - - // Act - DecisionResponse response = await client.GetResponseAsync(new DecisionRequest(s_state, [question]), options); - - // Assert - JsonNode sent = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!["duplicate"]!; - Assert.True(JsonNode.DeepEquals(JsonNode.Parse(instructions.GetRawText()), sent["instructions"]), sent.ToJsonString()); - Assert.Null(sent["criteria"]!["yes"]); - Assert.Equal("yes", Assert.IsType(Assert.Single(response.Answers)).SelectedCandidateId); - } - [Fact] public async Task GetResponseAsync_IncompleteDistribution_ThrowsDecisionProtocolExceptionAsync() { // Arrange: the answer leaves out a label, so its distribution is not complete. - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevResultJson.Result( "\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":0.9,\"probabilities\":{\"billing\":1.0}}")); using var httpClient = new HttpClient(handler); using JevDecisionClient client = CreateClient(httpClient); @@ -121,7 +95,7 @@ public async Task GetResponseAsync_IncompleteDistribution_ThrowsDecisionProtocol public async Task GetResponseAsync_DistributionRoundedToTwoDecimals_IsAcceptedAsync() { // Arrange: Jev rounds to two decimals, so three even probabilities sum to 0.99 rather than 1. - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevResultJson.Result( "\"department\":{\"type\":\"choice\",\"choice\":\"billing\",\"confidence\":0.01,\"probabilities\":{\"billing\":0.33,\"technical\":0.33,\"sales\":0.33}}")); using var httpClient = new HttpClient(handler); using JevDecisionClient client = CreateClient(httpClient); @@ -139,7 +113,7 @@ public async Task GetResponseAsync_DistributionRoundedToTwoDecimals_IsAcceptedAs public async Task GetResponseAsync_ValueWithinRoundingErrorOfItsBound_IsMovedOntoTheBoundAsync() { // Arrange - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("refund", 1.0000001))); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevResultJson.Result(JevResultJson.Noul("refund", 1.0000001))); using var httpClient = new HttpClient(handler); using JevDecisionClient client = CreateClient(httpClient); @@ -168,7 +142,7 @@ public async Task GetResponseAsync_StateThatIsNotTextObjectOrArray_ThrowsBeforeS public async Task GetResponseAsync_ModelInOptions_WinsOverTheClientModelAsync() { // Arrange - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("refund", 0.5))); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevResultJson.Result(JevResultJson.Noul("refund", 0.5))); using var httpClient = new HttpClient(handler); using JevDecisionClient client = CreateClient(httpClient); @@ -201,7 +175,7 @@ public void GetService_ReturnsTheMetadataAndTheClient() public async Task Constructor_WithApiKeyAndOptions_SendsToTheConfiguredEndpointAsync() { // Arrange - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("refund", 0.5))); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevResultJson.Result(JevResultJson.Noul("refund", 0.5))); using var httpClient = new HttpClient(handler); using var client = new JevDecisionClient(new ApiKeyCredential("test-key"), new JevDecisionClientOptions { @@ -229,24 +203,9 @@ public void Constructor_InvalidOptions_Throws() Assert.Throws(() => new JevDecisionClient(new ApiKeyCredential("key"), new JevDecisionClientOptions { Endpoint = new Uri("/jev", UriKind.Relative) })); } - [Fact] - public void ToDecisionQuestion_NoulWithOneCriterion_UsesItAsTheInstructions() - { - // Arrange: Jev accepts a Noul with only one outcome described, which a Binary question cannot express directly. - var noul = new JevNoulQuestion { Criteria = new JevNoulCriteria { True = "The customer is angry." } }; - - // Act - BinaryDecisionQuestion question = Assert.IsType(JevDecisionMapping.ToDecisionQuestion("angry", noul)); - - // Assert - Assert.Equal("The customer is angry.", question.Instructions); - Assert.Equal("The customer is angry.", question.TrueDescription); - Assert.Null(question.FalseDescription); - } - - private static JevDecisionClient CreateClient(HttpClient httpClient) + internal static JevDecisionClient CreateClient(HttpClient httpClient) { - var options = new JevChatClientOptions + var options = new JevDecisionClientOptions { Transport = new HttpClientPipelineTransport(httpClient), RetryPolicy = new JevRetryPolicy { RecordedWaitsForTests = [] }, diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs index 88d99d27b80..32f494c2211 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs @@ -18,11 +18,11 @@ namespace Microsoft.Agents.AI.TypeSafe.UnitTests; /// public sealed class JevLoopJudgeTests { - private static readonly Dictionary s_criteria = new() + private static readonly DecisionDefinition s_criteria = DecisionDefinition.Create(builder => { - ["blue_sky"] = new JevNoulQuestion { Instructions = "Does the assistant's latest response explain why the sky is blue?" }, - ["red_sunsets"] = new JevNoulQuestion { Instructions = "Does the assistant's latest response explain why sunsets are red?" }, - }; + builder.BinaryProbability(verdict => verdict.BlueSky, "Does `answer` explain why the sky is blue?"); + builder.BinaryProbability(verdict => verdict.RedSunsets, "Does `answer` explain why sunsets are red?"); + }); [Fact] public async Task RunAsync_DelegateEvaluatorWithJev_LoopsUntilEveryCriterionIsMetAsync() @@ -32,16 +32,26 @@ public async Task RunAsync_DelegateEvaluatorWithJev_LoopsUntilEveryCriterionIsMe _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Air scatters blue light.")), _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Air scatters blue light, and sunsets are red because light crosses more air."))); using var handler = new JevTestHttpHandler() - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("blue_sky", 0.98), JevChatTestData.Noul("red_sunsets", 0.03))) - .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("blue_sky", 0.97), JevChatTestData.Noul("red_sunsets", 0.95))); + .Reply(HttpStatusCode.OK, JevResultJson.Result(JevResultJson.Noul("blueSky", 0.98), JevResultJson.Noul("redSunsets", 0.03))) + .Reply(HttpStatusCode.OK, JevResultJson.Result(JevResultJson.Noul("blueSky", 0.97), JevResultJson.Noul("redSunsets", 0.95))); using var httpClient = new HttpClient(handler); - using JevChatClient judge = JevChatClientTests.CreateClient(httpClient); + using JevDecisionClient judge = JevDecisionClientTests.CreateClient(httpClient); var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => { - List conversation = [.. context.InitialMessages, new ChatMessage(ChatRole.Assistant, context.LastResponse.Text)]; - ChatResponse verdict = await judge.GetResponseAsync(conversation, new ChatOptions().WithJevQuestions(s_criteria), cancellationToken); - List missing = [.. verdict.GetJevResult()!.Answers.Where(answer => ((JevNoulResponse)answer.Value).Noul <= 0.5).Select(answer => answer.Key)]; + string request = string.Join("\n", context.InitialMessages.Select(message => message.Text)); + DecisionResponse decision = await judge.GetResponseAsync(new JudgedAnswer(request, context.LastResponse.Text), s_criteria, cancellationToken: cancellationToken); + List missing = []; + if (decision.Result.BlueSky <= 0.5) + { + missing.Add("blueSky"); + } + + if (decision.Result.RedSunsets <= 0.5) + { + missing.Add("redSunsets"); + } + return missing.Count == 0 ? LoopEvaluation.Stop() : LoopEvaluation.Continue($"Missing: {string.Join(", ", missing)}."); }); var loop = new LoopAgent(answerer.AsAIAgent(), evaluator, new LoopAgentOptions { MaxIterations = 3, NonStreamingReturnsLastResponseOnly = true }); @@ -51,31 +61,19 @@ public async Task RunAsync_DelegateEvaluatorWithJev_LoopsUntilEveryCriterionIsMe // Assert: two runs, and the second run received the missing criterion as feedback. Assert.Equal(2, answerer.Calls.Count); - Assert.Contains(answerer.Calls[1], message => message.Role == ChatRole.User && message.Text == "Missing: red_sunsets."); + Assert.Contains(answerer.Calls[1], message => message.Role == ChatRole.User && message.Text == "Missing: redSunsets."); Assert.Contains("sunsets are red", response.Text, StringComparison.Ordinal); - // Assert: Jev read the original request and the latest answer, and only the criteria questions. + // Assert: Jev judged the original request and the latest answer, with one Noul question per criterion. Assert.Equal(2, handler.Requests.Count); JsonNode first = JsonNode.Parse(handler.Requests[0].Body)!; - Assert.Equal(["blue_sky", "red_sunsets"], first["questions"]!.AsObject().Select(question => question.Key)); - JsonArray messages = first["state"]!["messages"]!.AsArray(); - Assert.Equal(["user", "assistant"], messages.Select(message => (string?)message!["role"])); - Assert.Equal("Air scatters blue light.", (string?)messages[1]!["contents"]![0]!["text"]); + Assert.Equal(["blueSky", "redSunsets"], first["questions"]!.AsObject().Select(question => question.Key)); + Assert.All(first["questions"]!.AsObject(), question => Assert.Equal("noul", (string?)question.Value!["type"])); + Assert.Equal("Explain why the sky is blue and why sunsets are red.", (string?)first["state"]!["request"]); + Assert.Equal("Air scatters blue light.", (string?)first["state"]!["answer"]); } +} - [Fact] - public async Task EvaluateAsync_AIJudgeLoopEvaluatorWithJev_IsNotSupportedAsync() - { - // Arrange: AIJudgeLoopEvaluator asks for a JudgeVerdict as JSON structured output, which Jev cannot write. - using var handler = new JevTestHttpHandler(); - using var httpClient = new HttpClient(handler); - using JevChatClient judge = JevChatClientTests.CreateClient(httpClient, new JevChatClientOptions { DefaultQuestions = s_criteria }); - using var answerer = new ScriptedChatClient(_ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Air scatters blue light."))); - var loop = new LoopAgent(answerer.AsAIAgent(), new AIJudgeLoopEvaluator(judge)); +public sealed record JudgedAnswer(string Request, string Answer); - // Act & Assert: the request fails before anything is sent, which is why the sample uses a delegate evaluator. - NotSupportedException exception = await Assert.ThrowsAsync(() => loop.RunAsync("Why is the sky blue?")); - Assert.Contains(nameof(ChatOptions.ResponseFormat), exception.Message, StringComparison.Ordinal); - Assert.Empty(handler.Requests); - } -} +public sealed record LoopVerdict(double BlueSky, double RedSunsets); diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevResultJson.cs similarity index 59% rename from dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs rename to dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevResultJson.cs index b73c95cbcd2..b83f93c972e 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevResultJson.cs @@ -1,39 +1,19 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; using System.Globalization; using System.Linq; namespace Microsoft.Agents.AI.TypeSafe.UnitTests; /// -/// Builds TypeSafe API responses and questions for the chat client tests. +/// Builds TypeSafe API responses as JSON, one answer at a time. /// -internal static class JevChatTestData +internal static class JevResultJson { - public const string Route = "__af_tool__.route"; - - /// Two questions about a support ticket: a Choice and a Noul. - public static Dictionary Questions() => new() - { - ["department"] = new JevChoiceQuestion - { - Instructions = "Which team should handle the request?", - Criteria = new Dictionary { ["billing"] = "Payments or refunds", ["technical"] = "Bugs or integrations" }, - }, - ["is_urgent"] = new JevNoulQuestion { Instructions = "Does the request need urgent attention?" }, - }; - - /// The answers to . - public static string[] UserAnswers(string department = "technical") => - [Choice("department", department, "billing", "technical"), Noul("is_urgent", 0.9)]; - public static string Result(params string[] answers) => "{\"model\":\"jev-test\",\"answers\":{" + string.Join(",", answers) + "},\"usage\":{\"input_tokens\":10,\"output_tokens\":2}}"; - public static string Result(IEnumerable answers, params string[] more) => Result([.. answers, .. more]); - public static string Choice(string id, string choice, params string[] labels) { string probabilities = string.Join(",", labels.Select(label => $"\"{label}\":{(label == choice ? "1.0" : "0.0")}")); @@ -59,7 +39,4 @@ public static string Score(string id, double score, int levels) })); return $"\"{id}\":{{\"type\":\"score\",\"score\":{score.ToString(CultureInfo.InvariantCulture)},\"confidence\":0.8,\"legend\":{{{legend}}},\"probabilities\":{{{probabilities}}}}}"; } - - /// A route answer that selects one of the route labels. - public static string RouteTo(string label, params string[] labels) => Choice(Route, label, labels); } diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs deleted file mode 100644 index df54fb10202..00000000000 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using Microsoft.Extensions.Logging; - -namespace Microsoft.Agents.AI.TypeSafe.UnitTests; - -/// -/// A logger factory that records every message, so tests can check what a client logs. -/// -internal sealed class ListLoggerFactory : ILoggerFactory -{ - public List<(LogLevel Level, string Message)> Entries { get; } = []; - - public ILogger CreateLogger(string categoryName) => new ListLogger(this.Entries); - - public void AddProvider(ILoggerProvider provider) - { - } - - public void Dispose() - { - } - - private sealed class ListLogger(List<(LogLevel Level, string Message)> entries) : ILogger - { - public IDisposable? BeginScope(TState state) - where TState : notnull => null; - - public bool IsEnabled(LogLevel logLevel) => true; - - public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => - entries.Add((logLevel, formatter(state, exception))); - } -}