[WIP][Agent Library]

src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al

@@ -43,5 +43,7 @@ permissionset 149031 "AI Test Toolkit - Obj"
         page "AIT Test Suite" = X,
         page "AIT Test Suite List" = X,
         page "AIT Test Suite Language Lookup" = X,
+        page "AIT Log Entry Outcome Part" = X,
+        page "AIT Agent Log Entry Part" = X,
         page "AIT Run History" = X;
 }

src/Tools/AI Test Toolkit/src/Agent/AITAgentLogEntryPart.Page.al

@@ -0,0 +1,55 @@
+// ------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+// ------------------------------------------------------------------------------------------------
+
+namespace System.TestTools.AITestToolkit;
+
+page 149050 "AIT Agent Log Entry Part"
+{
+    Caption = 'Agent Details';
+    PageType = CardPart;
+    Editable = false;
+    SourceTable = "AIT Log Entry";
+
+    layout
+    {
+        area(Content)
+        {
+            field("Agent Task IDs"; AgentTaskIDs)
+            {
+                ApplicationArea = All;
+                Caption = 'Agent Tasks Executed';
+                ToolTip = 'Specifies the comma-separated list of Agent Task IDs related to this log entry.';
+
+                trigger OnDrillDown()
+                begin
+                    AgentTestContextImpl.OpenAgentTaskList(AgentTaskIDs);
+                end;
+            }
+            field("Copilot Credits"; CopilotCredits)
+            {
+                ApplicationArea = All;
+                AutoFormatType = 0;
+                Caption = 'Copilot Credits Consumed';
+                ToolTip = 'Specifies the total Copilot Credits consumed by the Agent Tasks for this log entry.';
+
+                trigger OnDrillDown()
+                begin
+                    AgentTestContextImpl.OpenAgentConsumptionOverview(AgentTaskIDs);
+                end;
+            }
+        }
+    }
+
+    var
+        AgentTestContextImpl: Codeunit "Agent Test Context Impl.";
+        CopilotCredits: Decimal;
+        AgentTaskIDs: Text;
+
+    trigger OnAfterGetRecord()
+    begin
+        CopilotCredits := AgentTestContextImpl.GetCopilotCreditsForLogEntry(Rec."Entry No.");
+        AgentTaskIDs := AgentTestContextImpl.GetAgentTaskIDsForLogEntry(Rec."Entry No.");
+    end;
+}

src/Tools/AI Test Toolkit/src/Agent/AgentLogEntries.PageExt.al

@@ -39,6 +39,14 @@ pageextension 149030 "Agent Log Entries" extends "AIT Log Entries"
                 end;
             }
         }
+        addafter(TestOutcome)
+        {
+            part(AgentDetails; "AIT Agent Log Entry Part")
+            {
+                ApplicationArea = All;
+                SubPageLink = "Entry No." = field("Entry No.");
+            }
+        }
     }
 
     trigger OnAfterGetRecord()

src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al

@@ -186,6 +186,14 @@ page 149033 "AIT Log Entries"
                 }
             }
         }
+        area(FactBoxes)
+        {
+            part(TestOutcome; "AIT Log Entry Outcome Part")
+            {
+                ApplicationArea = All;
+                SubPageLink = "Entry No." = field("Entry No.");
+            }
+        }
     }
     actions
     {

src/Tools/AI Test Toolkit/src/Logs/AITLogEntryOutcomePart.Page.al

@@ -0,0 +1,110 @@
+// ------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+// ------------------------------------------------------------------------------------------------
+
+namespace System.TestTools.AITestToolkit;
+
+page 149049 "AIT Log Entry Outcome Part"
+{
+    ApplicationArea = All;
+    Caption = 'Test Outcome';
+    PageType = CardPart;
+    Editable = false;
+    SourceTable = "AIT Log Entry";
+    Extensible = true;
+
+    layout
+    {
+        area(Content)
+        {
+            field(Status; Rec.Status)
+            {
+                StyleExpr = StatusStyleExpr;
+            }
+            field(Accuracy; Rec."Test Method Line Accuracy")
+            {
+                Caption = 'Evaluation Result';
+                ToolTip = 'Specifies the accuracy of the eval line.';
+                AutoFormatType = 0;
+            }
+            field(TurnsText; TurnsText)
+            {
+                Caption = 'No. of Turns Passed';
+                ToolTip = 'Specifies the number of turns that passed out of the total number of turns.';
+                StyleExpr = TurnsStyleExpr;
+            }
+            group(ErrorMessageGroup)
+            {
+                Caption = 'Error Message';
+                field(ErrorMessage; ErrorMessage)
+                {
+                    ShowCaption = false;
+                    ToolTip = 'Specifies the error message from the eval.';
+                    Style = Unfavorable;
+                    Multiline = true;
+
+                    trigger OnDrillDown()
+                    begin
+                        Message(ErrorMessage);
+                    end;
+                }
+            }
+            field(Duration; Rec."Duration (ms)")
+            {
+                Caption = 'Duration (ms)';
+                ToolTip = 'Specifies the duration of the test execution in milliseconds.';
+                AutoFormatType = 0;
+            }
+        }
+    }
+
+    var
+        TurnsText: Text;
+        ErrorMessage: Text;
+        StatusStyleExpr: Text;
+        TurnsStyleExpr: Text;
+
+    trigger OnAfterGetRecord()
+    var
+        AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt.";
+    begin
+        TurnsText := AITTestSuiteMgt.GetTurnsAsText(Rec);
+        SetStatusStyleExpr();
+        SetTurnsStyleExpr();
+        SetErrorMessage();
+    end;
+
+    local procedure SetStatusStyleExpr()
+    begin
+        case Rec.Status of
+            Rec.Status::Success:
+                StatusStyleExpr := Format(PageStyle::Favorable);
+            Rec.Status::Error:
+                StatusStyleExpr := Format(PageStyle::Unfavorable);
+            Rec.Status::Skipped:
+                StatusStyleExpr := Format(PageStyle::Ambiguous);
+            else
+                StatusStyleExpr := '';
+        end;
+    end;
+
+    local procedure SetTurnsStyleExpr()
+    begin
+        case Rec."No. of Turns Passed" of
+            Rec."No. of Turns":
+                TurnsStyleExpr := Format(PageStyle::Favorable);
+            0:
+                TurnsStyleExpr := Format(PageStyle::Unfavorable);
+            else
+                TurnsStyleExpr := Format(PageStyle::Ambiguous);
+        end;
+    end;
+
+    local procedure SetErrorMessage()
+    begin
+        ErrorMessage := '';
+        if Rec.Status = Rec.Status::Error then
+            ErrorMessage := Rec.GetMessage();
+    end;
+}

src/Tools/Test Framework/Test Libraries/LibraryAgent/AI-TEST-AUTHORING.md

@@ -161,18 +161,24 @@ query:
   title: <task title>
   message: <task message body>
   attachments:
-    - file: <relative path inside .resources>
-    - file: <another path>
+    - file: <relative path inside .resources>   # static file
+    - file:                                      # OR: dynamically generated
+        action_type: <generator name>
+        action_data:
+          <key>: <value>                         # arbitrary data for the generator
 ```
 
+The `file` key supports two forms: a **scalar** value (static file path) or an **object** with `action_type` / `action_data` (dynamically generated file).
+
 How keys flow into library calls:
 
 | YAML key | Flows into |
 |---|---|
 | `query.title` | `AgentTaskBuilder.Initialize(AgentUserSecurityId, title)` — required, asserted via `Library Assert`. |
 | `query.from` | `AgentTaskMessageBuilder.Initialize(from, ...)`. If `from` is missing, no message is added (only the task title). |
 | `query.message` | `AgentTaskMessageBuilder.Initialize(..., message)`. Optional. |
-| `query.attachments[].file` | `IAgentTestResourceProvider.GetResource(file, ...)` → `AgentTaskMessageBuilder.AddAttachment(...)`. Use the `RunTurnAndWait` overload that accepts a provider when YAML uses attachments. |
+| `query.attachments[].file` (scalar) | `IAgentTestResourceProvider.GetResource(file, ...)` → `AgentTaskMessageBuilder.AddAttachment(...)`. Use the `RunTurnAndWait` overload that accepts a provider when YAML uses attachments. |
+| `query.attachments[].file` (object) | `IAgentTestResourceProvider.GenerateResource(action_type, action_data, ...)` → `AgentTaskMessageBuilder.AddAttachment(...)`. The `action_data` sub-object is extracted and passed as a `Test Input Json` codeunit; `action_type` is passed separately. |
 
 ### 7.3 Intervention continuation
 
@@ -212,6 +218,7 @@ expected_data:
     suggestions:                            # optional — list of suggestion codes that MUST be present
       - <CODE_A>
       - <CODE_B>
+    intent: "<agent intent for the request>"                   # optional — LLM judge validates the intervention message
   <agent_specific_count_key>: 1             # implemented per agent test app
   <agent_specific_status_key>: Released     # implemented per agent test app
 ```
@@ -222,10 +229,12 @@ expected_data:
 |---|---|
 | `expected_data.intervention_request.type` | `LibraryAgent.ParseUserInterventionRequestType(text)` → `Enum "Agent User Int Request Type"`. Values: `Assistance`, `Review`, `Message` (English ordinal names; no translation). |
 | `expected_data.intervention_request.suggestions[]` | Validated by `LibraryAgent.ValidateInterventionRequest` — every expected code must be present on the actual request. |
+| `expected_data.intervention_request.intent` | Validated by an LLM judge that evaluates whether the agent's intervention message semantically matches the declared intent. The judge returns a pass/fail verdict with reasoning. |
 
 Automatic validation in `LibraryAgent.FinalizeTurn`:
 
 - If `intervention_request` is declared in YAML: the agent must have paused for an intervention with the matching `type` and including every `suggestion` code listed.
+- If `intent` is declared: the framework calls an LLM judge to semantically validate that the intervention message matches the expected intent. This replaces brittle substring matching with semantic evaluation.
 - If `intervention_request` is **not** declared: the agent must **not** have paused for an intervention. Unexpected interventions fail the turn.
 
 So: declare `intervention_request` on every turn where you expect the

src/Tools/Test Framework/Test Libraries/LibraryAgent/README.md

@@ -2,6 +2,14 @@
 
 Test helpers for authoring AI agent tests in Business Central. The library provides helper methods to create and manage agent tasks, messages, and user interventions, drive YAML-described turn loops via `Library - Agent.RunTurnAndWait` / `FinalizeTurn`, and integrate with the AI Test Toolkit for evaluation.
 
+## Features
+
+- **Turn-loop driver** — `RunTurnAndWait` + `FinalizeTurn` handle the multi-turn lifecycle from YAML.
+- **Intervention validation** — type, suggestions, and semantic intent matching.
+- **LLM-as-judge** — when an `intent` key is declared in `intervention_request`, the framework uses GPT-4.1 to semantically evaluate whether the agent's intervention message matches the expected intent. Requires the `Agent Test LLM Judge` Copilot Capability (registered automatically by the library's install codeunit).
+- **Dynamic file generation** — `IAgentTestResourceProvider.GenerateResource` for test attachments that must be created at runtime.
+- **Placeholder engine** — date/time formula substitution in YAML values.
+
 ## Public documentation
 
 - [AI-TEST-AUTHORING.md](AI-TEST-AUTHORING.md) — YAML format reference for AI agent tests, the placeholder syntax, and how each YAML key maps to the library methods that consume it.

src/Tools/Test Framework/Test Libraries/LibraryAgent/src/AgentTestLLMJudgeCap.EnumExt.al

@@ -0,0 +1,15 @@
+// ------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+// ------------------------------------------------------------------------------------------------
+namespace System.TestLibraries.AI;
+
+using System.AI;
+
+enumextension 130566 "Agent Test LLM Judge Cap." extends "Copilot Capability"
+{
+    value(130566; "Agent Test LLM Judge")
+    {
+        Caption = 'Agent Test Library LLM Judge';
+    }
+}

src/Tools/Test Framework/Test Libraries/LibraryAgent/src/IAgentTestResourceProvider.Interface.al

@@ -5,6 +5,8 @@
 
 namespace System.TestLibraries.Agents;
 
+using System.TestTools.TestRunner;
+
 /// <summary>
 /// Interface for resolving test resource files from the consuming test app.
 /// Implement this in your test app to provide resource file access to the agent test library.
@@ -19,4 +21,19 @@ interface "IAgentTestResourceProvider"
     /// <param name="FileName">Returns the file name extracted from the path.</param>
     /// <param name="MIMEType">Returns the MIME type of the file.</param>
     procedure GetResource(ResourcePath: Text; var ResourceInStream: InStream; var FileName: Text[250]; var MIMEType: Text[100])
+
+    /// <summary>
+    /// Generates a resource dynamically from YAML-declared data.
+    /// Override this to support 'filegenerator' entries in YAML attachments.
+    /// The GeneratorData parameter contains the full filegenerator object from the YAML,
+    /// including a 'name' key for dispatch and any additional data keys the generator needs.
+    /// </summary>
+    /// <param name="GeneratorName">The name of the generator, from the 'name' key in the YAML filegenerator object.</param>
+    /// <param name="GeneratorData">The filegenerator object from the YAML, accessible via Test Input Json.</param>
+    /// <param name="ResourceInStream">Returns the generated file content as an InStream.</param>
+    /// <param name="FileName">Returns the generated file name.</param>
+    /// <param name="MIMEType">Returns the MIME type of the generated file.</param>
+    procedure GenerateResource(GeneratorName: Text; GeneratorData: Codeunit "Test Input Json"; var ResourceInStream: InStream; var FileName: Text[250]; var MIMEType: Text[100])
+    begin
+    end;
 }