diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/ApplicationAnalysisService.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/ApplicationAnalysisService.cs index 8f8a1e5c94..b2ce118946 100644 --- a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/ApplicationAnalysisService.cs +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/ApplicationAnalysisService.cs @@ -5,17 +5,15 @@ using Unity.AI.Models; using Unity.AI.Requests; using Unity.AI.Runtime; -using Unity.GrantManager.Applications; using Volo.Abp.DependencyInjection; namespace Unity.AI.Operations { public class ApplicationAnalysisService( - IApplicationRepository applicationRepository, IAIService aiService, IAIGenerationPrerequisiteValidator aiGenerationPrerequisiteValidator) : IApplicationAnalysisService, ITransientDependency { - public async Task RegenerateAndSaveAsync(ApplicationAnalysisOperationInputDto input, CancellationToken cancellationToken = default) + public async Task RegenerateAsync(ApplicationAnalysisOperationInputDto input, CancellationToken cancellationToken = default) { await aiGenerationPrerequisiteValidator.EnsureApplicationAnalysisAvailableAsync(input.ApplicationId); @@ -28,9 +26,6 @@ public async Task RegenerateAndSaveAsync(ApplicationAnalysisOperationInp }, cancellationToken); var analysisJson = JsonSerializer.Serialize(analysis, AIJsonDefaults.Indented); - var application = await applicationRepository.GetAsync(input.ApplicationId); - application.AIAnalysis = analysisJson; - await applicationRepository.UpdateAsync(application); return analysisJson; } diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/ApplicationScoringService.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/ApplicationScoringService.cs index 34856d162a..82b781d055 100644 --- a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/ApplicationScoringService.cs +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/ApplicationScoringService.cs @@ -9,18 +9,16 @@ using Unity.AI.Models; using Unity.AI.Requests; using Unity.AI.Runtime; -using Unity.GrantManager.Applications; using Volo.Abp.DependencyInjection; namespace Unity.AI.Operations { public class ApplicationScoringService( - IApplicationRepository applicationRepository, IAIService aiService, AIExecutionModeResolver executionModeResolver, ILogger logger) : IApplicationScoringService, ITransientDependency { - public async Task RegenerateAndSaveAsync(ApplicationScoringOperationInputDto input, CancellationToken cancellationToken = default) + public async Task RegenerateAsync(ApplicationScoringOperationInputDto input, CancellationToken cancellationToken = default) { var sections = input.Sections; var mode = executionModeResolver.ResolveMode(AIExecutionModeResolver.ApplicationScoringOperation); @@ -42,9 +40,6 @@ public async Task RegenerateAndSaveAsync(ApplicationScoringOperationInpu var combinedResults = JsonSerializer.Serialize(allSectionResults, AIJsonDefaults.Indented); var validatedJson = ValidateApplicationScoringJson(combinedResults); - var application = await applicationRepository.GetAsync(input.ApplicationId); - application.AIScoresheetAnswers = validatedJson; - await applicationRepository.UpdateAsync(application); return validatedJson; } @@ -140,6 +135,12 @@ private static JsonElement BuildBatchSectionSchema(IReadOnlyCollection(); foreach (var section in sections) { + if (section.SectionSchema.ValueKind != JsonValueKind.Array) + { + throw new InvalidOperationException( + $"Section schema for '{section.SectionName}' must be a JSON array."); + } + foreach (var question in section.SectionSchema.EnumerateArray()) { questions.Add(question.Clone()); diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/IApplicationAnalysisService.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/IApplicationAnalysisService.cs index 333817e90d..ddff668f03 100644 --- a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/IApplicationAnalysisService.cs +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/IApplicationAnalysisService.cs @@ -6,6 +6,6 @@ namespace Unity.AI.Operations { public interface IApplicationAnalysisService { - Task RegenerateAndSaveAsync(ApplicationAnalysisOperationInputDto input, CancellationToken cancellationToken = default); + Task RegenerateAsync(ApplicationAnalysisOperationInputDto input, CancellationToken cancellationToken = default); } } diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/IApplicationScoringService.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/IApplicationScoringService.cs index 9e22e8623f..a7890b76a3 100644 --- a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/IApplicationScoringService.cs +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Operations/IApplicationScoringService.cs @@ -6,6 +6,6 @@ namespace Unity.AI.Operations { public interface IApplicationScoringService { - Task RegenerateAndSaveAsync(ApplicationScoringOperationInputDto input, CancellationToken cancellationToken = default); + Task RegenerateAsync(ApplicationScoringOperationInputDto input, CancellationToken cancellationToken = default); } } diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Prompts/PromptDataPayloadBuilder.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Prompts/PromptDataPayloadBuilder.cs index 9788b9cd3f..ae9313f000 100644 --- a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Prompts/PromptDataPayloadBuilder.cs +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Prompts/PromptDataPayloadBuilder.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Linq; using System.Text.Json; -using System.Threading.Tasks; using Unity.AI.Models; using Unity.GrantManager.Applications; @@ -91,7 +90,7 @@ public static List BuildAttachmentSummaries( .ToList(); } - public static object BuildFormFieldConfigurationAsync( + public static object BuildFormFieldConfiguration( string? formSchema, ILogger logger) { diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIOperationResult.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIOperationResult.cs index d0c2a2ceec..efd964c51d 100644 --- a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIOperationResult.cs +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIOperationResult.cs @@ -8,9 +8,19 @@ public enum AIOperationOutcome InvalidOutput } + public enum AIFailureCategory + { + None, + ProviderUnavailable, + TransientProviderFailure, + PermanentProviderFailure, + InvalidOutput + } + public sealed record AIOperationResult( AIOperationOutcome Outcome, - AIProviderResult Response) + AIProviderResult Response, + AIFailureCategory FailureCategory = AIFailureCategory.None) { public string Content => Response.Content; @@ -20,14 +30,30 @@ public static AIOperationResult Success(AIProviderResult? response = null) => new(AIOperationOutcome.Success, response ?? AIProviderResult.Empty); public static AIOperationResult TransientFailure(AIProviderResult? response = null) => - new(AIOperationOutcome.TransientFailure, response ?? AIProviderResult.Empty); + new(AIOperationOutcome.TransientFailure, response ?? AIProviderResult.Empty, AIFailureCategory.TransientProviderFailure); public static AIOperationResult PermanentFailure(AIProviderResult? response = null) => - new(AIOperationOutcome.PermanentFailure, response ?? AIProviderResult.Empty); + new(AIOperationOutcome.PermanentFailure, response ?? AIProviderResult.Empty, AIFailureCategory.PermanentProviderFailure); + + public static AIOperationResult ProviderUnavailable(AIProviderResult? response = null) => + new(AIOperationOutcome.PermanentFailure, response ?? AIProviderResult.Empty, AIFailureCategory.ProviderUnavailable); public static AIOperationResult InvalidOutput(AIProviderResult? response = null) => - new(AIOperationOutcome.InvalidOutput, response ?? AIProviderResult.Empty); + new(AIOperationOutcome.InvalidOutput, response ?? AIProviderResult.Empty, AIFailureCategory.InvalidOutput); + + public AIOperationResult WithOutcome(AIOperationOutcome outcome, AIFailureCategory? failureCategory = null) => + new(outcome, Response, failureCategory ?? ResolveFailureCategory(outcome)); - public AIOperationResult WithOutcome(AIOperationOutcome outcome) => new(outcome, Response); + private static AIFailureCategory ResolveFailureCategory(AIOperationOutcome outcome) + { + return outcome switch + { + AIOperationOutcome.Success => AIFailureCategory.None, + AIOperationOutcome.TransientFailure => AIFailureCategory.TransientProviderFailure, + AIOperationOutcome.PermanentFailure => AIFailureCategory.PermanentProviderFailure, + AIOperationOutcome.InvalidOutput => AIFailureCategory.InvalidOutput, + _ => AIFailureCategory.None + }; + } } } diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIProviderPayloadValidator.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIProviderPayloadValidator.cs index 06016a9705..b49d718e15 100644 --- a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIProviderPayloadValidator.cs +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIProviderPayloadValidator.cs @@ -5,94 +5,92 @@ namespace Unity.AI.Runtime { - internal static class AIProviderPayloadValidator + public static class AIProviderPayloadValidator { - public static bool IsValidAttachmentSummaryText(string response) + public static AIResponseValidationResult ValidateAttachmentSummaryText(string response) { - return !string.IsNullOrWhiteSpace(response); + return !string.IsNullOrWhiteSpace(response) + ? AIResponseValidationResult.Success() + : AIResponseValidationResult.Invalid("Attachment summary response was empty."); } - public static bool IsValidApplicationAnalysisJson(string response) + public static AIResponseValidationResult ValidateApplicationAnalysisJson(string response) { if (!TryParseRootObject(response, out var root)) { - return false; + return AIResponseValidationResult.Invalid("Application analysis response was not valid JSON."); + } + + if (!root.TryGetProperty(AIJsonKeys.Decision, out var decision) || decision.ValueKind != JsonValueKind.String) + { + return AIResponseValidationResult.Invalid($"Application analysis response is missing or invalid required field '{AIJsonKeys.Decision}' (expected string)."); + } + + if (!root.TryGetProperty(AIJsonKeys.Errors, out var errors) || errors.ValueKind != JsonValueKind.Array) + { + return AIResponseValidationResult.Invalid($"Application analysis response is missing or invalid required field '{AIJsonKeys.Errors}' (expected array)."); + } + + if (!root.TryGetProperty(AIJsonKeys.Warnings, out var warnings) || warnings.ValueKind != JsonValueKind.Array) + { + return AIResponseValidationResult.Invalid($"Application analysis response is missing or invalid required field '{AIJsonKeys.Warnings}' (expected array)."); } - return HasStringProperty(root, AIJsonKeys.Decision) && - HasArrayProperty(root, AIJsonKeys.Errors) && - HasArrayProperty(root, AIJsonKeys.Warnings) && - HasArrayProperty(root, AIJsonKeys.Summaries) && - HasArrayProperty(root, AIJsonKeys.Recommendations); + if (!root.TryGetProperty(AIJsonKeys.Summaries, out var summaries) || summaries.ValueKind != JsonValueKind.Array) + { + return AIResponseValidationResult.Invalid($"Application analysis response is missing or invalid required field '{AIJsonKeys.Summaries}' (expected array)."); + } + + if (!root.TryGetProperty(AIJsonKeys.Recommendations, out var recommendations) || recommendations.ValueKind != JsonValueKind.Array) + { + return AIResponseValidationResult.Invalid($"Application analysis response is missing or invalid required field '{AIJsonKeys.Recommendations}' (expected array)."); + } + + return AIResponseValidationResult.Success(); } - public static bool IsValidApplicationScoringJson(string response, string sectionJson) + public static AIResponseValidationResult ValidateApplicationScoringJson(string response, string sectionJson) { if (!TryParseRootObject(response, out var root)) { - return false; + return AIResponseValidationResult.Invalid("Application scoring response was not valid JSON."); } var expectedQuestionIds = ExtractQuestionIds(sectionJson); if (expectedQuestionIds.Count == 0) { - return false; + return AIResponseValidationResult.Invalid("Application scoring section schema could not be parsed or did not contain any question ids."); } foreach (var questionId in expectedQuestionIds) { - if (!TryGetRequiredObject(root, questionId, out var answerObject)) + if (!root.TryGetProperty(questionId, out var answerObject) || answerObject.ValueKind != JsonValueKind.Object) { - return false; + return AIResponseValidationResult.Invalid( + $"Application scoring response is missing required answer object for question id '{questionId}'."); } - if (!HasPrimitiveProperty(answerObject, AIJsonKeys.Answer)) + if (!answerObject.TryGetProperty(AIJsonKeys.Answer, out var answerValue) + || answerValue.ValueKind == JsonValueKind.Null + || answerValue.ValueKind == JsonValueKind.Object + || answerValue.ValueKind == JsonValueKind.Array) { - return false; + return AIResponseValidationResult.Invalid( + $"Application scoring response is missing a valid answer for question id '{questionId}'."); } - if (!IsValidConfidenceProperty(answerObject, AIJsonKeys.Confidence)) + if (!answerObject.TryGetProperty(AIJsonKeys.Confidence, out var confidenceValue) + || confidenceValue.ValueKind != JsonValueKind.Number + || !confidenceValue.TryGetInt32(out var confidence) + || confidence < 0 + || confidence > 100) { - return false; + return AIResponseValidationResult.Invalid( + $"Application scoring response is missing a valid confidence score for question id '{questionId}'."); } } - return true; - } - - private static bool HasStringProperty(JsonElement element, string name) - { - return element.TryGetProperty(name, out var property) && - property.ValueKind == JsonValueKind.String; - } - - private static bool HasArrayProperty(JsonElement element, string name) - { - return element.TryGetProperty(name, out var property) && - property.ValueKind == JsonValueKind.Array; - } - - private static bool TryGetRequiredObject(JsonElement element, string name, out JsonElement value) - { - return element.TryGetProperty(name, out value) && - value.ValueKind == JsonValueKind.Object; - } - - private static bool HasPrimitiveProperty(JsonElement element, string name) - { - return element.TryGetProperty(name, out var property) && - property.ValueKind != JsonValueKind.Null && - property.ValueKind != JsonValueKind.Object && - property.ValueKind != JsonValueKind.Array; - } - - private static bool IsValidConfidenceProperty(JsonElement element, string name) - { - return element.TryGetProperty(name, out var property) && - property.ValueKind == JsonValueKind.Number && - property.TryGetInt32(out var confidence) && - confidence >= 0 && - confidence <= 100; + return AIResponseValidationResult.Success(); } private static HashSet ExtractQuestionIds(string sectionJson) @@ -169,5 +167,6 @@ private static bool TryParseRootObject(string response, out JsonElement root) return false; } } + } } diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIResponseValidationResult.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIResponseValidationResult.cs new file mode 100644 index 0000000000..ab0a542056 --- /dev/null +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/AIResponseValidationResult.cs @@ -0,0 +1,16 @@ +namespace Unity.AI.Runtime +{ + public sealed record AIResponseValidationResult( + bool IsValid, + AIFailureCategory FailureCategory = AIFailureCategory.None, + string? Reason = null) + { + public static AIResponseValidationResult Success() => + new(true); + + public static AIResponseValidationResult Invalid( + string reason, + AIFailureCategory failureCategory = AIFailureCategory.InvalidOutput) => + new(false, failureCategory, reason); + } +} diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/OpenAIRuntimeService.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/OpenAIRuntimeService.cs index e96efd23c7..b95cac4ef8 100644 --- a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/OpenAIRuntimeService.cs +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/AI/Runtime/OpenAIRuntimeService.cs @@ -25,8 +25,6 @@ public class OpenAIRuntimeService : IAIService, ITransientDependency private const string ApplicationScoringPromptType = AIPromptTypes.ApplicationScoring; private const int MaxAiAttempts = 3; - - public OpenAIRuntimeService( ILogger logger, OpenAITransportService openAITransportService, @@ -87,7 +85,7 @@ public async Task GenerateApplicationAnalysisAsync( settings, settings.CompletionTokens, cancellationToken: cancellationToken), - AIProviderPayloadValidator.IsValidApplicationAnalysisJson, + AIProviderPayloadValidator.ValidateApplicationAnalysisJson, "application analysis", cancellationToken); @@ -151,7 +149,7 @@ public async Task GenerateAttachmentSummaryAsync(Atta settings, settings.CompletionTokens, cancellationToken: cancellationToken), - AIProviderPayloadValidator.IsValidAttachmentSummaryText, + AIProviderPayloadValidator.ValidateAttachmentSummaryText, "attachment summary", cancellationToken); await _promptFileLogger.LogPromptOutputAsync(AttachmentSummaryPromptType, promptVersion, result.CaptureOutput, cancellationToken); @@ -220,13 +218,13 @@ public async Task GenerateApplicationScoringAsync(Ap await _promptFileLogger.LogPromptInputAsync(ApplicationScoringPromptType, promptVersion, systemPrompt, applicationScoringContent, cancellationToken); var result = await GenerateWithRetryAsync( - () => _openAITransportService.GenerateSummaryAsync( - applicationScoringContent, - systemPrompt, - settings, - settings.CompletionTokens, - cancellationToken: cancellationToken), - content => AIProviderPayloadValidator.IsValidApplicationScoringJson(content, section), + () => _openAITransportService.GenerateSummaryAsync( + applicationScoringContent, + systemPrompt, + settings, + settings.CompletionTokens, + cancellationToken: cancellationToken), + content => AIProviderPayloadValidator.ValidateApplicationScoringJson(content, section), $"application scoring section {request.SectionName}", cancellationToken); await _promptFileLogger.LogPromptOutputAsync(ApplicationScoringPromptType, promptVersion, result.CaptureOutput, cancellationToken); @@ -251,7 +249,7 @@ public async Task GenerateApplicationScoringAsync(Ap private async Task GenerateWithRetryAsync( Func> operation, - Func validator, + Func validator, string operationName, CancellationToken cancellationToken = default) { @@ -262,14 +260,23 @@ private async Task GenerateWithRetryAsync( cancellationToken.ThrowIfCancellationRequested(); lastResult = await operation(); - if (lastResult.Outcome == AIOperationOutcome.Success && validator(lastResult.Content)) - { - return lastResult; - } - if (lastResult.Outcome == AIOperationOutcome.Success) { - lastResult = lastResult.WithOutcome(AIOperationOutcome.InvalidOutput); + var validationResult = validator(lastResult.Content); + if (validationResult.IsValid) + { + return lastResult; + } + + lastResult = lastResult.WithOutcome(AIOperationOutcome.InvalidOutput, validationResult.FailureCategory); + + _logger.LogWarning( + "AI {OperationName} attempt {Attempt}/{MaxAttempts} returned invalid response shape ({FailureCategory}): {Reason}; will retry if attempts remain", + operationName, + attempt, + MaxAiAttempts, + validationResult.FailureCategory, + validationResult.Reason ?? "No validation reason provided"); } if (lastResult.Outcome == AIOperationOutcome.PermanentFailure) @@ -282,29 +289,45 @@ private async Task GenerateWithRetryAsync( if (lastResult.Outcome == AIOperationOutcome.TransientFailure) { _logger.LogWarning( - "AI {OperationName} attempt {Attempt}/{MaxAttempts} failed transiently; retrying", + "AI {OperationName} attempt {Attempt}/{MaxAttempts} failed transiently ({FailureCategory}); retrying", operationName, attempt, - MaxAiAttempts); + MaxAiAttempts, + lastResult.FailureCategory); } else if (lastResult.Outcome == AIOperationOutcome.InvalidOutput) { _logger.LogWarning( - "AI {OperationName} attempt {Attempt}/{MaxAttempts} returned invalid response shape; retrying", + "AI {OperationName} attempt {Attempt}/{MaxAttempts} returned invalid output ({FailureCategory}); retrying", operationName, attempt, - MaxAiAttempts); + MaxAiAttempts, + lastResult.FailureCategory); } } } _logger.LogWarning( - "AI {OperationName} exhausted retries with outcome {Outcome}; returning last result", + "AI {OperationName} exhausted retries with outcome {Outcome} and failure category {FailureCategory}; HTTP status {HttpStatusCode}; model {Model}; returning last result", operationName, - lastResult.Outcome); + lastResult.Outcome, + lastResult.FailureCategory, + lastResult.Response.HttpStatusCode, + lastResult.Response.Model); return lastResult; } + private static string ResolveNarrativeContent(AIOperationResult result) + { + return result.Outcome switch + { + AIOperationOutcome.Success => result.Content, + AIOperationOutcome.PermanentFailure => "AI service not available - service not configured.", + AIOperationOutcome.TransientFailure => "AI request failed - service temporarily unavailable.", + _ => "AI request failed - please try again later." + }; + } + private static bool TryParseJsonObjectFromResponse(string response, out JsonElement objectElement) { objectElement = default; diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/GrantApplications/ApplicationAnalysisAppService.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/GrantApplications/ApplicationAnalysisAppService.cs index 61e4f928b9..00403d3422 100644 --- a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/GrantApplications/ApplicationAnalysisAppService.cs +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/GrantApplications/ApplicationAnalysisAppService.cs @@ -46,7 +46,10 @@ await featureGuard.EnsureEnabledAsync( public virtual async Task GenerateApplicationAnalysisForPipelineAsync(Guid applicationId, string? promptVersion = null) { var input = await BuildInputAsync(applicationId, promptVersion); - await applicationAnalysisService.RegenerateAndSaveAsync(input); + var analysisJson = await applicationAnalysisService.RegenerateAsync(input); + var application = await applicationRepository.GetAsync(applicationId); + application.AIAnalysis = analysisJson; + await applicationRepository.UpdateAsync(application); return new ApplicationAnalysisResultDto { Completed = true }; } @@ -58,7 +61,7 @@ public virtual async Task GenerateApplicationAnaly var formSchema = await GetFormSchemaAsync(formSubmission?.ApplicationFormVersionId); var attachmentSummaries = PromptDataPayloadBuilder.BuildAttachmentSummaries(attachments); - var formFieldConfiguration = PromptDataPayloadBuilder.BuildFormFieldConfigurationAsync( + var formFieldConfiguration = PromptDataPayloadBuilder.BuildFormFieldConfiguration( formSchema, logger); diff --git a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/GrantApplications/ApplicationScoringAppService.cs b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/GrantApplications/ApplicationScoringAppService.cs index 2597a7c466..86da459c44 100644 --- a/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/GrantApplications/ApplicationScoringAppService.cs +++ b/applications/Unity.GrantManager/modules/Unity.AI/src/Unity.AI.Application/GrantApplications/ApplicationScoringAppService.cs @@ -44,7 +44,10 @@ await featureGuard.EnsureEnabledAsync( AILocalizationKeys.ScoringDisabled); var input = await BuildInputAsync(applicationId, promptVersion); - await applicationScoringService.RegenerateAndSaveAsync(input); + var scoresheetAnswers = await applicationScoringService.RegenerateAsync(input); + var application = await applicationRepository.GetAsync(applicationId); + application.AIScoresheetAnswers = scoresheetAnswers; + await applicationRepository.UpdateAsync(application); if (UnitOfWorkManager.Current != null) { @@ -77,7 +80,10 @@ await localEventBus.PublishAsync(new ApplicationAIScoringGeneratedEvent public virtual async Task GenerateApplicationScoringForPipelineAsync(Guid applicationId, string? promptVersion = null) { var input = await BuildInputAsync(applicationId, promptVersion); - await applicationScoringService.RegenerateAndSaveAsync(input); + var scoresheetAnswers = await applicationScoringService.RegenerateAsync(input); + var application = await applicationRepository.GetAsync(applicationId); + application.AIScoresheetAnswers = scoresheetAnswers; + await applicationRepository.UpdateAsync(application); return new ApplicationScoringResultDto { Completed = true }; } diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/Automation/BackgroundJobs/GenerateApplicationAnalysisJob.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/Automation/BackgroundJobs/GenerateApplicationAnalysisJob.cs index 76a40f7e72..26e6c54e7b 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/Automation/BackgroundJobs/GenerateApplicationAnalysisJob.cs +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Application/GrantApplications/Automation/BackgroundJobs/GenerateApplicationAnalysisJob.cs @@ -36,11 +36,8 @@ public override async Task ExecuteAsync(GenerateApplicationAnalysisBackgroundJob try { logger.LogInformation("Executing AI application analysis job for application {ApplicationId}.", args.ApplicationId); - var result = await applicationAnalysisAppService.GenerateApplicationAnalysisForPipelineAsync(args.ApplicationId, args.PromptVersion); - if (result.Completed) - { - logger.LogInformation("Completed AI application analysis job for application {ApplicationId}.", args.ApplicationId); - } + await applicationAnalysisAppService.GenerateApplicationAnalysisForPipelineAsync(args.ApplicationId, args.PromptVersion); + logger.LogInformation("Completed AI application analysis job for application {ApplicationId}.", args.ApplicationId); await AIGenerationRequestJobHelper.StampRateLimitBestEffortAsync(aiRateLimiter, logger, args.RequestedByUserId, args.ApplicationId, args.RequestKey); await AIGenerationRequestJobHelper.MarkCompletedInNewUowAsync(unitOfWorkManager, generationRequestRepository, args.RequestKey); diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Utilities/HtmlHelperExtensions.cs b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Utilities/HtmlHelperExtensions.cs new file mode 100644 index 0000000000..1f104ce2fb --- /dev/null +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Utilities/HtmlHelperExtensions.cs @@ -0,0 +1,16 @@ +using Ganss.Xss; +using Microsoft.AspNetCore.Html; +using Microsoft.AspNetCore.Mvc.Rendering; + +namespace Unity.GrantManager.Web.Utilities; + +public static class HtmlHelperExtensions +{ + private static readonly HtmlSanitizer _sanitizer = new(); + + public static IHtmlContent SanitizeRaw(this IHtmlHelper _, string? value) + { + if (string.IsNullOrEmpty(value)) return HtmlString.Empty; + return new HtmlString(_sanitizer.Sanitize(value)); + } +} diff --git a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Views/Shared/Components/AssessmentScoresWidget/Default.cshtml b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Views/Shared/Components/AssessmentScoresWidget/Default.cshtml index 9af7e0bc89..532ca7ebe5 100644 --- a/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Views/Shared/Components/AssessmentScoresWidget/Default.cshtml +++ b/applications/Unity.GrantManager/src/Unity.GrantManager.Web/Views/Shared/Components/AssessmentScoresWidget/Default.cshtml @@ -1,4 +1,5 @@ -@using Unity.Flex.Web.Views.Shared.Components.QuestionNumberWidget +@using Unity.GrantManager.Web.Utilities +@using Unity.Flex.Web.Views.Shared.Components.QuestionNumberWidget @using Unity.Flex.Web.Views.Shared.Components.QuestionTextAreaWidget @using Unity.Flex.Web.Views.Shared.Components.QuestionTextWidget @using Unity.Flex.Web.Views.Shared.Components.QuestionYesNoWidget @@ -76,7 +77,7 @@

-

@question.Description

+

@Html.SanitizeRaw(question.Description)

@switch (question.Type) { diff --git a/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Operations/ApplicationAnalysisServiceTests.cs b/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Operations/ApplicationAnalysisServiceTests.cs index 579cdf7b16..c13a000025 100644 --- a/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Operations/ApplicationAnalysisServiceTests.cs +++ b/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Operations/ApplicationAnalysisServiceTests.cs @@ -9,8 +9,6 @@ using Unity.AI.Operations; using Unity.AI.Requests; using Unity.AI.Responses; -using Unity.GrantManager.Applications; -using Volo.Abp.Domain.Entities; using Xunit; using Xunit.Abstractions; @@ -23,27 +21,21 @@ public ApplicationAnalysisServiceTests(ITestOutputHelper outputHelper) : base(ou } [Fact] - public async Task RegenerateAndSaveAsync_Uses_Input_Dto_And_Persists_Analysis() + public async Task RegenerateAsync_Uses_Input_Dto_And_Returns_Serialized_Response() { var applicationId = Guid.NewGuid(); - var application = WithId(new Application(), applicationId); ApplicationAnalysisRequest? capturedRequest = null; - var applicationRepository = Substitute.For(); - applicationRepository.GetAsync(applicationId).Returns(application); - var aiService = Substitute.For(); aiService.GenerateApplicationAnalysisAsync(Arg.Do(request => capturedRequest = request)) .Returns(new ApplicationAnalysisResponse { Decision = "ok" }); - var prerequisiteValidator = Substitute.For(); var service = new ApplicationAnalysisService( - applicationRepository, aiService, prerequisiteValidator); - var result = await service.RegenerateAndSaveAsync(new ApplicationAnalysisOperationInputDto + var result = await service.RegenerateAsync(new ApplicationAnalysisOperationInputDto { ApplicationId = applicationId, Schema = JsonSerializer.SerializeToElement(new { projectName = "Project Name" }), @@ -69,31 +61,25 @@ public async Task RegenerateAndSaveAsync_Uses_Input_Dto_And_Persists_Analysis() capturedRequest.Schema.GetProperty("projectName").GetString().ShouldBe("Project Name"); await prerequisiteValidator.Received(1).EnsureApplicationAnalysisAvailableAsync(applicationId); - await applicationRepository.Received(1).UpdateAsync(application); + await aiService.Received(1).GenerateApplicationAnalysisAsync(Arg.Any(), Arg.Any()); } [Fact] - public async Task RegenerateAndSaveAsync_Flows_Without_Repository_Loading_Inputs() + public async Task RegenerateAsync_Flows_Api_Request_To_The_Runtime_Without_Repository_Dependencies() { var applicationId = Guid.NewGuid(); - var application = WithId(new Application(), applicationId); ApplicationAnalysisRequest? capturedRequest = null; - var applicationRepository = Substitute.For(); - applicationRepository.GetAsync(applicationId).Returns(application); - var aiService = Substitute.For(); aiService.GenerateApplicationAnalysisAsync(Arg.Do(request => capturedRequest = request)) .Returns(new ApplicationAnalysisResponse()); - var prerequisiteValidator = Substitute.For(); var service = new ApplicationAnalysisService( - applicationRepository, aiService, prerequisiteValidator); - await service.RegenerateAndSaveAsync(new ApplicationAnalysisOperationInputDto + await service.RegenerateAsync(new ApplicationAnalysisOperationInputDto { ApplicationId = applicationId, Schema = JsonSerializer.SerializeToElement(new { }), @@ -107,12 +93,4 @@ await service.RegenerateAndSaveAsync(new ApplicationAnalysisOperationInputDto capturedRequest.Attachments.ShouldBeEmpty(); capturedRequest.PromptVersion.ShouldBeNull(); } - - private static T WithId(T entity, Guid id) where T : Entity - { - typeof(Entity) - .GetProperty(nameof(Entity.Id))! - .SetValue(entity, id); - return entity; - } } diff --git a/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Runtime/AIOperationResultTests.cs b/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Runtime/AIOperationResultTests.cs new file mode 100644 index 0000000000..5383ed9f58 --- /dev/null +++ b/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Runtime/AIOperationResultTests.cs @@ -0,0 +1,27 @@ +using Shouldly; +using Unity.AI.Runtime; +using Xunit; + +namespace Unity.GrantManager.AI.Runtime; + +public class AIOperationResultTests +{ + [Fact] + public void Failure_Factories_Should_Set_Consistent_Failure_Categories() + { + AIOperationResult.Success().FailureCategory.ShouldBe(AIFailureCategory.None); + AIOperationResult.ProviderUnavailable().FailureCategory.ShouldBe(AIFailureCategory.ProviderUnavailable); + AIOperationResult.TransientFailure().FailureCategory.ShouldBe(AIFailureCategory.TransientProviderFailure); + AIOperationResult.PermanentFailure().FailureCategory.ShouldBe(AIFailureCategory.PermanentProviderFailure); + AIOperationResult.InvalidOutput().FailureCategory.ShouldBe(AIFailureCategory.InvalidOutput); + } + + [Fact] + public void WithOutcome_Should_Resolve_Category_When_Not_Explicit() + { + var result = AIOperationResult.Success().WithOutcome(AIOperationOutcome.InvalidOutput); + + result.Outcome.ShouldBe(AIOperationOutcome.InvalidOutput); + result.FailureCategory.ShouldBe(AIFailureCategory.InvalidOutput); + } +} diff --git a/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Runtime/AIProviderPayloadValidatorTests.cs b/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Runtime/AIProviderPayloadValidatorTests.cs index d84d2e55a8..bbaf3981a5 100644 --- a/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Runtime/AIProviderPayloadValidatorTests.cs +++ b/applications/Unity.GrantManager/test/Unity.GrantManager.Application.Tests/AI/Runtime/AIProviderPayloadValidatorTests.cs @@ -1,4 +1,5 @@ using Shouldly; +using System.Text.Json; using Unity.AI.Runtime; using Xunit; @@ -6,118 +7,56 @@ namespace Unity.GrantManager.AI.Runtime; public class AIProviderPayloadValidatorTests { - [Theory] - [InlineData(null, false)] - [InlineData("", false)] - [InlineData(" ", false)] - [InlineData("Some summary text", true)] - public void IsValidAttachmentSummaryText_Should_RejectBlankAndAcceptContent(string? input, bool expected) + [Fact] + public void ValidateApplicationAnalysisJson_Should_Return_InvalidOutput_For_InvalidJson() { - AIProviderPayloadValidator.IsValidAttachmentSummaryText(input!).ShouldBe(expected); + var result = AIProviderPayloadValidator.ValidateApplicationAnalysisJson("not-json"); + + result.IsValid.ShouldBeFalse(); + result.FailureCategory.ShouldBe(AIFailureCategory.InvalidOutput); + result.Reason.ShouldContain("not valid JSON"); } [Fact] - public void IsValidApplicationAnalysisJson_Should_ReturnTrue_ForWellFormedPayload() + public void ValidateApplicationAnalysisJson_Should_Return_InvalidOutput_When_Decision_Is_Missing() { - var json = """ + var result = AIProviderPayloadValidator.ValidateApplicationAnalysisJson( + """ { - "decision": "Approved", "errors": [], "warnings": [], "summaries": [], "recommendations": [] } - """; - AIProviderPayloadValidator.IsValidApplicationAnalysisJson(json).ShouldBeTrue(); - } + """); - [Theory] - [InlineData(null)] - [InlineData("")] - [InlineData("not json")] - [InlineData("[]")] - public void IsValidApplicationAnalysisJson_Should_ReturnFalse_ForInvalidInput(string? input) - { - AIProviderPayloadValidator.IsValidApplicationAnalysisJson(input!).ShouldBeFalse(); + result.IsValid.ShouldBeFalse(); + result.FailureCategory.ShouldBe(AIFailureCategory.InvalidOutput); + result.Reason.ShouldContain("decision"); } [Fact] - public void IsValidApplicationAnalysisJson_Should_ReturnFalse_WhenDecisionMissing() + public void ValidateApplicationScoringJson_Should_Return_InvalidOutput_When_Answer_Is_Missing() { - var json = """{"errors":[],"warnings":[],"summaries":[],"recommendations":[]}"""; - AIProviderPayloadValidator.IsValidApplicationAnalysisJson(json).ShouldBeFalse(); - } - - [Fact] - public void IsValidApplicationAnalysisJson_Should_ReturnFalse_WhenErrorsIsNotArray() - { - var json = """{"decision":"ok","errors":"bad","warnings":[],"summaries":[],"recommendations":[]}"""; - AIProviderPayloadValidator.IsValidApplicationAnalysisJson(json).ShouldBeFalse(); - } + var sectionJson = JsonSerializer.Serialize(new[] + { + new { id = "q1" } + }); - [Fact] - public void IsValidApplicationAnalysisJson_Should_AcceptMarkdownWrappedJson() - { - var json = "```json\n{\"decision\":\"ok\",\"errors\":[],\"warnings\":[],\"summaries\":[],\"recommendations\":[]}\n```"; - AIProviderPayloadValidator.IsValidApplicationAnalysisJson(json).ShouldBeTrue(); - } + var result = AIProviderPayloadValidator.ValidateApplicationScoringJson("{}", sectionJson); - [Fact] - public void IsValidApplicationScoringJson_Should_ReturnTrue_ForWellFormedPayload() - { - var sectionJson = """[{"id":"q1"},{"id":"q2"}]"""; - var response = """ - { - "q1": {"answer": "Yes", "confidence": 85}, - "q2": {"answer": "No", "confidence": 42} - } - """; - AIProviderPayloadValidator.IsValidApplicationScoringJson(response, sectionJson).ShouldBeTrue(); + result.IsValid.ShouldBeFalse(); + result.FailureCategory.ShouldBe(AIFailureCategory.InvalidOutput); + result.Reason.ShouldContain("q1"); } [Fact] - public void IsValidApplicationScoringJson_Should_ReturnFalse_WhenSectionJsonIsEmpty() + public void ValidateAttachmentSummaryText_Should_Return_InvalidOutput_For_Empty_Text() { - AIProviderPayloadValidator.IsValidApplicationScoringJson("{}", "[]").ShouldBeFalse(); - } + var result = AIProviderPayloadValidator.ValidateAttachmentSummaryText(string.Empty); - [Fact] - public void IsValidApplicationScoringJson_Should_ReturnFalse_WhenAnswerMissing() - { - var sectionJson = """[{"id":"q1"}]"""; - var response = """{"q1": {"confidence": 50}}"""; - AIProviderPayloadValidator.IsValidApplicationScoringJson(response, sectionJson).ShouldBeFalse(); - } - - [Fact] - public void IsValidApplicationScoringJson_Should_ReturnFalse_WhenConfidenceOutOfRange() - { - var sectionJson = """[{"id":"q1"}]"""; - var response = """{"q1": {"answer": "Yes", "confidence": 150}}"""; - AIProviderPayloadValidator.IsValidApplicationScoringJson(response, sectionJson).ShouldBeFalse(); - } - - [Fact] - public void IsValidApplicationScoringJson_Should_ReturnFalse_WhenQuestionMissingFromResponse() - { - var sectionJson = """[{"id":"q1"},{"id":"q2"}]"""; - var response = """{"q1": {"answer": "Yes", "confidence": 80}}"""; - AIProviderPayloadValidator.IsValidApplicationScoringJson(response, sectionJson).ShouldBeFalse(); - } - - [Fact] - public void IsValidApplicationScoringJson_Should_AcceptQuestionsWrappedInObject() - { - var sectionJson = """{"questions":[{"id":"q1"}]}"""; - var response = """{"q1": {"answer": "Yes", "confidence": 75}}"""; - AIProviderPayloadValidator.IsValidApplicationScoringJson(response, sectionJson).ShouldBeTrue(); - } - - [Fact] - public void IsValidApplicationScoringJson_Should_ReturnFalse_WhenConfidenceIsNegative() - { - var sectionJson = """[{"id":"q1"}]"""; - var response = """{"q1": {"answer": "Yes", "confidence": -1}}"""; - AIProviderPayloadValidator.IsValidApplicationScoringJson(response, sectionJson).ShouldBeFalse(); + result.IsValid.ShouldBeFalse(); + result.FailureCategory.ShouldBe(AIFailureCategory.InvalidOutput); + result.Reason.ShouldContain("empty"); } }