Twilio: inbound calls on ConversationRelay and a response watchdog

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -154,6 +154,23 @@ public class ImageVariationSetting
 public class AudioSetting
 {
     public AudioTranscriptionSetting? Transcription { get; set; }
+    public AudioSynthesisSetting? Synthesis { get; set; }
+}
+
+/// <summary>
+/// Audio synthesis (text-to-speech) setting
+/// </summary>
+public class AudioSynthesisSetting
+{
+    /// <summary>
+    /// Timeout of a single request attempt, in seconds
+    /// </summary>
+    public int? TimeoutSeconds { get; set; }
+
+    /// <summary>
+    /// Number of retries after the first attempt fails or times out
+    /// </summary>
+    public int? MaxRetries { get; set; }
 }
 
 /// <summary>

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs

@@ -21,9 +21,20 @@ public void SetModelName(string model)
         _model = model;
     }
 
+    // A short phrase normally comes back in a few seconds, but some requests never finish;
+    // with the SDK default of 100s per attempt, four stalled attempts block the caller for ~7 minutes.
+    private const int DefaultTimeoutSeconds = 30;
+    private const int DefaultMaxRetries = 3;
+
     public async Task<BinaryData> GenerateAudioAsync(string text, string? voice = "alloy", string? format = "mp3", string? instructions = null)
     {
-        var audioClient = ProviderHelper.GetClient(Provider, _model, apiKey: null, _services)
+        var settingsService = _services.GetRequiredService<ILlmProviderService>();
+        var synthesis = settingsService.GetSetting(Provider, _model)?.Audio?.Synthesis;
+        var timeout = TimeSpan.FromSeconds(synthesis?.TimeoutSeconds ?? DefaultTimeoutSeconds);
+        var maxRetries = synthesis?.MaxRetries ?? DefaultMaxRetries;
+
+        var audioClient = ProviderHelper.GetClient(Provider, _model, apiKey: null, _services,
+                                            networkTimeout: timeout, maxRetries: maxRetries)
                                         .GetAudioClient(_model);
 
         var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format);

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.Chat.cs

@@ -56,7 +56,7 @@ private async Task<RoleDialogModel> InnerGetChatCompletions(Agent agent, List<Ro
         }
         else if (reason == ChatFinishReason.Length)
         {
-            _logger.LogWarning($"Action: {nameof(InnerGetChatCompletions)}, Reason: {reason}, Agent: {agent.Name}, MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{text}");
+            _logger.LogWarning($"Action: {nameof(InnerGetChatCompletions)}, Reason: {reason}, Agent: {agent.Name}, MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{text}, ToolCalls: {DescribeToolCalls(ToLlmToolCalls(value.ToolCalls))}");
 
             responseMessage = new RoleDialogModel(AgentRole.Assistant, $"AI response exceeded max output length")
             {
@@ -176,7 +176,7 @@ private async Task<bool> InnerGetChatCompletionsAsync(Agent agent,
         }
         else if (reason == ChatFinishReason.Length)
         {
-            _logger.LogWarning($"Action: {nameof(InnerGetChatCompletionsAsync)}, Reason: {reason}, Agent: {agent.Name}, MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{text}");
+            _logger.LogWarning($"Action: {nameof(InnerGetChatCompletionsAsync)}, Reason: {reason}, Agent: {agent.Name}, MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{text}, ToolCalls: {DescribeToolCalls(ToLlmToolCalls(value.ToolCalls))}");
 
             msg = new RoleDialogModel(AgentRole.Assistant, $"AI response exceeded max output length")
             {
@@ -321,6 +321,13 @@ private async Task<RoleDialogModel> InnerGetChatCompletionsStreamingAsync(Agent
                 }
                 else if (choice.FinishReason.HasValue)
                 {
+                    if (choice.FinishReason == ChatFinishReason.Length)
+                    {
+                        // What the model wrote before the limit is replaced by the notice below, so this is the only record of it
+                        _logger.LogWarning($"Action: {nameof(InnerGetChatCompletionsStreamingAsync)}, Reason: {choice.FinishReason}, Agent: {agent.Name}, " +
+                            $"MaxOutputTokens: {options.MaxOutputTokenCount}, Content:{textStream.GetText()}, ToolCalls: {DescribeToolCalls(ReconstructToolCalls(toolCalls))}");
+                    }
+
                     var text = choice.FinishReason == ChatFinishReason.Length ? "Model reached the maximum number of tokens allowed."
                         : choice.FinishReason == ChatFinishReason.ContentFilter ? "Content is omitted due to content filter rule."
                         : choice.FinishReason.Value.ToString();
@@ -759,6 +766,12 @@ private static List<LlmToolCall> ToLlmToolCalls(IEnumerable<ChatToolCall>? toolC
             .Select(x => new LlmToolCall(x.Id, x.FunctionName, x.FunctionArguments?.ToString()))
             .ToList();
 
+    /// <summary>
+    /// The tool calls of a reply cut off by the output limit, arguments as far as the model got
+    /// </summary>
+    private static string DescribeToolCalls(List<LlmToolCall> calls)
+        => string.Join(", ", calls.Select(x => $"{x.FunctionName}({x.FunctionArgs})"));
+
     /// <summary>
     /// Rebuilds the tool calls a streaming reply asked for.
     /// </summary>

src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs

@@ -1,16 +1,30 @@
 using OpenAI;
 using System.ClientModel;
+using System.ClientModel.Primitives;
 
 namespace BotSharp.Plugin.OpenAI.Providers;
 
 public class ProviderHelper
 {
-    public static OpenAIClient GetClient(string provider, string model, string? apiKey, IServiceProvider services)
+    public static OpenAIClient GetClient(string provider, string model, string? apiKey, IServiceProvider services,
+        TimeSpan? networkTimeout = null, int? maxRetries = null)
     {
         var settingsService = services.GetRequiredService<ILlmProviderService>();
         var settings = settingsService.GetSetting(provider, model);
-        var options = !string.IsNullOrEmpty(settings?.Endpoint) ?
-                        new OpenAIClientOptions { Endpoint = new Uri(settings.Endpoint) } : null;
+        var options = new OpenAIClientOptions();
+        if (!string.IsNullOrEmpty(settings?.Endpoint))
+        {
+            options.Endpoint = new Uri(settings.Endpoint);
+        }
+        // Left unset, the SDK defaults apply: 100s per attempt and 3 retries.
+        if (networkTimeout.HasValue)
+        {
+            options.NetworkTimeout = networkTimeout.Value;
+        }
+        if (maxRetries.HasValue)
+        {
+            options.RetryPolicy = new ClientRetryPolicy(maxRetries.Value);
+        }
         return new OpenAIClient(new ApiKeyCredential(apiKey ?? settings!.ApiKey), options);
     }
 

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs

@@ -0,0 +1,89 @@
+using BotSharp.Abstraction.Infrastructures;
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.Interfaces;
+using BotSharp.Plugin.Twilio.Models.Relay;
+using BotSharp.Plugin.Twilio.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace BotSharp.Plugin.Twilio.Controllers;
+
+public class TwilioConversationRelayController : TwilioController
+{
+    private readonly TwilioSetting _settings;
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+
+    public TwilioConversationRelayController(
+        TwilioSetting settings,
+        IServiceProvider services,
+        ILogger<TwilioConversationRelayController> logger)
+    {
+        _settings = settings;
+        _services = services;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Twilio calls this when a ConversationRelay session ends: we sent "end" (handoff), the caller hung up,
+    /// or the session failed.
+    /// </summary>
+    [ValidateRequest]
+    [HttpPost("twilio/relay/action")]
+    public async Task<TwiMLResult> SessionEnded(ConversationRelayActionRequest request)
+    {
+        _logger.LogInformation($"ConversationRelay session for conversation {request.ConversationId} ended: " +
+            $"{request.SessionStatus} {request.ErrorCode} {request.ErrorMessage}");
+
+        // Authenticate (synchronous, so identity writes survive into the async session hooks)
+        HookEmitter.Emit<ITwilioSessionHook>(_services, hook => hook.OnAuthenticate(request), request.AgentId);
+
+        var twilio = _services.GetRequiredService<TwilioService>();
+        var action = GetHandoffAction(request.HandoffData);
+
+        if (action == ConversationRelayHandoff.Transfer && !string.IsNullOrEmpty(_settings.CsrAgentNumber))
+        {
+            await HookEmitter.Emit<ITwilioSessionHook>(_services,
+                async hook => await hook.OnAgentTransferring(request, _settings), request.AgentId);
+            return TwiML(twilio.DialCsrAgent(string.Empty));
+        }
+
+        var callStillUp = !string.Equals(request.CallStatus, "completed", StringComparison.OrdinalIgnoreCase);
+        var canReconnect = request.Reconnects < _settings.ConversationRelay.MaxReconnects;
+        if (request.SessionStatus == "failed" && callStillUp && canReconnect)
+        {
+            _logger.LogWarning($"Reconnecting ConversationRelay for conversation {request.ConversationId} " +
+                $"(attempt {request.Reconnects + 1}).");
+            var relay = twilio.ReturnConversationRelay(request.AgentId, request.ConversationId,
+                welcomeGreeting: null, request.Reconnects + 1);
+            return TwiML(relay);
+        }
+
+        if (action == ConversationRelayHandoff.HangUp)
+        {
+            await HookEmitter.Emit<ITwilioSessionHook>(_services,
+                async hook => await hook.OnAgentHangUp(request), request.AgentId);
+        }
+
+        var response = new VoiceResponse();
+        response.Hangup();
+        return TwiML(response);
+    }
+
+    private static string? GetHandoffAction(string? handoffData)
+    {
+        if (string.IsNullOrWhiteSpace(handoffData))
+        {
+            return null;
+        }
+
+        try
+        {
+            using var doc = JsonDocument.Parse(handoffData);
+            return doc.RootElement.TryGetProperty("action", out var action) ? action.GetString() : null;
+        }
+        catch (JsonException)
+        {
+            return handoffData;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs

@@ -84,15 +84,32 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
             await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
                 async hook => await hook.OnVoicemailStarting(request), request.AgentId);
 
-            var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
-            response.Play(new Uri(url));
+            var voicemail = twilio.GetSavedSpeechFile(request.ConversationId, "voicemail");
+            twilio.AppendSpeech(response, request.ConversationId, voicemail);
         }
         else
         {
             if (agent.Labels.Contains("realtime"))
             {
                 response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction, agent);
             }
+            // A call that arrives with an intent already has its first message queued for the Gather flow
+            else if (string.IsNullOrWhiteSpace(request.Intent) && _settings.ConversationRelay.IsEnabledFor(request.AgentId))
+            {
+                // ConversationRelay speaks text only: a greeting a session hook wrote (e.g. with the caller's name) is
+                // said as it is, otherwise the prompts the session hooks queued become the greeting
+                var greeting = !string.IsNullOrWhiteSpace(instruction.Text)
+                    ? instruction.Text
+                    : string.Join(" ", instruction.SpeechPaths
+                        .Select(twilio.GetPromptText)
+                        .Where(text => !string.IsNullOrWhiteSpace(text)));
+                if (greeting.Length == 0 && instruction.SpeechPaths.Count > 0)
+                {
+                    _logger.LogWarning($"No prompt text for the welcome speech of agent {request.AgentId}; " +
+                        "starting ConversationRelay without a greeting.");
+                }
+                response = twilio.ReturnConversationRelay(request.AgentId, request.ConversationId, greeting);
+            }
             else
             {
                 if (string.IsNullOrWhiteSpace(request.Intent))

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs

@@ -38,8 +38,20 @@ public async Task<TwiMLResult> InitiateOutboundCall(ConversationalVoiceRequest r
             await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
                 async hook => await hook.OnVoicemailStarting(request), request.AgentId);
 
-            var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
-            response.Play(new Uri(url));
+            var voicemail = twilio.GetSavedSpeechFile(request.ConversationId, "voicemail");
+            twilio.AppendSpeech(response, request.ConversationId, voicemail);
+        }
+        else if (_settings.ConversationRelay.IsEnabledFor(request.AgentId))
+        {
+            // OutboundPhoneCallFn stored the initial message as reply 0
+            var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
+            var initial = await sessionManager.GetAssistantReplyAsync(request.ConversationId, 0);
+            if (string.IsNullOrWhiteSpace(initial?.Content))
+            {
+                _logger.LogWarning($"No initial message for conversation {request.ConversationId}; " +
+                    "starting ConversationRelay without a greeting.");
+            }
+            response = twilio.ReturnConversationRelay(request.AgentId, request.ConversationId, initial?.Content);
         }
         else
         {

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs

@@ -28,8 +28,7 @@ public async Task<TwiMLResult> Reconnect(ConversationalVoiceRequest request)
         if (!string.IsNullOrEmpty(request.InitAudioFile))
         {
             var twilio = _services.GetRequiredService<TwilioService>();
-            var audioUrl = twilio.GetSpeechPath(request.ConversationId, request.InitAudioFile);
-            response.Play(new Uri(audioUrl));
+            twilio.AppendSpeech(response, request.ConversationId, request.InitAudioFile);
         }
         else
         {

src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs

@@ -0,0 +1,30 @@
+using Task = System.Threading.Tasks.Task;
+
+namespace BotSharp.Plugin.Twilio.Models;
+
+/// <summary>
+/// What a phone flow wants to hear about while the agent works on one caller message.
+/// </summary>
+public class AgentTurnCallbacks
+{
+    /// <summary>
+    /// A tool-call indication, e.g. "#hold-on-1" or a sentence to say while the tool runs
+    /// </summary>
+    public Func<string, Task>? OnIndication { get; init; }
+
+    /// <summary>
+    /// When set, the turn streams: called for each piece of reply text as the model writes it
+    /// </summary>
+    public Func<string, Task>? OnStreamText { get; init; }
+
+    /// <summary>
+    /// Called when the model starts writing a new response, e.g. after a tool call
+    /// </summary>
+    public Func<Task>? OnStreamStarted { get; init; }
+
+    /// <summary>
+    /// Called when a response the model wrote ends in a tool call: what it said before the call is complete,
+    /// and the tool may take a while before anything else is said
+    /// </summary>
+    public Func<Task>? OnStreamEndedWithToolCall { get; init; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayActionRequest.cs

@@ -0,0 +1,33 @@
+using Microsoft.AspNetCore.Mvc;
+
+namespace BotSharp.Plugin.Twilio.Models.Relay;
+
+/// <summary>
+/// What Twilio posts to the Connect action URL when a ConversationRelay session ends
+/// </summary>
+public class ConversationRelayActionRequest : ConversationalVoiceRequest
+{
+    [FromForm]
+    public string? SessionId { get; set; }
+
+    /// <summary>
+    /// ended (we sent "end"), completed (caller hung up) or failed
+    /// </summary>
+    [FromForm]
+    public string? SessionStatus { get; set; }
+
+    [FromForm]
+    public string? SessionDuration { get; set; }
+
+    [FromForm]
+    public string? HandoffData { get; set; }
+
+    [FromForm]
+    public string? ErrorCode { get; set; }
+
+    [FromForm]
+    public string? ErrorMessage { get; set; }
+
+    [FromQuery(Name = "reconnects")]
+    public int Reconnects { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayHandoff.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Plugin.Twilio.Models.Relay;
+
+/// <summary>
+/// Values we put in the "end" message's handoffData, read back by the Connect action callback
+/// </summary>
+public static class ConversationRelayHandoff
+{
+    public const string HangUp = "hangup";
+    public const string Transfer = "transfer";
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayParameters.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Plugin.Twilio.Models.Relay;
+
+/// <summary>
+/// Custom TwiML Parameter names passed into the ConversationRelay "setup" message
+/// </summary>
+public static class ConversationRelayParameters
+{
+    public const string WelcomeGreetingLength = "welcome-greeting-length";
+}