Twilio: speak with TwiML Say, add ConversationRelay, bound OpenAI TTS timeout

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -154,6 +154,23 @@ public class ImageVariationSetting
 public class AudioSetting
 {
     public AudioTranscriptionSetting? Transcription { get; set; }
+    public AudioSynthesisSetting? Synthesis { get; set; }
+}
+
+/// <summary>
+/// Audio synthesis (text-to-speech) setting
+/// </summary>
+public class AudioSynthesisSetting
+{
+    /// <summary>
+    /// Timeout of a single request attempt, in seconds
+    /// </summary>
+    public int? TimeoutSeconds { get; set; }
+
+    /// <summary>
+    /// Number of retries after the first attempt fails or times out
+    /// </summary>
+    public int? MaxRetries { get; set; }
 }
 
 /// <summary>

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioSynthesisProvider.cs

@@ -21,9 +21,20 @@ public void SetModelName(string model)
         _model = model;
     }
 
+    // A short phrase normally comes back in a few seconds, but some requests never finish;
+    // with the SDK default of 100s per attempt, four stalled attempts block the caller for ~7 minutes.
+    private const int DefaultTimeoutSeconds = 30;
+    private const int DefaultMaxRetries = 3;
+
     public async Task<BinaryData> GenerateAudioAsync(string text, string? voice = "alloy", string? format = "mp3", string? instructions = null)
     {
-        var audioClient = ProviderHelper.GetClient(Provider, _model, apiKey: null, _services)
+        var settingsService = _services.GetRequiredService<ILlmProviderService>();
+        var synthesis = settingsService.GetSetting(Provider, _model)?.Audio?.Synthesis;
+        var timeout = TimeSpan.FromSeconds(synthesis?.TimeoutSeconds ?? DefaultTimeoutSeconds);
+        var maxRetries = synthesis?.MaxRetries ?? DefaultMaxRetries;
+
+        var audioClient = ProviderHelper.GetClient(Provider, _model, apiKey: null, _services,
+                                            networkTimeout: timeout, maxRetries: maxRetries)
                                         .GetAudioClient(_model);
 
         var (speechVoice, options) = PrepareGenerationOptions(voice: voice, format: format);

src/Plugins/BotSharp.Plugin.OpenAI/Providers/ProviderHelper.cs

@@ -1,16 +1,30 @@
 using OpenAI;
 using System.ClientModel;
+using System.ClientModel.Primitives;
 
 namespace BotSharp.Plugin.OpenAI.Providers;
 
 public class ProviderHelper
 {
-    public static OpenAIClient GetClient(string provider, string model, string? apiKey, IServiceProvider services)
+    public static OpenAIClient GetClient(string provider, string model, string? apiKey, IServiceProvider services,
+        TimeSpan? networkTimeout = null, int? maxRetries = null)
     {
         var settingsService = services.GetRequiredService<ILlmProviderService>();
         var settings = settingsService.GetSetting(provider, model);
-        var options = !string.IsNullOrEmpty(settings?.Endpoint) ?
-                        new OpenAIClientOptions { Endpoint = new Uri(settings.Endpoint) } : null;
+        var options = new OpenAIClientOptions();
+        if (!string.IsNullOrEmpty(settings?.Endpoint))
+        {
+            options.Endpoint = new Uri(settings.Endpoint);
+        }
+        // Left unset, the SDK defaults apply: 100s per attempt and 3 retries.
+        if (networkTimeout.HasValue)
+        {
+            options.NetworkTimeout = networkTimeout.Value;
+        }
+        if (maxRetries.HasValue)
+        {
+            options.RetryPolicy = new ClientRetryPolicy(maxRetries.Value);
+        }
         return new OpenAIClient(new ApiKeyCredential(apiKey ?? settings!.ApiKey), options);
     }
 

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioConversationRelayController.cs

@@ -0,0 +1,89 @@
+using BotSharp.Abstraction.Infrastructures;
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.Interfaces;
+using BotSharp.Plugin.Twilio.Models.Relay;
+using BotSharp.Plugin.Twilio.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace BotSharp.Plugin.Twilio.Controllers;
+
+public class TwilioConversationRelayController : TwilioController
+{
+    private readonly TwilioSetting _settings;
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+
+    public TwilioConversationRelayController(
+        TwilioSetting settings,
+        IServiceProvider services,
+        ILogger<TwilioConversationRelayController> logger)
+    {
+        _settings = settings;
+        _services = services;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Twilio calls this when a ConversationRelay session ends: we sent "end" (handoff), the caller hung up,
+    /// or the session failed.
+    /// </summary>
+    [ValidateRequest]
+    [HttpPost("twilio/relay/action")]
+    public async Task<TwiMLResult> SessionEnded(ConversationRelayActionRequest request)
+    {
+        _logger.LogInformation($"ConversationRelay session for conversation {request.ConversationId} ended: " +
+            $"{request.SessionStatus} {request.ErrorCode} {request.ErrorMessage}");
+
+        // Authenticate (synchronous, so identity writes survive into the async session hooks)
+        HookEmitter.Emit<ITwilioSessionHook>(_services, hook => hook.OnAuthenticate(request), request.AgentId);
+
+        var twilio = _services.GetRequiredService<TwilioService>();
+        var action = GetHandoffAction(request.HandoffData);
+
+        if (action == ConversationRelayHandoff.Transfer && !string.IsNullOrEmpty(_settings.CsrAgentNumber))
+        {
+            await HookEmitter.Emit<ITwilioSessionHook>(_services,
+                async hook => await hook.OnAgentTransferring(request, _settings), request.AgentId);
+            return TwiML(twilio.DialCsrAgent(string.Empty));
+        }
+
+        var callStillUp = !string.Equals(request.CallStatus, "completed", StringComparison.OrdinalIgnoreCase);
+        var canReconnect = request.Reconnects < _settings.ConversationRelay.MaxReconnects;
+        if (request.SessionStatus == "failed" && callStillUp && canReconnect)
+        {
+            _logger.LogWarning($"Reconnecting ConversationRelay for conversation {request.ConversationId} " +
+                $"(attempt {request.Reconnects + 1}).");
+            var relay = twilio.ReturnConversationRelay(request.AgentId, request.ConversationId,
+                welcomeGreeting: null, request.Reconnects + 1);
+            return TwiML(relay);
+        }
+
+        if (action == ConversationRelayHandoff.HangUp)
+        {
+            await HookEmitter.Emit<ITwilioSessionHook>(_services,
+                async hook => await hook.OnAgentHangUp(request), request.AgentId);
+        }
+
+        var response = new VoiceResponse();
+        response.Hangup();
+        return TwiML(response);
+    }
+
+    private static string? GetHandoffAction(string? handoffData)
+    {
+        if (string.IsNullOrWhiteSpace(handoffData))
+        {
+            return null;
+        }
+
+        try
+        {
+            using var doc = JsonDocument.Parse(handoffData);
+            return doc.RootElement.TryGetProperty("action", out var action) ? action.GetString() : null;
+        }
+        catch (JsonException)
+        {
+            return handoffData;
+        }
+    }
+}

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs

@@ -84,8 +84,8 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
             await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
                 async hook => await hook.OnVoicemailStarting(request), request.AgentId);
 
-            var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
-            response.Play(new Uri(url));
+            var voicemail = twilio.GetSavedSpeechFile(request.ConversationId, "voicemail");
+            twilio.AppendSpeech(response, request.ConversationId, voicemail);
         }
         else
         {

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs

@@ -38,8 +38,20 @@ public async Task<TwiMLResult> InitiateOutboundCall(ConversationalVoiceRequest r
             await HookEmitter.Emit<ITwilioCallStatusHook>(_services,
                 async hook => await hook.OnVoicemailStarting(request), request.AgentId);
 
-            var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3");
-            response.Play(new Uri(url));
+            var voicemail = twilio.GetSavedSpeechFile(request.ConversationId, "voicemail");
+            twilio.AppendSpeech(response, request.ConversationId, voicemail);
+        }
+        else if (_settings.ConversationRelay.IsEnabledFor(request.AgentId))
+        {
+            // OutboundPhoneCallFn stored the initial message as reply 0
+            var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
+            var initial = await sessionManager.GetAssistantReplyAsync(request.ConversationId, 0);
+            if (string.IsNullOrWhiteSpace(initial?.Content))
+            {
+                _logger.LogWarning($"No initial message for conversation {request.ConversationId}; " +
+                    "starting ConversationRelay without a greeting.");
+            }
+            response = twilio.ReturnConversationRelay(request.AgentId, request.ConversationId, initial?.Content);
         }
         else
         {

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioReconnectController.cs

@@ -28,8 +28,7 @@ public async Task<TwiMLResult> Reconnect(ConversationalVoiceRequest request)
         if (!string.IsNullOrEmpty(request.InitAudioFile))
         {
             var twilio = _services.GetRequiredService<TwilioService>();
-            var audioUrl = twilio.GetSpeechPath(request.ConversationId, request.InitAudioFile);
-            response.Play(new Uri(audioUrl));
+            twilio.AppendSpeech(response, request.ConversationId, request.InitAudioFile);
         }
         else
         {

src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs

@@ -0,0 +1,24 @@
+using Task = System.Threading.Tasks.Task;
+
+namespace BotSharp.Plugin.Twilio.Models;
+
+/// <summary>
+/// What a phone flow wants to hear about while the agent works on one caller message.
+/// </summary>
+public class AgentTurnCallbacks
+{
+    /// <summary>
+    /// A tool-call indication, e.g. "#hold-on-1" or a sentence to say while the tool runs
+    /// </summary>
+    public Func<string, Task>? OnIndication { get; init; }
+
+    /// <summary>
+    /// When set, the turn streams: called for each piece of reply text as the model writes it
+    /// </summary>
+    public Func<string, Task>? OnStreamText { get; init; }
+
+    /// <summary>
+    /// Called when the model starts writing a new response, e.g. after a tool call
+    /// </summary>
+    public Func<Task>? OnStreamStarted { get; init; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayActionRequest.cs

@@ -0,0 +1,33 @@
+using Microsoft.AspNetCore.Mvc;
+
+namespace BotSharp.Plugin.Twilio.Models.Relay;
+
+/// <summary>
+/// What Twilio posts to the Connect action URL when a ConversationRelay session ends
+/// </summary>
+public class ConversationRelayActionRequest : ConversationalVoiceRequest
+{
+    [FromForm]
+    public string? SessionId { get; set; }
+
+    /// <summary>
+    /// ended (we sent "end"), completed (caller hung up) or failed
+    /// </summary>
+    [FromForm]
+    public string? SessionStatus { get; set; }
+
+    [FromForm]
+    public string? SessionDuration { get; set; }
+
+    [FromForm]
+    public string? HandoffData { get; set; }
+
+    [FromForm]
+    public string? ErrorCode { get; set; }
+
+    [FromForm]
+    public string? ErrorMessage { get; set; }
+
+    [FromQuery(Name = "reconnects")]
+    public int Reconnects { get; set; }
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayHandoff.cs

@@ -0,0 +1,10 @@
+namespace BotSharp.Plugin.Twilio.Models.Relay;
+
+/// <summary>
+/// Values we put in the "end" message's handoffData, read back by the Connect action callback
+/// </summary>
+public static class ConversationRelayHandoff
+{
+    public const string HangUp = "hangup";
+    public const string Transfer = "transfer";
+}

src/Plugins/BotSharp.Plugin.Twilio/Models/Relay/ConversationRelayParameters.cs

@@ -0,0 +1,9 @@
+namespace BotSharp.Plugin.Twilio.Models.Relay;
+
+/// <summary>
+/// Custom TwiML Parameter names passed into the ConversationRelay "setup" message
+/// </summary>
+public static class ConversationRelayParameters
+{
+    public const string WelcomeGreetingLength = "welcome-greeting-length";
+}

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/HangupPhoneCallFn.cs

@@ -2,6 +2,7 @@
 using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+using BotSharp.Plugin.Twilio.Services;
 using Twilio.Rest.Api.V2010.Account;
 
 namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
@@ -28,7 +29,6 @@ public async Task<bool> Execute(RoleDialogModel message)
     {
         var args = JsonSerializer.Deserialize<HangupPhoneCallArgs>(message.FunctionArgs);
 
-        var fileStorage = _services.GetRequiredService<IFileStorageService>();
         var convService = _services.GetRequiredService<IConversationService>();
         var conversationId = convService.ConversationId;
         var states = _services.GetRequiredService<IConversationStateService>();
@@ -44,13 +44,10 @@ public async Task<bool> Execute(RoleDialogModel message)
         var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/hang-up?agent-id={message.CurrentAgentId}&conversation-id={conversationId}";
 
         // Generate initial assistant audio
-        string initAudioFile = null;
         if (!string.IsNullOrEmpty(args.ResponseContent) && _twilioSetting.GenerateEndingAudio)
         {
-            var completion = CompletionProvider.GetAudioSynthesizer(_services);
-            var data = await completion.GenerateAudioAsync(args.ResponseContent);
-            initAudioFile = "ending.mp3";
-            fileStorage.SaveSpeechFile(conversationId, initAudioFile, data);
+            var twilio = _services.GetRequiredService<TwilioService>();
+            var initAudioFile = await twilio.SaveSpeechAsync(conversationId, "ending", args.ResponseContent);
 
             processUrl += $"&init-audio-file={initAudioFile}";
         }

src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/LeaveVoicemailFn.cs

@@ -2,6 +2,7 @@
 using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+using BotSharp.Plugin.Twilio.Services;
 
 namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
 
@@ -28,7 +29,6 @@ public async Task<bool> Execute(RoleDialogModel message)
     {
         var args = JsonSerializer.Deserialize<LeaveVoicemailArgs>(message.FunctionArgs);
 
-        var fileStorage = _services.GetRequiredService<IFileStorageService>();
         var routing = _services.GetRequiredService<IRoutingService>();
         var conversationId = routing.Context.ConversationId;
         var states = _services.GetRequiredService<IConversationStateService>();
@@ -42,13 +42,10 @@ public async Task<bool> Execute(RoleDialogModel message)
         }
 
         // Generate voice message audio
-        string initAudioFile = null;
         if (!string.IsNullOrEmpty(args.VoicemailMessage))
         {
-            var completion = CompletionProvider.GetAudioSynthesizer(_services);
-            var data = await completion.GenerateAudioAsync(args.VoicemailMessage);
-            initAudioFile = "voicemail.mp3";
-            fileStorage.SaveSpeechFile(conversationId, initAudioFile, data);
+            var twilio = _services.GetRequiredService<TwilioService>();
+            await twilio.SaveSpeechAsync(conversationId, "voicemail", args.VoicemailMessage);
         }
 
         message.Content = args.VoicemailMessage;