InputAudioTranscribe

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -16,7 +16,7 @@ Task Connect(RealtimeHubConnection conn,
         Action<List<RoleDialogModel>> onModelResponseDone,
         Action<string> onConversationItemCreated,
         Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
-        Action onUserInterrupted);
+        Action onInterruptionDetected);
     Task AppenAudioBuffer(string message);
     Task AppenAudioBuffer(ArraySegment<byte> data, int length);
 

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs

@@ -1,5 +1,3 @@
-using System.Collections.Concurrent;
-
 namespace BotSharp.Abstraction.Realtime.Models;
 
 public class RealtimeHubConnection

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs

@@ -7,9 +7,11 @@ public class RealtimeModelSettings
     public bool InterruptResponse { get; set; } = true;
     public string InputAudioFormat { get; set; } = "g711_ulaw";
     public string OutputAudioFormat { get; set; } = "g711_ulaw";
+    public bool InputAudioTranscribe { get; set; } = false;
     public string Voice { get; set; } = "alloy";
     public float Temperature { get; set; } = 0.8f;
     public int MaxResponseOutputTokens { get; set; } = 512;
+    public int ModelResponseTimeout { get; set; } = 30;
     public AudioTranscription InputAudioTranscription { get; set; } = new();
     public ModelTurnDetection TurnDetection { get; set; } = new();
 }

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -139,7 +139,7 @@ await _completer.Connect(_conn,
                     await hook.OnMessageReceived(message);
                 }
             },
-            onUserInterrupted: async () =>
+            onInterruptionDetected: async () =>
             {
                 if (settings.InterruptResponse)
                 {

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -37,7 +37,7 @@ public async Task Connect(RealtimeHubConnection conn,
         Action<List<RoleDialogModel>> onModelResponseDone,
         Action<string> onConversationItemCreated,
         Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
-        Action onUserInterrupted)
+        Action onInterruptionDetected)
     {
         var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
         _model = realtimeModelSettings.Model;
@@ -62,7 +62,7 @@ public async Task Connect(RealtimeHubConnection conn,
                 onModelResponseDone,
                 onConversationItemCreated,
                 onInputAudioTranscriptionCompleted,
-                onUserInterrupted);
+                onInterruptionDetected);
         }
     }
 
@@ -139,11 +139,12 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
         Action<List<RoleDialogModel>> onModelResponseDone,
         Action<string> onConversationItemCreated,
         Action<RoleDialogModel> onUserAudioTranscriptionCompleted,
-        Action onUserInterrupted)
+        Action onInterruptionDetected)
     {
         var buffer = new byte[1024 * 32];
         // Model response timeout
-        var timeout = 30;
+        var settings = _services.GetRequiredService<RealtimeModelSettings>();
+        var timeout = settings.ModelResponseTimeout;
         WebSocketReceiveResult? result = default;
 
         do
@@ -241,7 +242,7 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             else if (response.Type == "input_audio_buffer.speech_started")
             {
                 // Handle user interuption
-                onUserInterrupted();
+                onInterruptionDetected();
             }
 
         } while (!result.CloseStatus.HasValue);
@@ -290,9 +291,6 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn)
             return fn;
         }).ToArray();
 
-        var words = new List<string>();
-        HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
-
         var realtimeModelSettings = _services.GetRequiredService<RealtimeModelSettings>();
 
         var sessionUpdate = new
@@ -302,12 +300,6 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn)
             {
                 InputAudioFormat = realtimeModelSettings.InputAudioFormat,
                 OutputAudioFormat = realtimeModelSettings.OutputAudioFormat,
-                /*InputAudioTranscription = new InputAudioTranscription
-                {
-                    Model = realtimeModelSettings.InputAudioTranscription.Model,
-                    Language = realtimeModelSettings.InputAudioTranscription.Language,
-                    Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
-                },*/
                 Voice = realtimeModelSettings.Voice,
                 Instructions = instruction,
                 ToolChoice = "auto",
@@ -329,6 +321,19 @@ public async Task<string> UpdateSession(RealtimeHubConnection conn)
             }
         };
 
+        if (realtimeModelSettings.InputAudioTranscribe)
+        {
+            var words = new List<string>();
+            HookEmitter.Emit<IRealtimeHook>(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)));
+
+            sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription
+            {
+                Model = realtimeModelSettings.InputAudioTranscription.Model,
+                Language = realtimeModelSettings.InputAudioTranscription.Language,
+                Prompt = string.Join(", ", words.Select(x => x.ToLower().Trim()).Distinct()).SubstringMax(1024)
+            };
+        }
+
         await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
         {
             await hook.OnSessionUpdated(agent, instruction, functions);

src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs

@@ -357,6 +357,14 @@ await HookEmitter.Emit<ITwilioCallStatusHook>(_services, async hook =>
         {
             await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallNoAnswerStatus(request));
         }
+        else if (request.CallStatus == "canceled")
+        {
+            await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallCanceledStatus(request));
+        }
+        else if (request.CallStatus == "failed")
+        {
+            await HookEmitter.Emit<ITwilioCallStatusHook>(_services, x => x.OnCallFailedStatus(request));
+        }
 
         return Ok();
     }

src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs

@@ -20,4 +20,8 @@ public interface ITwilioCallStatusHook
     Task OnCallBusyStatus(ConversationalVoiceRequest request);
 
     Task OnCallNoAnswerStatus(ConversationalVoiceRequest request);
+
+    Task OnCallCanceledStatus(ConversationalVoiceRequest request);
+
+    Task OnCallFailedStatus(ConversationalVoiceRequest request);
 }