Mak initial audio is not interruptable

src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs

@@ -21,28 +21,22 @@ public RealtimeHub(IServiceProvider services, ILogger<RealtimeHub> logger)
     public async Task Listen(WebSocket userWebSocket, 
         Func<string, RealtimeHubConnection> onUserMessageReceived)
     {
-        var buffer = new byte[1024 * 4];
+        var buffer = new byte[1024 * 16];
         WebSocketReceiveResult result;
 
-        var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
-        var model = llmProviderService.GetProviderModel("openai", "gpt-4",
-            realTime: true).Name;
-
         var completer = _services.GetServices<IRealTimeCompletion>().First(x => x.Provider == "openai");
-        completer.SetModelName(model);
 
         do
         {
             result = await userWebSocket.ReceiveAsync(new ArraySegment<byte>(buffer), CancellationToken.None);
             string receivedText = Encoding.UTF8.GetString(buffer, 0, result.Count);
-            _logger.LogDebug($"Received from user: {receivedText}");
+
             if (string.IsNullOrEmpty(receivedText))
             {
                 continue;
             }
 
             var conn = onUserMessageReceived(receivedText);
-            conn.Model = model;
 
             if (conn.Event == "user_connected")
             {
@@ -74,6 +68,17 @@ private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userW
         var agent = await agentService.LoadAgent(conversation.AgentId);
         conn.CurrentAgentId = agent.Id;
 
+        // Set model
+        var model = agent.LlmConfig.Model;
+        if (!model.Contains("-realtime-"))
+        {
+            var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
+            model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name;
+        }
+
+        completer.SetModelName(model);
+        conn.Model = model;
+
         var routing = _services.GetRequiredService<IRoutingService>();
         routing.Context.Push(agent.Id);
 
@@ -98,7 +103,7 @@ await completer.Connect(conn,
 
                 if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
                 {
-                    await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
+                    // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
                 }
                 else
                 {

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -128,9 +128,10 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
         Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
         Action onUserInterrupted)
     {
-        var buffer = new byte[1024 * 1024 * 1];
+        var buffer = new byte[1024 * 256];
         WebSocketReceiveResult result;
-        string lastAssistantItem = "";
+        string? lastAssistantItem = null;
+
         do
         {
             result = await _webSocket.ReceiveAsync(
@@ -166,15 +167,17 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
                 var data = JsonSerializer.Deserialize<ResponseAudioTranscript>(receivedText);
+                await Task.Delay(1000);
                 onAudioTranscriptDone(data.Transcript);
             }
             else if (response.Type == "response.audio.delta")
             {
                 var audio = JsonSerializer.Deserialize<ResponseAudioDelta>(receivedText);
-                lastAssistantItem = audio?.ItemId ?? "";
+                lastAssistantItem = audio?.ItemId;
 
                 if (audio != null && audio.Delta != null)
                 {
+                    _logger.LogDebug($"{response.Type}: {receivedText}");
                     onModelAudioDeltaReceived(audio.Delta);
                 }
             }
@@ -204,16 +207,19 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             {
                 // var elapsedTime = latestMediaTimestamp - responseStartTimestampTwilio;
                 // handle use interuption
-                var truncateEvent = new
+                if (!string.IsNullOrEmpty(lastAssistantItem))
                 {
-                    type = "conversation.item.truncate",
-                    item_id = lastAssistantItem,
-                    content_index = 0,
-                    audio_end_ms = 100
-                };
-
-                await SendEventToModel(truncateEvent);
-                onUserInterrupted();
+                    var truncateEvent = new
+                    {
+                        type = "conversation.item.truncate",
+                        item_id = lastAssistantItem,
+                        content_index = 0,
+                        audio_end_ms = 300
+                    };
+
+                    await SendEventToModel(truncateEvent);
+                    onUserInterrupted();
+                }
             }
 
         } while (!result.CloseStatus.HasValue);
@@ -612,7 +618,9 @@ public async Task<List<RoleDialogModel>> OnResponsedDone(RealtimeHubConnection c
 
                 outputs.Add(new RoleDialogModel(output.Role, content.Transcript)
                 {
-                    CurrentAgentId = conn.CurrentAgentId
+                    CurrentAgentId = conn.CurrentAgentId,
+                    MessageId = output.Id,
+                    MessageType = MessageTypeName.Plain
                 });
             }
         }

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -189,7 +189,14 @@ public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversa
         {
             foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
             {
-                response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
+                if (speechPath.StartsWith("twilio/"))
+                {
+                    response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
+                }
+                else
+                {
+                    response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"));
+                }
             }
         }
         var connect = new Connect();