Merge pull request #1019 from hchen2020/master

Optimize realtime route_to_agent

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs

@@ -12,7 +12,7 @@ public interface IAgentService
     Task<Agent> CreateAgent(Agent agent);
     Task<string> RefreshAgents();
     Task<PagedItems<Agent>> GetAgents(AgentFilter filter);
-    Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null);
+    Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null, bool byName = false);
 
     /// <summary>
     /// Load agent configurations and trigger hooks

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs

@@ -66,6 +66,12 @@ public class RoleDialogModel : ITrackableMessage
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? FunctionArgs { get; set; }
 
+    /// <summary>
+    /// Set this flag is in OnFunctionExecuting, if true, it won't be executed by InvokeFunction.
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    public bool Handled { get; set; } = false;
+
     /// <summary>
     /// Function execution structured data, this data won't pass to LLM.
     /// It's ideal to render in rich content in UI.

src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Utilities;
-using BotSharp.Core.Infrastructures;
 
 namespace BotSharp.Core.Realtime.Hooks;
 
@@ -40,33 +39,32 @@ public async Task OnFunctionExecuted(RoleDialogModel message)
 
         if (message.FunctionName == "route_to_agent")
         {
-            var inst = JsonSerializer.Deserialize<RoutingArgs>(message.FunctionArgs ?? "{}") ?? new();
-            message.Content = $"I'm your AI assistant '{inst.AgentName}' to help with: '{inst.NextActionReason}'";
             hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId();
 
-            var instruction = await hub.Completer.UpdateSession(hub.HubConn);
-            await hub.Completer.InsertConversationItem(message);
-            await hub.Completer.TriggerModelInference($"{instruction}\r\n\r\nAssist user task: {inst.NextActionReason}");
+            await hub.Completer.UpdateSession(hub.HubConn);
+            await hub.Completer.TriggerModelInference();
         }
         else if (message.FunctionName == "util-routing-fallback_to_router")
         {
-            var inst = JsonSerializer.Deserialize<FallbackArgs>(message.FunctionArgs ?? "{}") ?? new();
-            message.Content = $"Returned to Router due to {inst.Reason}";
             hub.HubConn.CurrentAgentId = routing.Context.GetCurrentAgentId();
 
-            var instruction = await hub.Completer.UpdateSession(hub.HubConn);
-            await hub.Completer.InsertConversationItem(message);
-            await hub.Completer.TriggerModelInference(instruction);
+            await hub.Completer.UpdateSession(hub.HubConn);
+            await hub.Completer.TriggerModelInference();
         }
         else
         {
-            // Clear cache to force to rebuild the agent instruction
-            Utilities.ClearCache();
-
             // Update session for changed states
             var instruction = await hub.Completer.UpdateSession(hub.HubConn);
             await hub.Completer.InsertConversationItem(message);
-            await hub.Completer.TriggerModelInference(instruction);
+
+            if (message.StopCompletion)
+            {
+                await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\"");
+            }
+            else
+            {
+                await hub.Completer.TriggerModelInference(instruction);
+            }
         }
     }
 }

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs

@@ -26,12 +26,18 @@ public async Task<PagedItems<Agent>> GetAgents(AgentFilter filter)
     }
 
     [SharpCache(10)]
-    public async Task<List<IdName>> GetAgentOptions(List<string>? agentIds)
+    public async Task<List<IdName>> GetAgentOptions(List<string>? agentIdsOrNames, bool byName = false)
     {
-        var agents = _db.GetAgents(new AgentFilter
-        {
-            AgentIds = !agentIds.IsNullOrEmpty() ? agentIds : null
-        });
+        var agents = byName ? 
+            _db.GetAgents(new AgentFilter
+            {
+                AgentNames = !agentIdsOrNames.IsNullOrEmpty() ? agentIdsOrNames : null
+            }) :
+            _db.GetAgents(new AgentFilter
+            {
+                AgentIds = !agentIdsOrNames.IsNullOrEmpty() ? agentIdsOrNames : null
+            });
+
         return agents?.Select(x => new IdName(x.Id, x.Name))?.OrderBy(x => x.Name)?.ToList() ?? [];
     }
 

src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs

@@ -86,10 +86,12 @@ public void Push(string agentId, string? reason = null, bool updateLazyRouting =
         if (!Guid.TryParse(agentId, out _))
         {
             var agentService = _services.GetRequiredService<IAgentService>();
-            agentId = agentService.GetAgents(new AgentFilter
+            var agents = agentService.GetAgentOptions([agentId], byName: true).Result;
+
+            if (agents.Count > 0)
             {
-                AgentNames = [agentId]
-            }).Result.Items.First().Id;
+                agentId = agents.First().Id;
+            }
         }
 
         if (_stack.Count == 0 || _stack.Peek() != agentId)

src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs

@@ -49,16 +49,23 @@ public async Task<bool> InvokeFunction(string name, RoleDialogModel message)
             await progressService.OnFunctionExecuting(clonedMessage);
         }
 
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var agent = await agentService.GetAgent(clonedMessage.CurrentAgentId);
         foreach (var hook in hooks)
         {
+            hook.SetAgent(agent);
             await hook.OnFunctionExecuting(clonedMessage);
         }
 
         bool result = false;
 
         try
         {
-            if (!isFillDummyContent)
+            if (clonedMessage.Handled)
+            {
+                clonedMessage.Content = clonedMessage.Content;
+            }
+            else if (!isFillDummyContent)
             {
                 result = await function.Execute(clonedMessage);
             }

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -162,7 +162,7 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             }
             else if (response.Type == "response.audio_transcript.delta")
             {
-
+                _logger.LogDebug($"{response.Type}: {receivedText}");
             }
             else if (response.Type == "response.audio_transcript.done")
             {
@@ -211,9 +211,14 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             }
             else if (response.Type == "input_audio_buffer.speech_started")
             {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
                 // Handle user interuption
                 onInterruptionDetected();
             }
+            else if (response.Type == "input_audio_buffer.speech_stopped")
+            {
+                _logger.LogInformation($"{response.Type}: {receivedText}");
+            }
         }
     }
 

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs

@@ -141,7 +141,6 @@ public VoiceResponse HangUp(ConversationalVoiceResponse voiceResponse)
         }
         else
         {
-            response.Pause(5);
             response.Say("Goodbye.");
         }
 

src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs

@@ -94,6 +94,13 @@ private async Task HandleWebSocket(IServiceProvider services, string conversatio
             }
             else if (eventType == "user_dtmf_receiving")
             {
+                // Send a Stop command to Twilio
+                string clearEvent = JsonSerializer.Serialize(new
+                {
+                    @event = "clear",
+                    streamSid = conn.StreamId
+                });
+                await SendEventToUser(webSocket, clearEvent);
             }
             else if (eventType == "user_dtmf_received")
             {
@@ -183,14 +190,6 @@ await hub.ConnectToModel(async data =>
                 streamSid = response.StreamSid
             });
 
-        /*if (response.Event == "dtmf")
-        {
-            // Send a Stop command to Twilio
-            string stopPlaybackCommand = "{ \"action\": \"stop_playback\" }";
-            var stopBytes = Encoding.UTF8.GetBytes(stopPlaybackCommand);
-            webSocket.SendAsync(new ArraySegment<byte>(stopBytes), WebSocketMessageType.Text, true, CancellationToken.None);
-        }*/
-
         return (eventType, data);
     }
 
@@ -225,7 +224,7 @@ private async Task HandleUserDtmfReceived(IServiceProvider _services, RealtimeHu
         var routing = _services.GetRequiredService<IRoutingService>();
         var hookProvider = _services.GetRequiredService<ConversationHookProvider>();
         var agentService = _services.GetRequiredService<IAgentService>();
-        var agent = await agentService.LoadAgent(conn.CurrentAgentId);
+        var agent = await agentService.GetAgent(conn.CurrentAgentId);
         var dialogs = routing.Context.GetDialogs();
         var convService = _services.GetRequiredService<IConversationService>();
         var conversation = await convService.GetConversation(conn.ConversationId);
@@ -248,7 +247,6 @@ private async Task HandleUserDtmfReceived(IServiceProvider _services, RealtimeHu
         }
 
         await completer.InsertConversationItem(message);
-        var instruction = await completer.UpdateSession(conn);
-        await completer.TriggerModelInference($"{instruction}\r\n\r\nReply based on the user input: {message.Content}");
+        await completer.TriggerModelInference($"Response based on the user input: {message.Content}");
     }
 }

src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-hangup_phone_call.json

@@ -8,12 +8,8 @@
       "reason": {
         "type": "string",
         "description": "The reason why user wants to end the phone call."
-      },
-      "response_content": {
-        "type": "string",
-        "description": "A response statement said to the user to politely and gratefully ending a conversation before hanging up."
       }
     },
-    "required": [ "reason", "response_content" ]
+    "required": [ "reason" ]
   }
 }