diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 55378b015..2d4b122ce 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -92,6 +92,7 @@ private async Task ConnectToModel(WebSocket userWebSocket) } routing.Context.SetDialogs(dialogs); + routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); var states = _services.GetRequiredService(); @@ -188,6 +189,7 @@ await _completer.Connect(_conn, // append input audio transcript to conversation dialogs.Add(message); storage.Append(_conn.ConversationId, message); + routing.Context.SetMessageId(_conn.ConversationId, message.MessageId); foreach (var hook in hookProvider.HooksOrderByPriority) { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index 8fe2eccee..105a7b4b6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -10,6 +10,9 @@ + + PreserveNewest + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs index 6960f4890..d5c8e713c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs @@ -35,6 +35,7 @@ public async Task InitiateStreamConversation(ConversationalVoiceReq throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); } + var twilio = _services.GetRequiredService(); VoiceResponse response = default!; if (request.AnsweredBy == "machine_start" && @@ -42,7 +43,8 @@ public async Task InitiateStreamConversation(ConversationalVoiceReq request.InitAudioFile != null) { response = new VoiceResponse(); - response.Play(new Uri(request.InitAudioFile)); + var url = twilio.GetSpeechPath(request.ConversationId, request.InitAudioFile); + response.Play(new Uri(url)); return TwiML(response); } @@ -68,9 +70,7 @@ await HookEmitter.Emit(_services, async hook => request.ConversationId = await InitConversation(request); - var twilio = _services.GetRequiredService(); - - response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction); + response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); await HookEmitter.Emit(_services, async hook => { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 079e13c51..b5749e0f6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -438,6 +438,26 @@ public async Task Hangup(ConversationalVoiceRequest request) return TwiML(response); } + [ValidateRequest] + [HttpPost("twilio/voice/transfer-call")] + public async Task TransferCall(ConversationalVoiceRequest request) + { + var instruction = new ConversationalVoiceResponse + { + ConversationId = request.ConversationId, + TransferTo = request.TransferTo + }; + + if (request.InitAudioFile != null) + { + instruction.SpeechPaths.Add(request.InitAudioFile); + } + + var twilio = _services.GetRequiredService(); + var response = twilio.TransferCall(instruction); + return TwiML(response); + } + [ValidateRequest] [HttpPost("twilio/voice/status")] public async Task PhoneCallStatus(ConversationalVoiceRequest request) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index 25e3ba0d3..46a2cb2b0 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -27,6 +27,9 @@ public class ConversationalVoiceRequest : VoiceRequest [FromForm] public string? CallbackSource { get; set; } + [FromQuery(Name = "transfer-to")] + public string? TransferTo { get; set; } + /// /// machine_start /// diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs index 7870f9615..f8f41579e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs @@ -13,4 +13,9 @@ public class ConversationalVoiceResponse public int Timeout { get; set; } = 3; public string Hints { get; set; } + + /// + /// The Phone Number to transfer to + /// + public string? TransferTo { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index 1e10a135d..73f164ade 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -108,6 +108,7 @@ public async Task Execute(RoleDialogModel message) statusCallback: new Uri(statusUrl), // https://www.twilio.com/docs/voice/answering-machine-detection machineDetection: _twilioSetting.MachineDetection, + machineDetectionSilenceTimeout: _twilioSetting.MachineDetectionSilenceTimeout, record: _twilioSetting.RecordingEnabled, recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}"); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs new file mode 100644 index 000000000..cc8e160cf --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs @@ -0,0 +1,78 @@ +using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; +using Twilio.Rest.Api.V2010.Account; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions; + +public class TransferPhoneCallFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + private readonly BotSharpOptions _options; + private readonly TwilioSetting _twilioSetting; + + public string Name => "util-twilio-transfer_phone_call"; + public string Indication => "Transferring the active line"; + + public TransferPhoneCallFn( + IServiceProvider services, + ILogger logger, + BotSharpOptions options, + TwilioSetting twilioSetting) + { + _services = services; + _logger = logger; + _options = options; + _twilioSetting = twilioSetting; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + + var fileStorage = _services.GetRequiredService(); + var states = _services.GetRequiredService(); + var routing = _services.GetRequiredService(); + var conversationId = routing.Context.ConversationId; + var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?conversation-id={conversationId}&transfer-to={args.PhoneNumber}"; + + // Generate initial assistant audio + string initAudioFile = null; + if (!string.IsNullOrEmpty(args.TransitionMessage)) + { + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(args.TransitionMessage); + initAudioFile = "transfer.mp3"; + fileStorage.SaveSpeechFile(conversationId, initAudioFile, data); + + processUrl += $"&init-audio-file={initAudioFile}"; + } + + if (!string.IsNullOrEmpty(initAudioFile)) + { + processUrl += $"&init-audio-file={initAudioFile}"; + } + + // Forward call + var sid = states.GetState("twilio_call_sid"); + + if (string.IsNullOrEmpty(sid)) + { + _logger.LogError("Twilio call sid is empty."); + message.Content = "There is an error when transferring the phone call."; + return false; + } + else + { + var call = CallResource.Update( + pathSid: sid, + url: new Uri(processUrl)); + + message.Content = args.TransitionMessage; + return true; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs index fa0b31798..14fce2901 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs @@ -7,8 +7,9 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook { private static string PREFIX = "util-twilio-"; private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call"; + private static string TRANSFER_PHONE_CALL_FN = $"{PREFIX}transfer_phone_call"; private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call"; - public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message"; + private static string TEXT_MESSAGE_FN = $"{PREFIX}text_message"; public void AddUtilities(List utilities) { @@ -18,6 +19,7 @@ public void AddUtilities(List utilities) Functions = [ new($"{OUTBOUND_PHONE_CALL_FN}"), + new($"{TRANSFER_PHONE_CALL_FN}"), new($"{HANGUP_PHONE_CALL_FN}"), new($"{TEXT_MESSAGE_FN}") ], diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs new file mode 100644 index 000000000..abd964b4a --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts; + +public class ForwardPhoneCallArgs +{ + [JsonPropertyName("phone_number")] + public string PhoneNumber { get; set; } = null!; + + [JsonPropertyName("transition_message")] + public string TransitionMessage { get; set; } = null!; +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 082a07217..3358e8acf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -160,6 +160,23 @@ public VoiceResponse DialCsrAgent(string speechPath) return response; } + public VoiceResponse TransferCall(ConversationalVoiceResponse conversationalVoiceResponse) + { + var response = new VoiceResponse(); + var conversationId = conversationalVoiceResponse.ConversationId; + if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) + { + foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) + { + var uri = GetSpeechPath(conversationId, speechPath); + response.Play(new Uri(uri)); + } + } + response.Dial(conversationalVoiceResponse.TransferTo, answerOnBridge: true); + + return response; + } + public VoiceResponse HoldOn(int interval, string message = null) { var twilioSetting = _services.GetRequiredService(); @@ -190,26 +207,17 @@ public VoiceResponse HoldOn(int interval, string message = null) /// /// /// - public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse) { var response = new VoiceResponse(); + var conversationId = conversationalVoiceResponse.ConversationId; if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any()) { foreach (var speechPath in conversationalVoiceResponse.SpeechPaths) { - if (speechPath.StartsWith("twilio/")) - { - response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); - } - else if (speechPath.StartsWith(_settings.CallbackHost)) - { - response.Play(new Uri(speechPath)); - } - else - { - response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationalVoiceResponse.ConversationId}/{speechPath}")); - } + var uri = GetSpeechPath(conversationId, speechPath); + response.Play(new Uri(uri)); } } @@ -220,4 +228,20 @@ public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversa return response; } + + public string GetSpeechPath(string conversationId, string speechPath) + { + if (speechPath.StartsWith("twilio/")) + { + return $"{_settings.CallbackHost}/{speechPath}"; + } + else if (speechPath.StartsWith(_settings.CallbackHost)) + { + return speechPath; + } + else + { + return $"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}"; + } + } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index 2de42ec8f..3b4907310 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -33,6 +33,7 @@ public class TwilioSetting public int MaxGatherAttempts { get; set; } = 4; public string? MachineDetection { get; set; } + public int MachineDetectionSilenceTimeout { get; set; } = 2500; public bool RecordingEnabled { get; set; } = false; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json new file mode 100644 index 000000000..7a8353864 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json @@ -0,0 +1,19 @@ +{ + "name": "util-twilio-transfer_phone_call", + "description": "When user wants to transfer the phone call", + "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "transition_message": { + "type": "string", + "description": "Transition message when forwarding." + }, + "phone_number": { + "type": "string", + "description": "Phone number transfer to." + } + }, + "required": [ "transition_message", "phone_number" ] + } +} \ No newline at end of file