add intermediate message

Author: Jicheng Lu

src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/ChatEvent.cs

@@ -5,6 +5,7 @@ public static class ChatEvent
     public const string OnConversationInitFromClient = nameof(OnConversationInitFromClient);
     public const string OnMessageReceivedFromClient = nameof(OnMessageReceivedFromClient);
     public const string OnMessageReceivedFromAssistant = nameof(OnMessageReceivedFromAssistant);
+    public const string OnIntermediateMessageReceivedFromAssistant = nameof(OnIntermediateMessageReceivedFromAssistant);
 
     public const string OnMessageDeleted = nameof(OnMessageDeleted);
     public const string OnNotificationGenerated = nameof(OnNotificationGenerated);

src/Infrastructure/BotSharp.Abstraction/Files/Models/FileSelectContext.cs

@@ -16,7 +16,4 @@ public class FileSelectItem
 
     [JsonPropertyName("file_source")]
     public string FileSource { get; set; }
-
-    [JsonPropertyName("file_name")]
-    public string? FileName { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/MessageHub/Models/HubObserveData.cs

@@ -2,5 +2,13 @@ namespace BotSharp.Abstraction.MessageHub.Models;
 
 public class HubObserveData<TData> : ObserveDataBase where TData : class, new()
 {
+    /// <summary>
+    /// The observed data
+    /// </summary>
     public TData Data { get; set; } = null!;
+
+    /// <summary>
+    /// Whether to save the observed data To Db
+    /// </summary>
+    public bool SaveDataToDb { get; set; }
 }

src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs

@@ -36,6 +36,17 @@ public async Task<bool> Execute(RoleDialogModel message)
 
         await Task.Delay(1500);
 
+#if DEBUG
+        var temp = RoleDialogModel.From(message, AgentRole.Assistant, $"Here is your weather in {args?.City}");
+        messageHub.Push(new()
+        {
+            EventName = ChatEvent.OnIntermediateMessageReceivedFromAssistant,
+            Data = temp,
+            RefId = conv.ConversationId,
+            SaveDataToDb = true
+        });
+#endif
+
         message.Indication = $"Still working on it... Hold on, {args?.City}";
         messageHub.Push(new()
         {

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Abstraction.Instructs;
+using BotSharp.Abstraction.Files.Converters;
 
 namespace BotSharp.Core.Files.Services;
 
@@ -22,13 +23,16 @@ public async Task<string> ReadPdf(string text, List<InstructFileModel> files, In
         {
             var provider = options?.Provider ?? "openai";
             var pdfFiles = await DownloadAndSaveFiles(sessionDir, files);
-
             var targetFiles = pdfFiles;
-            if (provider != "google-ai")
+
+            var converter = GetImageConverter(options?.ImageConvertProvider);
+            if (converter == null && provider == "openai")
             {
-                targetFiles = await ConvertPdfToImages(pdfFiles, options);
+                var fileCoreSettings = _services.GetRequiredService<FileCoreSettings>();
+                converter = GetImageConverter(fileCoreSettings?.ImageConverter?.Provider);
             }
 
+            targetFiles = await ConvertPdfToImages(converter, pdfFiles);
             if (targetFiles.IsNullOrEmpty())
             {
                 return content;
@@ -115,15 +119,12 @@ private async Task<IEnumerable<string>> DownloadAndSaveFiles(string dir, List<In
         return locs;
     }
 
-    private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> files, InstructOptions? options = null)
+    private async Task<IEnumerable<string>> ConvertPdfToImages(IImageConverter converter, IEnumerable<string> files)
     {
         var images = new List<string>();
-        var settings = _services.GetRequiredService<FileCoreSettings>();
-
-        var converter = GetImageConverter(options?.ImageConvertProvider);
         if (converter == null || files.IsNullOrEmpty())
         {
-            return images;
+            return files;
         }
 
         foreach (var file in files)

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs

@@ -91,9 +91,7 @@ private string BuildFileName(string? name, string? extension, string defaultName
 
     private IImageConverter? GetImageConverter(string? provider)
     {
-        var settings = _services.GetRequiredService<FileCoreSettings>();
-        var convertProvider = provider ?? settings?.ImageConverter?.Provider;
-        var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == convertProvider);
+        var converter = _services.GetServices<IImageConverter>().FirstOrDefault(x => x.Provider == provider);
         return converter;
     }
     #endregion

src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs

@@ -61,17 +61,6 @@ private async Task<string> GetAiResponse(Agent agent, string templateName)
                 new(AgentRole.User, text)
             });
 
-            await HookEmitter.Emit<IInstructHook>(_services, async hook =>
-                await hook.OnResponseGenerated(new InstructResponseModel
-                {
-                    AgentId = agent.Id,
-                    TemplateName = templateName,
-                    Provider = completion.Provider,
-                    Model = completion.Model,
-                    UserMessage = text,
-                    CompletionText = response.Content
-                }), agent.Id);
-
             return response.Content;
         }
         catch (Exception ex)

src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/instructions/instruction.liquid

@@ -1,5 +1,5 @@
 You're {{router.name}} ({{router.description}}). 
-You can understand messages sent by users in different languages, and route the request to appropriate agent.
+You can route the request to appropriate agent. Default language is English(US).
 Follow these steps to handle user request:
 1. Read the [CONVERSATION] content.
 2. Determine which agent is suitable to handle this conversation. Try to minimize the routing of human service.

src/Plugins/BotSharp.Plugin.ChatHub/Helpers/EventEmitter.cs

@@ -3,9 +3,9 @@
 
 namespace BotSharp.Plugin.ChatHub.Helpers;
 
-public class EventEmitter
+internal class EventEmitter
 {
-    public static async Task SendChatEvent<T>(
+    internal static async Task SendChatEvent<T>(
         IServiceProvider services,
         ILogger logger,
         string @event,

src/Plugins/BotSharp.Plugin.ChatHub/Observers/ChatHubObserver.cs

@@ -121,6 +121,32 @@ public override void OnNext(HubObserveData<RoleDialogModel> value)
                 _logger.LogCritical($"Receiving {value.EventName} ({value.Data.Indication}) in {nameof(ChatHubObserver)} - {conv.ConversationId}");
 #endif
                 break;
+            case ChatEvent.OnIntermediateMessageReceivedFromAssistant:
+                if (!AllowSendingMessage()) return;
+
+                model = new ChatResponseDto
+                {
+                    ConversationId = conv.ConversationId,
+                    MessageId = message.MessageId,
+                    MessageLabel = message.MessageLabel,
+                    Text = !string.IsNullOrEmpty(message.SecondaryContent) ? message.SecondaryContent : message.Content,
+                    Function = message.FunctionName,
+                    RichContent = message.SecondaryRichContent ?? message.RichContent,
+                    Data = message.Data,
+                    Sender = new()
+                    {
+                        FirstName = "AI",
+                        LastName = "Assistant",
+                        Role = AgentRole.Assistant
+                    }
+                };
+
+                if (value.SaveDataToDb)
+                {
+                    var storage = _services.GetRequiredService<IConversationStorage>();
+                    storage.Append(conv.ConversationId, message);
+                }
+                break;
         }
 
         SendEvent(value.EventName, model.ConversationId, model);