add chat files

src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs

@@ -3,7 +3,8 @@ namespace BotSharp.Abstraction.Files;
 public interface IBotSharpFileService
 {
     string GetDirectory(string conversationId);
-    IEnumerable<OutputFileModel> GetConversationFiles(string conversationId, string messageId);
+    IEnumerable<MessageFileModel> GetChatImages(string conversationId, List<RoleDialogModel> conversations, int offset = 2);
+    IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, bool imageOnly = false);
     string? GetMessageFile(string conversationId, string messageId, string fileName);
     void SaveMessageFiles(string conversationId, string messageId, List<BotSharpFile> files);
 
@@ -17,4 +18,11 @@ public interface IBotSharpFileService
     /// <returns></returns>
     bool DeleteMessageFiles(string conversationId, IEnumerable<string> messageIds, string targetMessageId, string? newMessageId = null);
     bool DeleteConversationFiles(IEnumerable<string> conversationIds);
+
+    /// <summary>
+    /// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
+    /// </summary>
+    /// <param name="data"></param>
+    /// <returns></returns>
+    (string, byte[]) GetFileInfoFromData(string data);
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs

@@ -4,14 +4,14 @@ namespace BotSharp.Abstraction.Files.Models;
 public class BotSharpFile
 {
     [JsonPropertyName("file_name")]
-    public string FileName { get; set; }
+    public string FileName { get; set; } = string.Empty;
 
+    /// <summary>
+    /// File data, e.g., "data:image/png;base64,aaaaaaaa"
+    /// </summary>
     [JsonPropertyName("file_data")]
-    public string FileData { get; set; }
+    public string FileData { get; set; } = string.Empty;
 
-    [JsonPropertyName("content_type")]
-    public string ContentType { get; set; }
-
-    [JsonPropertyName("file_size")]
-    public int FileSize { get; set; }
+    [JsonPropertyName("file_url")]
+    public string FileUrl { get; set; } = string.Empty;
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs

@@ -0,0 +1,32 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class MessageFileModel
+{
+    [JsonPropertyName("message_id")]
+    public string MessageId { get; set; }
+
+    [JsonPropertyName("file_url")]
+    public string FileUrl { get; set; }
+
+    [JsonPropertyName("file_storage_url")]
+    public string FileStorageUrl { get; set; }
+
+    [JsonPropertyName("file_name")]
+    public string FileName { get; set; }
+
+    [JsonPropertyName("file_type")]
+    public string FileType { get; set; }
+
+    [JsonPropertyName("content_type")]
+    public string ContentType { get; set; }
+
+    public MessageFileModel()
+    {
+
+    }
+
+    public override string ToString()
+    {
+        return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}";
+    }
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs

@@ -6,6 +6,6 @@ public interface ILlmProviderService
 {
     LlmModelSetting GetSetting(string provider, string model);
     List<string> GetProviders();
-    LlmModelSetting GetProviderModel(string provider, string id);
+    LlmModelSetting GetProviderModel(string provider, string id, bool multiModal = false);
     List<LlmModelSetting> GetProviderModels(string provider);
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -27,6 +27,11 @@ public class LlmModelSetting
     public string Endpoint { get; set; }
     public LlmModelType Type { get; set; } = LlmModelType.Chat;
 
+    /// <summary>
+    /// If true, allow sending images/vidoes to this model
+    /// </summary>
+    public bool MultiModal { get; set; }
+
     /// <summary>
     /// Prompt cost per 1K token
     /// </summary>

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -159,6 +159,7 @@
     <PackageReference Include="Colorful.Console" Version="1.2.15" />
     <PackageReference Include="EntityFrameworkCore.BootKit" Version="8.2.1" />
     <PackageReference Include="Fluid.Core" Version="2.8.0" />
+    <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
     <PackageReference Include="Nanoid" Version="3.0.0" />
     <PackageReference Include="RedLock.net" Version="2.3.2" />
   </ItemGroup>

src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs

@@ -1,3 +1,4 @@
+using Microsoft.AspNetCore.StaticFiles;
 using System.IO;
 using System.Threading;
 
@@ -7,16 +8,22 @@ public class BotSharpFileService : IBotSharpFileService
 {
     private readonly BotSharpDatabaseSettings _dbSettings;
     private readonly IServiceProvider _services;
+    private readonly ILogger<BotSharpFileService> _logger;
     private readonly string _baseDir;
+    private readonly IEnumerable<string> _allowedTypes = new List<string> { "image/png", "image/jpeg" };
 
     private const string CONVERSATION_FOLDER = "conversations";
     private const string FILE_FOLDER = "files";
+    private const int MIN_OFFSET = 1;
+    private const int MAX_OFFSET = 5;
 
     public BotSharpFileService(
         BotSharpDatabaseSettings dbSettings,
+        ILogger<BotSharpFileService> logger,
         IServiceProvider services)
     {
         _dbSettings = dbSettings;
+        _logger = logger;
         _services = services;
         _baseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository);
     }
@@ -31,29 +38,67 @@ public string GetDirectory(string conversationId)
         return dir;
     }
 
-    public IEnumerable<OutputFileModel> GetConversationFiles(string conversationId, string messageId)
+    public IEnumerable<MessageFileModel> GetChatImages(string conversationId, List<RoleDialogModel> conversations, int offset = 2)
     {
-        var outputFiles = new List<OutputFileModel>();
-        var dir = GetConversationFileDirectory(conversationId, messageId);
-        if (string.IsNullOrEmpty(dir))
+        var files = new List<MessageFileModel>();
+        if (string.IsNullOrEmpty(conversationId) || conversations.IsNullOrEmpty())
+        {
+            return files;
+        }
+
+        if (offset <= 0)
+        {
+            offset = MIN_OFFSET;
+        }
+        else if (offset > MAX_OFFSET)
         {
-            return outputFiles;
+            offset = MAX_OFFSET;
         }
 
-        foreach (var file in Directory.GetFiles(dir))
+        var messageIds = conversations.Select(x => x.MessageId).Distinct().TakeLast(offset).ToList();
+        files = GetMessageFiles(conversationId, messageIds, imageOnly: true).ToList();
+        return files;
+    }
+
+    public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, bool imageOnly = false)
+    {
+        var files = new List<MessageFileModel>();
+        if (messageIds.IsNullOrEmpty()) return files;
+
+        foreach (var messageId in messageIds)
         {
-            var fileName = Path.GetFileNameWithoutExtension(file);
-            var extension = Path.GetExtension(file);
-            var fileType = extension.Substring(1);
-            var model = new OutputFileModel()
+            var dir = GetConversationFileDirectory(conversationId, messageId);
+            if (string.IsNullOrEmpty(dir))
             {
-                FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}",
-                FileName = fileName,
-                FileType = fileType
-            };
-            outputFiles.Add(model);
+                continue;
+            }
+
+            foreach (var file in Directory.GetFiles(dir))
+            {
+                var contentType = GetFileContentType(file);
+                if (imageOnly && !_allowedTypes.Contains(contentType))
+                {
+                    continue;
+                }
+
+                var fileName = Path.GetFileNameWithoutExtension(file);
+                var extension = Path.GetExtension(file);
+                var fileType = extension.Substring(1);
+
+                var model = new MessageFileModel()
+                {
+                    MessageId = messageId,
+                    FileUrl = $"/conversation/{conversationId}/message/{messageId}/file/{fileName}",
+                    FileStorageUrl = file,
+                    FileName = fileName,
+                    FileType = fileType,
+                    ContentType = contentType
+                };
+                files.Add(model);
+            }
         }
-        return outputFiles;
+
+        return files;
     }
 
     public string? GetMessageFile(string conversationId, string messageId, string fileName)
@@ -75,19 +120,26 @@ public void SaveMessageFiles(string conversationId, string messageId, List<BotSh
         var dir = GetConversationFileDirectory(conversationId, messageId, createNewDir: true);
         if (string.IsNullOrEmpty(dir)) return;
 
-        for (int i = 0; i < files.Count; i++)
+        try
         {
-            var file = files[i];
-            if (string.IsNullOrEmpty(file.FileData))
+            for (int i = 0; i < files.Count; i++)
             {
-                continue;
-            }
+                var file = files[i];
+                if (string.IsNullOrEmpty(file.FileData))
+                {
+                    continue;
+                }
 
-            var bytes = GetFileBytes(file.FileData);
-            var fileType = Path.GetExtension(file.FileName);
-            var fileName = $"{i + 1}{fileType}";
-            Thread.Sleep(100);
-            File.WriteAllBytes(Path.Combine(dir, fileName), bytes);
+                var (_, bytes) = GetFileInfoFromData(file.FileData);
+                var fileType = Path.GetExtension(file.FileName);
+                var fileName = $"{i + 1}{fileType}";
+                Thread.Sleep(100);
+                File.WriteAllBytes(Path.Combine(dir, fileName), bytes);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError($"Error when saving conversation files: {ex.Message}");
         }
     }
 
@@ -137,6 +189,23 @@ public bool DeleteConversationFiles(IEnumerable<string> conversationIds)
         return true;
     }
 
+    public (string, byte[]) GetFileInfoFromData(string data)
+    {
+        if (string.IsNullOrEmpty(data))
+        {
+            return (string.Empty, new byte[0]);
+        }
+
+        var typeStartIdx = data.IndexOf(':');
+        var typeEndIdx = data.IndexOf(';');
+        var contentType = data.Substring(typeStartIdx + 1, typeEndIdx - typeStartIdx - 1);
+
+        var base64startIdx = data.IndexOf(',');
+        var base64Str = data.Substring(base64startIdx + 1);
+
+        return (contentType, Convert.FromBase64String(base64Str));
+    }
+
     #region Private methods
     private string GetConversationFileDirectory(string? conversationId, string? messageId, bool createNewDir = false)
     {
@@ -170,54 +239,16 @@ private string GetConversationFileDirectory(string? conversationId, string? mess
         return dir;
     }
 
-    private byte[] GetFileBytes(string data)
-    {
-        if (string.IsNullOrEmpty(data))
-        {
-            return new byte[0];
-        }
-
-        var startIdx = data.IndexOf(',');
-        var base64Str = data.Substring(startIdx + 1);
-        return Convert.FromBase64String(base64Str);
-    }
-
-    private string GetFileType(string data)
+    private string GetFileContentType(string filePath)
     {
-        if (string.IsNullOrEmpty(data))
+        string contentType;
+        var provider = new FileExtensionContentTypeProvider();
+        if (!provider.TryGetContentType(filePath, out contentType))
         {
-            return string.Empty;
+            contentType = string.Empty;
         }
 
-        var startIdx = data.IndexOf(':');
-        var endIdx = data.IndexOf(';');
-        var fileType = data.Substring(startIdx + 1, endIdx - startIdx - 1);
-        return fileType;
-    }
-
-    private string ParseFileFormat(string type)
-    {
-        var parsed = string.Empty;
-        switch (type)
-        {
-            case "image/png":
-                parsed = ".png";
-                break;
-            case "image/jpeg":
-            case "image/jpg":
-                parsed = ".jpeg";
-                break;
-            case "application/pdf":
-                parsed = ".pdf";
-                break;
-            case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
-                parsed = ".xlsx";
-                break;
-            case "text/plain":
-                parsed = ".txt";
-                break;
-        }
-        return parsed;
+        return contentType;
     }
     #endregion
 }