diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 5ce60519a..065cdc555 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -4,21 +4,27 @@ namespace BotSharp.Abstraction.Files; public interface IFileStorageService { + #region Common + string GetDirectory(string conversationId); + byte[] GetFileBytes(string fileStorageUrl); + bool SaveFileStreamToPath(string filePath, Stream stream); + bool SaveFileBytesToPath(string filePath, byte[] bytes); + string GetParentDir(string dir, int level = 1); + bool ExistDirectory(string? dir); + void CreateDirectory(string dir); + void DeleteDirectory(string dir); + string BuildDirectory(params string[] segments); + #endregion + + #region Conversation /// - /// Get the files that have been uploaded in the chat. - /// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file. + /// Get the message file screenshots for specific content types, e.g., pdf /// /// - /// - /// - /// - /// - /// + /// /// - Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes, - bool includeScreenShot = false, int? offset = null); + Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. @@ -45,20 +51,9 @@ Task> GetChatFiles(string conversationId, string s bool DeleteConversationFiles(IEnumerable conversationIds); #endregion + #region User string GetUserAvatar(); bool SaveUserAvatar(BotSharpFile file); #endregion - - #region Common - string GetDirectory(string conversationId); - byte[] GetFileBytes(string filePath); - bool SaveFileStreamToPath(string filePath, Stream stream); - bool SaveFileBytesToPath(string filePath, byte[] bytes); - string GetParentDir(string dir, int level = 1); - bool ExistDirectory(string? dir); - void CreateDirectory(string dir); - void DeleteDirectory(string dir); - string BuildDirectory(params string[] segments); - #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs index c54c31ff9..3483921af 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs @@ -40,7 +40,7 @@ public class FileBase /// /// File extension without dot /// - [JsonPropertyName("file_type")] + [JsonPropertyName("file_extension")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileType { get; set; } = string.Empty; + public string? FileExtension { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index e06e1a0f1..2a0128b66 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -15,6 +15,6 @@ public MessageFileModel() public override string ToString() { - return $"File name: {FileName}, File type: {FileType}, Content type: {ContentType}, Source: {FileSource}"; + return $"File name: {FileName}, File extension: {FileExtension}, Content type: {ContentType}, Source: {FileSource}"; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs index 6ba6cdd2d..d61c1b7c3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs @@ -2,13 +2,48 @@ namespace BotSharp.Abstraction.Files.Models; public class SelectFileOptions { + /// + /// Llm provider + /// public string? Provider { get; set; } + + /// + /// Llm model id + /// public string? ModelId { get; set; } + + /// + /// Agent id + /// public string? AgentId { get; set; } + + /// + /// Template (prompt) name + /// public string? Template { get; set; } + + /// + /// Description that user provides to select files + /// public string? Description { get; set; } + + /// + /// Whether include bot generated files + /// public bool IncludeBotFile { get; set; } + + /// + /// Conversation breakpoint + /// public bool FromBreakpoint { get; set; } + + /// + /// Message offset from last + /// public int? Offset { get; set; } + + /// + /// File content types. If null, all types of files will be retrived + /// public IEnumerable? ContentTypes { get; set; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index e10ba7408..df33906df 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -1,10 +1,4 @@ -using BotSharp.Abstraction.Repositories.Enums; using Microsoft.AspNetCore.StaticFiles; -using Microsoft.Extensions.DependencyInjection; -using System; -using System.IO; -using System.Net.Http; -using System.Net.Mime; namespace BotSharp.Abstraction.Files.Utilities; diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index dec4d9f67..dfa39aad2 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 300313625..41e0becc1 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -20,7 +20,7 @@ public async Task> SelectMessageFiles(string conve if (options.IncludeBotFile) { var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); - files = files.Concat(botFiles); + files = MergeMessageFiles(messageIds, files, botFiles); } if (files.IsNullOrEmpty()) @@ -31,6 +31,24 @@ public async Task> SelectMessageFiles(string conve return await SelectFiles(files, dialogs, options); } + private IEnumerable MergeMessageFiles(IEnumerable messageIds, IEnumerable userFiles, IEnumerable botFiles) + { + var files = new List(); + + if (messageIds.IsNullOrEmpty()) return files; + + foreach (var messageId in messageIds) + { + var users = userFiles.Where(x => x.MessageId == messageId).ToList(); + var bots = botFiles.Where(x => x.MessageId == messageId).ToList(); + + if (!users.IsNullOrEmpty()) files.AddRange(users); + if (!bots.IsNullOrEmpty()) files.AddRange(bots); + } + + return files; + } + private async Task> SelectFiles(IEnumerable files, IEnumerable dialogs, SelectFileOptions options) { if (files.IsNullOrEmpty()) return new List(); @@ -43,7 +61,7 @@ private async Task> SelectFiles(IEnumerable { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}, author: {x.FileSource}"; + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileExtension}, content_type: {x.ContentType}, author: {x.FileSource}"; }).ToList(); var agentId = !string.IsNullOrWhiteSpace(options.AgentId) ? options.AgentId : BuiltInAgentId.UtilityAssistant; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs similarity index 94% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs index c7a3cec97..c49edbce6 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -14,9 +14,9 @@ public string GetDirectory(string conversationId) return dir; } - public byte[] GetFileBytes(string filePath) + public byte[] GetFileBytes(string fileStorageUrl) { - using var stream = File.OpenRead(filePath); + using var stream = File.OpenRead(fileStorageUrl); var bytes = new byte[stream.Length]; stream.Read(bytes, 0, (int)stream.Length); return bytes; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs similarity index 71% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index bae18cae0..9e14677d1 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -6,22 +6,20 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public async Task> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes = null, - bool includeScreenShot = false, int? offset = null) + public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(dialogs, offset); + var source = FileSourceType.User; var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); foreach (var messageId in messageIds) { - var dir = Path.Combine(pathPrefix, messageId, source); + var dir = Path.Combine(pathPrefix, messageId, FileSourceType.User); if (!ExistDirectory(dir)) continue; foreach (var subDir in Directory.GetDirectories(dir)) @@ -29,22 +27,16 @@ public async Task> GetChatFiles(string conversatio var file = Directory.GetFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) - { - continue; - } + var screenshots = await GetScreenshots(file, subDir, messageId, source); + if (screenshots.IsNullOrEmpty()) continue; - var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); - if (foundFiles.IsNullOrEmpty()) continue; - - files.AddRange(foundFiles); + files.AddRange(screenshots); } } - return files; } + public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null) { @@ -72,14 +64,14 @@ public IEnumerable GetMessageFiles(string conversationId, IEnu } var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); + var fileExtension = Path.GetExtension(file).Substring(1); var model = new MessageFileModel() { MessageId = messageId, FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}", FileStorageUrl = file, FileName = fileName, - FileType = fileType, + FileExtension = fileExtension, ContentType = contentType, FileSource = source }; @@ -268,102 +260,80 @@ private IEnumerable GetMessageIds(IEnumerable dialogs, return messageIds; } + private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) + { + var converters = _services.GetServices(); + if (converters.IsNullOrEmpty()) return Enumerable.Empty(); + + var converter = GetPdf2ImageConverter(); + if (converter == null) + { + return Enumerable.Empty(); + } + return await converter.ConvertPdfToImages(pdfLoc, imageLoc); + } + + private IPdf2ImageConverter? GetPdf2ImageConverter() + { + var converters = _services.GetServices(); + return converters.FirstOrDefault(); + } - private async Task> GetMessageFiles(string file, string fileDir, string contentType, - string messageId, string source, bool includeScreenShot) + private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) { var files = new List(); try { - if (!_imageTypes.Contains(contentType) && includeScreenShot) + var contentType = FileUtility.GetFileContentType(file); + var screenshotDir = Path.Combine(parentDir, SCREENSHOT_FILE_FOLDER); + + if (ExistDirectory(screenshotDir) && !Directory.GetFiles(screenshotDir).IsNullOrEmpty()) { - var screenShotDir = Path.Combine(fileDir, SCREENSHOT_FILE_FOLDER); - if (ExistDirectory(screenShotDir) && !Directory.GetFiles(screenShotDir).IsNullOrEmpty()) + foreach (var screenshot in Directory.GetFiles(screenshotDir)) { - foreach (var screenShot in Directory.GetFiles(screenShotDir)) - { - contentType = FileUtility.GetFileContentType(screenShot); - if (!_imageTypes.Contains(contentType)) continue; - - var fileName = Path.GetFileNameWithoutExtension(screenShot); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = screenShot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenShotDir); - foreach (var image in images) + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var model = new MessageFileModel() { - contentType = FileUtility.GetFileContentType(image); - var fileName = Path.GetFileNameWithoutExtension(image); - var fileType = Path.GetExtension(image).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = screenshot, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); } } - else + else if (contentType == MediaTypeNames.Application.Pdf) { - var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileStorageUrl = file, - ContentType = contentType, - FileSource = source - }; - files.Add(model); + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel() + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileStorageUrl = image, + ContentType = screenshotContentType, + FileSource = source + }; + files.Add(model); + } } - return files; } catch (Exception ex) { - _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); return files; } } - - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) - { - var converters = _services.GetServices(); - if (converters.IsNullOrEmpty()) return Enumerable.Empty(); - - var converter = GetPdf2ImageConverter(); - if (converter == null) - { - return Enumerable.Empty(); - } - return await converter.ConvertPdfToImages(pdfLoc, imageLoc); - } - - private IPdf2ImageConverter? GetPdf2ImageConverter() - { - var converters = _services.GetServices(); - return converters.FirstOrDefault(); - } #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.User.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs similarity index 88% rename from src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs rename to src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 45d449bdc..d83cae037 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -9,11 +9,6 @@ public partial class LocalFileStorageService : IFileStorageService private readonly IUserIdentity _user; private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _imageTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg - }; private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs index 131a9baf3..787ab1472 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Files/MessageFileViewModel.cs @@ -10,8 +10,8 @@ public class MessageFileViewModel [JsonPropertyName("file_name")] public string FileName { get; set; } - [JsonPropertyName("file_type")] - public string FileType { get; set; } + [JsonPropertyName("file_extension")] + public string FileExtension { get; set; } [JsonPropertyName("content_type")] public string ContentType { get; set; } @@ -30,7 +30,7 @@ public static MessageFileViewModel Transform(MessageFileModel model) { FileUrl = model.FileUrl, FileName = model.FileName, - FileType = model.FileType, + FileExtension = model.FileExtension, ContentType = model.ContentType, FileSource = model.FileSource }; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 720343170..af82b0374 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -198,6 +198,7 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, List(); var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -262,13 +263,7 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, List GetChatCompletionsStreamingAsync(Agent agent, List(); var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); - builder.Attachments.Add($"{file.FileName}.{file.FileType}", fileBytes, ContentType.Parse(file.ContentType)); + builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 9d5b6ad02..a12bdfd4d 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -87,7 +87,7 @@ private async Task GetImageEditGeneration(RoleDialogModel message, strin stream.Close(); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); - return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted."; + return $"Image \"{image.FileName}.{image.FileExtension}\" is successfylly editted."; } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 183b775b2..a415207e8 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -8,12 +8,6 @@ public class ReadImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly IEnumerable _imageContentTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg, - }; - public ReadImageFn( IServiceProvider services, ILogger logger) @@ -29,7 +23,7 @@ public async Task Execute(RoleDialogModel message) var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); var agent = await agentService.LoadAgent(BuiltInAgentId.UtilityAssistant); var fileAgent = new Agent { @@ -44,7 +38,7 @@ public async Task Execute(RoleDialogModel message) return true; } - private async Task> AssembleFiles(string conversationId, List dialogs) + private List AssembleFiles(string conversationId, List dialogs) { if (dialogs.IsNullOrEmpty()) { @@ -52,7 +46,12 @@ private async Task> AssembleFiles(string conversationId, L } var fileStorage = _services.GetRequiredService(); - var images = await fileStorage.GetChatFiles(conversationId, FileSourceType.User, dialogs, _imageContentTypes); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var images = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, new List + { + MediaTypeNames.Image.Png, + MediaTypeNames.Image.Jpeg + }); foreach (var dialog in dialogs) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index ef34f96dd..e2b465c8e 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -51,11 +51,14 @@ private async Task> AssembleFiles(string conversationId, L } var fileStorage = _services.GetRequiredService(); - var files = await fileStorage.GetChatFiles(conversationId, FileSourceType.User, dialogs, _pdfContentTypes, includeScreenShot: true); + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); + + if (screenshots.IsNullOrEmpty()) return dialogs; foreach (var dialog in dialogs) { - var found = files.Where(x => x.MessageId == dialog.MessageId).ToList(); + var found = screenshots.Where(x => x.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index f12e8b34e..00f47149d 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -199,6 +199,7 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, List(); var state = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); var settingsService = _services.GetRequiredService(); var settings = settingsService.GetSetting(Provider, _model); var allowMultiModal = settings != null && settings.MultiModal; @@ -263,13 +264,7 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, List GetChatCompletionsStreamingAsync(Agent agent, List> GetChatFiles(string conversationId, string source, - IEnumerable dialogs, IEnumerable? contentTypes = null, - bool includeScreenShot = false, int? offset = null) + public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || dialogs.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) { return files; } - var messageIds = GetMessageIds(dialogs, offset); + var source = FileSourceType.User; var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; - foreach (var messageId in messageIds) { var dir = $"{pathPrefix}/{messageId}/{source}"; - foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) { var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); if (file == null) continue; - var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) - { - continue; - } - - var foundFiles = await GetMessageFiles(file, subDir, contentType, messageId, source, includeScreenShot); - if (foundFiles.IsNullOrEmpty()) continue; + var screenshots = await GetScreenshots(file, subDir, messageId, source); + if (screenshots.IsNullOrEmpty()) continue; - files.AddRange(foundFiles); + files.AddRange(screenshots); } } @@ -70,14 +60,14 @@ public IEnumerable GetMessageFiles(string conversationId, IEnu } var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); + var fileExtension = Path.GetExtension(file).Substring(1); var model = new MessageFileModel() { MessageId = messageId, FileUrl = BuilFileUrl(file), FileStorageUrl = file, FileName = fileName, - FileType = fileType, + FileExtension = fileExtension, ContentType = contentType, FileSource = source }; @@ -89,12 +79,13 @@ public IEnumerable GetMessageFiles(string conversationId, IEnu return files; } + + public string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName) { var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/"; var fileList = _cosClient.BucketClient.GetDirFiles(dir); - var found = fileList.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName)); return found; } @@ -246,88 +237,6 @@ private IEnumerable GetMessageIds(IEnumerable dialogs, } - private async Task> GetMessageFiles(string file, string fileDir, string contentType, - string messageId, string source, bool includeScreenShot) - { - var files = new List(); - try - { - if (!_imageTypes.Contains(contentType) && includeScreenShot) - { - var screenShotDir = $"{fileDir}/{SCREENSHOT_FILE_FOLDER}/"; - var fileList = _cosClient.BucketClient.GetDirFiles(screenShotDir); - - if (!fileList.IsNullOrEmpty()) - { - foreach (var screenShot in fileList) - { - contentType = FileUtility.GetFileContentType(screenShot); - if (!_imageTypes.Contains(contentType)) continue; - - var fileName = Path.GetFileNameWithoutExtension(screenShot); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileUrl = BuilFileUrl(screenShot), - FileStorageUrl = screenShot, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - else if (contentType == MediaTypeNames.Application.Pdf) - { - var images = await ConvertPdfToImages(file, screenShotDir); - foreach (var image in images) - { - contentType = FileUtility.GetFileContentType(image); - var fileName = Path.GetFileNameWithoutExtension(image); - var fileType = Path.GetExtension(image).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileUrl = BuilFileUrl(image), - FileStorageUrl = image, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - } - } - else - { - var fileName = Path.GetFileNameWithoutExtension(file); - var fileType = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() - { - MessageId = messageId, - FileName = fileName, - FileType = fileType, - FileUrl = BuilFileUrl(file), - FileStorageUrl = file, - ContentType = contentType, - FileSource = source - }; - files.Add(model); - } - - return files; - } - catch (Exception ex) - { - _logger.LogWarning($"Error when getting message files {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); - return files; - } - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) { var converters = _services.GetServices(); @@ -351,5 +260,64 @@ private string BuilFileUrl(string file) { return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}"; } + + private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + { + var files = new List(); + + try + { + var contentType = FileUtility.GetFileContentType(file); + var screenshotDir = $"{parentDir}/{SCREENSHOT_FILE_FOLDER}/"; + var screenshots = _cosClient.BucketClient.GetDirFiles(screenshotDir); + if (!screenshots.IsNullOrEmpty()) + { + foreach (var screenshot in screenshots) + { + var screenshotContentType = FileUtility.GetFileContentType(screenshot); + var fileName = Path.GetFileNameWithoutExtension(screenshot); + var fileExtension = Path.GetExtension(screenshot).Substring(1); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(screenshot), + FileStorageUrl = screenshot, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + else if (contentType == MediaTypeNames.Application.Pdf) + { + var images = await ConvertPdfToImages(file, screenshotDir); + foreach (var image in images) + { + var fileName = Path.GetFileNameWithoutExtension(image); + var fileExtension = Path.GetExtension(image).Substring(1); + var screenshotContentType = FileUtility.GetFileContentType(image); + var model = new MessageFileModel + { + MessageId = messageId, + FileName = fileName, + FileExtension = fileExtension, + FileUrl = BuilFileUrl(image), + FileStorageUrl = image, + ContentType = contentType, + FileSource = source + }; + files.Add(model); + } + } + return files; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting message file screenshots {file} (messageId: {messageId}), Error: {ex.Message}\r\n{ex.InnerException}"); + return files; + } + } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index 12bd9e173..ddb5e031c 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -40,7 +40,7 @@ public TencentCosService( _user = user; _logger = logger; _services = services; - _fullBuketName = $"{_settings.BucketName}-{_settings.AppId}"; + _fullBuketName = $"{settings.BucketName}-{settings.AppId}"; _cosClient = cosClient; } }