Skip to content

Commit 7d31df3

Browse files
Oceania2018Oceania2018
authored
Merge pull request #522 from iceljc/features/refine-image-generation
Features/refine image generation
2 parents 7df315c + fc5ef28 commit 7d31df3

File tree

15 files changed

+330
-78
lines changed

15 files changed

+330
-78
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using BotSharp.Abstraction.Functions.Models;
22
using BotSharp.Abstraction.Messaging;
33
using BotSharp.Abstraction.Messaging.Models.RichContent;
4+
using BotSharp.Abstraction.MLTasks;
45

56
namespace BotSharp.Abstraction.Conversations.Models;
67

@@ -87,6 +88,13 @@ public class RoleDialogModel : ITrackableMessage
8788

8889
public List<BotSharpFile> Files { get; set; } = new List<BotSharpFile>();
8990

91+
/// <summary>
92+
/// The images generated by AI
93+
/// </summary>
94+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
95+
[JsonPropertyName("generated_images")]
96+
public List<ImageGeneration> GeneratedImages { get; set; } = new List<ImageGeneration>();
97+
9098
private RoleDialogModel()
9199
{
92100
}

‎src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,14 @@ public interface IBotSharpFileService
2323
bool DeleteMessageFiles(string conversationId, IEnumerable<string> messageIds, string targetMessageId, string? newMessageId = null);
2424
bool DeleteConversationFiles(IEnumerable<string> conversationIds);
2525

26+
/// <summary>
27+
/// Take screenshots of pdf pages and get response from llm
28+
/// </summary>
29+
/// <param name="prompt"></param>
30+
/// <param name="files">Pdf files</param>
31+
/// <returns></returns>
32+
Task<string> InstructPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files);
33+
2634
/// <summary>
2735
/// Get file bytes and content type from data, e.g., "data:image/png;base64,aaaaaaaaa"
2836
/// </summary>

‎src/Infrastructure/BotSharp.Abstraction/Files/Models/ImageGeneration.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
namespace BotSharp.Abstraction.Files.Models;
2+
3+
public class ImageGeneration
4+
{
5+
[JsonPropertyName("image_url")]
6+
public string? ImageUrl { get; set; }
7+
8+
[JsonPropertyName("image_data")]
9+
public string? ImageData { get; set; }
10+
11+
[JsonPropertyName("description")]
12+
public string Description { get; set; } = string.Empty;
13+
}

‎src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -180,10 +180,8 @@
180180
<PackageReference Include="EntityFrameworkCore.BootKit" Version="8.4.2" />
181181
<PackageReference Include="Fluid.Core" Version="2.8.0" />
182182
<PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
183+
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
183184
<PackageReference Include="Nanoid" Version="3.0.0" />
184-
<PackageReference Include="PdfiumViewer" Version="2.13.0" />
185-
<PackageReference Include="PdfiumViewer.Native.x86.v8-xfa" Version="2018.4.8.256" />
186-
<PackageReference Include="PdfiumViewer.Native.x86_64.v8-xfa" Version="2018.4.8.256" />
187185
<PackageReference Include="RedLock.net" Version="2.3.2" />
188186
<PackageReference Include="System.Drawing.Common" Version="8.0.6" />
189187
</ItemGroup>

‎src/Infrastructure/BotSharp.Core/Files/Converters/PdfiumConverter.cs‎

Lines changed: 0 additions & 39 deletions
This file was deleted.

‎src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
using BotSharp.Abstraction.Files.Converters;
2-
using BotSharp.Core.Files.Converters;
31
using BotSharp.Core.Files.Hooks;
2+
using BotSharp.Core.Files.Services;
43
using Microsoft.Extensions.Configuration;
54

65
namespace BotSharp.Core.Files;
@@ -20,6 +19,5 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
2019

2120
services.AddScoped<IAgentHook, FileAnalyzerHook>();
2221
services.AddScoped<IAgentToolHook, FileAnalyzerToolHook>();
23-
services.AddScoped<IPdf2ImageConverter, PdfiumConverter>();
2422
}
2523
}

‎src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.Conversation.cs‎ renamed to ‎src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
using BotSharp.Abstraction.Files.Converters;
2-
using BotSharp.Core.Files.Converters;
32
using Microsoft.EntityFrameworkCore;
43
using System.IO;
5-
using System.Linq;
64
using System.Threading;
75

8-
namespace BotSharp.Core.Files;
6+
namespace BotSharp.Core.Files.Services;
97

108
public partial class BotSharpFileService
119
{
@@ -118,7 +116,7 @@ private async Task<List<MessageFileModel>> GetMessageFiles(string conversationId
118116
}
119117
catch (Exception ex)
120118
{
121-
_logger.LogWarning($"Error when reading conversation ({conversationId}) files: {ex.Message}");
119+
_logger.LogWarning($"Error when reading conversation ({conversationId}) files: {ex.Message}\r\n{ex.InnerException}\r\n{ex.StackTrace}");
122120
}
123121

124122
return files;
@@ -281,7 +279,7 @@ public bool DeleteConversationFiles(IEnumerable<string> conversationIds)
281279

282280
foreach (var conversationId in conversationIds)
283281
{
284-
var convDir = FindConversationDirectory(conversationId);
282+
var convDir = GetConversationDirectory(conversationId);
285283
if (!ExistDirectory(convDir)) continue;
286284

287285
Directory.Delete(convDir, true);
@@ -305,7 +303,7 @@ private string GetConversationFileDirectory(string? conversationId, string? mess
305303
return dir;
306304
}
307305

308-
private string? FindConversationDirectory(string conversationId)
306+
private string? GetConversationDirectory(string conversationId)
309307
{
310308
if (string.IsNullOrEmpty(conversationId)) return null;
311309

@@ -318,14 +316,18 @@ private async Task<IEnumerable<string>> ConvertPdfToImages(string pdfLoc, string
318316
var converters = _services.GetServices<IPdf2ImageConverter>();
319317
if (converters.IsNullOrEmpty()) return Enumerable.Empty<string>();
320318

321-
var converter = converters.FirstOrDefault(x => x.GetType().Name != typeof(PdfiumConverter).Name);
319+
var converter = GetPdf2ImageConverter();
322320
if (converter == null)
323321
{
324-
converter = converters.FirstOrDefault(x => x.GetType().Name == typeof(PdfiumConverter).Name);
325-
if (converter == null) return Enumerable.Empty<string>();
322+
return Enumerable.Empty<string>();
326323
}
327-
328324
return await converter.ConvertPdfToImages(pdfLoc, imageLoc);
329325
}
326+
327+
private IPdf2ImageConverter? GetPdf2ImageConverter()
328+
{
329+
var converters = _services.GetServices<IPdf2ImageConverter>();
330+
return converters.FirstOrDefault();
331+
}
330332
#endregion
331333
}

‎src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Pdf.cs‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
using System.IO;
2+
using System.Linq;
3+
using System.Net.Http;
4+
using System.Threading;
5+
6+
namespace BotSharp.Core.Files.Services;
7+
8+
public partial class BotSharpFileService
9+
{
10+
public async Task<string> InstructPdf(string? provider, string? model, string? modelId, string prompt, List<BotSharpFile> files)
11+
{
12+
var content = string.Empty;
13+
14+
if (string.IsNullOrWhiteSpace(prompt) || files.IsNullOrEmpty())
15+
{
16+
return content;
17+
}
18+
19+
var guid = Guid.NewGuid().ToString();
20+
var sessionDir = GetSessionDirectory(guid);
21+
if (!ExistDirectory(sessionDir))
22+
{
23+
Directory.CreateDirectory(sessionDir);
24+
}
25+
26+
try
27+
{
28+
var pdfFiles = await SaveFiles(sessionDir, files);
29+
var images = await ConvertPdfToImages(pdfFiles);
30+
if (images.IsNullOrEmpty()) return content;
31+
32+
var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai",
33+
model: model, modelId: modelId ?? "gpt-4", multiModal: true);
34+
var message = await completion.GetChatCompletions(new Agent()
35+
{
36+
Id = Guid.Empty.ToString(),
37+
}, new List<RoleDialogModel>
38+
{
39+
new RoleDialogModel(AgentRole.User, prompt)
40+
{
41+
Files = images.Select(x => new BotSharpFile { FileStorageUrl = x }).ToList()
42+
}
43+
});
44+
45+
content = message.Content;
46+
return content;
47+
}
48+
catch (Exception ex)
49+
{
50+
_logger.LogError($"Error when analyzing pdf in file service: {ex.Message}");
51+
return content;
52+
}
53+
finally
54+
{
55+
Directory.Delete(sessionDir, true);
56+
}
57+
}
58+
59+
#region Private methods
60+
private string GetSessionDirectory(string id)
61+
{
62+
var dir = Path.Combine(_baseDir, SESSION_FOLDER, id);
63+
return dir;
64+
}
65+
66+
private async Task<IEnumerable<string>> SaveFiles(string dir, List<BotSharpFile> files, string extension = "pdf")
67+
{
68+
if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
69+
{
70+
return Enumerable.Empty<string>();
71+
}
72+
73+
var locs = new List<string>();
74+
foreach (var file in files)
75+
{
76+
try
77+
{
78+
var bytes = new byte[0];
79+
if (!string.IsNullOrEmpty(file.FileUrl))
80+
{
81+
var http = _services.GetRequiredService<IHttpClientFactory>();
82+
using var client = http.CreateClient();
83+
bytes = await client.GetByteArrayAsync(file.FileUrl);
84+
}
85+
else if (!string.IsNullOrEmpty(file.FileData))
86+
{
87+
(_, bytes) = GetFileInfoFromData(file.FileData);
88+
}
89+
90+
if (!bytes.IsNullOrEmpty())
91+
{
92+
var guid = Guid.NewGuid().ToString();
93+
var fileDir = Path.Combine(dir, guid);
94+
if (!ExistDirectory(fileDir))
95+
{
96+
Directory.CreateDirectory(fileDir);
97+
}
98+
99+
var pdfDir = Path.Combine(fileDir, $"{guid}.{extension}");
100+
using (var fs = new FileStream(pdfDir, FileMode.Create))
101+
{
102+
fs.Write(bytes, 0, bytes.Length);
103+
fs.Close();
104+
locs.Add(pdfDir);
105+
Thread.Sleep(100);
106+
}
107+
}
108+
}
109+
catch (Exception ex)
110+
{
111+
_logger.LogWarning($"Error when saving pdf file: {ex.Message}");
112+
continue;
113+
}
114+
}
115+
return locs;
116+
}
117+
118+
private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> files)
119+
{
120+
var images = new List<string>();
121+
var converter = GetPdf2ImageConverter();
122+
if (converter == null || files.IsNullOrEmpty())
123+
{
124+
return images;
125+
}
126+
127+
foreach (var file in files)
128+
{
129+
try
130+
{
131+
var segs = file.Split(Path.DirectorySeparatorChar);
132+
var dir = string.Join(Path.DirectorySeparatorChar, segs.SkipLast(1));
133+
var folder = Path.Combine(dir, "screenshots");
134+
var urls = await converter.ConvertPdfToImages(file, folder);
135+
images.AddRange(urls);
136+
}
137+
catch (Exception ex)
138+
{
139+
_logger.LogWarning($"Error when converting pdf file to images ({file}): {ex.Message}");
140+
continue;
141+
}
142+
}
143+
return images;
144+
}
145+
#endregion
146+
}

‎src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.User.cs‎ renamed to ‎src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.User.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
using System.IO;
22

3-
namespace BotSharp.Core.Files;
3+
namespace BotSharp.Core.Files.Services;
44

55
public partial class BotSharpFileService
66
{

‎src/Infrastructure/BotSharp.Core/Files/BotSharpFileService.cs‎ renamed to ‎src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.cs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
using Microsoft.AspNetCore.StaticFiles;
2-
using System;
32
using System.IO;
4-
using System.Threading;
53

6-
namespace BotSharp.Core.Files;
4+
namespace BotSharp.Core.Files.Services;
75

86
public partial class BotSharpFileService : IBotSharpFileService
97
{
@@ -22,6 +20,7 @@ public partial class BotSharpFileService : IBotSharpFileService
2220
private const string BOT_FILE_FOLDER = "bot";
2321
private const string USERS_FOLDER = "users";
2422
private const string USER_AVATAR_FOLDER = "avatar";
23+
private const string SESSION_FOLDER = "sessions";
2524

2625
private const int MIN_OFFSET = 1;
2726
private const int MAX_OFFSET = 5;

0 commit comments

Comments
 (0)