Skip to content

Commit 9f8767b

Browse files
Oceania2018Oceania2018
authored
Merge pull request #1068 from iceljc/features/refine-file-instruct
Features/refine file instruct
2 parents 4977cc7 + 0c820aa commit 9f8767b

File tree

38 files changed

+410
-258
lines changed

38 files changed

+410
-258
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public interface IAgentService
1313
Task<string> RefreshAgents();
1414
Task<PagedItems<Agent>> GetAgents(AgentFilter filter);
1515
Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null, bool byName = false);
16+
Task<IEnumerable<AgentUtility>> GetAgentUtilityOptions();
1617

1718
/// <summary>
1819
/// Load agent configurations and trigger hooks

‎src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentUtility.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,7 @@ public override string ToString()
2626
public class UtilityItem
2727
{
2828
[JsonPropertyName("function_name")]
29-
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
30-
public string? FunctionName { get; set; }
29+
public string FunctionName { get; set; } = null!;
3130

3231
[JsonPropertyName("template_name")]
3332
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
@@ -36,4 +35,8 @@ public class UtilityItem
3635
[JsonPropertyName("visibility_expression")]
3736
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
3837
public string? VisibilityExpression { get; set; }
38+
39+
[JsonPropertyName("description")]
40+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
41+
public string? Description { get; set; }
3942
}

‎src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ public interface IFileStorageService
77
#region Common
88
string GetDirectory(string conversationId);
99
IEnumerable<string> GetFiles(string relativePath, string? searchQuery = null);
10-
byte[] GetFileBytes(string fileStorageUrl);
10+
BinaryData GetFileBytes(string fileStorageUrl);
1111
bool SaveFileStreamToPath(string filePath, Stream stream);
12-
bool SaveFileBytesToPath(string filePath, byte[] bytes);
12+
bool SaveFileBytesToPath(string filePath, BinaryData binary);
1313
string GetParentDir(string dir, int level = 1);
1414
bool ExistDirectory(string? dir);
1515
void CreateDirectory(string dir);

‎src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,23 @@ namespace BotSharp.Abstraction.Files.Models;
33
public class InstructFileModel : FileBase
44
{
55
/// <summary>
6-
/// File extension without dot
6+
/// File extension
77
/// </summary>
88
[JsonPropertyName("file_extension")]
99
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
1010
public string? FileExtension { get; set; } = string.Empty;
1111

1212
/// <summary>
13-
/// External file url
13+
/// File url
1414
/// </summary>
1515
[JsonPropertyName("file_url")]
1616
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
1717
public string? FileUrl { get; set; } = string.Empty;
18+
19+
/// <summary>
20+
/// File url
21+
/// </summary>
22+
[JsonPropertyName("content_type")]
23+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
24+
public string? ContentType { get; set; }
1825
}

‎src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,16 @@ public static class FileUtility
1111
/// </summary>
1212
/// <param name="data"></param>
1313
/// <returns></returns>
14-
public static (string, byte[]) GetFileInfoFromData(string data)
14+
public static (string?, BinaryData) GetFileInfoFromData(string data)
1515
{
1616
if (string.IsNullOrEmpty(data))
1717
{
18-
return (string.Empty, new byte[0]);
18+
return (null, BinaryData.Empty);
19+
}
20+
21+
if (!data.StartsWith("data:"))
22+
{
23+
return (null, BinaryData.FromBytes(Convert.FromBase64String(data)));
1924
}
2025

2126
var typeStartIdx = data.IndexOf(':');
@@ -25,13 +30,13 @@ public static (string, byte[]) GetFileInfoFromData(string data)
2530
var base64startIdx = data.IndexOf(',');
2631
var base64Str = data.Substring(base64startIdx + 1);
2732

28-
return (contentType, Convert.FromBase64String(base64Str));
33+
return (contentType, BinaryData.FromBytes(Convert.FromBase64String(base64Str)));
2934
}
3035

31-
public static string BuildFileDataFromFile(string fileName, byte[] bytes)
36+
public static string BuildFileDataFromFile(string fileName, BinaryData binary)
3237
{
3338
var contentType = GetFileContentType(fileName);
34-
var base64 = Convert.ToBase64String(bytes);
39+
var base64 = Convert.ToBase64String(binary.ToArray());
3540
return $"data:{contentType};base64,{base64}";
3641
}
3742

‎src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Utility.cs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
using BotSharp.Abstraction.Repositories.Settings;
2+
using System.IO;
3+
4+
namespace BotSharp.Core.Agents.Services;
5+
6+
public partial class AgentService
7+
{
8+
public async Task<IEnumerable<AgentUtility>> GetAgentUtilityOptions()
9+
{
10+
var utilities = new List<AgentUtility>();
11+
var hooks = _services.GetServices<IAgentUtilityHook>();
12+
foreach (var hook in hooks)
13+
{
14+
hook.AddUtilities(utilities);
15+
}
16+
17+
utilities = utilities.Where(x => !string.IsNullOrWhiteSpace(x.Category)
18+
&& !string.IsNullOrWhiteSpace(x.Name)
19+
&& !x.Items.IsNullOrEmpty()).ToList();
20+
21+
var allItems = utilities.SelectMany(x => x.Items).ToList();
22+
var functionNames = allItems.Select(x => x.FunctionName).Distinct().ToList();
23+
var mapper = await GetAgentDocs(functionNames);
24+
25+
allItems.ForEach(x =>
26+
{
27+
if (mapper.ContainsKey(x.FunctionName))
28+
{
29+
x.Description = mapper[x.FunctionName];
30+
}
31+
});
32+
33+
return utilities;
34+
}
35+
36+
#region Private methods
37+
private async ValueTask<IDictionary<string, string>> GetAgentDocs(IEnumerable<string> names)
38+
{
39+
var mapper = new Dictionary<string, string>();
40+
if (names.IsNullOrEmpty())
41+
{
42+
return mapper;
43+
}
44+
45+
var dir = GetAgentDocDir(BuiltInAgentId.UtilityAssistant);
46+
if (string.IsNullOrEmpty(dir))
47+
{
48+
return mapper;
49+
}
50+
51+
var matchDocs = Directory.GetFiles(dir, "*.md")
52+
.Where(x => names.Contains(Path.GetFileNameWithoutExtension(x)))
53+
.ToList();
54+
55+
if (matchDocs.IsNullOrEmpty())
56+
{
57+
return mapper;
58+
}
59+
60+
await foreach (var item in GetUtilityDescriptions(matchDocs))
61+
{
62+
mapper[item.Key] = item.Value;
63+
}
64+
65+
return mapper;
66+
}
67+
68+
private string GetAgentDocDir(string agentId)
69+
{
70+
var dbSettings = _services.GetRequiredService<BotSharpDatabaseSettings>();
71+
var agentSettings = _services.GetRequiredService<AgentSettings>();
72+
var dir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository, agentSettings.DataDir, agentId, "docs");
73+
if (!Directory.Exists(dir))
74+
{
75+
dir = string.Empty;
76+
}
77+
return dir;
78+
}
79+
80+
private async IAsyncEnumerable<KeyValuePair<string, string>> GetUtilityDescriptions(IEnumerable<string> docs)
81+
{
82+
foreach (var doc in docs)
83+
{
84+
var content = string.Empty;
85+
try
86+
{
87+
content = await File.ReadAllTextAsync(doc);
88+
}
89+
catch { }
90+
91+
if (string.IsNullOrWhiteSpace(content))
92+
{
93+
continue;
94+
}
95+
96+
var fileName = Path.GetFileNameWithoutExtension(doc);
97+
yield return new KeyValuePair<string, string>(fileName, content);
98+
}
99+
}
100+
#endregion
101+
}

‎src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -145,11 +145,7 @@ public IConversationStateService SetState<T>(string name, T value, bool isNeedVe
145145
newPair.Values = new List<StateValue> { newValue };
146146
_curStates[name] = newPair;
147147
}
148-
else if (isNoChange)
149-
{
150-
// do nothing
151-
}
152-
else
148+
else if (!isNoChange)
153149
{
154150
_curStates[name].Values.Add(newValue);
155151
}

‎src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
using BotSharp.Abstraction.Instructs.Models;
2-
using System.IO;
32

43
namespace BotSharp.Core.Files.Services;
54

@@ -14,12 +13,11 @@ public async Task<string> SpeechToText(InstructFileModel audio, string? text = n
1413
}
1514

1615
var completion = CompletionProvider.GetAudioTranscriber(_services, provider: options?.Provider, model: options?.Model);
17-
var audioBytes = await DownloadFile(audio);
18-
using var stream = new MemoryStream();
19-
stream.Write(audioBytes, 0, audioBytes.Length);
16+
var audioBinary = await DownloadFile(audio);
17+
using var stream = audioBinary.ToStream();
2018
stream.Position = 0;
2119

22-
var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}";
20+
var fileName = BuildFileName(audio.FileName, audio.FileExtension, "audio", "wav");
2321
var content = await completion.TranscriptTextAsync(stream, fileName, text ?? string.Empty);
2422
stream.Close();
2523
return content;

‎src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs‎

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
using BotSharp.Abstraction.Instructs.Models;
22
using BotSharp.Abstraction.Instructs;
3-
using System.IO;
4-
using BotSharp.Abstraction.Infrastructures;
53

64
namespace BotSharp.Core.Files.Services;
75

@@ -21,7 +19,12 @@ public async Task<string> ReadImages(string text, IEnumerable<InstructFileModel>
2119
{
2220
new RoleDialogModel(AgentRole.User, text)
2321
{
24-
Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? []
22+
Files = images?.Select(x => new BotSharpFile
23+
{
24+
FileUrl = x.FileUrl,
25+
FileData = x.FileData,
26+
ContentType = x.ContentType
27+
}).ToList() ?? []
2528
}
2629
});
2730

@@ -76,12 +79,11 @@ public async Task<RoleDialogModel> VaryImage(InstructFileModel image, InstructOp
7679

7780
var innerAgentId = options?.AgentId ?? Guid.Empty.ToString();
7881
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
79-
var bytes = await DownloadFile(image);
80-
using var stream = new MemoryStream();
81-
stream.Write(bytes, 0, bytes.Length);
82+
var binary = await DownloadFile(image);
83+
using var stream = binary.ToStream();
8284
stream.Position = 0;
8385

84-
var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
86+
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
8587
var message = await completion.GetImageVariation(new Agent()
8688
{
8789
Id = innerAgentId
@@ -113,12 +115,11 @@ public async Task<RoleDialogModel> EditImage(string text, InstructFileModel imag
113115
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
114116

115117
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
116-
var bytes = await DownloadFile(image);
117-
using var stream = new MemoryStream();
118-
stream.Write(bytes, 0, bytes.Length);
118+
var binary = await DownloadFile(image);
119+
using var stream = binary.ToStream();
119120
stream.Position = 0;
120121

121-
var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
122+
var fileName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
122123
var message = await completion.GetImageEdits(new Agent()
123124
{
124125
Id = innerAgentId
@@ -153,19 +154,17 @@ public async Task<RoleDialogModel> EditImage(string text, InstructFileModel imag
153154
var instruction = await GetAgentTemplate(innerAgentId, options?.TemplateName);
154155

155156
var completion = CompletionProvider.GetImageCompletion(_services, provider: options?.Provider ?? "openai", model: options?.Model ?? "dall-e-2");
156-
var imageBytes = await DownloadFile(image);
157-
var maskBytes = await DownloadFile(mask);
157+
var imageBinary = await DownloadFile(image);
158+
var maskBinary = await DownloadFile(mask);
158159

159-
using var imageStream = new MemoryStream();
160-
imageStream.Write(imageBytes, 0, imageBytes.Length);
160+
using var imageStream = imageBinary.ToStream();
161161
imageStream.Position = 0;
162162

163-
using var maskStream = new MemoryStream();
164-
maskStream.Write(maskBytes, 0, maskBytes.Length);
163+
using var maskStream = maskBinary.ToStream();
165164
maskStream.Position = 0;
166165

167-
var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}";
168-
var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}";
166+
var imageName = BuildFileName(image.FileName, image.FileExtension, "image", "png");
167+
var maskName = BuildFileName(image.FileName, image.FileExtension, "mask", "png");
169168
var message = await completion.GetImageEdits(new Agent()
170169
{
171170
Id = innerAgentId

‎src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs‎

Lines changed: 19 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public async Task<string> ReadPdf(string text, List<InstructFileModel> files, In
2222
try
2323
{
2424
var provider = options?.Provider ?? "openai";
25-
var pdfFiles = await DownloadFiles(sessionDir, files);
25+
var pdfFiles = await DownloadAndSaveFiles(sessionDir, files);
2626

2727
var targetFiles = pdfFiles;
2828
if (provider != "google-ai")
@@ -78,44 +78,38 @@ await hook.OnResponseGenerated(new InstructResponseModel
7878
}
7979

8080
#region Private methods
81-
private async Task<IEnumerable<string>> DownloadFiles(string dir, List<InstructFileModel> files, string extension = "pdf")
81+
private async Task<IEnumerable<string>> DownloadAndSaveFiles(string dir, List<InstructFileModel> files, string extension = "pdf")
8282
{
8383
if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty())
8484
{
8585
return Enumerable.Empty<string>();
8686
}
8787

88+
var downloadTasks = files.Select(x => DownloadFile(x));
89+
await Task.WhenAll(downloadTasks);
90+
8891
var locs = new List<string>();
89-
foreach (var file in files)
92+
for (int i = 0; i < files.Count; i++)
9093
{
91-
try
94+
var binary = downloadTasks.ElementAt(i).Result;
95+
if (binary == null || binary.IsEmpty)
9296
{
93-
var bytes = new byte[0];
94-
if (!string.IsNullOrEmpty(file.FileUrl))
95-
{
96-
var http = _services.GetRequiredService<IHttpClientFactory>();
97-
using var client = http.CreateClient();
98-
bytes = await client.GetByteArrayAsync(file.FileUrl);
99-
}
100-
else if (!string.IsNullOrEmpty(file.FileData))
101-
{
102-
(_, bytes) = FileUtility.GetFileInfoFromData(file.FileData);
103-
}
97+
continue;
98+
}
10499

105-
if (!bytes.IsNullOrEmpty())
106-
{
107-
var guid = Guid.NewGuid().ToString();
108-
var fileDir = _fileStorage.BuildDirectory(dir, guid);
109-
DeleteIfExistDirectory(fileDir, true);
100+
try
101+
{
102+
var guid = Guid.NewGuid().ToString();
103+
var fileDir = _fileStorage.BuildDirectory(dir, guid);
104+
DeleteIfExistDirectory(fileDir, createNew: true);
110105

111-
var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
112-
_fileStorage.SaveFileBytesToPath(outputDir, bytes);
113-
locs.Add(outputDir);
114-
}
106+
var outputDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
107+
_fileStorage.SaveFileBytesToPath(outputDir, binary);
108+
locs.Add(outputDir);
115109
}
116110
catch (Exception ex)
117111
{
118-
_logger.LogWarning(ex, $"Error when saving pdf file.");
112+
_logger.LogWarning(ex, $"Error when saving #{i + 1} {extension} file.");
119113
continue;
120114
}
121115
}

0 commit comments

Comments
 (0)