From 64dca85d77650112ecc80d6b3b4b445b39539e3a Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 14 Jun 2024 15:39:56 -0500 Subject: [PATCH 1/2] add translation memory --- .../Repositories/IBotSharpRepository.cs | 7 + .../Translation/Models/TranslationMemory.cs | 25 +++ .../Models/TranslationMemoryQuery.cs | 43 +++++ .../BotSharp.Core/BotSharp.Core.csproj | 4 + .../Infrastructures/Utilities.cs | 17 +- .../Repository/BotSharpDbContext.cs | 8 + .../FileRepository.Translation.cs | 156 ++++++++++++++++++ .../FileRepository/FileRepository.cs | 6 +- .../TwoStagePlanner/TwoStagePlanner.cs | 2 +- .../Translation/TranslationService.cs | 83 +++++++--- .../Users/Services/UserService.cs | 2 +- .../Repository/MongoRepository.Translation.cs | 14 ++ .../BotSharp.Plugin.MongoStorage/Using.cs | 1 + 13 files changed, 339 insertions(+), 29 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs create mode 100644 src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 6adcc8ad8..8c3a0e81b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -2,6 +2,7 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Tasks.Models; +using BotSharp.Abstraction.Translation.Models; using BotSharp.Abstraction.Users.Models; namespace BotSharp.Abstraction.Repositories; @@ -88,4 +89,10 @@ public interface IBotSharpRepository #region Statistics void IncrementConversationCount(); #endregion + + #region Translation + IEnumerable GetTranslationMemories(IEnumerable queries); + bool SaveTranslationMemories(IEnumerable inputs); + + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs new file mode 100644 index 000000000..73e1f9c0d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs @@ -0,0 +1,25 @@ +namespace BotSharp.Abstraction.Translation.Models; + +public class TranslationMemory +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("original_text")] + public string OriginalText { get; set; } + + [JsonPropertyName("hash_text")] + public string HashText { get; set; } + + [JsonPropertyName("memories")] + public List Memories { get; set; } = new List(); +} + +public class TranslationMemoryItem +{ + [JsonPropertyName("translated_text")] + public string TranslatedText { get; set; } + + [JsonPropertyName("language")] + public string Language { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs new file mode 100644 index 000000000..66f1df8e4 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs @@ -0,0 +1,43 @@ +namespace BotSharp.Abstraction.Translation.Models; + +public class TranslationMemoryQuery +{ + public string OriginalText { get; set; } + public string HashText { get; set; } + public string Language { get; set; } + + public TranslationMemoryQuery() + { + + } +} + +public class TranslationMemoryInput : TranslationMemoryQuery +{ + public string TranslatedText { get; set; } + + public TranslationMemoryInput() + { + + } + + public override string ToString() + { + return $"Origin: {OriginalText} -> Translation: {TranslatedText} (Language: {Language})"; + } +} + +public class TranslationMemoryOutput: TranslationMemoryQuery +{ + public string TranslatedText { get; set; } + + public TranslationMemoryOutput() + { + + } + + public override string ToString() + { + return $"Origin: {OriginalText} -> Translation: {TranslatedText} (Language: {Language})"; + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 7bdd84645..eb01f6a07 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -172,4 +172,8 @@ + + + + diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs index 449d2c3fa..4cb268889 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs @@ -4,11 +4,11 @@ namespace BotSharp.Core.Infrastructures; public static class Utilities { - public static string HashText(string password, string salt) + public static string HashTextMd5(string text) { using var md5 = System.Security.Cryptography.MD5.Create(); - var data = md5.ComputeHash(Encoding.UTF8.GetBytes(password + salt)); + var data = md5.ComputeHash(Encoding.UTF8.GetBytes(text)); var sb = new StringBuilder(); foreach (var c in data) { @@ -17,6 +17,19 @@ public static string HashText(string password, string salt) return sb.ToString(); } + public static string HashTextSha256(string text) + { + using var sha256 = System.Security.Cryptography.SHA256.Create(); + + var data = sha256.ComputeHash(Encoding.UTF8.GetBytes(text)); + var sb = new StringBuilder(); + foreach(var c in data) + { + sb.Append(c.ToString("x2")); + } + return sb.ToString(); + } + public static (string, string) SplitAsTuple(this string str, string sep) { var splits = str.Split(sep); diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 6bba6c2a2..5ce769012 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Tasks.Models; +using BotSharp.Abstraction.Translation.Models; using BotSharp.Abstraction.Users.Models; using Microsoft.EntityFrameworkCore.Infrastructure; @@ -224,4 +225,11 @@ public void IncrementConversationCount() throw new NotImplementedException(); } #endregion + + #region Translation + public IEnumerable GetTranslationMemories(IEnumerable queries) + => throw new NotImplementedException(); + public bool SaveTranslationMemories(IEnumerable inputs) => + throw new NotImplementedException(); + #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs new file mode 100644 index 000000000..8d8a14e77 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs @@ -0,0 +1,156 @@ +using BotSharp.Abstraction.Translation.Models; +using System.IO; + +namespace BotSharp.Core.Repository; + +public partial class FileRepository +{ + public IEnumerable GetTranslationMemories(IEnumerable queries) + { + var list = new List(); + if (queries.IsNullOrEmpty()) + { + return list; + } + + var dir = Path.Combine(_dbSettings.FileRepository, "translation"); + var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE); + if (!Directory.Exists(dir) || !File.Exists(file)) + { + return list; + } + + var content = File.ReadAllText(file); + if (string.IsNullOrWhiteSpace(content)) + { + return list; + } + + var memories = ReadTranslationMemoryContent(content); + foreach (var query in queries) + { + if (string.IsNullOrWhiteSpace(query.HashText) || string.IsNullOrWhiteSpace(query.Language)) + { + continue; + } + + var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText)); + if (foundMemory == null) continue; + + var foundItem = foundMemory.Memories?.FirstOrDefault(x => x.Language.Equals(query.Language)); + if (foundItem == null) continue; + + list.Add(new TranslationMemoryOutput + { + OriginalText = query.OriginalText, + TranslatedText = foundItem.TranslatedText, + HashText = foundMemory.HashText, + Language = foundItem.Language, + }); + } + + return list; + } + + public bool SaveTranslationMemories(IEnumerable inputs) + { + if (inputs.IsNullOrEmpty()) return false; + + try + { + var dir = Path.Combine(_dbSettings.FileRepository, "translation"); + var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE); + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + var content = string.Empty; + if (File.Exists(file)) + { + content = File.ReadAllText(file); + } + + var memories = ReadTranslationMemoryContent(content); + + foreach (var input in inputs) + { + if (string.IsNullOrWhiteSpace(input.OriginalText) || + string.IsNullOrWhiteSpace(input.TranslatedText) || + string.IsNullOrWhiteSpace(input.HashText) || + string.IsNullOrWhiteSpace(input.Language)) + { + continue; + } + + var newItem = new TranslationMemoryItem + { + TranslatedText = input.TranslatedText, + Language = input.Language + }; + + var foundMemory = memories?.FirstOrDefault(x => x.HashText.Equals(input.HashText)); + if (foundMemory == null) + { + var newMemory = new TranslationMemory + { + Id = Guid.NewGuid().ToString(), + OriginalText = input.OriginalText, + HashText = input.HashText, + Memories = new List { newItem } + }; + + if (memories == null) + { + memories = new List { newMemory }; + } + else + { + memories.Add(newMemory); + } + } + else + { + var foundItem = foundMemory.Memories?.FirstOrDefault(x => x.Language.Equals(input.Language)); + if (foundItem != null) continue; + + if (foundMemory.Memories == null) + { + foundMemory.Memories = new List { newItem }; + } + else + { + foundMemory.Memories.Add(newItem); + } + } + } + + var json = JsonSerializer.Serialize(memories, _options); + File.WriteAllText(file, json); + return true; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving translation memories: {ex.Message}"); + return false; + } + } + + private List ReadTranslationMemoryContent(string? content) + { + var memories = new List(); + + try + { + if (string.IsNullOrWhiteSpace(content)) + { + return memories; + } + + memories = JsonSerializer.Deserialize>(content, _options) ?? new List(); + } + catch {} + + return memories; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index 2094b821f..a30142be3 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -17,6 +17,7 @@ public partial class FileRepository : IBotSharpRepository private readonly AgentSettings _agentSettings; private readonly ConversationSetting _conversationSettings; private readonly StatisticsSettings _statisticsSetting; + private readonly ILogger _logger; private JsonSerializerOptions _options; private const string AGENT_FILE = "agent.json"; @@ -34,19 +35,22 @@ public partial class FileRepository : IBotSharpRepository private const string PLUGIN_CONFIG_FILE = "config.json"; private const string AGENT_TASK_PREFIX = "#metadata"; private const string AGENT_TASK_SUFFIX = "/metadata"; + private const string TRANSLATION_MEMORY_FILE = "memory.json"; public FileRepository( IServiceProvider services, BotSharpDatabaseSettings dbSettings, AgentSettings agentSettings, ConversationSetting conversationSettings, - StatisticsSettings statisticsSettings) + StatisticsSettings statisticsSettings, + ILogger logger) { _services = services; _dbSettings = dbSettings; _agentSettings = agentSettings; _conversationSettings = conversationSettings; _statisticsSetting = statisticsSettings; + _logger = logger; _options = new JsonSerializerOptions { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs index 6df03a089..d815ee5a7 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs @@ -32,7 +32,7 @@ public async Task GetNextInstruction(Agent router, string m if (_plan1st.IsNullOrEmpty() && _plan2nd.IsNullOrEmpty()) { Directory.CreateDirectory(tempDir); - _md5 = Utilities.HashText(string.Join(".", dialogs.Where(x => x.Role == AgentRole.User)), "botsharp"); + _md5 = Utilities.HashTextMd5($"{string.Join(".", dialogs.Where(x => x.Role == AgentRole.User))}{"botsharp"}"); var filePath = Path.Combine(tempDir, $"{_md5}-1st.json"); FirstStagePlan[] items = new FirstStagePlan[0]; if (File.Exists(filePath)) diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs index ec33f609d..d72bef2a6 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -1,4 +1,3 @@ -using Amazon.Runtime.Internal.Transform; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Options; @@ -13,17 +12,21 @@ namespace BotSharp.Core.Translation; public class TranslationService : ITranslationService { private readonly IServiceProvider _services; + private readonly IBotSharpRepository _db; private readonly ILogger _logger; private readonly BotSharpOptions _options; private Agent _router; private string _messageId; private IChatCompletion _completion; - public TranslationService(IServiceProvider services, + public TranslationService( + IServiceProvider services, + IBotSharpRepository db, ILogger logger, BotSharpOptions options) { _services = services; + _db = db; _logger = logger; _options = options; } @@ -56,41 +59,73 @@ public async Task Translate(Agent router, string messageId, T data, string model: _router?.LlmConfig?.Model); var template = _router.Templates.First(x => x.Name == "translation_prompt").Content; + var map = new Dictionary(); var keys = unique.ToArray(); - var texts = unique.ToArray() + + #region Search memory + var queries = keys.Select(x => new TranslationMemoryQuery + { + OriginalText = x, + HashText = Utilities.HashTextSha256(x), + Language = language + }).ToList(); + var memories = _db.GetTranslationMemories(queries); + var memoryHashes = memories.Select(x => x.HashText).ToList(); + + foreach (var memory in memories) + { + map[memory.OriginalText] = memory.TranslatedText; + } + + var outOfMemoryList = queries.Where(x => !memoryHashes.Contains(x.HashText)).ToList(); + #endregion + + var texts = outOfMemoryList.ToArray() .Select((text, i) => new TranslationInput { Id = i + 1, - Text = text + Text = text.OriginalText }).ToList(); try { - var translatedStringList = await InnerTranslate(texts, language, template); - - int retry = 0; - while (translatedStringList.Texts.Length != texts.Count && retry < 3) + if (!texts.IsNullOrEmpty()) { - translatedStringList = await InnerTranslate(texts, language, template); - retry++; - } + var translatedStringList = await InnerTranslate(texts, language, template); - // Override language if it's Unknown, it's used to output the corresponding language. - var states = _services.GetRequiredService(); - if (!states.ContainsState(StateConst.LANGUAGE)) - { - var inputLanguage = string.IsNullOrEmpty(translatedStringList.InputLanguage) ? LanguageType.ENGLISH : translatedStringList.InputLanguage; - states.SetState(StateConst.LANGUAGE, inputLanguage, activeRounds: 1); - } + int retry = 0; + while (translatedStringList.Texts.Length != texts.Count && retry < 3) + { + translatedStringList = await InnerTranslate(texts, language, template); + retry++; + } - var translatedTexts = translatedStringList.Texts; - var map = new Dictionary(); + // Override language if it's Unknown, it's used to output the corresponding language. + var states = _services.GetRequiredService(); + if (!states.ContainsState(StateConst.LANGUAGE)) + { + var inputLanguage = string.IsNullOrEmpty(translatedStringList.InputLanguage) ? LanguageType.ENGLISH : translatedStringList.InputLanguage; + states.SetState(StateConst.LANGUAGE, inputLanguage, activeRounds: 1); + } - for (var i = 0; i < texts.Count; i++) - { - map[keys[i]] = translatedTexts[i].Text; - } + var translatedTexts = translatedStringList.Texts; + var memoryInputs = new List(); + for (var i = 0; i < texts.Count; i++) + { + map[outOfMemoryList[i].OriginalText] = translatedTexts[i].Text; + memoryInputs.Add(new TranslationMemoryInput + { + OriginalText = outOfMemoryList[i].OriginalText, + HashText = outOfMemoryList[i].HashText, + TranslatedText = translatedTexts[i].Text, + Language = language + }); + } +; + _db.SaveTranslationMemories(memoryInputs); + } + clonedData = Assign(clonedData, map); } catch (Exception ex) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index 81a7a79c6..aadeb2e17 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -61,7 +61,7 @@ record = user; record.Phone = "+" + Regex.Match(user.Phone, @"\d+").Value; } record.Salt = Guid.NewGuid().ToString("N"); - record.Password = Utilities.HashText(user.Password, record.Salt); + record.Password = Utilities.HashTextMd5($"{user.Password}{record.Salt}"); if (_setting.NewUserVerification) { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs new file mode 100644 index 000000000..9f2f98a16 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs @@ -0,0 +1,14 @@ +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + public IEnumerable GetTranslationMemories(IEnumerable queries) + { + throw new NotImplementedException(); + } + + public bool SaveTranslationMemories(IEnumerable inputs) + { + throw new NotImplementedException(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs index ca13f204b..f40df94a9 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs @@ -7,6 +7,7 @@ global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Utilities; global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Translation.Models; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; global using MongoDB.Bson; From 54b551ae4473203b7396c035d9eb3f3ff4512a14 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 14 Jun 2024 17:10:25 -0500 Subject: [PATCH 2/2] add mongo translation memory --- .../Translation/Models/TranslationMemory.cs | 4 +- .../Models/TranslationMemoryQuery.cs | 9 +- .../FileRepository.Translation.cs | 12 +- .../Translation/TranslationService.cs | 2 +- .../Collections/TranslationMemoryDocument.cs | 8 ++ .../Models/TranslationMemoryMongoElement.cs | 25 ++++ .../MongoDbContext.cs | 5 +- .../Repository/MongoRepository.Agent.cs | 2 - .../Repository/MongoRepository.AgentTask.cs | 1 - .../MongoRepository.Conversation.cs | 4 - .../Repository/MongoRepository.Log.cs | 2 - .../Repository/MongoRepository.Plugin.cs | 1 - .../Repository/MongoRepository.Transaction.cs | 3 - .../Repository/MongoRepository.Translation.cs | 115 +++++++++++++++++- .../Repository/MongoRepository.User.cs | 1 - .../Repository/MongoRepository.cs | 8 +- .../BotSharp.Plugin.MongoStorage/Using.cs | 4 +- 17 files changed, 175 insertions(+), 31 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Collections/TranslationMemoryDocument.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs index 73e1f9c0d..88460268d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs @@ -11,8 +11,8 @@ public class TranslationMemory [JsonPropertyName("hash_text")] public string HashText { get; set; } - [JsonPropertyName("memories")] - public List Memories { get; set; } = new List(); + [JsonPropertyName("translations")] + public List Translations { get; set; } = new List(); } public class TranslationMemoryItem diff --git a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs index 66f1df8e4..be5191f85 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs @@ -10,6 +10,11 @@ public TranslationMemoryQuery() { } + + public override string ToString() + { + return $"[Origin: {OriginalText}] translates to [{Language}]"; + } } public class TranslationMemoryInput : TranslationMemoryQuery @@ -23,7 +28,7 @@ public TranslationMemoryInput() public override string ToString() { - return $"Origin: {OriginalText} -> Translation: {TranslatedText} (Language: {Language})"; + return $"[Origin: {OriginalText}] -> [Translation: {TranslatedText}] (Language: {Language})"; } } @@ -38,6 +43,6 @@ public TranslationMemoryOutput() public override string ToString() { - return $"Origin: {OriginalText} -> Translation: {TranslatedText} (Language: {Language})"; + return $"[Origin: {OriginalText}] -> [Translation: {TranslatedText}] (Language: {Language})"; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs index 8d8a14e77..e52e09c24 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs @@ -37,7 +37,7 @@ public IEnumerable GetTranslationMemories(IEnumerable x.HashText.Equals(query.HashText)); if (foundMemory == null) continue; - var foundItem = foundMemory.Memories?.FirstOrDefault(x => x.Language.Equals(query.Language)); + var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(query.Language)); if (foundItem == null) continue; list.Add(new TranslationMemoryOutput @@ -97,7 +97,7 @@ public bool SaveTranslationMemories(IEnumerable inputs) Id = Guid.NewGuid().ToString(), OriginalText = input.OriginalText, HashText = input.HashText, - Memories = new List { newItem } + Translations = new List { newItem } }; if (memories == null) @@ -111,16 +111,16 @@ public bool SaveTranslationMemories(IEnumerable inputs) } else { - var foundItem = foundMemory.Memories?.FirstOrDefault(x => x.Language.Equals(input.Language)); + var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(input.Language)); if (foundItem != null) continue; - if (foundMemory.Memories == null) + if (foundMemory.Translations == null) { - foundMemory.Memories = new List { newItem }; + foundMemory.Translations = new List { newItem }; } else { - foundMemory.Memories.Add(newItem); + foundMemory.Translations.Add(newItem); } } } diff --git a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs index d72bef2a6..399f9c0d4 100644 --- a/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs +++ b/src/Infrastructure/BotSharp.Core/Translation/TranslationService.cs @@ -122,7 +122,7 @@ public async Task Translate(Agent router, string messageId, T data, string Language = language }); } -; + _db.SaveTranslationMemories(memoryInputs); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/TranslationMemoryDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/TranslationMemoryDocument.cs new file mode 100644 index 000000000..54c11b3e7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/TranslationMemoryDocument.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class TranslationMemoryDocument : MongoBase +{ + public string OriginalText { get; set; } + public string HashText { get; set; } + public List Translations { get; set; } = new List(); +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs new file mode 100644 index 000000000..0d090b78f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/TranslationMemoryMongoElement.cs @@ -0,0 +1,25 @@ +namespace BotSharp.Plugin.MongoStorage.Models; + +public class TranslationMemoryMongoElement +{ + public string TranslatedText { get; set; } + public string Language { get; set; } + + public static TranslationMemoryMongoElement ToMongoElement(TranslationMemoryItem item) + { + return new TranslationMemoryMongoElement + { + TranslatedText = item.TranslatedText, + Language = item.Language + }; + } + + public static TranslationMemoryItem ToDomainElement(TranslationMemoryMongoElement element) + { + return new TranslationMemoryItem + { + TranslatedText = element.TranslatedText, + Language = element.Language + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index ca691e058..dfbbfbaaf 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.MongoStorage.Collections; - namespace BotSharp.Plugin.MongoStorage; public class MongoDbContext @@ -117,4 +115,7 @@ public IMongoCollection UserAgents public IMongoCollection Plugins => Database.GetCollection($"{_collectionPrefix}_Plugins"); + + public IMongoCollection TranslationMemories + => Database.GetCollection($"{_collectionPrefix}_TranslationMemories"); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 97ceee9f5..167f4b794 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -2,8 +2,6 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Routing.Models; -using BotSharp.Plugin.MongoStorage.Collections; -using BotSharp.Plugin.MongoStorage.Models; namespace BotSharp.Plugin.MongoStorage.Repository; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs index a3df61da8..a00858c4b 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Tasks.Models; -using BotSharp.Plugin.MongoStorage.Collections; namespace BotSharp.Plugin.MongoStorage.Repository; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index aefd0db55..bed3a255f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,9 +1,5 @@ using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Repositories.Filters; -using BotSharp.Abstraction.Repositories.Models; -using BotSharp.Plugin.MongoStorage.Collections; -using BotSharp.Plugin.MongoStorage.Models; namespace BotSharp.Plugin.MongoStorage.Repository; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs index 94adacc3b..316312306 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs @@ -1,6 +1,4 @@ using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Plugin.MongoStorage.Collections; -using BotSharp.Plugin.MongoStorage.Models; namespace BotSharp.Plugin.MongoStorage.Repository; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs index c6bc60b3a..b26da408e 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Plugin.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Plugins.Models; -using BotSharp.Plugin.MongoStorage.Collections; namespace BotSharp.Plugin.MongoStorage.Repository; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs index 470b1fb19..44320bd91 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs @@ -1,8 +1,5 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Users.Models; -using BotSharp.Plugin.MongoStorage.Collections; -using BotSharp.Plugin.MongoStorage.Models; - namespace BotSharp.Plugin.MongoStorage.Repository; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs index 9f2f98a16..5f0e987e3 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Translation.cs @@ -1,14 +1,125 @@ +using Microsoft.Extensions.Logging; +using System.Threading; + namespace BotSharp.Plugin.MongoStorage.Repository; public partial class MongoRepository { public IEnumerable GetTranslationMemories(IEnumerable queries) { - throw new NotImplementedException(); + var list = new List(); + if (queries.IsNullOrEmpty()) + { + return list; + } + + var hashTexts = queries.Where(x => !string.IsNullOrEmpty(x.HashText)).Select(x => x.HashText).ToList(); + var filter = Builders.Filter.In(x => x.HashText, hashTexts); + var memories = _dc.TranslationMemories.Find(filter).ToList(); + if (memories.IsNullOrEmpty()) return list; + + foreach (var query in queries) + { + if (string.IsNullOrWhiteSpace(query.HashText) || string.IsNullOrWhiteSpace(query.Language)) + { + continue; + } + + var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText)); + if (foundMemory == null) continue; + + var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(query.Language)); + if (foundItem == null) continue; + + list.Add(new TranslationMemoryOutput + { + OriginalText = query.OriginalText, + TranslatedText = foundItem.TranslatedText, + HashText = foundMemory.HashText, + Language = foundItem.Language, + }); + } + + return list; } public bool SaveTranslationMemories(IEnumerable inputs) { - throw new NotImplementedException(); + if (inputs.IsNullOrEmpty()) return false; + + var hashTexts = inputs.Where(x => !string.IsNullOrEmpty(x.HashText)).Select(x => x.HashText).ToList(); + var filter = Builders.Filter.In(x => x.HashText, hashTexts); + var memories = _dc.TranslationMemories.Find(filter)?.ToList() ?? new List(); + + var newMemories = new List(); + var updateMemories = new List(); + + try + { + foreach (var input in inputs) + { + if (string.IsNullOrWhiteSpace(input.OriginalText) || + string.IsNullOrWhiteSpace(input.TranslatedText) || + string.IsNullOrWhiteSpace(input.HashText) || + string.IsNullOrWhiteSpace(input.Language)) + { + continue; + } + + var newItem = new TranslationMemoryMongoElement + { + TranslatedText = input.TranslatedText, + Language = input.Language + }; + + var foundMemory = memories?.FirstOrDefault(x => x.HashText.Equals(input.HashText)); + if (foundMemory == null) + { + newMemories.Add(new TranslationMemoryDocument + { + Id = Guid.NewGuid().ToString(), + OriginalText = input.OriginalText, + HashText = input.HashText, + Translations = new List { newItem } + }); + } + else + { + var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(input.Language)); + if (foundItem != null) continue; + + if (foundMemory.Translations == null) + { + foundMemory.Translations = new List { newItem }; + } + else + { + foundMemory.Translations.Add(newItem); + } + updateMemories.Add(foundMemory); + } + } + + if (!newMemories.IsNullOrEmpty()) + { + _dc.TranslationMemories.InsertMany(newMemories); + } + + if (!updateMemories.IsNullOrEmpty()) + { + foreach (var mem in updateMemories) + { + var updateFilter = Builders.Filter.Eq(x => x.Id, mem.Id); + _dc.TranslationMemories.ReplaceOne(updateFilter, mem); + Thread.Sleep(50); + } + } + return true; + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving translation memories: {ex.Message}"); + return false; + } } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 9a3ce3f47..11f482871 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Users.Models; -using BotSharp.Plugin.MongoStorage.Collections; namespace BotSharp.Plugin.MongoStorage.Repository; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 02d21c439..133edba14 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Users.Models; +using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -8,12 +9,17 @@ public partial class MongoRepository : IBotSharpRepository { private readonly MongoDbContext _dc; private readonly IServiceProvider _services; + private readonly ILogger _logger; private UpdateOptions _options; - public MongoRepository(MongoDbContext dc, IServiceProvider services) + public MongoRepository( + MongoDbContext dc, + IServiceProvider services, + ILogger logger) { _dc = dc; _services = services; + _logger = logger; _options = new UpdateOptions { IsUpsert = true, diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs index f40df94a9..7c74b648f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Using.cs @@ -12,4 +12,6 @@ global using Microsoft.Extensions.DependencyInjection; global using MongoDB.Bson; global using MongoDB.Driver; -global using MongoDB.Bson.Serialization.Attributes; \ No newline at end of file +global using MongoDB.Bson.Serialization.Attributes; +global using BotSharp.Plugin.MongoStorage.Collections; +global using BotSharp.Plugin.MongoStorage.Models; \ No newline at end of file