Features/add translation memory

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -2,6 +2,7 @@
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Tasks.Models;
+using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Models;
 
 namespace BotSharp.Abstraction.Repositories;
@@ -88,4 +89,10 @@ public interface IBotSharpRepository
     #region Statistics
     void IncrementConversationCount();
     #endregion
+
+    #region Translation
+    IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries);
+    bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs);
+
+    #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemory.cs

@@ -0,0 +1,25 @@
+namespace BotSharp.Abstraction.Translation.Models;
+
+public class TranslationMemory
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; }
+
+    [JsonPropertyName("original_text")]
+    public string OriginalText { get; set; }
+
+    [JsonPropertyName("hash_text")]
+    public string HashText { get; set; }
+
+    [JsonPropertyName("translations")]
+    public List<TranslationMemoryItem> Translations { get; set; } = new List<TranslationMemoryItem>();
+}
+
+public class TranslationMemoryItem
+{
+    [JsonPropertyName("translated_text")]
+    public string TranslatedText { get; set; }
+
+    [JsonPropertyName("language")]
+    public string Language { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Translation/Models/TranslationMemoryQuery.cs

@@ -0,0 +1,48 @@
+namespace BotSharp.Abstraction.Translation.Models;
+
+public class TranslationMemoryQuery
+{
+    public string OriginalText { get; set; }
+    public string HashText { get; set; }
+    public string Language { get; set; }
+
+    public TranslationMemoryQuery()
+    {
+
+    }
+
+    public override string ToString()
+    {
+        return $"[Origin: {OriginalText}] translates to [{Language}]";
+    }
+}
+
+public class TranslationMemoryInput : TranslationMemoryQuery
+{
+    public string TranslatedText { get; set; }
+
+    public TranslationMemoryInput()
+    {
+
+    }
+
+    public override string ToString()
+    {
+        return $"[Origin: {OriginalText}] -> [Translation: {TranslatedText}] (Language: {Language})";
+    }
+}
+
+public class TranslationMemoryOutput: TranslationMemoryQuery
+{
+    public string TranslatedText { get; set; }
+
+    public TranslationMemoryOutput()
+    {
+
+    }
+
+    public override string ToString()
+    {
+        return $"[Origin: {OriginalText}] -> [Translation: {TranslatedText}] (Language: {Language})";
+    }
+}

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -172,4 +172,8 @@
     <ProjectReference Include="..\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Translation\Models\" />
+  </ItemGroup>
+
 </Project>

src/Infrastructure/BotSharp.Core/Infrastructures/Utilities.cs

@@ -4,11 +4,11 @@ namespace BotSharp.Core.Infrastructures;
 
 public static class Utilities
 {
-    public static string HashText(string password, string salt)
+    public static string HashTextMd5(string text)
     {
         using var md5 = System.Security.Cryptography.MD5.Create();
 
-        var data = md5.ComputeHash(Encoding.UTF8.GetBytes(password + salt));
+        var data = md5.ComputeHash(Encoding.UTF8.GetBytes(text));
         var sb = new StringBuilder();
         foreach (var c in data)
         {
@@ -17,6 +17,19 @@ public static string HashText(string password, string salt)
         return sb.ToString();
     }
 
+    public static string HashTextSha256(string text)
+    {
+        using var sha256 = System.Security.Cryptography.SHA256.Create();
+
+        var data = sha256.ComputeHash(Encoding.UTF8.GetBytes(text));
+        var sb = new StringBuilder();
+        foreach(var c in data)
+        {
+            sb.Append(c.ToString("x2"));
+        }
+        return sb.ToString();
+    }
+
     public static (string, string) SplitAsTuple(this string str, string sep)
     {
         var splits = str.Split(sep);

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Tasks.Models;
+using BotSharp.Abstraction.Translation.Models;
 using BotSharp.Abstraction.Users.Models;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 
@@ -224,4 +225,11 @@ public void IncrementConversationCount()
         throw new NotImplementedException();
     }
     #endregion
+
+    #region Translation
+    public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
+        => throw new NotImplementedException();
+    public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs) =>
+        throw new NotImplementedException();
+    #endregion
 }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Translation.cs

@@ -0,0 +1,156 @@
+using BotSharp.Abstraction.Translation.Models;
+using System.IO;
+
+namespace BotSharp.Core.Repository;
+
+public partial class FileRepository
+{
+    public IEnumerable<TranslationMemoryOutput> GetTranslationMemories(IEnumerable<TranslationMemoryQuery> queries)
+    {
+        var list = new List<TranslationMemoryOutput>();
+        if (queries.IsNullOrEmpty())
+        {
+            return list;
+        }
+
+        var dir = Path.Combine(_dbSettings.FileRepository, "translation");
+        var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE);
+        if (!Directory.Exists(dir) || !File.Exists(file))
+        {
+            return list;
+        }
+
+        var content = File.ReadAllText(file);
+        if (string.IsNullOrWhiteSpace(content))
+        {
+            return list;
+        }
+
+        var memories = ReadTranslationMemoryContent(content);
+        foreach (var query in queries)
+        {
+            if (string.IsNullOrWhiteSpace(query.HashText) || string.IsNullOrWhiteSpace(query.Language))
+            {
+                continue;
+            }
+
+            var foundMemory = memories.FirstOrDefault(x => x.HashText.Equals(query.HashText));
+            if (foundMemory == null) continue;
+
+            var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(query.Language));
+            if (foundItem == null) continue;
+
+            list.Add(new TranslationMemoryOutput
+            {
+                OriginalText = query.OriginalText,
+                TranslatedText = foundItem.TranslatedText,
+                HashText = foundMemory.HashText,
+                Language = foundItem.Language,
+            });
+        }
+
+        return list;
+    }
+
+    public bool SaveTranslationMemories(IEnumerable<TranslationMemoryInput> inputs)
+    {
+        if (inputs.IsNullOrEmpty()) return false;
+
+        try
+        {
+            var dir = Path.Combine(_dbSettings.FileRepository, "translation");
+            var file = Path.Combine(dir, TRANSLATION_MEMORY_FILE);
+            if (!Directory.Exists(dir))
+            {
+                Directory.CreateDirectory(dir);
+            }
+
+            var content = string.Empty;
+            if (File.Exists(file))
+            {
+                content = File.ReadAllText(file);
+            }
+
+            var memories = ReadTranslationMemoryContent(content);
+
+            foreach (var input in inputs)
+            {
+                if (string.IsNullOrWhiteSpace(input.OriginalText) ||
+                    string.IsNullOrWhiteSpace(input.TranslatedText) ||
+                    string.IsNullOrWhiteSpace(input.HashText) ||
+                    string.IsNullOrWhiteSpace(input.Language))
+                {
+                    continue;
+                }
+
+                var newItem = new TranslationMemoryItem
+                {
+                    TranslatedText = input.TranslatedText,
+                    Language = input.Language
+                };
+
+                var foundMemory = memories?.FirstOrDefault(x => x.HashText.Equals(input.HashText));
+                if (foundMemory == null)
+                {
+                    var newMemory = new TranslationMemory
+                    {
+                        Id = Guid.NewGuid().ToString(),
+                        OriginalText = input.OriginalText,
+                        HashText = input.HashText,
+                        Translations = new List<TranslationMemoryItem> { newItem }
+                    };
+
+                    if (memories == null)
+                    {
+                        memories = new List<TranslationMemory> { newMemory };
+                    }
+                    else
+                    {
+                        memories.Add(newMemory);
+                    }
+                }
+                else
+                {
+                    var foundItem = foundMemory.Translations?.FirstOrDefault(x => x.Language.Equals(input.Language));
+                    if (foundItem != null) continue;
+
+                    if (foundMemory.Translations == null)
+                    {
+                        foundMemory.Translations = new List<TranslationMemoryItem> { newItem };
+                    }
+                    else
+                    {
+                        foundMemory.Translations.Add(newItem);
+                    }
+                }
+            }
+
+            var json = JsonSerializer.Serialize(memories, _options);
+            File.WriteAllText(file, json);
+            return true;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when saving translation memories: {ex.Message}");
+            return false;
+        }
+    }
+
+    private List<TranslationMemory> ReadTranslationMemoryContent(string? content)
+    {
+        var memories = new List<TranslationMemory>();
+
+        try
+        {
+            if (string.IsNullOrWhiteSpace(content))
+            {
+                return memories;
+            }
+
+           memories = JsonSerializer.Deserialize<List<TranslationMemory>>(content, _options) ?? new List<TranslationMemory>();
+        }
+        catch {}
+
+        return memories;
+    }
+}

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs

@@ -17,6 +17,7 @@ public partial class FileRepository : IBotSharpRepository
     private readonly AgentSettings _agentSettings;
     private readonly ConversationSetting _conversationSettings;
     private readonly StatisticsSettings _statisticsSetting;
+    private readonly ILogger<FileRepository> _logger;
     private JsonSerializerOptions _options;
 
     private const string AGENT_FILE = "agent.json";
@@ -34,19 +35,22 @@ public partial class FileRepository : IBotSharpRepository
     private const string PLUGIN_CONFIG_FILE = "config.json";
     private const string AGENT_TASK_PREFIX = "#metadata";
     private const string AGENT_TASK_SUFFIX = "/metadata";
+    private const string TRANSLATION_MEMORY_FILE = "memory.json";
 
     public FileRepository(
         IServiceProvider services,
         BotSharpDatabaseSettings dbSettings,
         AgentSettings agentSettings,
         ConversationSetting conversationSettings,
-        StatisticsSettings statisticsSettings)
+        StatisticsSettings statisticsSettings,
+         ILogger<FileRepository> logger)
     {
         _services = services;
         _dbSettings = dbSettings;
         _agentSettings = agentSettings;
         _conversationSettings = conversationSettings;
         _statisticsSetting = statisticsSettings;
+        _logger = logger;
 
         _options = new JsonSerializerOptions
         {

src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/TwoStagePlanner.cs

@@ -32,7 +32,7 @@ public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string m
         if (_plan1st.IsNullOrEmpty() && _plan2nd.IsNullOrEmpty())
         {
             Directory.CreateDirectory(tempDir);
-            _md5 = Utilities.HashText(string.Join(".", dialogs.Where(x => x.Role == AgentRole.User)), "botsharp");
+            _md5 = Utilities.HashTextMd5($"{string.Join(".", dialogs.Where(x => x.Role == AgentRole.User))}{"botsharp"}");
             var filePath = Path.Combine(tempDir, $"{_md5}-1st.json");
             FirstStagePlan[] items = new FirstStagePlan[0];
             if (File.Exists(filePath))