Merge pull request #928 from iceljc/master

iceljc · web-flow · commit f41820f83097 · 2025-03-10T12:21:11.000-05:00
refine state key search
diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs
@@ -12,7 +12,7 @@ public interface IAgentService
     Task<Agent> CreateAgent(Agent agent);
     Task<string> RefreshAgents();
     Task<PagedItems<Agent>> GetAgents(AgentFilter filter);
-    Task<List<IdName>> GetAgentOptions();
+    Task<List<IdName>> GetAgentOptions(List<string>? agentIds = null);
 
     /// <summary>
     /// Load agent configurations and trigger hooks
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Enums/MessageTypeName.cs
@@ -6,4 +6,5 @@ public static class MessageTypeName
     public const string Notification = "notification";
     public const string FunctionCall = "function";
     public const string Audio = "audio";
+    public const string Error = "error";
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs
@@ -62,14 +62,7 @@ Task<bool> SendMessage(string agentId,
 
     void SaveStates();
 
-    /// <summary>
-    /// Get conversation keys for searching
-    /// </summary>
-    /// <param name="query">search query</param>
-    /// <param name="convLimit">conversation limit</param>
-    /// <param name="preLoad">if pre-loading, then keys are not filter by the search query</param>
-    /// <returns></returns>
-    Task<List<string>> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false);
+    Task<List<string>> GetConversationStateSearhKeys(ConversationStateKeysFilter filter);
 
     Task<bool> MigrateLatestStates(int batchSize = 100, int errorLimit = 10);
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs
@@ -1,5 +1,5 @@
-using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Abstraction.Loggers.Models;
+using BotSharp.Abstraction.Repositories.Filters;
 
 namespace BotSharp.Abstraction.Loggers.Services;
 
@@ -12,5 +12,6 @@ public interface ILoggerService
 
     #region Instruction
     Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter);
+    Task<List<string>> GetInstructionLogSearchKeys(InstructLogKeysFilter filter);
     #endregion
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationFilter.cs
@@ -23,9 +23,9 @@ public class ConversationFilter
     /// <summary>
     /// Check whether each key in the list is in the conversation states and its value equals to target value if not empty 
     /// </summary>
-    public IEnumerable<KeyValue>? States { get; set; } = [];
+    public List<KeyValue>? States { get; set; }
 
-    public IEnumerable<string>? Tags { get; set; } = [];
+    public List<string>? Tags { get; set; }
 
     public static ConversationFilter Empty()
     {
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/ConversationStateKeysFilter.cs
@@ -0,0 +1,21 @@
+namespace BotSharp.Abstraction.Repositories.Filters;
+
+public class ConversationStateKeysFilter
+{
+    public string? Query { get; set; }
+    public int KeyLimit { get; set; } = 10;
+    public int ConvLimit { get; set; } = 100;
+    public bool PreLoad { get; set; }
+    public List<string>? AgentIds { get; set; }
+    public List<string>? UserIds { get; set; }
+
+    public ConversationStateKeysFilter()
+    {
+        
+    }
+
+    public static ConversationStateKeysFilter Empty()
+    {
+        return new();   
+    }
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogFilter.cs
@@ -1,4 +1,4 @@
-namespace BotSharp.Abstraction.Instructs.Models;
+namespace BotSharp.Abstraction.Repositories.Filters;
 
 public class InstructLogFilter : Pagination
 {
@@ -11,6 +11,6 @@ public class InstructLogFilter : Pagination
 
     public static InstructLogFilter Empty()
     {
-        return new InstructLogFilter();
+        return new();
     }
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/InstructLogKeysFilter.cs
@@ -0,0 +1,21 @@
+namespace BotSharp.Abstraction.Repositories.Filters;
+
+public class InstructLogKeysFilter
+{
+    public string? Query { get; set; }
+    public int KeyLimit { get; set; } = 10;
+    public int LogLimit { get; set; } = 100;
+    public bool PreLoad { get; set; }
+    public List<string>? AgentIds { get; set; }
+    public List<string>? UserIds { get; set; }
+
+    public InstructLogKeysFilter()
+    {
+        
+    }
+
+    public static InstructLogKeysFilter Empty()
+    {
+        return new();
+    }
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Repositories.Filters;
@@ -149,7 +148,7 @@ List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHou
          => throw new NotImplementedException();
     List<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
          => throw new NotImplementedException();
-    List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100)
+    List<string> GetConversationStateSearchKeys(ConversationStateKeysFilter filter)
          => throw new NotImplementedException();
     List<string> GetConversationsToMigrate(int batchSize = 100)
         => throw new NotImplementedException();
@@ -182,6 +181,9 @@ bool SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
 
     PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filter)
         => throw new NotImplementedException();
+
+    List<string> GetInstructionLogSearchKeys(InstructLogKeysFilter filter)
+         => throw new NotImplementedException();
     #endregion
 
     #region Statistics
diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs
@@ -10,7 +10,7 @@ public interface IUserService
     Task<PagedItems<User>> GetUsers(UserFilter filter);
     Task<List<User>> SearchLoginUsers(User filter);
     Task<User?> GetUserDetails(string userId, bool includeAgent = false);
-    Task<bool> IsAdminUser(string userId);
+    Task<(bool, User?)> IsAdminUser(string userId);
     Task<UserAuthorization> GetUserAuthorizations(IEnumerable<string>? agentIds = null);
     Task<bool> UpdateUser(User user, bool isUpdateUserAgents = false);
     Task<User> CreateUser(User user);
diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs
@@ -30,9 +30,12 @@ public async Task<PagedItems<Agent>> GetAgents(AgentFilter filter)
 #if !DEBUG
     [SharpCache(10, perInstanceCache: true)]
 #endif
-    public async Task<List<IdName>> GetAgentOptions()
+    public async Task<List<IdName>> GetAgentOptions(List<string>? agentIds)
     {
-        var agents = _db.GetAgents(AgentFilter.Empty());
+        var agents = _db.GetAgents(new AgentFilter
+        {
+            AgentIds = !agentIds.IsNullOrEmpty() ? agentIds : null
+        });
         return agents?.Select(x => new IdName(x.Id, x.Name))?.OrderBy(x => x.Name)?.ToList() ?? [];
     }
 
diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs
@@ -18,7 +18,7 @@ public async Task<string> RefreshAgents()
 
         var userIdentity = _services.GetRequiredService<IUserIdentity>();
         var userService = _services.GetRequiredService<IUserService>();
-        var isValid = await userService.IsAdminUser(userIdentity.Id);
+        var (isValid, _) = await userService.IsAdminUser(userIdentity.Id);
         if (!isValid)
         {
             return "Unauthorized user.";
diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs
@@ -222,17 +222,26 @@ public void SaveStates()
         _state.Save();
     }
 
-    public async Task<List<string>> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false)
+    public async Task<List<string>> GetConversationStateSearhKeys(ConversationStateKeysFilter filter)
     {
+        if (filter == null)
+        {
+            filter = ConversationStateKeysFilter.Empty();
+        }
+
         var keys = new List<string>();
-        if (!preload && string.IsNullOrWhiteSpace(query))
+        if (!filter.PreLoad && string.IsNullOrWhiteSpace(filter.Query))
         {
             return keys;
         }
 
+        var userService = _services.GetRequiredService<IUserService>();
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        keys = db.GetConversationStateSearchKeys(convUpperLimit: convLimit);
-        keys = preload ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList();
-        return keys.OrderBy(x => x).Take(keyLimit).ToList();
+
+        var (isAdmin, user) = await userService.IsAdminUser(_user.Id);
+        filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : [];
+        keys = db.GetConversationStateSearchKeys(filter);
+        keys = filter.PreLoad ? keys : keys.Where(x => x.Contains(filter.Query ?? string.Empty, StringComparison.OrdinalIgnoreCase)).ToList();
+        return keys.OrderBy(x => x).Take(filter.KeyLimit).ToList();
     }
 }
diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs
@@ -1,6 +1,4 @@
-using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Abstraction.Loggers.Models;
-using BotSharp.Abstraction.Users.Enums;
 using BotSharp.Abstraction.Users.Models;
 
 namespace BotSharp.Core.Loggers.Services;
@@ -15,11 +13,10 @@ public async Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLo
         }
 
         var userService = _services.GetRequiredService<IUserService>();
-        var user = await userService.GetUser(_user.Id);
-        var isAdmin = UserConstant.AdminRoles.Contains(user?.Role);
+        var (isAdmin, user) = await userService.IsAdminUser(_user.Id);
         if (!isAdmin && user?.Id == null) return new();
 
-        filter.UserIds = isAdmin ? [] : user?.Id != null ? [user.Id] : [];
+        filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null;
 
         var agents = new List<Agent>();
         var users = new List<User>();
@@ -61,4 +58,27 @@ public async Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLo
             Count = logs.Count
         };
     }
+
+    public async Task<List<string>> GetInstructionLogSearchKeys(InstructLogKeysFilter filter)
+    {
+        if (filter == null)
+        {
+            filter = InstructLogKeysFilter.Empty();
+        }
+
+        var keys = new List<string>();
+        if (!filter.PreLoad && string.IsNullOrWhiteSpace(filter.Query))
+        {
+            return keys;
+        }
+
+        var userService = _services.GetRequiredService<IUserService>();
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+
+        var (isAdmin, user) = await userService.IsAdminUser(_user.Id);
+        filter.UserIds = !isAdmin && user?.Id != null ? [user.Id] : null;
+        keys = db.GetInstructionLogSearchKeys(filter);
+        keys = filter.PreLoad ? keys : keys.Where(x => x.Contains(filter.Query ?? string.Empty, StringComparison.OrdinalIgnoreCase)).ToList();
+        return keys.OrderBy(x => x).Take(filter.KeyLimit).ToList();
+    }
 }
diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs
@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Loggers.Models;
+using BotSharp.Abstraction.Users.Models;
 using System.IO;
 
 namespace BotSharp.Core.Repository;
@@ -647,7 +648,7 @@ public List<string> TruncateConversation(string conversationId, string messageId
 #if !DEBUG
     [SharpCache(10)]
 #endif
-    public List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100)
+    public List<string> GetConversationStateSearchKeys(ConversationStateKeysFilter filter)
     {
         var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir);
         if (!Directory.Exists(dir)) return [];
@@ -658,26 +659,29 @@ public List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, in
         foreach (var d in Directory.GetDirectories(dir))
         {
             var convFile = Path.Combine(d, CONVERSATION_FILE);
-            var stateFile = Path.Combine(d, STATE_FILE);
-            if (!File.Exists(convFile) || !File.Exists(stateFile))
+            var latestStateFile = Path.Combine(d, CONV_LATEST_STATE_FILE);
+            if (!File.Exists(convFile) || !File.Exists(latestStateFile))
             {
                 continue;
             }
 
             var convJson = File.ReadAllText(convFile);
-            var stateJson = File.ReadAllText(stateFile);
+            var stateJson = File.ReadAllText(latestStateFile);
             var conv = JsonSerializer.Deserialize<Conversation>(convJson, _options);
-            var states = JsonSerializer.Deserialize<List<StateKeyValue>>(stateJson, _options);
-            if (conv == null || conv.DialogCount < messageLowerLimit)
+            var states = JsonSerializer.Deserialize<Dictionary<string, JsonDocument>>(stateJson, _options);
+            if (conv == null
+                || states.IsNullOrEmpty()
+                || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(conv.AgentId))
+                || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(conv.UserId)))
             {
                 continue;
             }
 
-            var stateKeys = states?.Select(x => x.Key)?.Distinct()?.ToList() ?? [];
+            var stateKeys = states?.Select(x => x.Key)?.ToList() ?? [];
             keys.AddRange(stateKeys);
             count++;
 
-            if (count >= convUpperLimit)
+            if (count >= filter.ConvLimit)
             {
                 break;
             }
diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs
@@ -1,4 +1,3 @@
-using BotSharp.Abstraction.Instructs.Models;
 using BotSharp.Abstraction.Loggers.Models;
 using System.IO;
 
@@ -271,6 +270,44 @@ public PagedItems<InstructionLogModel> GetInstructionLogs(InstructLogFilter filt
                 Count = logs.Count()
             };
         }
+
+        public List<string> GetInstructionLogSearchKeys(InstructLogKeysFilter filter)
+        {
+            var keys = new List<string>();
+            var baseDir = Path.Combine(_dbSettings.FileRepository, INSTRUCTION_LOG_FOLDER);
+            if (!Directory.Exists(baseDir))
+            {
+                return keys;
+            }
+
+            var count = 0;
+            var files = Directory.GetFiles(baseDir);
+            foreach (var file in files)
+            {
+                var json = File.ReadAllText(file);
+                var log = JsonSerializer.Deserialize<InstructionLogModel>(json, _options);
+                if (log == null) continue;
+
+                if (log == null
+                    || log.InnerStates.IsNullOrEmpty()
+                    || (!filter.UserIds.IsNullOrEmpty() && !filter.UserIds.Contains(log.UserId))
+                    || (!filter.AgentIds.IsNullOrEmpty() && !filter.AgentIds.Contains(log.AgentId)))
+                {
+                    continue;
+                }
+
+                var stateKeys = log.InnerStates.Select(x => x.Key)?.ToList() ?? [];
+                keys.AddRange(stateKeys);
+                count++;
+
+                if (count >= filter.LogLimit)
+                {
+                    break;
+                }
+            }
+
+            return keys.Distinct().ToList();
+        }
         #endregion
 
         #region Private methods
diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs
@@ -418,23 +418,24 @@ public async Task<PagedItems<User>> GetUsers(UserFilter filter)
         return users;
     }
 
-    public async Task<bool> IsAdminUser(string userId)
+    [SharpCache(10, perInstanceCache: true)]
+    public async Task<(bool, User?)> IsAdminUser(string userId)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
         var user = db.GetUserById(userId);
-        return user != null && UserConstant.AdminRoles.Contains(user.Role);
+        var isAdmin = user != null && UserConstant.AdminRoles.Contains(user.Role);
+        return (isAdmin, user);
     }
 
     public async Task<UserAuthorization> GetUserAuthorizations(IEnumerable<string>? agentIds = null)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var user = db.GetUserById(_user.Id);
+        var (isAdmin, user) = await IsAdminUser(_user.Id);
         var auth = new UserAuthorization();
 
         if (user == null) return auth;
 
-        auth.IsAdmin = UserConstant.AdminRoles.Contains(user.Role);
-
+        auth.IsAdmin = isAdmin;
         var role = db.GetRoles(new RoleFilter { Names = [user.Role] }).FirstOrDefault();
         var permissions = user.Permissions?.Any() == true ? user.Permissions : role?.Permissions ?? [];
         auth.Permissions = permissions;
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/PluginController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/PluginController.cs
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/RoleController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/RoleController.cs
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs

Original file line number	Diff line number	Diff line change
`@@ -6,4 +6,5 @@ public static class MessageTypeName`
`6`	`6`	`public const string Notification = "notification";`
`7`	`7`	`public const string FunctionCall = "function";`
`8`	`8`	`public const string Audio = "audio";`
	`9`	`+ public const string Error = "error";`
`9`	`10`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-namespace BotSharp.Abstraction.Instructs.Models;`
	`1`	`+namespace BotSharp.Abstraction.Repositories.Filters;`
`2`	`2`
`3`	`3`	`public class InstructLogFilter : Pagination`
`4`	`4`	`{`
`@@ -11,6 +11,6 @@ public class InstructLogFilter : Pagination`
`11`	`11`
`12`	`12`	`public static InstructLogFilter Empty()`
`13`	`13`	`{`
`14`		`- return new InstructLogFilter();`
	`14`	`+ return new();`
`15`	`15`	`}`
`16`	`16`	`}`