Merge pull request #361 from iceljc/features/refine-conversation-functionality

Oceania2018 · web-flow · commit d33bd37615fd · 2024-03-25T17:31:54.000-05:00
Features/refine conversation functionality
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs
@@ -24,6 +24,8 @@ public class Conversation
 
     public string Channel { get; set; } = ConversationChannel.OpenAPI;
 
+    public int DialogCount { get; set; }
+
     public DateTime UpdatedTime { get; set; } = DateTime.UtcNow;
     public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
 
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/RichContentJsonConverter .cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/RichContentJsonConverter .cs
@@ -16,7 +16,7 @@ public class RichContentJsonConverter : JsonConverter<IRichMessage>
         if (root.TryGetProperty("rich_type", out JsonElement element))
         {
             var richType = element.GetString();
-            res = parser.ParseRichMessage(richType, jsonText, options);
+            res = parser.ParseRichMessage(richType, jsonText, root, options);
         }
 
         return res;
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/TemplateMessageJsonConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/TemplateMessageJsonConverter.cs
@@ -16,7 +16,7 @@ public class TemplateMessageJsonConverter : JsonConverter<ITemplateMessage>
         if (root.TryGetProperty("template_type", out JsonElement element))
         {
             var templateType = element.GetString();
-            res = parser.ParseTemplateMessage(templateType, jsonText, options);
+            res = parser.ParseTemplateMessage(templateType, jsonText, root, options);
         }
 
         return res;
diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/MessageParser.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/MessageParser.cs
@@ -12,7 +12,7 @@ public MessageParser()
     {
     }
 
-    public IRichMessage? ParseRichMessage(string richType, string jsonText, JsonSerializerOptions options)
+    public IRichMessage? ParseRichMessage(string richType, string jsonText, JsonElement root, JsonSerializerOptions options)
     {
         IRichMessage? res = null;
 
@@ -36,11 +36,22 @@ public MessageParser()
         {
             res = JsonSerializer.Deserialize<TextMessage>(jsonText, options);
         }
+        else if (richType == RichTypeEnum.GenericTemplate)
+        {
+            if (root.TryGetProperty("element_type", out var element))
+            {
+                var elementType = element.GetString();
+                if (elementType == typeof(GenericElement).Name)
+                {
+                    res = JsonSerializer.Deserialize<GenericTemplateMessage<GenericElement>>(jsonText, options);
+                }
+            }
+        }
 
         return res;
     }
 
-    public ITemplateMessage? ParseTemplateMessage(string templateType, string jsonText, JsonSerializerOptions options)
+    public ITemplateMessage? ParseTemplateMessage(string templateType, string jsonText, JsonElement root, JsonSerializerOptions options)
     {
         ITemplateMessage? res = null;
 
@@ -60,6 +71,17 @@ public MessageParser()
         {
             res = JsonSerializer.Deserialize<ProductTemplateMessage>(jsonText, options);
         }
+        else if (templateType == TemplateTypeEnum.Generic)
+        {
+            if (root.TryGetProperty("element_type", out var element))
+            {
+                var elementType = element.GetString();
+                if (elementType == typeof(GenericElement).Name)
+                {
+                    res = JsonSerializer.Deserialize<GenericTemplateMessage<GenericElement>>(jsonText, options);
+                }
+            }
+        }
 
         return res;
     }
diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs
@@ -46,7 +46,7 @@ public void Append(string conversationId, RoleDialogModel dialog)
                 AgentId = agentId,
                 MessageId = dialog.MessageId,
                 FunctionName = dialog.FunctionName,
-                CreateTime = dialog.CreatedAt
+                CreateTime = DateTime.UtcNow
             }; 
             
             var content = dialog.Content.RemoveNewLine();
@@ -65,7 +65,7 @@ public void Append(string conversationId, RoleDialogModel dialog)
                 MessageId = dialog.MessageId,
                 SenderId = dialog.SenderId,
                 FunctionName = dialog.FunctionName,
-                CreateTime = dialog.CreatedAt
+                CreateTime = DateTime.UtcNow
             };
             
             var content = dialog.Content.RemoveNewLine();
diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs
@@ -117,6 +117,7 @@ public void AppendConversationDialogs(string conversationId, List<DialogElement>
                     var conv = JsonSerializer.Deserialize<Conversation>(json, _options);
                     if (conv != null)
                     {
+                        conv.DialogCount += dialogs.Count();
                         conv.UpdatedTime = DateTime.UtcNow;
                         File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options));
                     }
@@ -353,23 +354,40 @@ public List<string> GetIdleConversations(int batchSize, int messageLimit, int bu
                 batchSize = batchLimit;
             }
 
+            if (bufferHours <= 0)
+            {
+                bufferHours = 12;
+            }
+
+            if (messageLimit <= 0)
+            {
+                messageLimit = 2;
+            }
+
             foreach (var d in Directory.GetDirectories(dir))
             {
                 var convFile = Path.Combine(d, CONVERSATION_FILE);
                 if (!File.Exists(convFile))
                 {
+                    Directory.Delete(d, true);
                     continue;
                 }
 
                 var json = File.ReadAllText(convFile);
                 var conv = JsonSerializer.Deserialize<Conversation>(json, _options);
-                if (conv == null || conv.UpdatedTime > utcNow.AddHours(-bufferHours))
+
+                if (conv == null)
                 {
+                    Directory.Delete(d, true);
                     continue;
                 }
 
-                var dialogs = GetConversationDialogs(conv.Id);
-                if (dialogs.Count <= messageLimit)
+                if (conv.UpdatedTime > utcNow.AddHours(-bufferHours))
+                {
+                    continue;
+                }
+
+                if (conv.DialogCount <= messageLimit)
                 {
                     ids.Add(conv.Id);
                     if (ids.Count >= batchSize)
@@ -398,7 +416,7 @@ public bool TruncateConversation(string conversationId, string messageId, bool c
             if (foundIdx < 0) return false;
 
             // Handle truncated dialogs
-            var isSaved = HandleTruncatedDialogs(dialogDir, dialogs, foundIdx);
+            var isSaved = HandleTruncatedDialogs(convDir, dialogDir, dialogs, foundIdx);
             if (!isSaved) return false;
 
             // Handle truncated states
@@ -489,10 +507,18 @@ private List<StateKeyValue> CollectConversationStates(string stateFile)
             return states ?? new List<StateKeyValue>();
         }
 
-        private bool HandleTruncatedDialogs(string dialogDir, List<DialogElement> dialogs, int foundIdx)
+        private bool HandleTruncatedDialogs(string convDir, string dialogDir, List<DialogElement> dialogs, int foundIdx)
         {
             var truncatedDialogs = dialogs.Where((x, idx) => idx < foundIdx).ToList();
             var isSaved = SaveTruncatedDialogs(dialogDir, truncatedDialogs);
+            var convFile = Path.Combine(convDir, CONVERSATION_FILE);
+            var convJson = File.ReadAllText(convFile);
+            var conv = JsonSerializer.Deserialize<Conversation>(convJson, _options);
+            if (conv != null)
+            {
+                conv.DialogCount = truncatedDialogs.Count;
+                File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options));
+            }
             return isSaved;
         }
 
diff --git a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs
@@ -15,7 +15,18 @@ public ConversationTimeoutService(IServiceProvider services, ILogger<Conversatio
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
-            _logger.LogInformation("Conversation Timeout Service is running.");
+            _logger.LogInformation("Conversation Timeout Service is running...");
+
+            _ = Task.Run(async () =>
+            {
+                await DoWork(stoppingToken);
+            });
+        }
+
+        private async Task DoWork(CancellationToken stoppingToken)
+        {
+            _logger.LogInformation("Conversation Timeout Service is doing work...");
+
             try
             {
                 while (true)
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs
@@ -8,6 +8,7 @@ public class ConversationDocument : MongoBase
     public string Title { get; set; }
     public string Channel { get; set; }
     public string Status { get; set; }
+    public int DialogCount { get; set; }
     public DateTime CreatedTime { get; set; }
     public DateTime UpdatedTime { get; set; }
     public DateTime Breakpoint { get; set; }
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs
@@ -123,7 +123,8 @@ public void AppendConversationDialogs(string conversationId, List<DialogElement>
         var filterDialog = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
         var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList();
         var updateDialog = Builders<ConversationDialogDocument>.Update.PushEach(x => x.Dialogs, dialogElements);
-        var updateConv = Builders<ConversationDocument>.Update.Set(x => x.UpdatedTime, DateTime.UtcNow);
+        var updateConv = Builders<ConversationDocument>.Update.Set(x => x.UpdatedTime, DateTime.UtcNow)
+                                                              .Inc(x => x.DialogCount, dialogs.Count);
 
         _dc.ConversationDialogs.UpdateOne(filterDialog, updateDialog);
         _dc.Conversations.UpdateOne(filterConv, updateConv);
@@ -168,13 +169,16 @@ public ConversationState GetConversationStates(string conversationId)
 
     public void UpdateConversationStates(string conversationId, List<StateKeyValue> states)
     {
-        if (string.IsNullOrEmpty(conversationId) || states.IsNullOrEmpty()) return;
+        if (string.IsNullOrEmpty(conversationId) || states == null) return;
 
+        var filterConv = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
         var filterStates = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
         var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList();
         var updateStates = Builders<ConversationStateDocument>.Update.Set(x => x.States, saveStates);
+        var updateConv = Builders<ConversationDocument>.Update.Set(x => x.UpdatedTime, DateTime.UtcNow);
 
         _dc.ConversationStates.UpdateOne(filterStates, updateStates);
+        _dc.Conversations.UpdateOne(filterConv, updateConv);
     }
 
     public void UpdateConversationStatus(string conversationId, string status)
@@ -220,6 +224,7 @@ public Conversation GetConversation(string conversationId)
             Status = conv.Status,
             Dialogs = dialogElements,
             States = curStates,
+            DialogCount = conv.DialogCount,
             CreatedTime = conv.CreatedTime,
             UpdatedTime = conv.UpdatedTime,
             Breakpoint = conv.Breakpoint
@@ -308,6 +313,7 @@ public PagedItems<Conversation> GetConversations(ConversationFilter filter)
                 Title = conv.Title,
                 Channel = conv.Channel,
                 Status = conv.Status,
+                DialogCount = conv.DialogCount,
                 CreatedTime = conv.CreatedTime,
                 UpdatedTime = conv.UpdatedTime,
                 Breakpoint = conv.Breakpoint
@@ -335,6 +341,7 @@ public List<Conversation> GetLastConversations()
             Title = c.Title,
             Channel = c.Channel,
             Status = c.Status,
+            DialogCount = c.DialogCount,
             CreatedTime = c.CreatedTime,
             UpdatedTime = c.UpdatedTime,
             Breakpoint = c.Breakpoint  
@@ -353,11 +360,21 @@ public List<string> GetIdleConversations(int batchSize, int messageLimit, int bu
             batchSize = batchLimit;
         }
 
+        if (bufferHours <= 0)
+        {
+            bufferHours = 12;
+        }
+
+        if (messageLimit <= 0)
+        {
+            messageLimit = 2;
+        }
+
         while (true)
         {
             var skip = (page - 1) * batchSize;
             var candidates = _dc.Conversations.AsQueryable()
-                                              .Where(x => x.UpdatedTime <= utcNow.AddHours(-bufferHours))
+                                              .Where(x => (x.DialogCount <= messageLimit) && x.UpdatedTime <= utcNow.AddHours(-bufferHours))
                                               .Skip(skip)
                                               .Take(batchSize)
                                               .Select(x => x.Id)
@@ -367,13 +384,8 @@ public List<string> GetIdleConversations(int batchSize, int messageLimit, int bu
             {
                 break;
             }
-
-            var targets = _dc.ConversationDialogs.AsQueryable()
-                                                 .Where(x => candidates.Contains(x.ConversationId) && x.Dialogs != null && x.Dialogs.Count <= messageLimit)
-                                                 .Select(x => x.ConversationId)
-                                                 .ToList();
-
-            conversationIds = conversationIds.Concat(targets).ToList();
+            
+            conversationIds = conversationIds.Concat(candidates).Distinct().ToList();
             if (conversationIds.Count >= batchSize)
             {
                 break;
@@ -420,10 +432,16 @@ public bool TruncateConversation(string conversationId, string messageId, bool c
             _dc.ConversationStates.ReplaceOne(stateFilter, foundStates);
         }
 
-        // Save
+        // Save dialogs
         foundDialog.Dialogs = truncatedDialogs;
         _dc.ConversationDialogs.ReplaceOne(dialogFilter, foundDialog);
 
+        // Update conversation
+        var convFilter = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
+        var updateConv = Builders<ConversationDocument>.Update.Set(x => x.UpdatedTime, DateTime.UtcNow)
+                                                              .Set(x => x.DialogCount, truncatedDialogs.Count);
+        _dc.Conversations.UpdateOne(convFilter, updateConv);
+
         // Remove logs
         if (cleanLog)
         {

Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ public class RichContentJsonConverter : JsonConverter<IRichMessage>`
`16`	`16`	`if (root.TryGetProperty("rich_type", out JsonElement element))`
`17`	`17`	`{`
`18`	`18`	`var richType = element.GetString();`
`19`		`- res = parser.ParseRichMessage(richType, jsonText, options);`
	`19`	`+ res = parser.ParseRichMessage(richType, jsonText, root, options);`
`20`	`20`	`}`
`21`	`21`
`22`	`22`	`return res;`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ public class TemplateMessageJsonConverter : JsonConverter<ITemplateMessage>`
`16`	`16`	`if (root.TryGetProperty("template_type", out JsonElement element))`
`17`	`17`	`{`
`18`	`18`	`var templateType = element.GetString();`
`19`		`- res = parser.ParseTemplateMessage(templateType, jsonText, options);`
	`19`	`+ res = parser.ParseTemplateMessage(templateType, jsonText, root, options);`
`20`	`20`	`}`
`21`	`21`
`22`	`22`	`return res;`
Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@ public MessageParser()`
`12`	`12`	`{`
`13`	`13`	`}`
`14`	`14`
`15`		`- public IRichMessage? ParseRichMessage(string richType, string jsonText, JsonSerializerOptions options)`
	`15`	`+ public IRichMessage? ParseRichMessage(string richType, string jsonText, JsonElement root, JsonSerializerOptions options)`
`16`	`16`	`{`
`17`	`17`	`IRichMessage? res = null;`
`18`	`18`
`@@ -36,11 +36,22 @@ public MessageParser()`
`36`	`36`	`{`
`37`	`37`	`res = JsonSerializer.Deserialize<TextMessage>(jsonText, options);`
`38`	`38`	`}`
	`39`	`+ else if (richType == RichTypeEnum.GenericTemplate)`
	`40`	`+ {`
	`41`	`+ if (root.TryGetProperty("element_type", out var element))`
	`42`	`+ {`
	`43`	`+ var elementType = element.GetString();`
	`44`	`+ if (elementType == typeof(GenericElement).Name)`
	`45`	`+ {`
	`46`	`+ res = JsonSerializer.Deserialize<GenericTemplateMessage<GenericElement>>(jsonText, options);`
	`47`	`+ }`
	`48`	`+ }`
	`49`	`+ }`
`39`	`50`
`40`	`51`	`return res;`
`41`	`52`	`}`
`42`	`53`
`43`		`- public ITemplateMessage? ParseTemplateMessage(string templateType, string jsonText, JsonSerializerOptions options)`
	`54`	`+ public ITemplateMessage? ParseTemplateMessage(string templateType, string jsonText, JsonElement root, JsonSerializerOptions options)`
`44`	`55`	`{`
`45`	`56`	`ITemplateMessage? res = null;`
`46`	`57`
`@@ -60,6 +71,17 @@ public MessageParser()`
`60`	`71`	`{`
`61`	`72`	`res = JsonSerializer.Deserialize<ProductTemplateMessage>(jsonText, options);`
`62`	`73`	`}`
	`74`	`+ else if (templateType == TemplateTypeEnum.Generic)`
	`75`	`+ {`
	`76`	`+ if (root.TryGetProperty("element_type", out var element))`
	`77`	`+ {`
	`78`	`+ var elementType = element.GetString();`
	`79`	`+ if (elementType == typeof(GenericElement).Name)`
	`80`	`+ {`
	`81`	`+ res = JsonSerializer.Deserialize<GenericTemplateMessage<GenericElement>>(jsonText, options);`
	`82`	`+ }`
	`83`	`+ }`
	`84`	`+ }`
`63`	`85`
`64`	`86`	`return res;`
`65`	`87`	`}`