Skip to content

refactor file dialog structure #440

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 7, 2024
Merged

refactor file dialog structure #440

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,22 @@ public class Conversation

public class DialogElement
{
[JsonPropertyName("meta_data")]
public DialogMetaData MetaData { get; set; }

[JsonPropertyName("content")]
public string Content { get; set; }

[JsonPropertyName("secondary_content")]
public string? SecondaryContent { get; set; }

[JsonPropertyName("rich_content")]
public string? RichContent { get; set; }

[JsonPropertyName("secondary_rich_content")]
public string? SecondaryRichContent { get; set; }

[JsonPropertyName("payload")]
public string? Payload { get; set; }

public DialogElement()
Expand All @@ -63,10 +74,21 @@ public override string ToString()

public class DialogMetaData
{
[JsonPropertyName("role")]
public string Role { get; set; }

[JsonPropertyName("agent_id")]
public string AgentId { get; set; }

[JsonPropertyName("message_id")]
public string MessageId { get; set; }

[JsonPropertyName("function_name")]
public string? FunctionName { get; set; }

[JsonPropertyName("sender_id")]
public string? SenderId { get; set; }

[JsonPropertyName("create_at")]
public DateTime CreateTime { get; set; }
}
1 change: 0 additions & 1 deletion src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ public interface IBotSharpRepository
void CreateNewConversation(Conversation conversation);
bool DeleteConversations(IEnumerable<string> conversationIds);
List<DialogElement> GetConversationDialogs(string conversationId);
void UpdateConversationDialogElements(string conversationId, List<DialogContentUpdateModel> updateElements);
void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs);
ConversationState GetConversationStates(string conversationId);
void UpdateConversationStates(string conversationId, List<StateKeyValue> states);
Expand Down
3 changes: 0 additions & 3 deletions src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,6 @@ public List<string> GetIdleConversations(int batchSize, int messageLimit, int bu
public List<DialogElement> GetConversationDialogs(string conversationId)
=> throw new NotImplementedException();

public void UpdateConversationDialogElements(string conversationId, List<DialogContentUpdateModel> updateElements)
=> new NotImplementedException();

public ConversationState GetConversationStates(string conversationId)
=> throw new NotImplementedException();

Expand Down
118 changes: 28 additions & 90 deletions src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public void CreateNewConversation(Conversation conversation)
var dialogFile = Path.Combine(dir, DIALOG_FILE);
if (!File.Exists(dialogFile))
{
File.WriteAllText(dialogFile, string.Empty);
File.WriteAllText(dialogFile, "[]");
}

var stateFile = Path.Combine(dir, STATE_FILE);
Expand Down Expand Up @@ -65,37 +65,18 @@ public List<DialogElement> GetConversationDialogs(string conversationId)
if (!string.IsNullOrEmpty(convDir))
{
var dialogDir = Path.Combine(convDir, DIALOG_FILE);
dialogs = CollectDialogElements(dialogDir);
}

return dialogs;
}

public void UpdateConversationDialogElements(string conversationId, List<DialogContentUpdateModel> updateElements)
{
var dialogElements = GetConversationDialogs(conversationId);
if (dialogElements.IsNullOrEmpty() || updateElements.IsNullOrEmpty()) return;

var convDir = FindConversationDirectory(conversationId);
if (!string.IsNullOrEmpty(convDir))
{
var dialogDir = Path.Combine(convDir, DIALOG_FILE);
if (File.Exists(dialogDir))
var texts = File.ReadAllText(dialogDir);
try
{
var updated = dialogElements.Select((x, idx) =>
{
var found = updateElements.FirstOrDefault(e => e.Index == idx);
if (found != null)
{
x.Content = found.UpdateContent;
}
return x;
}).ToList();

var texts = ParseDialogElements(updated);
File.WriteAllLines(dialogDir, texts);
dialogs = JsonSerializer.Deserialize<List<DialogElement>>(texts, _options) ?? new List<DialogElement>();
}
catch
{
dialogs = new List<DialogElement>();
}
}

return dialogs;
}

public void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs)
Expand All @@ -106,8 +87,18 @@ public void AppendConversationDialogs(string conversationId, List<DialogElement>
var dialogFile = Path.Combine(convDir, DIALOG_FILE);
if (File.Exists(dialogFile))
{
var texts = ParseDialogElements(dialogs);
File.AppendAllLines(dialogFile, texts);
var prevDialogs = File.ReadAllText(dialogFile);
var elements = JsonSerializer.Deserialize<List<DialogElement>>(prevDialogs, _options);
if (elements != null)
{
elements.AddRange(dialogs);
}
else
{
elements = elements ?? new List<DialogElement>();
}

File.WriteAllText(dialogFile, JsonSerializer.Serialize(elements, _options));
}

var convFile = Path.Combine(convDir, CONVERSATION_FILE);
Expand Down Expand Up @@ -519,69 +510,16 @@ private List<DialogElement> CollectDialogElements(string dialogDir)

if (!File.Exists(dialogDir)) return dialogs;

var rawDialogs = File.ReadAllLines(dialogDir);
if (!rawDialogs.IsNullOrEmpty())
{
for (int i = 0; i < rawDialogs.Count(); i += 5)
{
var blocks = rawDialogs[i].Split("|");
var content = rawDialogs[i + 2];
var trimmedContent = content.Substring(4);
var secondaryContent = rawDialogs[i + 4];
var trimmedSecondaryContent = string.IsNullOrEmpty(secondaryContent) ? null : secondaryContent.Substring(4);
var payload = blocks.Count() > 6 ? blocks[6] : null;

var meta = new DialogMetaData
{
Role = blocks[1],
AgentId = blocks[2],
MessageId = blocks[3],
SenderId = !string.IsNullOrWhiteSpace(blocks[4]) ? blocks[4] : null,
FunctionName = !string.IsNullOrWhiteSpace(blocks[5]) ? blocks[5] : null,
CreateTime = DateTime.Parse(blocks[0])
};

var richContent = DecodeText(rawDialogs[i + 1]);
var secondaryRichContent = DecodeText(rawDialogs[i + 3]);
dialogs.Add(new DialogElement
{
MetaData = meta,
Content = trimmedContent,
SecondaryContent = trimmedSecondaryContent,
RichContent = richContent,
SecondaryRichContent = secondaryRichContent,
Payload = payload
});
}
}
var texts = File.ReadAllText(dialogDir);
dialogs = JsonSerializer.Deserialize<List<DialogElement>>(texts) ?? new List<DialogElement>();
return dialogs;
}

private List<string> ParseDialogElements(List<DialogElement> dialogs)
private string ParseDialogElements(List<DialogElement> dialogs)
{
var dialogTexts = new List<string>();
if (dialogs.IsNullOrEmpty()) return dialogTexts;

foreach (var element in dialogs)
{
var meta = element.MetaData;
var createTime = meta.CreateTime.ToString("MM/dd/yyyy hh:mm:ss.ffffff tt", CultureInfo.InvariantCulture);
var encodedRichContent = EncodeText(element.RichContent);
var encodedSecondaryRichContent = EncodeText(element.SecondaryRichContent);
var payload = element.Payload;
var metaStr = $"{createTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{meta.SenderId}|{meta.FunctionName}|{payload}";
dialogTexts.Add(metaStr);

dialogTexts.Add(encodedRichContent);
var content = $" - {element.Content}";
dialogTexts.Add(content);

dialogTexts.Add(encodedSecondaryRichContent);
var secondaryContent = element.SecondaryContent == null ? null : $" - {element.SecondaryContent}";
dialogTexts.Add(secondaryContent);
}
if (dialogs.IsNullOrEmpty()) return "[]";

return dialogTexts;
return JsonSerializer.Serialize(dialogs, _options) ?? "[]";
}

private List<StateKeyValue> CollectConversationStates(string stateFile)
Expand Down Expand Up @@ -701,7 +639,7 @@ private bool SaveTruncatedDialogs(string dialogDir, List<DialogElement> dialogs)
if (!File.Exists(dialogDir)) File.Create(dialogDir);

var texts = ParseDialogElements(dialogs);
File.WriteAllLines(dialogDir, texts);
File.WriteAllText(dialogDir, texts);
return true;
}

Expand Down
2 changes: 1 addition & 1 deletion src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public partial class FileRepository : IBotSharpRepository
private const string USER_AGENT_FILE = "agents.json";
private const string CONVERSATION_FILE = "conversation.json";
private const string STATS_FILE = "stats.json";
private const string DIALOG_FILE = "dialogs.txt";
private const string DIALOG_FILE = "dialogs.json";
private const string STATE_FILE = "state.json";
private const string BREAKPOINT_FILE = "breakpoint.json";
private const string EXECUTION_LOG_FILE = "execution.log";
Expand Down
21 changes: 0 additions & 21 deletions src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,27 +85,6 @@ public List<DialogElement> GetConversationDialogs(string conversationId)
return formattedDialog ?? new List<DialogElement>();
}

public void UpdateConversationDialogElements(string conversationId, List<DialogContentUpdateModel> updateElements)
{
if (string.IsNullOrEmpty(conversationId) || updateElements.IsNullOrEmpty()) return;

var filterDialog = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
var foundDialog = _dc.ConversationDialogs.Find(filterDialog).FirstOrDefault();
if (foundDialog == null || foundDialog.Dialogs.IsNullOrEmpty()) return;

foundDialog.Dialogs = foundDialog.Dialogs.Select((x, idx) =>
{
var found = updateElements.FirstOrDefault(e => e.Index == idx);
if (found != null)
{
x.Content = found.UpdateContent;
}
return x;
}).ToList();

_dc.ConversationDialogs.ReplaceOne(filterDialog, foundDialog);
}

public void AppendConversationDialogs(string conversationId, List<DialogElement> dialogs)
{
if (string.IsNullOrEmpty(conversationId)) return;
Expand Down