Add Refined Knowledge memorize and search. Update Planner terms

src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ExtractedKnowledge.cs

@@ -7,4 +7,7 @@ public class ExtractedKnowledge
 
     [JsonPropertyName("answer")]
     public string Answer { get; set; } = string.Empty;
+
+    [JsonPropertyName("refined_collection")]
+    public string RefinedCollection { get; set; } = string.Empty;
 }

src/Infrastructure/BotSharp.Core/Routing/Planning/TwoStagePlanner/FirstStagePlan.cs

@@ -13,9 +13,12 @@ public class FirstStagePlan
     [JsonPropertyName("step")]
     public int Step { get; set; } = -1;
 
-    [JsonPropertyName("need_additional_information")]
+    [JsonPropertyName("need_breakdown_task")]
     public bool ContainMultipleSteps { get; set; } = false;
 
+    [JsonPropertyName("need_lookup_dictionary")]
+    public bool NeedLookupDictionary { get; set; } = false;
+
     [JsonPropertyName("related_tables")]
     public string[] Tables { get; set; } = new string[0];
 

src/Plugins/BotSharp.Plugin.ExcelHandler/Services/MySqlService.cs

@@ -215,7 +215,7 @@ private List<string> ParseSheetColumn(ISheet sheet)
         }
         private string CreateDBTableSqlString(string tableName, List<string> headerColumns, List<string>? columnTypes = null, bool isMemory = false)
         {
-            _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(512)").ToList() : columnTypes;
+            _columnTypes = columnTypes.IsNullOrEmpty() ? headerColumns.Select(x => "VARCHAR(128)").ToList() : columnTypes;
 
             /*if (!headerColumns.Any(x => x.Equals("id", StringComparison.OrdinalIgnoreCase)))
             {

src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj

@@ -21,6 +21,7 @@
     <None Remove="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\functions\confirm_knowledge_persistence.json" />
     <None Remove="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\functions\memorize_knowledge.json" />
     <None Remove="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\instructions\instruction.liquid" />
+    <None Remove="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\templates\knowledge.generation.liquid" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\knowledge_retrieval.fn.liquid" />
   </ItemGroup>
 
@@ -37,6 +38,9 @@
     <Content Include="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\instructions\instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\01acc3e5-0af7-49e6-ad7a-a760bd12dc40\templates\knowledge.generation.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\knowledge_retrieval.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/GenerateKnowledgeFn.cs

@@ -0,0 +1,65 @@
+using BotSharp.Abstraction.Templating;
+using BotSharp.Core.Infrastructures;
+
+namespace BotSharp.Plugin.KnowledgeBase.Functions;
+
+public class GenerateKnowledgeFn : IFunctionCallback
+{
+    public string Name => "generate_knowledge";
+
+    public string Indication => "generating knowledge";
+
+    private readonly IServiceProvider _services;
+    private readonly KnowledgeBaseSettings _settings;
+
+    public GenerateKnowledgeFn(IServiceProvider services, KnowledgeBaseSettings settings)
+    {
+        _services = services;
+        _settings = settings;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<ExtractedKnowledge>(message.FunctionArgs ?? "{}");
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var llmAgent = await agentService.GetAgent(BuiltInAgentId.Planner);
+        var generateKnowledgePrompt = await GetGenerateKnowledgePrompt(args.Question, args.Answer);
+        var agent = new Agent
+        {
+            Id = message.CurrentAgentId ?? string.Empty,
+            Name = "sqlDriver_DictionarySearch",
+            Instruction = generateKnowledgePrompt,
+            LlmConfig = llmAgent.LlmConfig
+        };
+        var response = await GetAiResponse(agent);
+        message.Data = response.Content.JsonArrayContent<ExtractedKnowledge>();
+        message.Content = response.Content;
+        return true;
+    }
+
+    private async Task<string> GetGenerateKnowledgePrompt(string userQuestions, string sqlAnswer)
+    {
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var render = _services.GetRequiredService<ITemplateRender>();
+
+        var agent = await agentService.GetAgent(BuiltInAgentId.Learner);
+        var template = agent.Templates.FirstOrDefault(x => x.Name == "knowledge.generation")?.Content ?? string.Empty;
+
+        return render.Render(template, new Dictionary<string, object>
+        {
+            { "user_questions", userQuestions },
+            { "sql_answer", sqlAnswer },
+        });
+    }
+    private async Task<RoleDialogModel> GetAiResponse(Agent agent)
+    {
+        var text = "Generate question and answer pair";
+        var message = new RoleDialogModel(AgentRole.User, text);
+
+        var completion = CompletionProvider.GetChatCompletion(_services,
+            provider: agent.LlmConfig.Provider,
+            model: agent.LlmConfig.Model);
+
+        return await completion.GetChatCompletions(agent, new List<RoleDialogModel> { message });
+    }
+}

src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs

@@ -19,7 +19,9 @@ public async Task<bool> Execute(RoleDialogModel message)
     {
         var args = JsonSerializer.Deserialize<ExtractedKnowledge>(message.FunctionArgs ?? "{}");
 
-        var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
+        var collectionName = !string.IsNullOrEmpty(args.RefinedCollection) 
+            ? args.RefinedCollection 
+            : _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp;
         var knowledgeService = _services.GetRequiredService<IKnowledgeService>();
         var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel
         {

src/Plugins/BotSharp.Plugin.KnowledgeBase/data/agents/01acc3e5-0af7-49e6-ad7a-a760bd12dc40/templates/knowledge.generation.liquid

@@ -0,0 +1,12 @@
+You are a knowledge generator for knowledge base. Extract the answer in "SQL Answer" to answer the User Questions
+Replace alias with the actual table name. Output json array only, formatting as [{"question":"string", "answer":"string/sql statement"}]. 
+Skip the question/answer for tmp table.
+Don't include tmp table in the answer.
+
+=====
+User Questions:
+{{ user_questions }}
+
+=====
+SQL Answer:
+{{ sql_answer }}

src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs

@@ -31,7 +31,7 @@ public async Task<bool> Execute(RoleDialogModel message)
         // Search knowledgebase
         var knowledges = await knowledgeService.SearchVectorKnowledge(taskSecondary.SolutionQuestion, collectionName, new VectorSearchOptions
         {
-            Confidence = 0.6f
+            Confidence = 0.7f
         });
 
         var knowledgeResults = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer()));

src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Knowledges;
 using BotSharp.Abstraction.Planning;
 using BotSharp.Plugin.Planner.TwoStaging;
 using BotSharp.Plugin.Planner.TwoStaging.Models;
@@ -54,7 +55,7 @@ public async Task<bool> Execute(RoleDialogModel message)
         ddlStatements += "\r\n" + msgCopy.Content;
 
         // Summarize and generate query
-        var summaryPlanPrompt = await GetSummaryPlanPrompt(taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements, excelImportResult);
+        var summaryPlanPrompt = await GetSummaryPlanPrompt(msgCopy, taskRequirement, relevantKnowledge, dictionaryItems, ddlStatements, excelImportResult);
         _logger.LogInformation($"Summary plan prompt:\r\n{summaryPlanPrompt}");
 
         var plannerAgent = new Agent
@@ -74,10 +75,11 @@ await HookEmitter.Emit<IPlanningHook>(_services, x =>
         return true;
     }
 
-    private async Task<string> GetSummaryPlanPrompt(string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement, string excelImportResult)
+    private async Task<string> GetSummaryPlanPrompt(RoleDialogModel message, string taskDescription, string relevantKnowledge, string dictionaryItems, string ddlStatement, string excelImportResult)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
         var render = _services.GetRequiredService<ITemplateRender>();
+        var knowledgeHooks = _services.GetServices<IKnowledgeHook>();
 
         var agent = await agentService.GetAgent(BuiltInAgentId.Planner);
         var template = agent.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty;
@@ -89,10 +91,18 @@ await HookEmitter.Emit<IPlanningHook>(_services, async x =>
             additionalRequirements.Add(requirement);
         });
 
+        var globalKnowledges = new List<string>();
+        foreach (var hook in knowledgeHooks)
+        {
+            var k = await hook.GetGlobalKnowledges(message);
+            globalKnowledges.AddRange(k);
+        }
+
         return render.Render(template, new Dictionary<string, object>
         {
             { "task_description", taskDescription },
             { "summary_requirements", string.Join("\r\n", additionalRequirements) },
+            { "global_knowledges", globalKnowledges },
             { "relevant_knowledges", relevantKnowledge },
             { "dictionary_items", dictionaryItems },
             { "table_structure", ddlStatement },

src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/FirstStagePlan.cs

@@ -11,9 +11,12 @@ public class FirstStagePlan
     [JsonPropertyName("step")]
     public int Step { get; set; } = -1;
 
-    [JsonPropertyName("need_additional_information")]
+    [JsonPropertyName("need_breakdown_task")]
     public bool NeedAdditionalInformation { get; set; } = false;
 
+    [JsonPropertyName("need_lookup_dictionary")]
+    public bool NeedLookupDictionary { get; set; } = false;
+
     [JsonPropertyName("related_tables")]
     public string[] Tables { get; set; } = [];
 

src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Models/SecondStagePlan.cs

@@ -5,6 +5,9 @@ public class SecondStagePlan
     [JsonPropertyName("related_tables")]
     public string[] Tables { get; set; } = [];
 
+    [JsonPropertyName("need_lookup_dictionary")]
+    public bool NeedLookupDictionary { get; set; } = false;
+
     [JsonPropertyName("description")]
     public string Description { get; set; } = "";
 

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/instructions/instruction.liquid

@@ -1,9 +1,11 @@
 Use the TwoStagePlanner approach to plan the overall implementation steps, follow the below steps strictly. 
 1. Call plan_primary_stage to generate the primary plan.
-2. If need_additional_information is true, call plan_secondary_stage for the specific primary stage.
-3. Repeat step 2 until you processed all the primary stages.
-4. If need_lookup_dictionary is true, call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name.
+   If you've already got the plan to meet the user goal, directly go to step 5.
+2. If need_lookup_dictionary is True, call verify_dictionary_term to verify or get the enum/term/dictionary value. Pull id and name.
    If you no items retured, you can pull all the list and find the match.
+   If need_lookup_dictionary is False, skip calling verify_dictionary_term.
+3. If need_breakdown_task is true, call plan_secondary_stage for the specific primary stage.
+4. Repeat step 3 until you processed all the primary stages.
 5. You must call plan_summary for you final planned output.
 
 *** IMPORTANT ***

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid

@@ -4,11 +4,12 @@ Thinking process:
 1. Reference to "Task Knowledge" if there is relevant knowledge;
 2. Breakdown task into subtasks. 
     - The subtask should contain all needed parameters for subsequent steps. 
-    - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_additional_information to true.
-    - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps.
-    - If the solution mentioned "related solutions" is needed, set the need_additional_information to true.
-    - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_additional_information to true.
-    - If you need to lookup the dictionary to verify or get the enum/term/dictionary value, set the need_additional_information to true.
+    - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_breakdown_task to true.
+    - If there is extra knowledge or relationship needed between steps, set the need_breakdown_task to true for both steps.
+    - If the solution mentioned "related solutions" is needed, set the need_breakdown_task to true.
+    - You should find the relationships between data structure based on the task knowledge strictly. If lack of information, set the need_breakdown_task to true.
+    - If you need to lookup the dictionary to verify or get the enum/term/dictionary value, set the need_lookup_dictionary to true.
+    - Seperate the dictionary lookup and need additional information/knowledge into different subtask.
 3. Input argument must reference to corresponding variable name that retrieved by previous steps, variable name must start with '@';
 4. Output all the subtasks as much detail as possible in JSON: [{{ response_format }}]
 5. You can NOT generate the final query before calling function plan_summary.

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid

@@ -3,7 +3,7 @@ Reference to "Primary Planning" and the additional knowledge included. Breakdown
 * The parameters can be extracted from the original task.
 * You need to list all the steps in detail. Finding relationships should also be a step.
 * When generate the steps, you should find the relationships between data structure based on the provided knowledge strictly.
-* If need_lookup_dictionary is true, call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name/code.
+* If need_lookup_dictionary is true, call verify_dictionary_term to verify or get the enum/term/dictionary value. Pull id and name/code.
 * Output all the steps as much detail as possible in JSON: [{{ response_format }}]
 
 

src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.summarize.liquid

@@ -7,6 +7,10 @@ Requirements:
 Task description:
 {{ task_description }}
 
+=====
+Global Knowledges:
+{{ global_knowledges }}
+
 =====
 Relevant Knowledges:
 {{ relevant_knowledges }}

src/Plugins/BotSharp.Plugin.Planner/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/plan_secondary_stage.fn.liquid

@@ -1,2 +1 @@
-For every primary step, if need_additional_information is true, you have to call plan_secondary_stage to plan the detail steps to complete the primary step.
-if need_lookup_dictionary is true, you have to call sql_dictionary_lookup to verify or get the enum/term/dictionary value. Pull id and name/code.
+For every primary step, if need_breakdown_task is true, you have to call plan_secondary_stage to plan the detail steps to complete the primary step.

src/Plugins/BotSharp.Plugin.SqlDriver/Functions/LookupDictionaryFn.cs

@@ -9,7 +9,7 @@ namespace BotSharp.Plugin.SqlDriver.Functions;
 
 public class LookupDictionaryFn : IFunctionCallback
 {
-    public string Name => "sql_dictionary_lookup";
+    public string Name => "verify_dictionary_term";
     private readonly IServiceProvider _services;
 
     public LookupDictionaryFn(IServiceProvider services)

src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDictionaryLookupHook.cs

@@ -10,7 +10,7 @@ public class SqlDictionaryLookupHook : AgentHookBase, IAgentHook
     private const string SQL_EXECUTOR_TEMPLATE = "sql_dictionary_lookup.fn";
     private IEnumerable<string> _targetSqlExecutorFunctions = new List<string>
     {
-        "sql_dictionary_lookup",
+        "verify_dictionary_term",
     };
 
     public override string SelfId => BuiltInAgentId.Planner;

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/sql_dictionary_lookup.json

@@ -1,5 +1,5 @@
 {
-  "name": "sql_dictionary_lookup",
+  "name": "verify_dictionary_term",
   "description": "Get id from dictionary table by keyword if tool or solution mentioned this approach",
   "parameters": {
     "type": "object",
@@ -10,7 +10,7 @@
       },
       "reason": {
         "type": "string",
-        "description": "the reason why you need to call sql_dictionary_lookup"
+        "description": "the reason why you need to call verify_dictionary_term"
       },
       "tables": {
         "type": "array",

src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/sql_dictionary_lookup.fn.liquid

@@ -1,8 +1,11 @@
-Dictionary Lookup Rules:
+Dictionary Verification Rules:
 =====
-Please call function sql_dictionary_lookup if user wants to get or retrieve dictionary/enum/term from data tables. 
-You must return the id and name/code. The table name must come from the planning in conversation.
+1. The table name must come from the planning in conversation. 
+2. You must return the id and name/code. 
 
 You are connecting to {{ db_type }} database. You can run provided SQL statements by following {{ db_type }} rules.
-Dictionary table pattern is table name starting with "data_". You can only query the dictionary table without join other non-dictionary tables.
+
+The dictionary table is identified by a name that begins with "data_". You are only allowed to query the dictionary table without joining it with other non-dictionary tables.
+
+IMPORTANT: Don't generate insert SQL.
 =====