From 78a95914b13e8a16c1d17060ebde5831e71c90bc Mon Sep 17 00:00:00 2001
From: Jicheng Lu <103353@smsassist.com>
Date: Wed, 5 Feb 2025 17:50:32 -0600
Subject: [PATCH] add max output token
---
.../Agents/Constants/LlmConstant.cs | 6 ++++++
.../Agents/Models/AgentLlmConfig.cs | 10 ++++++++++
.../Statistics/Models/BotSharpStats.cs | 3 ++-
.../ViewModels/Agents/AgentCreationModel.cs | 2 +-
.../ViewModels/Agents/AgentUpdateModel.cs | 2 +-
.../Providers/ChatCompletionProvider.cs | 6 ++++--
src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs | 1 +
.../Providers/Chat/ChatCompletionProvider.cs | 5 ++++-
src/Plugins/BotSharp.Plugin.AzureOpenAI/Using.cs | 1 +
.../Providers/Chat/ChatCompletionProvider.cs | 4 +++-
src/Plugins/BotSharp.Plugin.DeepSeekAI/Using.cs | 1 +
.../Providers/Chat/GeminiChatCompletionProvider.cs | 13 ++++++++++++-
src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs | 1 +
.../Models/AgentLlmConfigMongoElement.cs | 3 +++
.../Providers/Chat/ChatCompletionProvider.cs | 4 +++-
.../Realtime/RealTimeCompletionProvider.cs | 4 +++-
src/Plugins/BotSharp.Plugin.OpenAI/Using.cs | 1 +
17 files changed, 57 insertions(+), 10 deletions(-)
create mode 100644 src/Infrastructure/BotSharp.Abstraction/Agents/Constants/LlmConstant.cs
diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Constants/LlmConstant.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Constants/LlmConstant.cs
new file mode 100644
index 000000000..3919ca571
--- /dev/null
+++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Constants/LlmConstant.cs
@@ -0,0 +1,6 @@
+namespace BotSharp.Abstraction.Agents.Constants;
+
+public static class LlmConstant
+{
+ public const int DEFAULT_MAX_OUTPUT_TOKEN = 1024;
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs
index 8025c2b86..f1df3a012 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs
@@ -22,6 +22,16 @@ public class AgentLlmConfig
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? Model { get; set; }
+ ///
+ /// Max recursion depth
+ ///
[JsonPropertyName("max_recursion_depth")]
public int MaxRecursionDepth { get; set; } = 3;
+
+ ///
+ /// Max output token
+ ///
+ [JsonPropertyName("max_output_tokens")]
+ [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+ public int? MaxOutputTokens { get; set; }
}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs
index 33178d6a5..c0163a7e4 100644
--- a/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs
+++ b/src/Infrastructure/BotSharp.Abstraction/Statistics/Models/BotSharpStats.cs
@@ -52,7 +52,7 @@ public override string ToString()
public static (DateTime, DateTime) BuildTimeInterval(DateTime recordTime, StatsInterval interval)
{
DateTime startTime = recordTime;
- DateTime endTime = DateTime.UtcNow;
+ DateTime endTime = startTime;
switch (interval)
{
@@ -70,6 +70,7 @@ public static (DateTime, DateTime) BuildTimeInterval(DateTime recordTime, StatsI
break;
}
+ endTime = endTime.AddSeconds(-1);
startTime = DateTime.SpecifyKind(startTime, DateTimeKind.Utc);
endTime = DateTime.SpecifyKind(endTime, DateTimeKind.Utc);
return (startTime, endTime);
diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs
index bb26836c3..8e4d915d4 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs
@@ -80,7 +80,7 @@ public Agent ToAgent()
MaxMessageCount = MaxMessageCount,
Profiles = Profiles,
Labels = Labels,
- LlmConfig = LlmConfig,
+ LlmConfig = LlmConfig ?? new(),
KnowledgeBases = KnowledgeBases,
Rules = Rules,
RoutingRules = RoutingRules?.Select(x => RoutingRuleUpdateModel.ToDomainElement(x))?.ToList() ?? [],
diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs
index 611d633ff..ac2749b55 100644
--- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs
+++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs
@@ -105,7 +105,7 @@ public Agent ToAgent()
Utilities = Utilities ?? [],
KnowledgeBases = KnowledgeBases ?? [],
Rules = Rules ?? [],
- LlmConfig = LlmConfig
+ LlmConfig = LlmConfig ?? new()
};
return agent;
diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs
index 8657b31b2..b71306cf5 100644
--- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs
@@ -170,12 +170,14 @@ public Task GetChatCompletionsStreamingAsync(Agent agent, List();
var temperature = decimal.Parse(state.GetState("temperature", "0.0"));
- var maxToken = int.Parse(state.GetState("max_tokens", "512"));
+ var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
+ ? tokens
+ : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var parameters = new MessageParameters()
{
Messages = messages,
- MaxTokens = maxToken,
+ MaxTokens = maxTokens,
Model = settings.Name,
Stream = false,
Temperature = temperature,
diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs
index d00446fc5..b2e4484c7 100644
--- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs
+++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Using.cs
@@ -8,6 +8,7 @@
global using Anthropic.SDK.Constants;
global using Anthropic.SDK.Messaging;
global using BotSharp.Abstraction.Agents;
+global using BotSharp.Abstraction.Agents.Constants;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.Conversations.Models;
diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
index 9879e4fb4..3fef7c0cd 100644
--- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
@@ -225,7 +225,10 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, List();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
- var maxTokens = int.Parse(state.GetState("max_tokens", "1024"));
+ var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
+ ? tokens
+ : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
+
var options = new ChatCompletionOptions()
{
Temperature = temperature,
diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Using.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Using.cs
index 71f5272d2..2cc3faf0a 100644
--- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Using.cs
+++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Using.cs
@@ -5,6 +5,7 @@
global using System.Threading.Tasks;
global using Microsoft.Extensions.DependencyInjection;
global using Microsoft.Extensions.Logging;
+global using BotSharp.Abstraction.Agents.Constants;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.Conversations;
diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs
index 474c73507..ef4d0c8aa 100644
--- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs
@@ -201,7 +201,9 @@ public void SetModelName(string model)
var messages = new List();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
- var maxTokens = int.Parse(state.GetState("max_tokens", "1024"));
+ var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
+ ? tokens
+ : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var options = new ChatCompletionOptions()
{
Temperature = temperature,
diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Using.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Using.cs
index 4fa278e60..751f1aec8 100644
--- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Using.cs
+++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Using.cs
@@ -10,6 +10,7 @@
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.MLTasks;
global using BotSharp.Abstraction.Agents;
+global using BotSharp.Abstraction.Agents.Constants;
global using BotSharp.Abstraction.Agents.Enums;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Loggers;
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
index 5930a6b93..9567ae578 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs
@@ -1,5 +1,6 @@
using BotSharp.Abstraction.Agents;
using BotSharp.Abstraction.Agents.Enums;
+using BotSharp.Abstraction.Conversations;
using BotSharp.Abstraction.Loggers;
using Google.Protobuf.WellKnownTypes;
using Microsoft.Extensions.Logging;
@@ -188,10 +189,20 @@ public void SetModelName(string model)
}
}
+ var state = _services.GetRequiredService();
+ var temperature = float.Parse(state.GetState("temperature", "0.0"));
+ var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
+ ? tokens
+ : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var request = new GenerateContentRequest
{
Contents = contents,
- Tools = tools
+ Tools = tools,
+ GenerationConfig = new()
+ {
+ Temperature = temperature,
+ MaxOutputTokens = maxTokens
+ }
};
var prompt = GetPrompt(systemPrompts, funcPrompts, convPrompts);
diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs
index 17152bf4b..c7f6c176b 100644
--- a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs
+++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs
@@ -5,6 +5,7 @@
global using System.Linq;
global using System.Text.Json;
global using BotSharp.Abstraction.Conversations.Models;
+global using BotSharp.Abstraction.Agents.Constants;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.MLTasks;
global using Microsoft.Extensions.Configuration;
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs
index 5251b5324..32989ad18 100644
--- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs
+++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs
@@ -8,6 +8,7 @@ public class AgentLlmConfigMongoElement
public string? Model { get; set; }
public bool IsInherit { get; set; }
public int MaxRecursionDepth { get; set; }
+ public int? MaxOutputTokens { get; set; }
public static AgentLlmConfigMongoElement? ToMongoElement(AgentLlmConfig? config)
{
@@ -19,6 +20,7 @@ public class AgentLlmConfigMongoElement
Model = config.Model,
IsInherit = config.IsInherit,
MaxRecursionDepth = config.MaxRecursionDepth,
+ MaxOutputTokens = config.MaxOutputTokens,
};
}
@@ -32,6 +34,7 @@ public class AgentLlmConfigMongoElement
Model = config.Model,
IsInherit = config.IsInherit,
MaxRecursionDepth = config.MaxRecursionDepth,
+ MaxOutputTokens = config.MaxOutputTokens,
};
}
}
diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
index 4eae55321..15eb04c57 100644
--- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
@@ -202,7 +202,9 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, List();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
- var maxTokens = int.Parse(state.GetState("max_tokens", "1024"));
+ var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
+ ? tokens
+ : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var options = new ChatCompletionOptions()
{
Temperature = temperature,
diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
index 473894993..eb678c5d8 100644
--- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
+++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs
@@ -70,7 +70,9 @@ public async Task CreateSession(Agent agent, List();
var temperature = float.Parse(state.GetState("temperature", "0.0"));
- var maxTokens = int.Parse(state.GetState("max_tokens", "1024"));
+ var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens)
+ ? tokens
+ : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN;
var options = new ChatCompletionOptions()
{
ToolChoice = ChatToolChoice.CreateAutoChoice(),
diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs
index fe9a02b49..ac1022a23 100644
--- a/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs
+++ b/src/Plugins/BotSharp.Plugin.OpenAI/Using.cs
@@ -6,6 +6,7 @@
global using Microsoft.Extensions.DependencyInjection;
global using Microsoft.Extensions.Logging;
global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Agents.Constants;
global using BotSharp.Abstraction.Agents.Models;
global using BotSharp.Abstraction.Conversations;
global using BotSharp.Abstraction.Conversations.Models;