From 8c460d54b28a284667acb8684bff320094a43980 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 23 Sep 2024 20:33:42 -0500 Subject: [PATCH 01/14] return not found if no records. --- .../Functions/ExecuteQueryFn.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs index 7833bfb4b..0806c82cc 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/ExecuteQueryFn.cs @@ -29,7 +29,15 @@ public async Task Execute(RoleDialogModel message) _ => throw new NotImplementedException($"Database type {settings.DatabaseType} is not supported.") }; - message.Content = JsonSerializer.Serialize(results); + if (results.Count() == 0) + { + message.Content = "No record found"; + } + else + { + message.Content = JsonSerializer.Serialize(results); + } + return true; } From 3f24bdd0f24f61776d9a50866f6c44622a2ce4e0 Mon Sep 17 00:00:00 2001 From: "jason.wang" Date: Tue, 24 Sep 2024 15:05:24 +0800 Subject: [PATCH 02/14] add affiliateId claim & add redis expire --- .../BotSharp.Abstraction/Users/IUserIdentity.cs | 1 + .../BotSharp.Core/Users/Services/UserIdentity.cs | 3 +++ .../BotSharp.Core/Users/Services/UserService.cs | 9 +++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs index 4d32d241b..347e75545 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserIdentity.cs @@ -10,4 +10,5 @@ public interface IUserIdentity string FullName { get; } string? UserLanguage { get; } string? Phone { get; } + string? AffiliateId { get; } } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs index ee3b5c755..531fd2865 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs @@ -69,4 +69,7 @@ public string? UserLanguage [JsonPropertyName("phone")] public string? Phone => _claims?.FirstOrDefault(x => x.Type == "phone")?.Value; + + [JsonPropertyName("affiliateId")] + public string? AffiliateId => _claims?.FirstOrDefault(x => x.Type == "affiliateId")?.Value; } diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index e4f5fc317..f8df34dc1 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -254,7 +254,8 @@ private string GenerateJwtToken(User user) new Claim("type", user.Type ?? UserType.Client), new Claim("role", user.Role ?? UserRole.User), new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()), - new Claim("phone", user.Phone ?? string.Empty) + new Claim("phone", user.Phone ?? string.Empty), + new Claim("affiliateId", user.AffiliateId ?? string.Empty) }; var validators = _services.GetServices(); @@ -280,14 +281,14 @@ private string GenerateJwtToken(User user) }; var tokenHandler = new JwtSecurityTokenHandler(); var token = tokenHandler.CreateToken(tokenDescriptor); - SaveUserTokenExpiresCache(user.Id, expires).GetAwaiter().GetResult(); + SaveUserTokenExpiresCache(user.Id, expires, expireInMinutes).GetAwaiter().GetResult(); return tokenHandler.WriteToken(token); } - private async Task SaveUserTokenExpiresCache(string userId, DateTime expires) + private async Task SaveUserTokenExpiresCache(string userId, DateTime expires, int expireInMinutes) { var _cacheService = _services.GetRequiredService(); - await _cacheService.SetAsync(GetUserTokenExpiresCacheKey(userId), expires, null); + await _cacheService.SetAsync(GetUserTokenExpiresCacheKey(userId), expires, TimeSpan.FromMinutes(expireInMinutes)); } private string GetUserTokenExpiresCacheKey(string userId) From feb48a11e547aa40d46288d0f839bd21c0195080 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 24 Sep 2024 23:09:56 -0500 Subject: [PATCH 03/14] login confict --- .../BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs index 6e0a6b2f0..824c95711 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs @@ -37,7 +37,8 @@ public void OnAuthorization(AuthorizationFilterContext context) if (validTo != currentExpires) { Serilog.Log.Warning($"Token expired. Token expires at {validTo}, current expires at {currentExpires}"); - context.Result = new UnauthorizedResult(); + // login confict + context.Result = new ConflictResult(); } } } From a38c27718a3e08d34c9e827a9c75fb46a3b78819 Mon Sep 17 00:00:00 2001 From: "jason.wang" Date: Wed, 25 Sep 2024 22:09:43 +0800 Subject: [PATCH 04/14] fix get affiliate token user issue --- .../BotSharp.Core/Users/Services/UserService.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index f8df34dc1..a46e16cff 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -112,7 +112,11 @@ public async Task GetAffiliateToken(string authorization) var base64 = Encoding.UTF8.GetString(Convert.FromBase64String(authorization)); var (id, password) = base64.SplitAsTuple(":"); var db = _services.GetRequiredService(); - var record = db.GetUserByPhone(id); + var record = db.GetAffiliateUserByPhone(id); + if (record == null) + { + record = db.GetUserByPhone(id); + } var isCanLoginAffiliateRoleType = record != null && !record.IsDisabled && record.Type != UserType.Client; if (!isCanLoginAffiliateRoleType) From 8f6fbed7ef939b12136c6673485bdff7e2f7e5d1 Mon Sep 17 00:00:00 2001 From: "jason.wang" Date: Wed, 25 Sep 2024 22:15:52 +0800 Subject: [PATCH 05/14] UserSingleLoginFilter add token expired don't valid single login --- .../BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs index 824c95711..bd6f2dd6a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs @@ -34,7 +34,7 @@ public void OnAuthorization(AuthorizationFilterContext context) var validTo = token.ValidTo.ToLongTimeString(); var currentExpires = GetUserExpires().ToLongTimeString(); - if (validTo != currentExpires) + if (token.ValidTo > DateTime.UtcNow && validTo != currentExpires) { Serilog.Log.Warning($"Token expired. Token expires at {validTo}, current expires at {currentExpires}"); // login confict From c77b575e2faaf7de14d52861ca3ed5eac3cd8946 Mon Sep 17 00:00:00 2001 From: "jason.wang" Date: Wed, 25 Sep 2024 22:25:37 +0800 Subject: [PATCH 06/14] AllowAnonymous api no valid single login --- .../BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs index bd6f2dd6a..e8db0801b 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Filters/UserSingleLoginFilter.cs @@ -20,6 +20,14 @@ public UserSingleLoginFilter(IUserService userService, IServiceProvider services public void OnAuthorization(AuthorizationFilterContext context) { + var isAllowAnonymous = context.ActionDescriptor.EndpointMetadata + .Any(em => em.GetType() == typeof(AllowAnonymousAttribute)); + + if (isAllowAnonymous) + { + return; + } + var bearerToken = GetBearerToken(context); if (!string.IsNullOrWhiteSpace(bearerToken)) { @@ -34,7 +42,7 @@ public void OnAuthorization(AuthorizationFilterContext context) var validTo = token.ValidTo.ToLongTimeString(); var currentExpires = GetUserExpires().ToLongTimeString(); - if (token.ValidTo > DateTime.UtcNow && validTo != currentExpires) + if (validTo != currentExpires) { Serilog.Log.Warning($"Token expired. Token expires at {validTo}, current expires at {currentExpires}"); // login confict From ebeccad54278921df36ef84b4ce64e818dc76cde Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 26 Sep 2024 17:22:06 -0500 Subject: [PATCH 07/14] Add RemoveAsync cache --- .../Infrastructures/ICacheService.cs | 1 + .../Infrastructures/SharpCacheAttribute.cs | 13 ++++++++++++- .../Infrastructures/MemoryCacheService.cs | 5 +++++ .../Infrastructures/RedisCacheService.cs | 16 ++++++++++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/ICacheService.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/ICacheService.cs index 692e9e4af..ef9c7d8e1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/ICacheService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/ICacheService.cs @@ -5,4 +5,5 @@ public interface ICacheService Task GetAsync(string key); Task GetAsync(string key, Type type); Task SetAsync(string key, T value, TimeSpan? expiry); + Task RemoveAsync(string key); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/SharpCacheAttribute.cs b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/SharpCacheAttribute.cs index 17ddb8303..dd30f4916 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Infrastructures/SharpCacheAttribute.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Infrastructures/SharpCacheAttribute.cs @@ -31,7 +31,13 @@ public override void OnEntry(MethodContext context) var value = cache.GetAsync(key, context.TaskReturnType).Result; if (value != null) { - context.ReplaceReturnValue(this, value); + // check if the cache is out of date + var isOutOfDate = IsOutOfDate(context, value).Result; + + if (!isOutOfDate) + { + context.ReplaceReturnValue(this, value); + } } } @@ -58,6 +64,11 @@ public override void OnSuccess(MethodContext context) } } + public virtual Task IsOutOfDate(MethodContext context, object value) + { + return Task.FromResult(false); + } + private string GetCacheKey(SharpCacheSettings settings, MethodContext context) { var key = settings.Prefix + "-" + context.Method.Name; diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/MemoryCacheService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/MemoryCacheService.cs index 2d2fe13ba..a775ab6ac 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/MemoryCacheService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/MemoryCacheService.cs @@ -32,4 +32,9 @@ public async Task SetAsync(string key, T value, TimeSpan? expiry) AbsoluteExpirationRelativeToNow = expiry }); } + + public async Task RemoveAsync(string key) + { + _cache.Remove(key); + } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/RedisCacheService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/RedisCacheService.cs index 279f90148..bde2708e9 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/RedisCacheService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/RedisCacheService.cs @@ -76,4 +76,20 @@ public async Task SetAsync(string key, T value, TimeSpan? expiry) var db = redis.GetDatabase(); await db.StringSetAsync(key, JsonConvert.SerializeObject(value), expiry); } + + public async Task RemoveAsync(string key) + { + if (string.IsNullOrEmpty(_settings.Redis)) + { + return; + } + + if (redis == null) + { + redis = ConnectionMultiplexer.Connect(_settings.Redis); + } + + var db = redis.GetDatabase(); + await db.KeyDeleteAsync(key); + } } From 08d1a4091646db874a9fc600c6af755d8bef2efc Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 27 Sep 2024 21:05:30 -0500 Subject: [PATCH 08/14] =?UTF-8?q?AI=E6=90=9C=E7=B4=A2=E9=83=BD=E5=9F=BA?= =?UTF-8?q?=E4=BA=8E=E5=95=86=E5=93=81=E6=9D=A5=E5=B1=95=E5=BC=80=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BotSharp.Core/Users/Services/UserService.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index a46e16cff..e937ffcaf 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -291,8 +291,13 @@ private string GenerateJwtToken(User user) private async Task SaveUserTokenExpiresCache(string userId, DateTime expires, int expireInMinutes) { - var _cacheService = _services.GetRequiredService(); - await _cacheService.SetAsync(GetUserTokenExpiresCacheKey(userId), expires, TimeSpan.FromMinutes(expireInMinutes)); + var config = _services.GetService(); + var enableSingleLogin = bool.Parse(config["Jwt:EnableSingleLogin"] ?? "false"); + if (enableSingleLogin) + { + var _cacheService = _services.GetRequiredService(); + await _cacheService.SetAsync(GetUserTokenExpiresCacheKey(userId), expires, TimeSpan.FromMinutes(expireInMinutes)); + } } private string GetUserTokenExpiresCacheKey(string userId) From af484579f5b0927b45465bf826ec54c832fda81e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 28 Sep 2024 20:00:35 -0500 Subject: [PATCH 09/14] Support more data type in Qdrant payload. --- .../VectorStorage/IVectorDb.cs | 2 +- .../Models/VectorCollectionData.cs | 2 +- .../VectorStorage/Models/VectorCreateModel.cs | 2 +- .../VectorKnowledgeCreateRequest.cs | 2 +- .../Knowledges/VectorKnowledgeViewModel.cs | 2 +- .../Functions/MemorizeKnowledgeFn.cs | 2 +- .../MemVecDb/MemoryVectorDb.cs | 4 +- .../Services/KnowledgeService.Document.cs | 2 +- .../Services/KnowledgeService.Vector.cs | 2 +- .../Functions/PrimaryStagePlanFn.cs | 11 ++-- .../BotSharp.Plugin.Qdrant.csproj | 2 +- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 63 +++++++++++++++++-- .../Services/DbKnowledgeService.cs | 2 +- 13 files changed, 76 insertions(+), 22 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index ae0828de1..66f7727a9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -12,7 +12,7 @@ public interface IVectorDb Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false); Task CreateCollection(string collectionName, int dimension); Task DeleteCollection(string collectionName); - Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); + Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, List ids); Task DeleteCollectionAllData(string collectionName); diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs index ba614c2c2..0b075f7e7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCollectionData.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.VectorStorage.Models; public class VectorCollectionData { public string Id { get; set; } - public Dictionary Data { get; set; } = new(); + public Dictionary Data { get; set; } = new(); public double? Score { get; set; } [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs index 59199fb91..65641d849 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorCreateModel.cs @@ -6,5 +6,5 @@ public class VectorCreateModel { public string Text { get; set; } public string DataSource { get; set; } = VectorDataSource.Api; - public Dictionary? Payload { get; set; } + public Dictionary? Payload { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs index fa1c1fb30..9477d65d8 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeCreateRequest.cs @@ -11,5 +11,5 @@ public class VectorKnowledgeCreateRequest public string DataSource { get; set; } = VectorDataSource.Api; [JsonPropertyName("payload")] - public Dictionary? Payload { get; set; } + public Dictionary? Payload { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs index dcdf2e57a..bbd7dc3ab 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Knowledges/VectorKnowledgeViewModel.cs @@ -9,7 +9,7 @@ public class VectorKnowledgeViewModel public string Id { get; set; } [JsonPropertyName("data")] - public IDictionary Data { get; set; } + public IDictionary Data { get; set; } [JsonPropertyName("score")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 62388f197..0af59975d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -24,7 +24,7 @@ public async Task Execute(RoleDialogModel message) var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel { Text = args.Question, - Payload = new Dictionary + Payload = new Dictionary { { KnowledgePayloadName.Answer, args.Answer } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 41331c2b1..f766bb7e5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -59,7 +59,7 @@ public async Task> Search(string collectionNam .Take(limit) .Select(i => new VectorCollectionData { - Data = new Dictionary { { "text", _vectors[collectionName][i].Text } }, + Data = new Dictionary { { "text", _vectors[collectionName][i].Text } }, Score = similarities[i], Vector = withVector ? _vectors[collectionName][i].Vector : null, }) @@ -68,7 +68,7 @@ public async Task> Search(string collectionNam return await Task.FromResult(results); } - public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) + public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) { _vectors[collectionName].Add(new VecRecord { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs index bd851eb86..744691779 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs @@ -398,7 +398,7 @@ private async Task> SaveToVectorDb( var vectorDb = GetVectorDb(); var textEmbedding = GetTextEmbedding(collectionName); - var payload = new Dictionary + var payload = new Dictionary { { KnowledgePayloadName.DataSource, vectorDataSource }, { KnowledgePayloadName.FileId, fileId.ToString() }, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs index ee46b13e2..903eaf695 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Vector.cs @@ -196,7 +196,7 @@ public async Task UpsertVectorCollectionData(string collectionName, Vector withPayload: true); if (!found.IsNullOrEmpty()) { - if (found.First().Data["text"] == update.Text) + if (found.First().Data["text"].ToString() == update.Text) { // Only update payload return await db.Upsert(collectionName, guid, found.First().Vector, update.Text, update.Payload); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index 02d4f3a5c..e92c3e8ed 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -19,23 +19,23 @@ public async Task Execute(RoleDialogModel message) { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - var knowledgeService = _services.GetRequiredService(); - var knowledgeSettings = _services.GetRequiredService(); + // var knowledgeService = _services.GetRequiredService(); + // var knowledgeSettings = _services.GetRequiredService(); state.SetState("max_tokens", "4096"); var task = JsonSerializer.Deserialize(message.FunctionArgs); - var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + // var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; // Get knowledge from vectordb var hooks = _services.GetServices(); var knowledges = new List(); foreach (var question in task.Questions) { - var list = await knowledgeService.SearchVectorKnowledge(question, collectionName, new VectorSearchOptions + /*var list = await knowledgeService.SearchVectorKnowledge(question, collectionName, new VectorSearchOptions { Confidence = 0.4f }); - knowledges.Add(string.Join("\r\n\r\n=====\r\n", list.Select(x => x.ToQuestionAnswer()))); + knowledges.Add(string.Join("\r\n\r\n=====\r\n", list.Select(x => x.ToQuestionAnswer())));*/ foreach (var hook in hooks) { @@ -43,6 +43,7 @@ public async Task Execute(RoleDialogModel message) knowledges.AddRange(k); } } + knowledges = knowledges.Distinct().ToList(); // Get first stage planning prompt var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj b/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj index 33fae3f96..fb19756b5 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj +++ b/src/Plugins/BotSharp.Plugin.Qdrant/BotSharp.Plugin.Qdrant.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 6c54f60b9..9dde5b355 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -142,7 +142,13 @@ public async Task> GetPagedCollectionDa var points = response?.Result?.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, - Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), + Data = x.Payload.ToDictionary(p => p.Key, p => p.Value.KindCase switch + { + Value.KindOneofCase.StringValue => p.Value.StringValue, + Value.KindOneofCase.BoolValue => p.Value.BoolValue, + Value.KindOneofCase.IntegerValue => p.Value.IntegerValue, + _ => new object() + }), Vector = filter.WithVector ? x.Vectors?.Vector?.Data?.ToArray() : null })?.ToList() ?? new List(); @@ -175,12 +181,18 @@ public async Task> GetCollectionData(string co return points.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, - Data = x.Payload?.ToDictionary(x => x.Key, x => x.Value.StringValue) ?? new(), + Data = x.Payload?.ToDictionary(p => p.Key, p => p.Value.KindCase switch + { + Value.KindOneofCase.StringValue => p.Value.StringValue, + Value.KindOneofCase.BoolValue => p.Value.BoolValue, + Value.KindOneofCase.IntegerValue => p.Value.IntegerValue, + _ => new object() + }) ?? new(), Vector = x.Vectors?.Vector?.Data?.ToArray() }); } - public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) + public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) { // Insert vectors var point = new PointStruct() @@ -200,7 +212,42 @@ public async Task Upsert(string collectionName, Guid id, float[] vector, s { foreach (var item in payload) { - point.Payload[item.Key] = item.Value; + if (item.Value is string str) + { + point.Payload[item.Key] = str; + } + else if (item.Value is bool b) + { + point.Payload[item.Key] = b; + } + else if (item.Value is byte int8) + { + point.Payload[item.Key] = int8; + } + else if (item.Value is short int16) + { + point.Payload[item.Key] = int16; + } + else if (item.Value is int int32) + { + point.Payload[item.Key] = int32; + } + else if (item.Value is long int64) + { + point.Payload[item.Key] = int64; + } + else if (item.Value is float f32) + { + point.Payload[item.Key] = f32; + } + else if (item.Value is double f64) + { + point.Payload[item.Key] = f64; + } + else if (item.Value is DateTime dt) + { + point.Payload[item.Key] = dt.ToUniversalTime().ToString("o"); + } } } @@ -241,7 +288,13 @@ public async Task> Search(string collectionNam results = points.Select(x => new VectorCollectionData { Id = x.Id.Uuid, - Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), + Data = x.Payload.ToDictionary(p => p.Key, p => p.Value.KindCase switch + { + Value.KindOneofCase.StringValue => p.Value.StringValue, + Value.KindOneofCase.BoolValue => p.Value.BoolValue, + Value.KindOneofCase.IntegerValue => p.Value.IntegerValue, + _ => new object() + }), Score = x.Score, Vector = x.Vectors?.Vector?.Data?.ToArray() }).ToList(); diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs index 129da04c9..2402f7ad0 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs @@ -69,7 +69,7 @@ public async Task Import(string provider, string model, string schema) await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel { Text = item.Question, - Payload = new Dictionary + Payload = new Dictionary { { KnowledgePayloadName.Answer, item.Answer } } From 9125db9c0d6b432dad0e24874669a5a9eb5b2a0d Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 28 Sep 2024 20:01:16 -0500 Subject: [PATCH 10/14] Handle content filter error for AzureOpenAI. --- .../Providers/Chat/ChatCompletionProvider.cs | 77 ++++++++++++------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 678ee82be..d4a8a7133 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Files.Utilities; using OpenAI.Chat; +using System.ClientModel; namespace BotSharp.Plugin.AzureOpenAI.Providers.Chat; @@ -37,43 +38,67 @@ public async Task GetChatCompletions(Agent agent, List GetChatCompletions(Agent agent, List Date: Sun, 29 Sep 2024 12:48:32 +0800 Subject: [PATCH 11/14] fix: modify the return result status code --- .../BotSharp.OpenAPI/Controllers/UserController.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index 6a9b00556..a9d67c5d7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -68,7 +68,8 @@ public async Task> ActivateUser(UserActivationModel model) var token = await _userService.ActiveUser(model); if (token == null) { - return Unauthorized(); + //return Unauthorized(); + return BadRequest(); } return Ok(token); } From 2c77ad0798d4ef30e4a27e74d2001862af7eba8b Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Sun, 29 Sep 2024 13:14:52 +0800 Subject: [PATCH 12/14] remove Unauthorized --- .../BotSharp.OpenAPI/Controllers/UserController.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index a9d67c5d7..eb6458a29 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -68,7 +68,6 @@ public async Task> ActivateUser(UserActivationModel model) var token = await _userService.ActiveUser(model); if (token == null) { - //return Unauthorized(); return BadRequest(); } return Ok(token); From 8e060c9ca2383981e4d5496daab95bc940e91221 Mon Sep 17 00:00:00 2001 From: AnonymousDotNet <18776095145@163.com> Date: Sun, 29 Sep 2024 19:26:17 +0800 Subject: [PATCH 13/14] feat: add del user api --- .../Repositories/IBotSharpRepository.cs | 9 ++++---- .../Users/IAuthenticationHook.cs | 1 + .../Users/IUserService.cs | 1 + .../Users/Services/UserService.cs | 22 +++++++++++++++++-- .../Controllers/UserController.cs | 6 +++++ .../Repository/MongoRepository.User.cs | 8 +++++++ 6 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index ced839d04..de3516499 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -14,7 +14,7 @@ public interface IBotSharpRepository void Add(object entity); #region Plugin - PluginConfig GetPluginConfig(); + PluginConfig GetPluginConfig(); void SavePluginConfig(PluginConfig config); #endregion @@ -22,7 +22,7 @@ public interface IBotSharpRepository User? GetUserByEmail(string email) => throw new NotImplementedException(); User? GetUserByPhone(string phone) => throw new NotImplementedException(); User? GetAffiliateUserByPhone(string phone) => throw new NotImplementedException(); - User? GetUserById(string id) => throw new NotImplementedException(); + User? GetUserById(string id) => throw new NotImplementedException(); List GetUserByIds(List ids) => throw new NotImplementedException(); User? GetUserByAffiliateId(string affiliateId) => throw new NotImplementedException(); User? GetUserByUserName(string userName) => throw new NotImplementedException(); @@ -30,9 +30,10 @@ public interface IBotSharpRepository void UpdateUserVerified(string userId) => throw new NotImplementedException(); void UpdateUserVerificationCode(string userId, string verficationCode) => throw new NotImplementedException(); void UpdateUserPassword(string userId, string password) => throw new NotImplementedException(); - void UpdateUserEmail(string userId, string email)=> throw new NotImplementedException(); + void UpdateUserEmail(string userId, string email) => throw new NotImplementedException(); void UpdateUserPhone(string userId, string Iphone) => throw new NotImplementedException(); void UpdateUserIsDisable(string userId, bool isDisable) => throw new NotImplementedException(); + void UpdateUsersIsDisable(List userIds, bool isDisable) => throw new NotImplementedException(); #endregion #region Agent @@ -76,7 +77,7 @@ public interface IBotSharpRepository List GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable excludeAgentIds); IEnumerable TruncateConversation(string conversationId, string messageId, bool cleanLog = false); #endregion - + #region Execution Log void AddExecutionLogs(string conversationId, List logs); List GetExecutionLogs(string conversationId); diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs index cbc18dd58..572f57fd8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs @@ -10,4 +10,5 @@ public interface IAuthenticationHook void BeforeSending(Token token); Task UserCreated(User user); Task VerificationCodeResetPassword(User user); + Task DelUsers(List userIds); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs index 10c449059..750858a78 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IUserService.cs @@ -19,4 +19,5 @@ public interface IUserService Task ModifyUserPhone(string phone); Task UpdatePassword(string newPassword, string verificationCode); Task GetUserTokenExpires(); + Task UpdateUsersIsDisable(List userIds, bool isDisable); } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index e937ffcaf..c0cf95888 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -1,5 +1,5 @@ -using BotSharp.Abstraction.Users.Enums; using BotSharp.Abstraction.Infrastructures; +using BotSharp.Abstraction.Users.Enums; using BotSharp.Abstraction.Users.Models; using BotSharp.Abstraction.Users.Settings; using BotSharp.OpenAPI.ViewModels.Users; @@ -9,7 +9,6 @@ using System.IdentityModel.Tokens.Jwt; using System.Security.Claims; using System.Text.RegularExpressions; -using System.Net; namespace BotSharp.Core.Users.Services; @@ -524,4 +523,23 @@ public async Task ModifyUserPhone(string phone) db.UpdateUserPhone(record.Id, phone); return true; } + + public async Task UpdateUsersIsDisable(List userIds, bool isDisable) + { + var db = _services.GetRequiredService(); + db.UpdateUsersIsDisable(userIds, isDisable); + + if (!isDisable) + { + return true; + } + + // del membership + var hooks = _services.GetServices(); + foreach (var hook in hooks) + { + await hook.DelUsers(userIds); + } + return true; + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index eb6458a29..281de3e85 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -143,6 +143,12 @@ public async Task ModifyUserPhone([FromQuery] string phone) return await _userService.ModifyUserPhone(phone); } + [HttpPost("/user/update/isdisable")] + public async Task UpdateUsersIsDisable([FromQuery] List userIds, [FromQuery] bool isDisable) + { + return await _userService.UpdateUsersIsDisable(userIds, isDisable); + } + #region Avatar [HttpPost("/user/avatar")] public bool UploadUserAvatar([FromBody] UserAvatarModel input) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs index 3a3cf5434..9b6b62c1a 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs @@ -126,4 +126,12 @@ public void UpdateUserIsDisable(string userId, bool isDisable) .Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.Users.UpdateOne(filter, update); } + + public void UpdateUsersIsDisable(List userIds, bool isDisable) + { + foreach (var userId in userIds) + { + UpdateUserIsDisable(userId, isDisable); + } + } } From dcd3fc5ea90b5d553192448574611678e70816f6 Mon Sep 17 00:00:00 2001 From: "jason.wang" Date: Tue, 1 Oct 2024 20:21:35 +0800 Subject: [PATCH 14/14] Issues-321 --- .../Browsing/Models/BrowserActionResult.cs | 1 + .../Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs index b3576b90c..bd34e9364 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Browsing/Models/BrowserActionResult.cs @@ -2,6 +2,7 @@ namespace BotSharp.Abstraction.Browsing.Models; public class BrowserActionResult { + public int ResponseStatusCode { get; set; } public bool IsSuccess { get; set; } public string? Message { get; set; } public string? StackTrace { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index a7e5f27f5..cc1a00094 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -64,6 +64,7 @@ await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCall await Task.Delay(args.WaitTime * 1000); } + result.ResponseStatusCode = response.Status; if (response.Status == 200) { // Disable this due to performance issue, some page is too large @@ -71,7 +72,7 @@ await _instance.NewPage(message, enableResponseCallback: args.EnableResponseCall result.IsSuccess = true; } else - { + { result.Message = response.StatusText; } }