Merge branch 'SciSharp:master' into master

Author: hchen2020

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs

@@ -63,4 +63,13 @@ Task<bool> SendMessage(string agentId,
     bool IsConversationMode();
 
     void SaveStates();
+
+    /// <summary>
+    /// Get conversation keys for searching
+    /// </summary>
+    /// <param name="query">search query</param>
+    /// <param name="convLimit">conversation limit</param>
+    /// <param name="preLoad">if pre-loading, then keys are not filter by the search query</param>
+    /// <returns></returns>
+    Task<List<string>> GetConversationSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false);
 }

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -146,7 +146,9 @@ List<Conversation> GetLastConversations()
         => throw new NotImplementedException();
     List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours, IEnumerable<string> excludeAgentIds)
          => throw new NotImplementedException();
-    IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+    List<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+         => throw new NotImplementedException();
+    List<string> GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100)
          => throw new NotImplementedException();
     #endregion
 

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -221,4 +221,18 @@ public void SaveStates()
     {
         _state.Save();
     }
+
+    public async Task<List<string>> GetConversationSearhKeys(string query, int convlimit = 100, int keyLimit = 10, bool preLoad = false)
+    {
+        var keys = new List<string>();
+        if (!preLoad && string.IsNullOrWhiteSpace(query))
+        {
+            return keys;
+        }
+
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        keys = db.GetConversationSearchKeys(convlimit: convlimit);
+        keys = preLoad ? keys : keys.Where(x => x.Contains(query, StringComparison.OrdinalIgnoreCase)).ToList();
+        return keys.OrderBy(x => x).Take(keyLimit).ToList();
+    }
 }

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -131,7 +131,7 @@ public void UpdateConversationStates(string conversationId, List<StateKeyValue>
     public void UpdateConversationStatus(string conversationId, string status)
         => throw new NotImplementedException();
 
-    public IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+    public List<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
         => throw new NotImplementedException();
     #endregion
 

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -547,7 +547,7 @@ public List<string> GetIdleConversations(int batchSize, int messageLimit, int bu
         }
 
 
-        public IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+        public List<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
         {
             var deletedMessageIds = new List<string>();
             if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId))
@@ -603,6 +603,46 @@ public IEnumerable<string> TruncateConversation(string conversationId, string me
         }
 
 
+        public List<string> GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100)
+        {
+            var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir);
+            if (!Directory.Exists(dir)) return [];
+
+            var count = 0;
+            var keys = new List<string>();
+
+            foreach (var d in Directory.GetDirectories(dir))
+            {
+                var convFile = Path.Combine(d, CONVERSATION_FILE);
+                var stateFile = Path.Combine(d, STATE_FILE);
+                if (!File.Exists(convFile) || !File.Exists(stateFile))
+                {
+                    continue;
+                }
+
+                var convJson = File.ReadAllText(convFile);
+                var stateJson = File.ReadAllText(stateFile);
+                var conv = JsonSerializer.Deserialize<Conversation>(convJson, _options);
+                var states = JsonSerializer.Deserialize<List<StateKeyValue>>(stateJson, _options);
+                if (conv == null || conv.DialogCount < messageLimit)
+                {
+                    continue;
+                }
+
+                var stateKeys = states?.Select(x => x.Key)?.Distinct()?.ToList() ?? [];
+                keys.AddRange(stateKeys);
+                count++;
+
+                if (count > convlimit)
+                {
+                    break;
+                }
+            }
+
+            return keys.Distinct().ToList();
+        }
+
+
         #region Private methods
         private string? FindConversationDirectory(string conversationId)
         {

src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs

@@ -553,6 +553,16 @@ public async Task<bool> UnpinConversationFromDashboard([FromRoute] string agentI
     }
     #endregion
 
+    #region Search state keys
+    [HttpGet("/conversation/state/keys")]
+    public async Task<List<string>> GetConversationStateKeys([FromQuery] string query, [FromQuery] int keyLimit = 10, [FromQuery] bool preLoad = false)
+    {
+        var convService = _services.GetRequiredService<IConversationService>();
+        var keys = await convService.GetConversationSearhKeys(query, keyLimit: keyLimit, preLoad: preLoad);
+        return keys;
+    }
+    #endregion
+
     #region Private methods
     private void SetStates(IConversationService conv, NewMessageModel input)
     {

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDialogDocument.cs

@@ -4,5 +4,6 @@ public class ConversationDialogDocument : MongoBase
 {
     public string ConversationId { get; set; }
     public string AgentId { get; set; }
+    public DateTime UpdatedTime { get; set; }
     public List<DialogMongoElement> Dialogs { get; set; }
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs

@@ -4,6 +4,7 @@ public class ConversationStateDocument : MongoBase
 {
     public string ConversationId { get; set; }
     public string AgentId { get; set; }
+    public DateTime UpdatedTime { get; set; }
     public List<StateMongoElement> States { get; set; } = new List<StateMongoElement>();
     public List<BreakpointMongoElement> Breakpoints { get; set; } = new List<BreakpointMongoElement>();
 }

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -30,16 +30,18 @@ public void CreateNewConversation(Conversation conversation)
             Id = Guid.NewGuid().ToString(),
             ConversationId = convDoc.Id,
             AgentId = conversation.AgentId,
-            Dialogs = new List<DialogMongoElement>()
+            Dialogs = [],
+            UpdatedTime = utcNow
         };
 
         var stateDoc = new ConversationStateDocument
         {
             Id = Guid.NewGuid().ToString(),
             ConversationId = convDoc.Id,
             AgentId = conversation.AgentId,
-            States = new List<StateMongoElement>(),
-            Breakpoints = new List<BreakpointMongoElement>()
+            States = [],
+            Breakpoints = [],
+            UpdatedTime = utcNow
         };
 
         _dc.Conversations.InsertOne(convDoc);
@@ -97,7 +99,8 @@ public void AppendConversationDialogs(string conversationId, List<DialogElement>
         var filterConv = Builders<ConversationDocument>.Filter.Eq(x => x.Id, conversationId);
         var filterDialog = Builders<ConversationDialogDocument>.Filter.Eq(x => x.ConversationId, conversationId);
         var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList();
-        var updateDialog = Builders<ConversationDialogDocument>.Update.PushEach(x => x.Dialogs, dialogElements);
+        var updateDialog = Builders<ConversationDialogDocument>.Update.PushEach(x => x.Dialogs, dialogElements)
+                                                                      .Set(x => x.UpdatedTime, DateTime.UtcNow);
         var updateConv = Builders<ConversationDocument>.Update.Set(x => x.UpdatedTime, DateTime.UtcNow)
                                                               .Inc(x => x.DialogCount, dialogs.Count);
 
@@ -190,7 +193,8 @@ public bool UpdateConversationMessage(string conversationId, UpdateMessageReques
             found.SecondaryRichContent = request.Message.RichContent;
         }
 
-        var update = Builders<ConversationDialogDocument>.Update.Set(x => x.Dialogs, dialogs);
+        var update = Builders<ConversationDialogDocument>.Update.Set(x => x.Dialogs, dialogs)
+                                                                .Set(x => x.UpdatedTime, DateTime.UtcNow);
         _dc.ConversationDialogs.UpdateOne(filter, update);
         return true;
     }
@@ -208,7 +212,8 @@ public void UpdateConversationBreakpoint(string conversationId, ConversationBrea
             Reason = breakpoint.Reason
         };
         var filterState = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
-        var updateState = Builders<ConversationStateDocument>.Update.Push(x => x.Breakpoints, newBreakpoint);
+        var updateState = Builders<ConversationStateDocument>.Update.Push(x => x.Breakpoints, newBreakpoint)
+                                                                    .Set(x => x.UpdatedTime, DateTime.UtcNow);
 
         _dc.ConversationStates.UpdateOne(filterState, updateState);
     }
@@ -258,7 +263,8 @@ public void UpdateConversationStates(string conversationId, List<StateKeyValue>
 
         var filterStates = Builders<ConversationStateDocument>.Filter.Eq(x => x.ConversationId, conversationId);
         var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList();
-        var updateStates = Builders<ConversationStateDocument>.Update.Set(x => x.States, saveStates);
+        var updateStates = Builders<ConversationStateDocument>.Update.Set(x => x.States, saveStates)
+                                                                     .Set(x => x.UpdatedTime, DateTime.UtcNow);
 
         _dc.ConversationStates.UpdateOne(filterStates, updateStates);
     }
@@ -500,7 +506,7 @@ public List<string> GetIdleConversations(int batchSize, int messageLimit, int bu
         return conversationIds.Take(batchSize).ToList();
     }
 
-    public IEnumerable<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
+    public List<string> TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
     {
         var deletedMessageIds = new List<string>();
         if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId))
@@ -566,11 +572,13 @@ public IEnumerable<string> TruncateConversation(string conversationId, string me
             }
 
             // Update
+            foundStates.UpdatedTime = DateTime.UtcNow;
             _dc.ConversationStates.ReplaceOne(stateFilter, foundStates);
         }
 
         // Save dialogs
         foundDialog.Dialogs = truncatedDialogs;
+        foundDialog.UpdatedTime = DateTime.UtcNow;
         _dc.ConversationDialogs.ReplaceOne(dialogFilter, foundDialog);
 
         // Update conversation
@@ -603,6 +611,27 @@ public IEnumerable<string> TruncateConversation(string conversationId, string me
         return deletedMessageIds;
     }
 
+
+    public List<string> GetConversationSearchKeys(int messageLimit = 2, int convlimit = 100)
+    {
+        var convFilter = Builders<ConversationDocument>.Filter.Gte(x => x.DialogCount, messageLimit);
+        var conversations = _dc.Conversations.Find(convFilter)
+                                             .SortByDescending(x => x.UpdatedTime)
+                                             .Limit(convlimit)
+                                             .ToList();
+
+        if (conversations.IsNullOrEmpty()) return [];
+
+        var convIds = conversations.Select(x => x.Id).ToList();
+        var stateFilter = Builders<ConversationStateDocument>.Filter.In(x => x.ConversationId, convIds);
+
+        var states = _dc.ConversationStates.Find(stateFilter).ToList();
+        var keys = states.SelectMany(x => x.States.Select(x => x.Key)).Distinct().ToList();
+        return keys;
+    }
+
+
+
     private string ConvertSnakeCaseToPascalCase(string snakeCase)
     {
         string[] words = snakeCase.Split('_');

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs

@@ -429,6 +429,11 @@ public bool UpdateUser(User user, bool updateUserAgents = false)
         return true;
     }
 
+    public Dashboard? GetDashboard(string userId = null)
+    {
+        return null;
+    }
+
     public void AddDashboardConversation(string userId, string conversationId)
     {
         var user = _dc.Users.AsQueryable()