KnowPro.NET Language Search (#1760)

* Hooking up natural language search pipeline
* Memory<T> updates
* Standard mechanism for Noise terms
* Refactor, Bug fixes

Author: umeshma

dotnet/typeagent/examples/examplesLib/CommandLine/NamedArgs.cs

@@ -3,7 +3,7 @@
 
 namespace TypeAgent.ExamplesLib.CommandLine;
 
-public class NamedArgs
+public class NamedArgs : INamedArgs
 {
     string _argPrefix;
     ParseResult _args;

dotnet/typeagent/examples/knowProConsole/MemoryCommands.cs

@@ -19,6 +19,7 @@ public IList<Command> GetCommands()
             SearchTermsDef(),
             MessagesDef(),
             AliasesDef(),
+            SearchDef(),
             SearchRagDef()
         ];
     }
@@ -108,6 +109,22 @@ private async Task AliasesAsync(ParseResult parseResult, CancellationToken cance
         }
     }
 
+    private Command SearchDef()
+    {
+        Command command = new("kpSearch")
+        {
+            Options.Arg<string>("query"),
+        };
+        command.SetAction(this.SearchAsync);
+        return command;
+    }
+    private Task SearchAsync(ParseResult args, CancellationToken cancellationToken)
+    {
+        IConversation conversation = EnsureConversation();
+        KnowProWriter.WriteError("Not impl");
+        return Task.CompletedTask;
+    }
+
     private Command SearchRagDef()
     {
         Command command = new("kpSearchRag")

dotnet/typeagent/examples/knowProConsole/TestCommands.cs

@@ -22,6 +22,7 @@ public IList<Command> GetCommands()
             SearchMessagesTermsDef(),
             TestEmbeddingsDef(),
             SearchQueryTermsDef(),
+            SearchLangDef(),
             KnowledgeDef(),
             BuildIndexDef(),
         ];
@@ -295,7 +296,7 @@ private Command SearchQueryTermsDef()
 
     private async Task SearchQueryTermsAsync(ParseResult args, CancellationToken cancellationToken)
     {
-       // IConversation conversation = EnsureConversation();
+        // IConversation conversation = EnsureConversation();
 
         NamedArgs namedArgs = new NamedArgs(args);
         var query = namedArgs.Get("query");
@@ -309,6 +310,38 @@ private async Task SearchQueryTermsAsync(ParseResult args, CancellationToken can
         KnowProWriter.WriteJson(result);
     }
 
+    private Command SearchLangDef()
+    {
+        Command cmd = new("kpTestSearchLang")
+        {
+            Args.Arg<string>("query")
+        };
+        cmd.TreatUnmatchedTokensAsErrors = false;
+        cmd.SetAction(this.SearchLangAsync);
+        return cmd;
+    }
+
+    private async Task SearchLangAsync(ParseResult args, CancellationToken cancellationToken)
+    {
+        IConversation conversation = EnsureConversation();
+
+        NamedArgs namedArgs = new NamedArgs(args);
+        var query = namedArgs.Get("query");
+        if (string.IsNullOrEmpty(query))
+        {
+            return;
+        }
+        if (conversation is IMemory memory)
+        {
+            IList<ConversationSearchResult> results = await memory.SearchAsync(query, null, null, null, cancellationToken);
+            foreach(var result in results)
+            {
+                await KnowProWriter.WriteConversationSearchResultsAsync(conversation, result);
+            }
+        }
+    }
+
+
     private Command KnowledgeDef()
     {
         Command cmd = new("kpTestKnowledge")

dotnet/typeagent/src/common/INamedArgs.cs

@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace TypeAgent.Common;
+
+public interface INamedArgs
+{
+    string? Get(string name);
+
+    string GetRequired(string name);
+
+    T? Get<T>(string name);
+}

dotnet/typeagent/src/common/ListExtensions.cs

@@ -21,6 +21,16 @@ public static int GetCount<T>(this IList<T>? list)
         return list is not null ? list.Count : 0;
     }
 
+    public static void AddRange<T>(this IList<T> list, IEnumerable<T> items)
+    {
+        ArgumentVerify.ThrowIfNull(items, nameof(items));
+
+        foreach (var item in items)
+        {
+            list.Add(item);
+        }
+    }
+
     public static void Shift<T>(this IList<T> list)
     {
         if (list.Count > 0)

dotnet/typeagent/src/common/SetExtensions.cs

@@ -5,6 +5,11 @@ namespace TypeAgent.Common;
 
 public static class SetExtensions
 {
+    public static bool IsNullOrEmpty<T>(this ISet<T> set)
+    {
+        return set is null || set.Count == 0;
+    }
+
     public static void LoadFromFile(this HashSet<string> set, string filePath)
     {
         foreach(var line in File.ReadLines(filePath))

dotnet/typeagent/src/conversationMemory/IMemory.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace TypeAgent.ConversationMemory;
+
+public interface IMemory
+{
+    ValueTask<IList<ConversationSearchResult>> SearchAsync(
+        string searchText,
+        LangSearchOptions? options = null,
+        LangSearchFilter? filter = null,
+        LangSearchDebugContext? debugContext = null,
+        CancellationToken cancellationToken = default
+    );
+}

dotnet/typeagent/src/conversationMemory/Includes.cs

@@ -8,6 +8,8 @@
 global using System.Text.Json;
 global using System.Text.Json.Serialization;
 
+global using Microsoft.TypeChat;
+
 global using TypeAgent.Common;
 global using TypeAgent.KnowPro;
 global using TypeAgent.KnowPro.Lang;

dotnet/typeagent/src/conversationMemory/Memory.cs

@@ -3,7 +3,7 @@
 
 namespace TypeAgent.ConversationMemory;
 
-public class Memory<TMessage> : Conversation<TMessage>
+public class Memory<TMessage> : Conversation<TMessage>, IMemory
     where TMessage : class, IMessage, new()
 {
 
@@ -18,4 +18,69 @@ public Memory(MemorySettings settings, IStorageProvider<TMessage> storageProvide
     public string Name { get; set; }
 
     public IList<string> Tags { get; set; }
+
+    public NoiseText NoiseTerms { get; set; }
+
+    private bool UseScoped => Settings.UseScopedSearch is not null && Settings.UseScopedSearch.Value;
+
+    public async ValueTask<IList<ConversationSearchResult>> SearchAsync(
+        string searchText,
+        LangSearchOptions? options = null,
+        LangSearchFilter? filter = null,
+        LangSearchDebugContext? debugContext = null,
+        CancellationToken cancellationToken = default
+    )
+    {
+        options = AdjustLanguageSearchOptions(options);
+        if (UseScoped)
+        {
+            // Using Structured Tags for scoping
+            throw new NotImplementedException();
+        }
+        else
+        {
+            IConversation conversation = this;
+            return await conversation.SearchAsync(
+                searchText,
+                Settings.QueryTranslator,
+                options,
+                filter,
+                debugContext,
+                cancellationToken
+            ).ConfigureAwait(false);
+        }
+    }
+
+    public virtual IList<PromptSection>? GetModelInstructions()
+    {
+        return null;
+    }
+
+    private LangSearchOptions AdjustLanguageSearchOptions(LangSearchOptions? options)
+    {
+        // Clone options so we can edit them
+        options = options is null
+            ? LangSearchOptions.CreateTypical()
+            : new LangSearchOptions(options);
+
+        var instructions = GetModelInstructions();
+        if (!instructions.IsNullOrEmpty())
+        {
+            if (!options.ModelInstructions.IsNullOrEmpty())
+            {
+                options.ModelInstructions.AddRange(instructions);
+            }
+            else
+            {
+                options.ModelInstructions = instructions;
+            }
+        }
+        // Filter noise terms
+        options.CompilerSettings.TermFilter = (t) =>
+        {
+            return NoiseTerms.IsNullOrEmpty() || !NoiseTerms.Contains(t);
+        };
+
+        return options;
+    }
 }

dotnet/typeagent/src/conversationMemory/MemorySettings.cs

@@ -16,7 +16,8 @@ public MemorySettings(
         ConversationSettings conversationSettings,
         int embeddingCacheSize = 64,
         IChatModel? chatModel = null,
-        ITextEmbeddingModel? embeddingModel = null
+        ITextEmbeddingModel? embeddingModel = null,
+        NoiseText? noiseTerms = null
     )
     {
         ArgumentVerify.ThrowIfNull(conversationSettings, nameof(conversationSettings));
@@ -26,6 +27,12 @@ public MemorySettings(
         EmbeddingModel = new TextEmbeddingModelWithCache(embeddingCacheSize);
         QueryTranslator = new SearchQueryTranslator(ChatModel);
         KnowledgeExtractor = new KnowledgeExtractor(ChatModel);
+
+        NoiseTerms = noiseTerms ?? new NoiseText(
+            typeof(MemorySettings).Assembly,
+            "TypeAgent.ConversationMemory.noiseTerms.txt"
+        );
+
     }
 
     public IChatModel ChatModel { get; }
@@ -37,4 +44,9 @@ public MemorySettings(
     public ISearchQueryTranslator? QueryTranslator { get; set; }
 
     public IKnowledgeExtractor KnowledgeExtractor { get; set; }
+
+    public NoiseText? NoiseTerms { get; set; }
+
+    // Setting this to true leverages Structured Tags
+    public bool? UseScopedSearch { get; set; } = null;
 }