KnowPro.NET updates (#1761)

* Entity merging
  * For distinct operator etc. 
 * For answer generation
* Updated test app

Author: umeshma

dotnet/typeagent/examples/examplesLib/KnowProWriter.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Threading.Tasks;
 using TypeAgent.ExamplesLib.CommandLine;
 
 namespace TypeAgent.ExamplesLib;
@@ -35,16 +36,16 @@ public static async Task WriteSemanticRefsAsync(IConversation conversation)
         }
     }
 
-public static async Task WriteMessagesAsync(IConversation conversation)
-{
-    await foreach (var message in conversation.Messages)
+    public static async Task WriteMessagesAsync(IConversation conversation)
     {
-        WriteMessage(message);
-        WriteLine();
+        await foreach (var message in conversation.Messages)
+        {
+            WriteMessage(message);
+            WriteLine();
+        }
     }
-}
 
-public static void WriteMessage(IMessage message)
+    public static void WriteMessage(IMessage message)
     {
         PushColor(ConsoleColor.Cyan);
         WriteNameValue("Timestamp", message.Timestamp);
@@ -71,7 +72,7 @@ public static void WriteMetadata(IMessage message)
         }
     }
 
-public static void WriteEntity(ConcreteEntity? entity)
+    public static void WriteEntity(ConcreteEntity? entity)
     {
         if (entity is not null)
         {
@@ -85,10 +86,36 @@ public static void WriteEntity(ConcreteEntity? entity)
         }
     }
 
+    public static void WriteAction(TypeAgent.KnowPro.Action? action)
+    {
+        if (action is not null)
+        {
+            WriteLine(action.ToString());
+        }
+    }
+
+    public static void WriteTopic(Topic? topic)
+    {
+        if (topic is not null)
+        {
+            WriteLine(topic.Text);
+        }
+    }
+
+    public static void WriteTag(Tag tag)
+    {
+        if (tag is not null)
+        {
+            WriteLine(tag.Text);
+        }
+    }
+
     public static async Task WriteConversationSearchResultsAsync(
-        IConversation conversation,
-        ConversationSearchResult? searchResult,
-        bool verbose = false
+            IConversation conversation,
+            ConversationSearchResult? searchResult,
+            bool showKnowledge,
+            bool showMessages,
+            bool verbose = false
     )
     {
         if (searchResult is null)
@@ -118,7 +145,7 @@ public static async Task WriteConversationSearchResultsAsync(
         if (!searchResult.KnowledgeMatches.IsNullOrEmpty())
         {
             WriteLineHeading("Knowledge");
-            WriteKnowledgeSearchResults(conversation, searchResult.KnowledgeMatches);
+            await WriteKnowledgeSearchResultsAsync(conversation, searchResult.KnowledgeMatches);
         }
     }
 
@@ -131,9 +158,11 @@ IList<ScoredMessageOrdinal> messageOrdinals
         WriteJson(messageOrdinals);
     }
 
-    public static void WriteKnowledgeSearchResults(
+    public static async Task WriteKnowledgeSearchResultsAsync(
         IConversation conversation,
-        IDictionary<KnowledgeType, SemanticRefSearchResult>? results
+        IDictionary<KnowledgeType, SemanticRefSearchResult>? results,
+        int? maxToDisplay = null,
+        bool isAsc = false
     )
     {
         if (results.IsNullOrEmpty())
@@ -144,15 +173,17 @@ public static void WriteKnowledgeSearchResults(
 
         foreach (var kv in results!)
         {
-            WriteKnowledgeSearchResult(conversation, kv.Key, kv.Value);
+            await WriteKnowledgeSearchResultAsync(conversation, kv.Key, kv.Value, maxToDisplay, isAsc);
             WriteLine();
         }
     }
 
-    public static void WriteKnowledgeSearchResult(
+    public static async Task WriteKnowledgeSearchResultAsync(
         IConversation conversation,
         KnowledgeType kType,
-        SemanticRefSearchResult result
+        SemanticRefSearchResult result,
+        int? maxToDisplay = null,
+        bool isAsc = true
     )
     {
         WriteLineUnderline(kType.ToString().ToUpper());
@@ -162,7 +193,102 @@ SemanticRefSearchResult result
             ListType.Ol)
         );
         WriteLine($"{result.SemanticRefMatches.Count} matches");
-        WriteJson(result.SemanticRefMatches);
+
+        await WriteScoredSemanticRefsAsync(
+            result.SemanticRefMatches,
+            conversation.SemanticRefs,
+            kType,
+            maxToDisplay is not null ? maxToDisplay.Value : result.SemanticRefMatches.Count,
+            isAsc
+        );
+    }
+
+    public static void WriteSemanticRef(SemanticRef sr)
+    {
+        switch (sr.KnowledgeType)
+        {
+            default:
+                break;
+
+            case KnowledgeType.EntityTypeName:
+            case KnowledgeType.STagTypeName:
+                WriteEntity(sr.AsEntity());
+                break;
+
+            case KnowledgeType.ActionTypeName:
+                WriteAction(sr.AsAction());
+                break;
+
+            case KnowledgeType.TopicTypeName:
+                WriteTopic(sr.AsTopic());
+                break;
+
+            case KnowledgeType.TagTypeName:
+                WriteTag(sr.AsTag());
+                break;
+        }
+    }
+
+    public static async Task WriteScoredSemanticRefsAsync(
+        IList<ScoredSemanticRefOrdinal> semanticRefMatches,
+        ISemanticRefCollection semanticRefCollection,
+        KnowledgeType kType,
+        int maxToDisplay,
+        bool isAsc = true
+    )
+    {
+        if (isAsc)
+        {
+            WriteLine("Sorted in ascending order(lowest first)");
+        }
+
+        var matchesToDisplay = semanticRefMatches.Slice(0, maxToDisplay);
+        WriteLine($"Displaying {matchesToDisplay.Count} matches of total {semanticRefMatches.Count}");
+
+        if (kType == KnowledgeType.Entity)
+        {
+            IList<Scored<ConcreteEntity>> entities = await semanticRefCollection.GetDistinctEntitiesAsync(matchesToDisplay);
+            for (int i = 0; i < entities.Count; ++i)
+            {
+                var pos = isAsc ? matchesToDisplay.Count - (i + 1) : i;
+                WriteLine(
+                    ConsoleColor.Green,
+                    $"{pos + 1} / {matchesToDisplay.Count}: [{entities[i].Score}]"
+                );
+                WriteEntity(entities[i]);
+                WriteLine();
+            }
+        }
+        else
+        {
+            IList<SemanticRef> semanticRefs = await semanticRefCollection.GetAsync(matchesToDisplay);
+            for (int i = 0; i < matchesToDisplay.Count; ++i)
+            {
+                var pos = isAsc ? matchesToDisplay.Count - (i + 1) : i;
+                WriteScoredRef(
+                    pos,
+                    matchesToDisplay.Count,
+                    matchesToDisplay[pos],
+                    semanticRefs[pos]
+                );
+            }
+        }
+
+    }
+
+    public static void WriteScoredRef(
+        int matchNumber,
+        int totalMatches,
+        ScoredSemanticRefOrdinal scoredRef,
+        SemanticRef semanticRef
+    )
+    {
+        WriteLine(
+            ConsoleColor.Green,
+            $"#{matchNumber + 1} / {totalMatches}: <{scoredRef.SemanticRefOrdinal}::{semanticRef.Range.Start.MessageOrdinal}> {semanticRef.KnowledgeType} [{scoredRef.Score}]"
+        );
+        WriteSemanticRef(semanticRef);
+        WriteLine();
     }
 
     public static void WriteDataFileStats<TMessage>(ConversationData<TMessage> data)

dotnet/typeagent/examples/knowProConsole/MemoryCommands.cs

@@ -154,7 +154,12 @@ private async Task SearchRagAsync(ParseResult args, CancellationToken cancellati
             namedArgs.Get<int>("budget"),
             cancellationToken
         );
-        await KnowProWriter.WriteConversationSearchResultsAsync(conversation, matches, true);
+        await KnowProWriter.WriteConversationSearchResultsAsync(
+            conversation,
+            matches,
+            true,
+            true
+        );
     }
 
     private IConversation EnsureConversation()

dotnet/typeagent/examples/knowProConsole/TestCommands.cs

@@ -124,7 +124,12 @@ private async Task SearchMessagesAsync(ParseResult result, CancellationToken can
             null,
             cancellationToken
         );
-        await KnowProWriter.WriteConversationSearchResultsAsync(conversation, searchResults);
+        await KnowProWriter.WriteConversationSearchResultsAsync(
+            conversation,
+            searchResults,
+            true,
+            true
+        );
 
         DateRange? conversationDateRange = await conversation.GetDateRangeAsync();
         if (conversationDateRange is not null)
@@ -148,7 +153,7 @@ private async Task SearchMessagesAsync(ParseResult result, CancellationToken can
             },
             cancellationToken
         );
-        await KnowProWriter.WriteConversationSearchResultsAsync(conversation, searchResults, true);
+        await KnowProWriter.WriteConversationSearchResultsAsync(conversation, searchResults, true, true);
     }
 
     private Command TestEmbeddingsDef()
@@ -280,7 +285,7 @@ async Task TestSearchKnowledgeAsync(IConversation conversation, SearchTermGroup
             cancellationToken
         ).ConfigureAwait(false);
 
-        KnowProWriter.WriteKnowledgeSearchResults(_kpContext.Conversation!, results);
+        await KnowProWriter.WriteKnowledgeSearchResultsAsync(_kpContext.Conversation!, results);
     }
 
     private Command SearchQueryTermsDef()
@@ -334,9 +339,14 @@ private async Task SearchLangAsync(ParseResult args, CancellationToken cancellat
         if (conversation is IMemory memory)
         {
             IList<ConversationSearchResult> results = await memory.SearchAsync(query, null, null, null, cancellationToken);
-            foreach(var result in results)
+            foreach (var result in results)
             {
-                await KnowProWriter.WriteConversationSearchResultsAsync(conversation, result);
+                await KnowProWriter.WriteConversationSearchResultsAsync(
+                    conversation,
+                    result,
+                    true,
+                    false
+                );
             }
         }
     }

dotnet/typeagent/src/common/EnumerationExtensions.cs

@@ -78,4 +78,13 @@ public static IEnumerable<List<T>> Batch<T>(this IEnumerable<T> items, int batch
             yield return batch;
         }
     }
+
+    public static List<Scored<T>> GetTopK<T>(this IEnumerable<Scored<T>> items, int topk)
+    {
+        ArgumentVerify.ThrowIfNull(items, nameof(items));
+
+        var topNList = new TopNCollection<T>(topk);
+        topNList.Add(items);
+        return topNList.ByRankAndClear();
+    }
 }

dotnet/typeagent/src/common/ListExtensions.cs

@@ -140,13 +140,6 @@ public static int BinarySearchFirst<T, TSearchValue>(
         return lo;
     }
 
-    public static List<Scored<T>> GetTopK<T>(this IEnumerable<Scored<T>> list, int topK)
-    {
-        var topNList = new TopNCollection<T>(topK);
-        topNList.Add(list);
-        return topNList.ByRankAndClear();
-    }
-
     public static void Fill<T>(this IList<T> list, T value, int count)
     {
         for (int i = 0; i < count; ++i)

dotnet/typeagent/src/common/Multiset.cs

@@ -73,6 +73,23 @@ public void Add(IEnumerable<KeyValuePair<TKey, TValue>> keyValues)
         }
     }
 
+    public void AddUnique(TKey key, TValue value)
+    {
+        List<TValue>? values = Get(key);
+        if (values is null)
+        {
+            Add(key, value);
+        }
+        else
+        {
+            int pos = values.IndexOf(value);
+            if (pos < 0)
+            {
+                Add(key, value);
+            }
+        }
+    }
+
     public void Remove(TKey key, TValue value)
     {
         if (TryGetValue(key, out var valueList))

dotnet/typeagent/src/common/StringExtensions.cs

@@ -47,4 +47,16 @@ int maxCharsPerChunk
             yield return chunk;
         }
     }
+
+    public static List<string> LowerAndSort(this List<string> list)
+    {
+        int count = list.Count;
+        for (int i = 0; i < count; ++i)
+        {
+            list[i] = list[i].ToLower();
+        }
+        list.Sort();
+        return list;
+    }
+
 }