KnowPro.NET: Answers Part 3, indexing improvements (#1769)

* Answer Generation
  * Select relevant messages 
  * Include Messages in Context 
* Indexing
  * Auto Alias mapping for names
  * Auto load Synonym tables
 * Storage improvements
*  Bug fixes
* Refactor

Author: umeshma

dotnet/typeagent/examples/knowProConsole/PodcastCommands.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Threading.Tasks;
 using TypeAgent.KnowPro.Storage;
 
 namespace KnowProConsole;
@@ -48,11 +49,11 @@ private Command PodcastLoadDef()
         {
             Args.Arg<string>("name", "Name of existing podcast index"),
         };
-        cmd.SetAction(this.PodcastLoad);
+        cmd.SetAction(this.PodcastLoadAsync);
         return cmd;
     }
 
-    private void PodcastLoad(ParseResult args)
+    private async Task PodcastLoadAsync(ParseResult args)
     {
         NamedArgs namedArgs = new(args);
         string name = namedArgs.GetRequired("name");
@@ -68,6 +69,8 @@ private void PodcastLoad(ParseResult args)
 
         SetCurrent(podcast);
         KnowProWriter.WriteLine(ConsoleColor.Cyan, $"Loaded {name}");
+
+        //await podcast.BuildSecondaryIndexesAsync();
     }
 
     private Command PodcastImportIndexDef()
@@ -121,7 +124,7 @@ await podcast.ImportTranscriptAsync(
 
         if (namedArgs.Get<bool>("buildIndex"))
         {
-            await podcast.UpdateIndexAsync(cancellationToken);
+            await podcast.BuildIndexAsync(cancellationToken);
         }
     }
 

dotnet/typeagent/src/common/ListExtensions.cs

@@ -21,6 +21,12 @@ public static int GetCount<T>(this IList<T>? list)
         return list is not null ? list.Count : 0;
     }
 
+    public static T? GetOrNull<T>(this IList<T> list, int index)
+        where T : class
+    {
+        return (index < list.Count) ? list[index] : null;
+    }
+
     public static void AddRange<T>(this IList<T> list, IEnumerable<T> items)
     {
         ArgumentVerify.ThrowIfNull(items, nameof(items));

dotnet/typeagent/src/common/Multiset.cs renamed to dotnet/typeagent/src/common/MultiMap.cs

@@ -6,11 +6,11 @@ namespace TypeAgent.Common;
 /// <summary>
 /// A dictionary that permits DUPLICATES.
 /// </summary>
-public class Multiset<TKey, TValue> : Dictionary<TKey, List<TValue>>, IEnumerable<TValue>
+public class MultiMap<TKey, TValue> : Dictionary<TKey, List<TValue>>, IEnumerable<TValue>
 {
     Func<List<TValue>> _allocator = NewList;
 
-    public Multiset(IEnumerable<KeyValuePair<TKey, TValue>>? values = null)
+    public MultiMap(IEnumerable<KeyValuePair<TKey, TValue>>? values = null)
         : base()
     {
         if (values is not null)
@@ -19,12 +19,12 @@ public Multiset(IEnumerable<KeyValuePair<TKey, TValue>>? values = null)
         }
     }
 
-    public Multiset(IEqualityComparer<TKey> comparer)
+    public MultiMap(IEqualityComparer<TKey> comparer)
         : base(comparer)
     {
     }
 
-    public Multiset(
+    public MultiMap(
         int capacity,
         IEqualityComparer<TKey> comparer,
         Func<List<TValue>>? allocator

dotnet/typeagent/src/common/StringExtensions.cs

@@ -58,6 +58,15 @@ int maxCharsPerChunk
         }
     }
 
+    public static void ToLower(this IList<string> list)
+    {
+        int count = list.Count;
+        for (int i = 0; i < count; ++i)
+        {
+            list[i] = list[i].ToLower();
+        }
+    }
+
     public static List<string> LowerAndSort(this List<string> list)
     {
         int count = list.Count;
@@ -77,6 +86,15 @@ public static List<string> LowerAndSort(this List<string> list)
     public static IList<string> SplitLines(this string text, StringSplitOptions options = default)
         => text.Split(s_lineSplitter, options);
 
+    [GeneratedRegex(@"(""[^""]+""|'[^']+'|\b\S+\b)")]
+    private static partial Regex s_wordBreakRegEx();
+
+    private static readonly Regex s_wordSplitter = s_wordBreakRegEx();
+
+    public static IList<string> SplitWords(this string text, StringSplitOptions options = default)
+    => text.Split(s_wordSplitter, options);
+
+
     // Split using a Regex
     public static IList<string> Split(this string text, Regex regex, StringSplitOptions options = default)
     {

dotnet/typeagent/src/common/TopNCollection.cs

@@ -217,7 +217,7 @@ public void Add(IEnumerable<Scored<T>> items)
     public List<Scored<T>> ByRankAndClear()
     {
         var results = _items ?? [];
-        results.Sort();
+        results.Sort((x, y) => x.CompareTo(y));
         _items = null;
         return results;
     }

dotnet/typeagent/src/conversationMemory/Memory.cs

@@ -81,4 +81,13 @@ private LangSearchOptions AdjustLanguageSearchOptions(LangSearchOptions? options
 
         return options;
     }
+
+    protected void BeginIndexing()
+    {
+    }
+
+    protected void EndIndexing()
+    {
+
+    }
 }

dotnet/typeagent/src/conversationMemory/PersonName.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace TypeAgent.ConversationMemory;
+
+public class PersonName
+{
+    public PersonName(string fullName)
+    {
+        ArgumentVerify.ThrowIfNullOrEmpty(fullName, nameof(fullName));
+
+        Names = fullName.SplitWords(StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+    }
+
+    public IList<string> Names { get; }
+
+    public bool HasNames => !Names.IsNullOrEmpty();
+
+    public string? FirstName => Names.GetOrNull(0);
+
+    public string? LastName => Names.Count > 1 ? Names[^1] : null;
+}

dotnet/typeagent/src/conversationMemory/Podcast.cs

@@ -10,6 +10,36 @@ public Podcast(MemorySettings settings, IStorageProvider<PodcastMessage> provide
     {
     }
 
+    public async ValueTask BuildIndexAsync(CancellationToken cancellationToken)
+    {
+        BeginIndexing();
+        try
+        {
+            await this.UpdateIndexAsync(
+                cancellationToken
+            ).ConfigureAwait(false);
+
+            await BuildSecondaryIndexesAsync(
+                cancellationToken
+            ).ConfigureAwait(false);
+        }
+        finally
+        {
+            EndIndexing();
+        }
+    }
+
+    public async ValueTask BuildSecondaryIndexesAsync(CancellationToken cancellationToken = default)
+    {
+        await BuildParticipantAliasesAsync(
+            cancellationToken
+        ).ConfigureAwait(false);
+
+        await AddSynonymsAsync(
+            cancellationToken
+        ).ConfigureAwait(false);
+    }
+
     public async ValueTask ImportTranscriptAsync(
         string filePath,
         string? name = null,
@@ -35,6 +65,42 @@ await Messages.AppendAsync(
         ).ConfigureAwait(false);
     }
 
+    private async ValueTask AddSynonymsAsync(CancellationToken cancellationToken)
+    {
+        AliasMap aliases = AliasMap.LoadResource(
+            typeof(Podcast).Assembly,
+            "TypeAgent.ConversationMemory.podcastVerbs.json"
+        );
+
+        await SecondaryIndexes.TermToRelatedTermsIndex.Aliases.AddAsync(
+            aliases,
+            cancellationToken
+        ).ConfigureAwait(false);
+    }
+
+    private async ValueTask BuildParticipantAliasesAsync(CancellationToken cancellationToken = default)
+    {
+        var aliases = await CollectParticipantAliasesAsync(
+            cancellationToken
+        ).ConfigureAwait(false);
+
+        await SecondaryIndexes.TermToRelatedTermsIndex.Aliases.AddAsync(
+            aliases,
+            cancellationToken
+        ).ConfigureAwait(false);
+    }
+
+    private async ValueTask<AliasMap> CollectParticipantAliasesAsync(CancellationToken cancellationToken = default)
+    {
+        AliasMap aliases = [];
+        await foreach (var message in Messages)
+        {
+            PodcastMessageMeta metadata = message.Metadata;
+            metadata.CollectAliases(aliases);
+        }
+        return aliases;
+    }
+
     private void AssignMessageListeners(IList<PodcastMessage> messages, ISet<string> participants)
     {
         foreach (var message in messages)

dotnet/typeagent/src/conversationMemory/PodcastMessage.cs

@@ -23,6 +23,34 @@ public PodcastMessageMeta(string? speaker)
 
     public override string? Source => Speaker;
     public override IList<string>? Dest => Listeners;
+
+    internal void CollectAliases(AliasMap aliasMap)
+    {
+        CollectAlias(Speaker, aliasMap);
+        if (!Listeners.IsNullOrEmpty())
+        {
+            foreach (var listener in Listeners)
+            {
+                CollectAlias(listener, aliasMap);
+            }
+        }
+    }
+
+    private void CollectAlias(string? fullName, AliasMap aliasMap)
+    {
+        if (string.IsNullOrEmpty(fullName))
+        {
+            return;
+        }
+
+        PersonName person = new PersonName(fullName);
+        if (person.HasNames && person.Names.Count == 2)
+        {
+            // If participantName is a full name, then associate firstName with the full name
+            aliasMap.AddUnique(person.FirstName, fullName);
+            aliasMap.AddUnique(fullName, person.FirstName);
+        }
+    }
 }
 
 public class PodcastMessage : Message<PodcastMessageMeta>, ITranscriptMessage

dotnet/typeagent/src/conversationMemory/conversationMemory.csproj

@@ -10,10 +10,12 @@
 
   <ItemGroup>
     <None Remove="noiseTerms.txt" />
+    <None Remove="podcastVerbs.json" />
   </ItemGroup>
 
   <ItemGroup>
     <EmbeddedResource Include="noiseTerms.txt" />
+    <EmbeddedResource Include="podcastVerbs.json" />
   </ItemGroup>
 
   <ItemGroup>