KnowPro.NET: Fixed SqliteProvider ordering bug (#1771)

SqliteProvider Bug:
- For bulk lookups (list of ordinals etc.) was not returning values in
the order of asked for
- The "IN" clause does not guarantee order
- Collection semantics require values to be returned in the precise
order ids were provided

Author: umeshma

dotnet/typeagent/examples/knowProConsole/PodcastCommands.cs

@@ -53,7 +53,7 @@ private Command PodcastLoadDef()
         return cmd;
     }
 
-    private async Task PodcastLoadAsync(ParseResult args)
+    private Task PodcastLoadAsync(ParseResult args)
     {
         NamedArgs namedArgs = new(args);
         string name = namedArgs.GetRequired("name");
@@ -71,6 +71,7 @@ private async Task PodcastLoadAsync(ParseResult args)
         KnowProWriter.WriteLine(ConsoleColor.Cyan, $"Loaded {name}");
 
         //await podcast.BuildSecondaryIndexesAsync();
+        return Task.CompletedTask;
     }
 
     private Command PodcastImportIndexDef()
@@ -165,6 +166,9 @@ private async Task ImportExistingIndexAsync(NamedArgs namedArgs, string filePath
             count = await podcast.ImportPropertyIndexAsync(data.SemanticRefs, cancellationToken);
             KnowProWriter.WriteLine($"{count} properties imported");
 
+            await podcast.UpdateMessageIndexAsync(false, cancellationToken);
+            await podcast.BuildSecondaryIndexesAsync(cancellationToken);
+
             SetCurrent(podcast);
         }
         catch

dotnet/typeagent/examples/knowProConsole/TestCommands.cs

@@ -387,14 +387,15 @@ private Command AnswerDef()
     {
         Command cmd = new("kpTestAnswer")
         {
-            Args.Arg<string>("query")
+            Args.Arg<string>("query"),
+            Options.Arg<bool>("debug", false)
         };
         cmd.TreatUnmatchedTokensAsErrors = false;
         cmd.SetAction(this.AnswerAsync);
         return cmd;
     }
 
-    private async Task AnswerAsync(ParseResult args, CancellationToken cancellationToken)
+    private async Task AnswerAsync(ParseResult args)
     {
         IConversation conversation = EnsureConversation();
 
@@ -404,6 +405,7 @@ private async Task AnswerAsync(ParseResult args, CancellationToken cancellationT
         {
             return;
         }
+        /*
         IList<AnswerResponse> answers = await conversation.AnswerQuestionAsync(
             query,
             null,
@@ -413,23 +415,19 @@ private async Task AnswerAsync(ParseResult args, CancellationToken cancellationT
             cancellationToken
         ).ConfigureAwait(false);
         KnowProWriter.WriteJson(answers);
-        /*
-        AnswerContext context = new AnswerContext();
-
-        IList<ConcreteEntity> entities = await conversation.SemanticRefs.GetAllEntitiesAsync(cancellationToken);
-        entities = [.. entities.ToDistinct()];
-
-        IList<Topic> topics = await conversation.SemanticRefs.GetAllTopicsAsync(cancellationToken);
-        topics = [.. topics.ToDistinct()];
-
-        context.Entities = entities.Map((e) => new RelevantEntity { Entity = e });
-        context.Topics = topics.Map((t) => new RelevantTopic { Topic = t });
-
-        List<IMessage> messages = await conversation.Messages.GetAllAsync(cancellationToken);
-        context.Messages = messages.Map((m) => new RelevantMessage(m));
-        string prompt = context.ToPromptString();
-        ConsoleWriter.WriteLine(prompt);
         */
+        var searchResults = await conversation.SearchAsync(query);
+        foreach (var searchResult in searchResults)
+        {
+            AnswerContext context = await AnswerContext.FromSearchResultAsync(conversation, searchResult);
+            if (namedArgs.Get<bool>("debug"))
+            {
+                KnowProWriter.WriteLine(ConsoleColor.Cyan, context.ToJson());
+            }
+
+            var answer = await conversation.AnswerQuestionAsync(query, context);
+            KnowProWriter.WriteJson(answer);
+        }
     }
 
     private IConversation EnsureConversation()

dotnet/typeagent/src/common/DictionaryExtensions.cs

@@ -14,4 +14,18 @@ public static TValue GetValueOrDefault<TKey, TValue>(this IDictionary<TKey, TVal
     {
         return dictionary.TryGetValue(key, out TValue? value) ? value : defaultValue;
     }
+
+    // Gets values in the same order as the keys
+    public static IEnumerable<TValue> GetValues<TKey, TValue>(
+        this IDictionary<TKey, TValue> dictionary,
+        IEnumerable<TKey> keysInOrder
+    )
+    {
+        ArgumentVerify.ThrowIfNull(keysInOrder, nameof(keysInOrder));
+
+        foreach (TKey key in keysInOrder)
+        {
+            yield return dictionary[key];
+        }
+    }
 }

dotnet/typeagent/src/knowpro/Answer/AnswerContextBuilder.cs

@@ -253,11 +253,11 @@ CancellationToken cancellationToken
         return rangeOrdinals.Count != meta.Count || rangeOrdinals.Count != timestamps.Count
             ? throw new InvalidOperationException("ordinal list to meta list mismatch")
             : new EnclosingMetadata
-        {
-            Ordinals = rangeOrdinals,
-            Meta = meta,
-            Timestamps = timestamps
-        };
+            {
+                Ordinals = rangeOrdinals,
+                Meta = meta,
+                Timestamps = timestamps
+            };
     }
 
     private TimestampRange? GetTimeRange(string? min, string? max)
@@ -271,7 +271,7 @@ private struct EnclosingMetadata
     {
         public List<int> Ordinals { get; set; }
 
-        public IList<IMessageMetadata> Meta { get; set;}
+        public IList<IMessageMetadata> Meta { get; set; }
 
         public IList<string> Timestamps { get; set; }
     }

dotnet/typeagent/src/knowpro/Answer/AnswerContextSchema.cs

@@ -49,11 +49,14 @@ public partial class AnswerContext
 {
     // Relevant entities
     // Use the 'name' and 'type' properties of entities to PRECISELY identify those that answer the user question.
+    [JsonPropertyName("entities")]
     public IList<RelevantEntity>? Entities { get; set; }
 
     // Relevant topics
+    [JsonPropertyName("topics")]
     public IList<RelevantTopic> Topics { get; set; }
 
     // Relevant messages
+    [JsonPropertyName("messages")]
     public IList<RelevantMessage>? Messages { get; set; }
 };

dotnet/typeagent/src/knowpro/Answer/AnswerContextSchemaImpl.cs

@@ -30,6 +30,11 @@ public RelevantMessage(IMessage message)
 
 public partial class AnswerContext
 {
+    public string ToJson()
+    {
+        return Serializer.ToJsonIndented(this);
+    }
+
     public string ToPromptString()
     {
         var json = new StringBuilder();
@@ -53,6 +58,23 @@ public string ToPromptString()
         return json.ToString();
     }
 
+    public static ValueTask<AnswerContext> FromSearchResultAsync(
+        IConversation conversation,
+        ConversationSearchResult searchResult,
+        AnswerContextOptions? contextOptions = null,
+        CancellationToken cancellationToken = default
+    )
+    {
+        AnswerContextBuilder contextBuilder = new AnswerContextBuilder(conversation);
+        return contextBuilder.FromSearchResultAsync(
+            searchResult,
+            contextOptions,
+            cancellationToken
+        );
+
+
+    }
+
     private int AddPrompt(StringBuilder text, int propertyCount, string name, object value)
     {
         if (propertyCount > 0)

dotnet/typeagent/src/knowpro/ConversationAnswer.cs

@@ -9,22 +9,13 @@ namespace TypeAgent.KnowPro;
 public static class ConversationAnswer
 {
     public static async ValueTask<AnswerResponse> AnswerQuestionAsync(
-        this IConversation conversation,
-        string question,
-        ConversationSearchResult searchResult,
-        AnswerContextOptions? contextOptions = null,
-        CancellationToken cancellationToken = default
+    this IConversation conversation,
+    string question,
+    AnswerContext context,
+    CancellationToken cancellationToken = default
     )
     {
-        ArgumentVerify.ThrowIfNull(searchResult, nameof(searchResult));
-
-        AnswerContextBuilder contextBuilder = new AnswerContextBuilder(conversation);
-
-        AnswerContext context = await contextBuilder.FromSearchResultAsync(
-            searchResult,
-            contextOptions,
-            cancellationToken
-        ).ConfigureAwait(false);
+        ArgumentVerify.ThrowIfNull(context, nameof(context));
 
         IAnswerGenerator generator = conversation.Settings.AnswerGenerator;
 
@@ -47,6 +38,30 @@ public static async ValueTask<AnswerResponse> AnswerQuestionAsync(
         throw new NotImplementedException("Answer chunking");
     }
 
+
+    public static async ValueTask<AnswerResponse> AnswerQuestionAsync(
+        this IConversation conversation,
+        string question,
+        ConversationSearchResult searchResult,
+        AnswerContextOptions? contextOptions = null,
+        CancellationToken cancellationToken = default
+    )
+    {
+        ArgumentVerify.ThrowIfNull(searchResult, nameof(searchResult));
+        AnswerContext context = await AnswerContext.FromSearchResultAsync(
+            conversation,
+            searchResult,
+            contextOptions,
+            cancellationToken
+        ).ConfigureAwait(false);
+
+        return await conversation.AnswerQuestionAsync(
+            question,
+            context,
+            cancellationToken
+        ).ConfigureAwait(false);
+    }
+
     public static async ValueTask<IList<AnswerResponse>> AnswerQuestionAsync(
         this IConversation conversation,
         string question,
@@ -87,5 +102,4 @@ public static async ValueTask<IList<AnswerResponse>> AnswerQuestionAsync(
 
         return answerResponses;
     }
-
 }

dotnet/typeagent/src/knowproStorage/Sqlite/SqliteDatabase.cs

@@ -118,6 +118,29 @@ public IEnumerable<T> Enumerate<T>(string sql, Func<SqliteDataReader, T> rowDese
         return Enumerate<T>(sql, null, rowDeserializer);
     }
 
+    public Dictionary<TKey, TValue> GetKeyValues<TKey, TValue>(
+        string sql,
+        Action<SqliteCommand> addParams,
+        Func<SqliteDataReader, KeyValuePair<TKey, TValue>> rowDeserializer,
+        Dictionary<TKey, TValue>? keyValues = null
+    )
+    {
+        using var cmd = _connection.CreateCommand();
+        cmd.CommandText = sql;
+        if (addParams is not null)
+        {
+            addParams(cmd);
+        }
+        using var reader = cmd.ExecuteReader();
+        keyValues ??= [];
+        while (reader.Read())
+        {
+            var kv = rowDeserializer(reader);
+            keyValues.Add(kv.Key, kv.Value);
+        }
+        return keyValues;
+    }
+
     public IEnumerable<KeyValuePair<int, NormalizedEmbeddingB>> EnumerateEmbeddings(string sql)
     {
         return Enumerate<KeyValuePair<int, NormalizedEmbeddingB>>(