KnowPro.NET: Bug fixes and improvements (#1764)

umeshma · web-flow · commit 2f8f0b26c4ec · 2025-11-06T01:56:38.000Z
* Fixed bug recently introduced query compiler. Terms were being
over-written.
* More efficient Topic and Tag Matching; support wildcards
diff --git a/dotnet/typeagent/examples/knowProConsole/ConversationEventHandler.cs b/dotnet/typeagent/examples/knowProConsole/ConversationEventHandler.cs
@@ -6,6 +6,7 @@ namespace KnowProConsole;
 public class ConversationEventHandler
 {
     InplaceText _inplaceUpdate;
+    string _prevEventType = string.Empty;
 
     public ConversationEventHandler()
     {
@@ -47,6 +48,11 @@ private void KnowledgeExtractor_OnExtracted(BatchProgress item)
 
     private void WriteProgress(BatchProgress progress, string label)
     {
+        if (_prevEventType != label)
+        {
+            ConsoleWriter.WriteLine();
+            _prevEventType = label;
+        }
         _inplaceUpdate.Write($"[{label}: {progress.CountCompleted} / {progress.Count}]");
     }
 }
diff --git a/dotnet/typeagent/examples/knowProConsole/TestCommands.cs b/dotnet/typeagent/examples/knowProConsole/TestCommands.cs
@@ -401,16 +401,10 @@ private async Task AnswerAsync(ParseResult args, CancellationToken cancellationT
         NamedArgs namedArgs = new NamedArgs(args);
         AnswerContext context = new AnswerContext();
 
-        List<ConcreteEntity> entities = await conversation.SemanticRefs.SelectAsync<SemanticRef, ConcreteEntity>(
-            (sr) => sr.KnowledgeType == KnowledgeType.Entity ? sr.AsEntity() : null,
-            cancellationToken
-        );
+        IList<ConcreteEntity> entities = await conversation.SemanticRefs.GetAllEntitiesAsync(cancellationToken);
         entities = [.. entities.ToDistinct()];
 
-        List<Topic> topics = await conversation.SemanticRefs.SelectAsync<SemanticRef, Topic>(
-            (sr) => sr.KnowledgeType == KnowledgeType.Topic ? sr.AsTopic() : null,
-            cancellationToken
-        );
+        IList<Topic> topics = await conversation.SemanticRefs.GetAllTopicsAsync(cancellationToken);
         topics = [.. topics.ToDistinct()];
 
         context.Entities = entities.Map((e) => new RelevantEntity { Entity = e });
diff --git a/dotnet/typeagent/src/aiclient/EnvVars.cs b/dotnet/typeagent/src/aiclient/EnvVars.cs
@@ -40,7 +40,7 @@ public static string Get(string key, string? keySuffix = null, string? defaultVa
 
     public static int GetInt(string key, string? keySuffix = null, int? defaultValue = null)
     {
-        var numString = Get(key, keySuffix, string.Empty);
+        var numString = Get(key, keySuffix, defaultValue?.ToString());
         if (string.IsNullOrEmpty(numString) && defaultValue is not null)
         {
             return defaultValue.Value;
@@ -69,4 +69,4 @@ public static string ToVarName(string key, string? keySuffix = null)
     {
         return !string.IsNullOrEmpty(keySuffix) ? key + "_" + keySuffix : key;
     }
-}
+}
diff --git a/dotnet/typeagent/src/knowpro/ISemanticRefCollection.cs b/dotnet/typeagent/src/knowpro/ISemanticRefCollection.cs
@@ -6,14 +6,16 @@ namespace TypeAgent.KnowPro;
 public interface ISemanticRefCollection : IAsyncCollection<SemanticRef>
 {
     ValueTask<TextRange> GetTextRangeAsync(int ordinal, CancellationToken cancellationToken = default);
+
     ValueTask<IList<TextRange>> GetTextRangeAsync(IList<int> ordinals, CancellationToken cancellationToken = default);
 
     ValueTask<KnowledgeType> GetKnowledgeTypeAsync(int ordinal, CancellationToken cancellation = default);
+
     ValueTask<IList<KnowledgeType>> GetKnowledgeTypeAsync(IList<int> ordinal, CancellationToken cancellation = default);
 
-    // TODO
-    // Add methods to enumerate by knowledge Type, casting appropriately.
-    // More efficient than looping over all 
+    ValueTask<IList<SemanticRef>> GetAllAsync(KnowledgeType? kType = null, CancellationToken cancellationToken = default);
+
+    ValueTask<IList<ScoredSemanticRefOrdinal>> GetAllOrdinalsAsync(KnowledgeType? kType = null, CancellationToken cancellationToken = default);
 
     event Action<BatchProgress> OnKnowledgeExtracted;
     void NotifyKnowledgeProgress(BatchProgress progress);
@@ -90,4 +92,30 @@ public static async ValueTask<IList<Scored<ConcreteEntity>>> GetDistinctEntities
             ? entitites.GetTopK(topK.Value)
             : [.. entitites];
     }
+
+    public static async ValueTask<IList<ConcreteEntity>> GetAllEntitiesAsync(
+        this ISemanticRefCollection semanticRefs,
+        CancellationToken cancellation = default
+    )
+    {
+        var list = await semanticRefs.GetAllAsync(
+            KnowledgeType.Entity,
+            cancellation
+        ).ConfigureAwait(false);
+
+        return [.. list.Select((sr) => sr.AsEntity())];
+    }
+
+    public static async ValueTask<IList<Topic>> GetAllTopicsAsync(
+        this ISemanticRefCollection semanticRefs,
+        CancellationToken cancellation = default
+    )
+    {
+        var list = await semanticRefs.GetAllAsync(
+            KnowledgeType.Topic,
+            cancellation
+        ).ConfigureAwait(false);
+
+        return [.. list.Select((sr) => sr.AsTopic())];
+    }
 }
diff --git a/dotnet/typeagent/src/knowpro/Query/LookupExtensions.cs b/dotnet/typeagent/src/knowpro/Query/LookupExtensions.cs
@@ -130,4 +130,38 @@ public static async ValueTask<IList<ScoredSemanticRefOrdinal>> LookupPropertyAsy
         }
         return scoredRefs;
     }
+
+    public static async ValueTask<IList<ScoredSemanticRefOrdinal>?> LookupAllAsync(
+        this ISemanticRefCollection semanticRefs,
+        QueryEvalContext context,
+        KnowledgeType knowledgeType,
+        TextRangesInScope? rangesInScope,
+        ScoreBooster? scoreBooster = null
+    )
+    {
+        IList<ScoredSemanticRefOrdinal> scoredOrdinals = await semanticRefs.GetAllOrdinalsAsync(
+            knowledgeType
+        ).ConfigureAwait(false);
+
+        if (rangesInScope is null)
+        {
+            return scoredOrdinals;
+        }
+
+        // TODO: avoid this double alloction
+        IList<TextRange> ranges = await semanticRefs.GetTextRangeAsync(
+            scoredOrdinals.ToOrdinals()
+        ).ConfigureAwait(false);
+
+        List<ScoredSemanticRefOrdinal> scoredRefs = [];
+        int count = ranges.Count;
+        for (int i = 0; i < count; ++i)
+        {
+            if (rangesInScope.IsRangeInScope(ranges[i]))
+            {
+                scoredRefs.Add(scoredOrdinals[i]);
+            }
+        }
+        return scoredRefs;
+    }
 }
diff --git a/dotnet/typeagent/src/knowpro/Query/MatchTermExpr.cs b/dotnet/typeagent/src/knowpro/Query/MatchTermExpr.cs
@@ -115,7 +115,7 @@ Term relatedTerm
         }
     }
 
-    private ValueTask<IList<ScoredSemanticRefOrdinal>?> LookupTermAsync(QueryEvalContext context, Term term)
+    protected virtual ValueTask<IList<ScoredSemanticRefOrdinal>?> LookupTermAsync(QueryEvalContext context, Term term)
     {
         return context.SemanticRefIndex.LookupTermAsync(
             context,
@@ -292,3 +292,36 @@ string propertyValue
         );
     }
 }
+
+internal class MatchKnowledgeTypeExpr : MatchSearchTermExpr
+{
+    KnowledgeType _kType;
+
+
+    public MatchKnowledgeTypeExpr(SearchTerm searchTerm, KnowledgeType kType)
+        : base(searchTerm)
+    {
+        _kType = kType;
+    }
+
+    protected override ValueTask<IList<ScoredSemanticRefOrdinal>?> LookupTermAsync(QueryEvalContext context, Term term)
+    {
+        if (SearchTerm.IsWildcard())
+        {
+            return context.Conversation.SemanticRefs.LookupAllAsync(
+                context,
+                _kType,
+                context.TextRangesInScope,
+                ScoreBooster
+            );
+        }
+
+        return context.SemanticRefIndex.LookupTermAsync(
+            context,
+            term,
+            context.TextRangesInScope,
+            _kType,
+            ScoreBooster
+        );
+    }
+}
diff --git a/dotnet/typeagent/src/knowpro/Query/QueryCompiler.cs b/dotnet/typeagent/src/knowpro/Query/QueryCompiler.cs
@@ -157,7 +157,7 @@ public async ValueTask<QueryOpExpr<IList<ScoredMessageOrdinal>>> CompileMessageS
                         matchFilter
                     );
                     termExpressions.Add(searchTermExpr);
-                    compiledTerms[0].Terms.Add(searchTerm.Clone());
+                    compiledTerms[0].Terms.Add(searchTerm);
                     break;
 
             }
@@ -215,7 +215,7 @@ public async ValueTask<QueryOpExpr<IList<ScoredMessageOrdinal>>> CompileMessageS
                         matchFilter
                     );
                     termExpressions.Add(searchTermExpr);
-                    compiledTerms[0].Terms.Add(searchTerm.Clone());
+                    compiledTerms[0].Terms.Add(searchTerm);
                     break;
             }
         }
@@ -275,10 +275,12 @@ private QueryOpExpr<SemanticRefAccumulator> CompileSelect(
                         propertyTerm.PropertyValue.Term.Weight ??= Settings.EntityTermMatchWeight;
                     }
                     return new MatchPropertySearchTermExpr(propertyTerm);
+
                 case "tag":
+                    return new MatchKnowledgeTypeExpr(propertyTerm.PropertyValue, KnowledgeType.Tag);
+
                 case "topic":
-                    // TODO
-                    throw new NotImplementedException();
+                    return new MatchKnowledgeTypeExpr(propertyTerm.PropertyValue, KnowledgeType.Topic);
             }
         }
         else
@@ -372,7 +374,8 @@ private async ValueTask<QueryOpExpr<MessageAccumulator>> CompileMessageReRankAsy
         QueryOpExpr<MessageAccumulator> srcExpr,
         string? rawQueryText,
         SearchOptions? options
-    ) {
+    )
+    {
         var messageIndex = _conversation.SecondaryIndexes.MessageIndex;
         int messageCount = await messageIndex.GetCountAsync(
             _cancellationToken
diff --git a/dotnet/typeagent/src/knowpro/SearchTerm.cs b/dotnet/typeagent/src/knowpro/SearchTerm.cs
@@ -61,16 +61,16 @@ public void AddRelated(Term term)
 
     internal bool RelatedTermsRequired { get; set; }
 
+    /*
     internal SearchTerm Clone()
     {
         return new SearchTerm(Term);
     }
-
+    */
     internal SearchTerm ToRequired()
     {
-        var copy = this.Clone();
-        copy.RelatedTermsRequired = true;
-        return copy;
+        RelatedTermsRequired = true;
+        return this;
     }
 
     public static implicit operator SearchTerm(string value)
diff --git a/dotnet/typeagent/src/knowproStorage/Sqlite/SqliteDatabase.cs b/dotnet/typeagent/src/knowproStorage/Sqlite/SqliteDatabase.cs
@@ -71,6 +71,20 @@ public List<T> GetList<T>(string commandText, Func<SqliteDataReader, T> cb)
         return reader.GetList<T>(cb);
     }
 
+    public List<T> GetList<T>(string commandText, Action<SqliteCommand> addParams, Func<SqliteDataReader, T> cb)
+    {
+        ArgumentVerify.ThrowIfNullOrEmpty(commandText, nameof(commandText));
+
+        using var command = CreateCommand(commandText);
+        if (addParams is not null)
+        {
+            addParams(command);
+        }
+        using var reader = command.ExecuteReader();
+
+        return reader.GetList<T>(cb);
+    }
+
     public List<T>? GetListOrNull<T>(string commandText, Func<SqliteDataReader, T> cb)
     {
         ArgumentVerify.ThrowIfNullOrEmpty(commandText, nameof(commandText));
diff --git a/dotnet/typeagent/src/knowproStorage/Sqlite/SqliteMessageTextIndex.cs b/dotnet/typeagent/src/knowproStorage/Sqlite/SqliteMessageTextIndex.cs
@@ -208,7 +208,7 @@ public IEnumerable<KeyValuePair<int, NormalizedEmbeddingB>> GetSubset(IList<int>
             string[] placeholderIds = SqliteDatabase.MakeInPlaceholderParamIds(batch.Count);
             var rows = _db.Enumerate(
                 $@"
-SELECT msg_id, embedding FROM MessageTextIndex
+SELECT msg_id, embedding
 FROM MessageTextIndex WHERE msg_id IN ({SqliteDatabase.MakeInStatement(placeholderIds)})
 ORDER BY msg_id",
                 (cmd) => cmd.AddPlaceholderParameters(placeholderIds, batch),
diff --git a/dotnet/typeagent/src/knowproStorage/Sqlite/SqliteSemanticRefCollection.cs b/dotnet/typeagent/src/knowproStorage/Sqlite/SqliteSemanticRefCollection.cs
@@ -242,6 +242,60 @@ public ValueTask<IList<SemanticRef>> GetSliceAsync(int startOrdinal, int endOrdi
         return ValueTask.FromResult(GetSlice(startOrdinal, endOrdinal));
     }
 
+    public IList<SemanticRef> GetAll(KnowledgeType? kType = null)
+    {
+        return kType is null
+            ? _db.GetList(
+                "SELECT * FROM SemanticRefs",
+                ReadSemanticRef
+            )
+            : _db.GetList(
+@"SELECT * from SemanticRefs
+WHERE knowledge_type = @kType",
+            (cmd) => cmd.AddParameter("@kType", kType.ToString()),
+            ReadSemanticRef
+        );
+    }
+
+    public ValueTask<IList<SemanticRef>> GetAllAsync(KnowledgeType? kType = null, CancellationToken cancellationToken = default)
+    {
+        return ValueTask.FromResult(GetAll());
+    }
+
+    public IList<ScoredSemanticRefOrdinal> GetAllOrdinals(KnowledgeType? kType = null)
+    {
+        return kType is null
+            ? _db.GetList(
+                "SELECT semref_id, 1.0 FROM SemanticRefs",
+                (reader) =>
+                {
+                    return new ScoredSemanticRefOrdinal
+                    {
+                        SemanticRefOrdinal = reader.GetInt32(0),
+                        Score = reader.GetInt32(1)
+                    };
+                }
+            )
+            : _db.GetList(
+@"SELECT semref_id, 1.0 from SemanticRefs
+WHERE knowledge_type = @kType",
+            (cmd) => cmd.AddParameter("@kType", kType.ToString()),
+                (reader) =>
+                {
+                    return new ScoredSemanticRefOrdinal
+                    {
+                        SemanticRefOrdinal = reader.GetInt32(0),
+                        Score = reader.GetInt32(1)
+                    };
+                }
+        );
+    }
+
+    public ValueTask<IList<ScoredSemanticRefOrdinal>> GetAllOrdinalsAsync(KnowledgeType? kType = null, CancellationToken cancellationToken = default)
+    {
+        return ValueTask.FromResult(GetAllOrdinals(kType));
+    }
+
     public event Action<BatchProgress> OnKnowledgeExtracted;
 
     public void NotifyKnowledgeProgress(BatchProgress progress)

Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,7 @@ namespace KnowProConsole;`
`6`	`6`	`public class ConversationEventHandler`
`7`	`7`	`{`
`8`	`8`	`InplaceText _inplaceUpdate;`
	`9`	`+ string _prevEventType = string.Empty;`
`9`	`10`
`10`	`11`	`public ConversationEventHandler()`
`11`	`12`	`{`
`@@ -47,6 +48,11 @@ private void KnowledgeExtractor_OnExtracted(BatchProgress item)`
`47`	`48`
`48`	`49`	`private void WriteProgress(BatchProgress progress, string label)`
`49`	`50`	`{`
	`51`	`+ if (_prevEventType != label)`
	`52`	`+ {`
	`53`	`+ ConsoleWriter.WriteLine();`
	`54`	`+ _prevEventType = label;`
	`55`	`+ }`
`50`	`56`	`_inplaceUpdate.Write($"[{label}: {progress.CountCompleted} / {progress.Count}]");`
`51`	`57`	`}`
`52`	`58`	`}`
Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,7 @@ public static string Get(string key, string? keySuffix = null, string? defaultVa`
`40`	`40`
`41`	`41`	`public static int GetInt(string key, string? keySuffix = null, int? defaultValue = null)`
`42`	`42`	`{`
`43`		`- var numString = Get(key, keySuffix, string.Empty);`
	`43`	`+ var numString = Get(key, keySuffix, defaultValue?.ToString());`
`44`	`44`	`if (string.IsNullOrEmpty(numString) && defaultValue is not null)`
`45`	`45`	`{`
`46`	`46`	`return defaultValue.Value;`
`@@ -69,4 +69,4 @@ public static string ToVarName(string key, string? keySuffix = null)`
`69`	`69`	`{`
`70`	`70`	`return !string.IsNullOrEmpty(keySuffix) ? key + "_" + keySuffix : key;`
`71`	`71`	`}`
`72`		`-}`
	`72`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -61,16 +61,16 @@ public void AddRelated(Term term)`
`61`	`61`
`62`	`62`	`internal bool RelatedTermsRequired { get; set; }`
`63`	`63`
	`64`	`+ /*`
`64`	`65`	`internal SearchTerm Clone()`
`65`	`66`	`{`
`66`	`67`	`return new SearchTerm(Term);`
`67`	`68`	`}`
`68`		`-`
	`69`	`+ */`
`69`	`70`	`internal SearchTerm ToRequired()`
`70`	`71`	`{`
`71`		`- var copy = this.Clone();`
`72`		`- copy.RelatedTermsRequired = true;`
`73`		`- return copy;`
	`72`	`+ RelatedTermsRequired = true;`
	`73`	`+ return this;`
`74`	`74`	`}`
`75`	`75`
`76`	`76`	`public static implicit operator SearchTerm(string value)`