Skip to content

Commit 4cc4fc8

Browse files
umeshmarobgruen
umeshma
and
robgruen
authored
KnowPro.NET updates (#1744)
* Related Term resolution * Indexers * ConversationIndexer (starting) * Parallelization support * Setting up for knowledge indexing * Cleaner indexing event mechanism * Test app improvements * Refactoring * Bugs --------- Co-authored-by: robgruen <robgruen@microsoft.com>
1 parent 9277593 commit 4cc4fc8

32 files changed

+1006
-594
lines changed

‎dotnet/typeagent/examples/examplesLib/CommandLine/ConsoleApp.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,12 @@ public virtual async Task<int> ProcessCommandAsync(string[] cmdLine, Cancellatio
129129
return CommandResult.NotHandled;
130130
}
131131
var parseResult = _allCommands.Parse(cmdLine);
132+
if (parseResult.Errors.Count > 0)
133+
{
134+
WriteArgErrors(parseResult);
135+
return CommandResult.NotHandled;
136+
}
137+
132138
return await parseResult.InvokeAsync(null, cancellationToken).ConfigureAwait(false);
133139
}
134140

@@ -217,6 +223,14 @@ protected virtual void WriteTitle()
217223
}
218224
}
219225

226+
protected void WriteArgErrors(ParseResult parseResult)
227+
{
228+
foreach (var err in parseResult.Errors)
229+
{
230+
ConsoleWriter.WriteError(err.Message);
231+
}
232+
}
233+
220234
public void AddModule(ICommandModule module)
221235
{
222236
_allCommands.AddModule(module);

‎dotnet/typeagent/examples/examplesLib/CommandLine/UI.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ public ConsoleControl(CursorPos topLeft)
5555

5656
public CursorPos TopLeft { get; }
5757

58+
public ConsoleColor? Color { get; set; }
59+
5860
protected void Erase()
5961
{
6062
if (!string.IsNullOrEmpty(_lastText))
@@ -66,10 +68,33 @@ protected void Erase()
6668

6769
protected void WriteInPlace(string text)
6870
{
71+
if (Color is not null)
72+
{
73+
ConsoleWriter.PushColor(Color.Value);
74+
}
75+
6976
var curPos = TopLeft.Apply();
7077
ConsoleWriter.WriteInPlace(text, _lastText);
7178
_lastText = text;
7279
curPos.Apply();
80+
81+
if (Color is not null)
82+
{
83+
ConsoleWriter.PopColor();
84+
}
85+
}
86+
}
87+
88+
public class InplaceText : ConsoleControl
89+
{
90+
public InplaceText(bool slideDown = true)
91+
: base(slideDown ? CursorPos.CaptureLeft() : CursorPos.Capture())
92+
{
93+
}
94+
95+
public void Write(string text)
96+
{
97+
base.WriteInPlace(text);
7398
}
7499
}
75100

‎dotnet/typeagent/examples/knowProConsole/ConversationEventHandler.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace KnowProConsole;
5+
6+
public class ConversationEventHandler
7+
{
8+
InplaceText _inplaceUpdate;
9+
10+
public ConversationEventHandler()
11+
{
12+
_inplaceUpdate = new InplaceText();
13+
}
14+
15+
public InplaceText Progress => _inplaceUpdate;
16+
17+
public void Subscribe(IConversation conversation)
18+
{
19+
conversation.SecondaryIndexes.TermToRelatedTermsIndex.FuzzyIndex.OnIndexed += this.FuzzyIndex_OnIndexed;
20+
}
21+
22+
public void Unsubscribe(IConversation conversation)
23+
{
24+
conversation.SecondaryIndexes.TermToRelatedTermsIndex.FuzzyIndex.OnIndexed -= this.FuzzyIndex_OnIndexed;
25+
}
26+
27+
private void FuzzyIndex_OnIndexed(BatchProgress item)
28+
{
29+
WriteProgress(item, "Fuzzy");
30+
}
31+
32+
private void WriteProgress(BatchProgress progress, string label)
33+
{
34+
_inplaceUpdate.Write($"[{label}: {progress.CountCompleted} / {progress.Count}]");
35+
}
36+
}

‎dotnet/typeagent/examples/knowProConsole/KnowProConsole.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,28 @@ public class KnowProConsoleContext : KnowProContext
77
{
88
public KnowProConsoleContext()
99
{
10+
EventHandler = new ConversationEventHandler();
11+
}
12+
13+
public ConversationEventHandler EventHandler { get; }
14+
15+
public void UnloadCurrent()
16+
{
17+
if (Conversation is not null)
18+
{
19+
EventHandler.Unsubscribe(Conversation);
20+
Conversation = null;
21+
}
22+
}
23+
24+
public void SetCurrent(IConversation conversation)
25+
{
26+
UnloadCurrent();
27+
if (conversation is not null)
28+
{
29+
Conversation = conversation;
30+
EventHandler.Subscribe(conversation);
31+
}
1032
}
1133
}
1234

‎dotnet/typeagent/examples/knowProConsole/PodcastCommands.cs‎

Lines changed: 9 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -127,32 +127,6 @@ private async Task PodcastImportIndexAsync(ParseResult args, CancellationToken c
127127
podcast.Dispose();
128128
throw;
129129
}
130-
/*
131-
var messages = await podcast.Messages.GetAsync([1, 2, 3, 4], cancellationToken);
132-
KnowProWriter.WriteJson(messages);
133-
134-
await KnowProWriter.WriteSemanticRefsAsync(podcast);
135-
136-
IList<ScoredSemanticRefOrdinal>? matches;
137-
if (data.SemanticIndexData is not null)
138-
{
139-
await podcast.ImportTermToSemanticRefIndexAsync(data.SemanticIndexData.Items, cancellationToken);
140-
count = await podcast.SemanticRefIndex.GetCountAsync(cancellationToken);
141-
KnowProWriter.WriteLine($"{count} index entries imported");
142-
143-
matches = await podcast.SemanticRefIndex.LookupTermAsync("Children of Time", cancellationToken);
144-
KnowProWriter.WriteLine($"{matches?.Count ?? 0} matches");
145-
}
146-
147-
count = await podcast.ImportPropertyIndexAsync(data.SemanticRefs, cancellationToken);
148-
KnowProWriter.WriteLine($"{count} properties imported");
149-
matches = await podcast.SecondaryIndexes.PropertyToSemanticRefIndex.LookupPropertyAsync(
150-
KnowledgePropertyName.EntityName,
151-
"Children of Time",
152-
cancellationToken
153-
);
154-
KnowProWriter.WriteLine($"{matches.Count} matches");
155-
*/
156130
}
157131

158132

@@ -161,35 +135,21 @@ private Podcast CreatePodcast(string name, bool createNew)
161135
// TODO: standardize this boilerplate, esp the cache binding
162136
var model = new TextEmbeddingModelWithCache(256);
163137
ConversationSettings settings = new ConversationSettings(model);
164-
var provider = CreateStorageProvider(settings, name, createNew);
138+
var provider = _kpContext.CreateStorageProvider<PodcastMessage, PodcastMessageMeta>(settings, name, createNew);
165139
model.Cache.PersistentCache = provider.GetEmbeddingCache();
166140

167-
var podcast = new Podcast(settings, CreateStorageProvider(settings, name, false));
141+
var podcast = new Podcast(settings, provider);
168142
return podcast;
169143
}
170144

171-
private SqliteStorageProvider<PodcastMessage, PodcastMessageMeta> CreateStorageProvider(
172-
ConversationSettings settings,
173-
string name,
174-
bool createNew
175-
)
176-
{
177-
// TODO: make this a standard factory method
178-
var provider = new SqliteStorageProvider<PodcastMessage, PodcastMessageMeta>(
179-
settings,
180-
_kpContext.DotnetPath,
181-
name,
182-
createNew
183-
);
184-
185-
return provider;
186-
}
187-
188145
private void UnloadCurrent()
189146
{
190-
_podcast?.Dispose();
191-
_podcast = null;
192-
_kpContext.Conversation = null;
147+
_kpContext.UnloadCurrent();
148+
if (_podcast is not null)
149+
{
150+
_podcast.Dispose();
151+
_podcast = null;
152+
}
193153
}
194154

195155
private void SetCurrent(Podcast? podcast)
@@ -198,7 +158,7 @@ private void SetCurrent(Podcast? podcast)
198158
if (podcast is not null)
199159
{
200160
_podcast = podcast;
201-
_kpContext.Conversation = podcast;
161+
_kpContext.SetCurrent(podcast);
202162
}
203163
}
204164
}

‎dotnet/typeagent/examples/knowProConsole/TestCommands.cs‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,14 @@ public TestCommands(KnowProConsoleContext context)
1515
}
1616

1717
public IList<Command> GetCommands()
18-
{
18+
{
1919
return [
2020
SearchTermsDef(),
2121
SearchPropertyTermsDef(),
2222
SearchMessagesTermsDef(),
2323
TestEmbeddingsDef(),
2424
SearchQueryTermsDef(),
25+
BuildIndexDef(),
2526
];
2627
}
2728

@@ -205,6 +206,48 @@ private async Task TestEmbeddingsAsync(ParseResult args, CancellationToken cance
205206
}
206207
}
207208

209+
private Command BuildIndexDef()
210+
{
211+
Command cmd = new("kpTestBuildIndex")
212+
{
213+
Options.Arg<bool>("related", "index related terms", false)
214+
};
215+
cmd.TreatUnmatchedTokensAsErrors = false;
216+
cmd.SetAction(this.BuildIndexAsync);
217+
return cmd;
218+
}
219+
220+
private async Task BuildIndexAsync(ParseResult args, CancellationToken cancellationToken)
221+
{
222+
IConversation conversation = EnsureConversation();
223+
224+
NamedArgs namedArgs = new NamedArgs(args);
225+
226+
var cachingModel = conversation.Settings.RelatedTermIndexSettings.EmbeddingIndexSetting.EmbeddingModel as TextEmbeddingModelWithCache;
227+
try
228+
{
229+
if (cachingModel is not null)
230+
{
231+
cachingModel.CacheEnabled = false;
232+
}
233+
if (namedArgs.Get<bool>("related"))
234+
{
235+
await conversation.SecondaryIndexes.TermToRelatedTermsIndex.FuzzyIndex.ClearAsync(cancellationToken);
236+
237+
await conversation.BuildRelatedTermsIndexAsync(cancellationToken);
238+
}
239+
KnowProWriter.WriteLine();
240+
}
241+
finally
242+
{
243+
if (cachingModel is not null)
244+
{
245+
cachingModel.CacheEnabled = true;
246+
}
247+
}
248+
}
249+
250+
208251
async Task TestSearchKnowledgeAsync(IConversation conversation, SearchTermGroup searchGroup, CancellationToken cancellationToken)
209252
{
210253
KnowProWriter.WriteLine(searchGroup);

‎dotnet/typeagent/src/aiclient/ITextEmbeddingModel.cs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,18 +32,29 @@ public static async Task<List<float[]>> GenerateInBatchesAsync(
3232
int batchSize,
3333
int maxCharsPerChunk,
3434
int concurrency = 1,
35+
Action<BatchProgress>? progress = null,
3536
CancellationToken cancellationToken = default
3637
)
3738
{
3839
batchSize = Math.Min(batchSize, model.MaxBatchSize);
3940
if (batchSize > 1)
4041
{
4142
List<List<string>> chunks = [.. texts.GetStringChunks(batchSize, maxCharsPerChunk)];
43+
int rawCompleted = 0;
44+
Action<BatchProgress>? notifyProgress = progress is null
45+
? null
46+
: (batch) =>
47+
{
48+
rawCompleted += chunks[batch.CountCompleted - 1].Count;
49+
progress(new BatchProgress(rawCompleted, texts.Count));
50+
};
51+
4252
var embeddingChunks = await chunks.MapAsync(
4353
concurrency,
4454
(chunk) => model.GenerateAsync(chunk, cancellationToken),
55+
notifyProgress,
4556
cancellationToken
46-
);
57+
).ConfigureAwait(false);
4758

4859
return embeddingChunks.Flat();
4960
}
@@ -52,8 +63,9 @@ public static async Task<List<float[]>> GenerateInBatchesAsync(
5263
return await texts.MapAsync(
5364
concurrency,
5465
(value) => model.GenerateAsync(value, cancellationToken),
66+
progress,
5567
cancellationToken
56-
);
68+
).ConfigureAwait(false);
5769
}
5870
}
5971
}

0 commit comments

Comments
 (0)