Skip to content

Commit 7e6790d

Browse files
umeshmaumeshma
authored
KnowPro.NET Answer generation Part 1 (#1763)
* Answer Context * OneOrMany union type * Serialization for Typescript style unions for answers * Testing out C# equivalent * Entity and topic merging * Bugs * Refactor
1 parent c0b753f commit 7e6790d

24 files changed

+698
-55
lines changed

‎dotnet/typeagent/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ This **sample code** is in early stage experimental developement with **frequent
44

55
Working towards .NET versions of the following **TypeAgent** packages:
66
* [KnowPro](../../ts/packages/knowPro/README.md)
7+
* [Memory](../../ts/packages/memory/README.md)
78
* [AIClient](../../ts/packages/aiclient/README.md)
89
* [TypeAgent Common Libs](../../ts/packages/typeagent/README.md)
910

@@ -13,11 +14,19 @@ TypeAgent.NET also incorporates [TypeChat.NET](https://github.com/microsoft/type
1314
KnowPro.NET will implement Structured RAG in C# for .NET platforms. This work is in curently in progress
1415

1516
It will improve on the Typescript implementation in the following ways:
17+
* Storage and indexing using Storage providers
1618
* Fully asynchronous
1719
* Operators are/will be reworked for more efficient async operation
1820
* Asynchronous storage providers with improved Sql schemas
1921
* Larger index sizes
2022

23+
Libraries:
24+
* [KnowPro](./src/knowpro): KnowPro Core
25+
* [KnowProStorage](./src/knowproStorage): KnowPro Storage Providers
26+
* [Conversation Memory](./src/conversationMemory): Implementations of memory types using KnowPro.NET
27+
* [Vector](./src/vector)
28+
* [AIClient](./src/aiclient)
29+
2130
## Trademarks
2231

2332
This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft

‎dotnet/typeagent/examples/knowProConsole/Includes.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
global using TypeAgent.AIClient;
1919
global using TypeAgent.KnowPro;
2020
global using TypeAgent.KnowPro.KnowledgeExtractor;
21+
global using TypeAgent.KnowPro.Answer;
2122
global using TypeAgent.KnowPro.Storage.Local;
2223
global using TypeAgent.KnowPro.Storage.Sqlite;
2324
global using TypeAgent.ConversationMemory;

‎dotnet/typeagent/examples/knowProConsole/TestCommands.cs‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT License.
33

4+
using TypeAgent.KnowPro.Answer;
45
using TypeAgent.KnowPro.Lang;
56

67
namespace KnowProConsole;
@@ -25,6 +26,7 @@ public IList<Command> GetCommands()
2526
SearchLangDef(),
2627
KnowledgeDef(),
2728
BuildIndexDef(),
29+
AnswerDef()
2830
];
2931
}
3032

@@ -351,7 +353,6 @@ await KnowProWriter.WriteConversationSearchResultsAsync(
351353
}
352354
}
353355

354-
355356
private Command KnowledgeDef()
356357
{
357358
Command cmd = new("kpTestKnowledge")
@@ -382,6 +383,45 @@ private async Task KnowledgeAsync(ParseResult args, CancellationToken cancellati
382383
}
383384
}
384385

386+
private Command AnswerDef()
387+
{
388+
Command cmd = new("kpTestAnswer")
389+
{
390+
Args.Arg<string>("text")
391+
};
392+
cmd.TreatUnmatchedTokensAsErrors = false;
393+
cmd.SetAction(this.AnswerAsync);
394+
return cmd;
395+
}
396+
397+
private async Task AnswerAsync(ParseResult args, CancellationToken cancellationToken)
398+
{
399+
IConversation conversation = EnsureConversation();
400+
401+
NamedArgs namedArgs = new NamedArgs(args);
402+
AnswerContext context = new AnswerContext();
403+
404+
List<ConcreteEntity> entities = await conversation.SemanticRefs.SelectAsync<SemanticRef, ConcreteEntity>(
405+
(sr) => sr.KnowledgeType == KnowledgeType.Entity ? sr.AsEntity() : null,
406+
cancellationToken
407+
);
408+
entities = [.. entities.ToDistinct()];
409+
410+
List<Topic> topics = await conversation.SemanticRefs.SelectAsync<SemanticRef, Topic>(
411+
(sr) => sr.KnowledgeType == KnowledgeType.Topic ? sr.AsTopic() : null,
412+
cancellationToken
413+
);
414+
topics = [.. topics.ToDistinct()];
415+
416+
context.Entities = entities.Map((e) => new RelevantEntity { Entity = e });
417+
context.Topics = topics.Map((t) => new RelevantTopic { Topic = t });
418+
419+
List<IMessage> messages = await conversation.Messages.GetAllAsync(cancellationToken);
420+
context.Messages = messages.Map((m) => new RelevantMessage(m));
421+
string prompt = context.ToPromptString();
422+
ConsoleWriter.WriteLine(prompt);
423+
}
424+
385425
private IConversation EnsureConversation()
386426
{
387427
return (_kpContext.Conversation is not null)

‎dotnet/typeagent/src/common/DateTimeExtensions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@ public static class DateTimeExtensions
77
{
88
public static string ToISOString(this DateTimeOffset dt)
99
{
10-
return dt.ToString("o", System.Globalization.CultureInfo.InvariantCulture);
10+
return dt.ToString("o");
1111
}
1212
}

‎dotnet/typeagent/src/common/EnumerationExtensions.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,4 +87,52 @@ public static List<Scored<T>> GetTopK<T>(this IEnumerable<Scored<T>> items, int
8787
topNList.Add(items);
8888
return topNList.ByRankAndClear();
8989
}
90+
91+
public static async ValueTask<List<T>> ToListAsync<T>(
92+
this IAsyncEnumerable<T> source,
93+
CancellationToken cancellationToken = default)
94+
{
95+
List<T> list = [];
96+
await foreach (var item in source.WithCancellation(cancellationToken))
97+
{
98+
list.Add(item);
99+
}
100+
return list;
101+
}
102+
103+
public static async ValueTask<List<TSelect>> SelectAsync<T, TSelect>(
104+
this IAsyncEnumerable<T> source,
105+
Func<T, TSelect?> selector,
106+
CancellationToken cancellationToken = default)
107+
{
108+
ArgumentVerify.ThrowIfNull(selector, nameof(selector));
109+
110+
List<TSelect> list = [];
111+
await foreach (var item in source.WithCancellation(cancellationToken))
112+
{
113+
TSelect? selected = selector(item);
114+
if (selected is not null)
115+
{
116+
list.Add(selected);
117+
}
118+
}
119+
return list;
120+
}
121+
122+
public static async Task<List<T>> WhereAsync<T>(
123+
this IAsyncEnumerable<T> source,
124+
Func<T, bool> predicate,
125+
CancellationToken cancellationToken = default)
126+
{
127+
ArgumentVerify.ThrowIfNull(predicate, nameof(predicate));
128+
var list = new List<T>();
129+
await foreach (var item in source.WithCancellation(cancellationToken))
130+
{
131+
if (predicate(item))
132+
{
133+
list.Add(item);
134+
}
135+
}
136+
return list;
137+
}
90138
}

‎dotnet/typeagent/src/common/IsoDateJsonConvertor.cs‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using System.Globalization;
5+
6+
namespace TypeAgent.Common;
7+
8+
/// <summary>
9+
/// Forces DateTimeOffset serialization/deserialization to a stable ISO 8601 (round‑trip) string ("o").
10+
/// Example: 2025-11-05T14:23:17.1234567+00:00
11+
/// </summary>
12+
public class IsoDateJsonConverter : JsonConverter<DateTimeOffset>
13+
{
14+
public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
15+
{
16+
if (reader.TokenType == JsonTokenType.String)
17+
{
18+
string? s = reader.GetString();
19+
if (!string.IsNullOrEmpty(s))
20+
{
21+
if (DateTimeOffset.TryParse(s, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var value))
22+
{
23+
return value;
24+
}
25+
}
26+
throw new JsonException($"Invalid DateTimeOffset value: '{s}'.");
27+
}
28+
throw new JsonException("Invalid DateTimeOffset value");
29+
}
30+
31+
public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options)
32+
{
33+
writer.WriteStringValue(value.ToISOString());
34+
}
35+
}

‎dotnet/typeagent/src/common/OneOrMany.cs‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace TypeAgent.Common;
5+
6+
public abstract class OneOrManyItem
7+
{
8+
[JsonIgnore]
9+
public abstract bool IsSingle { get; }
10+
11+
public static OneOrManyItem<T>? Create<T>(T? value)
12+
{
13+
return value is not null ? new SingleItem<T>(value) : null;
14+
}
15+
16+
public static OneOrManyItem<T>? Create<T>(IList<T>? value)
17+
{
18+
return value.IsNullOrEmpty()
19+
? null
20+
: value.Count == 1
21+
? new SingleItem<T>(value[0])
22+
: new ListItem<T>(value);
23+
}
24+
}
25+
26+
[JsonConverter(typeof(OneOrManyJsonConverterFactory))]
27+
public abstract class OneOrManyItem<T> : OneOrManyItem
28+
{
29+
}
30+
31+
public class SingleItem<T> : OneOrManyItem<T>
32+
{
33+
public SingleItem() { }
34+
35+
public SingleItem(T value)
36+
{
37+
Value = value;
38+
}
39+
40+
[JsonIgnore]
41+
public override bool IsSingle => true;
42+
43+
public T Value { get; set; }
44+
45+
public static implicit operator T(SingleItem<T> item)
46+
{
47+
return item.Value;
48+
}
49+
}
50+
51+
public class ListItem<T> : OneOrManyItem<T>
52+
{
53+
public ListItem()
54+
{
55+
56+
}
57+
58+
public ListItem(IList<T> value)
59+
{
60+
Value = value;
61+
}
62+
63+
[JsonIgnore]
64+
public override bool IsSingle => false;
65+
66+
public IList<T> Value { get; set; }
67+
}
68+
69+
public class OneOrManyJsonConverterFactory : JsonConverterFactory
70+
{
71+
public override bool CanConvert(Type typeToConvert) => true;
72+
73+
public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options)
74+
{
75+
var elementType = typeToConvert.GetGenericArguments()[0];
76+
var converterType = typeof(OneOrManyJsonConverter<>).MakeGenericType(elementType);
77+
return (JsonConverter)Activator.CreateInstance(converterType)!;
78+
}
79+
}
80+
81+
public class OneOrManyJsonConverter<T> : JsonConverter<OneOrManyItem<T>>
82+
{
83+
public override OneOrManyItem<T>? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
84+
{
85+
if (reader.TokenType == JsonTokenType.Null)
86+
{
87+
return null;
88+
}
89+
90+
if (reader.TokenType == JsonTokenType.StartArray)
91+
{
92+
var list = JsonSerializer.Deserialize<List<T>>(ref reader, options);
93+
return new ListItem<T>()
94+
{
95+
Value = list
96+
};
97+
}
98+
99+
T value = JsonSerializer.Deserialize<T>(ref reader, options);
100+
return new SingleItem<T>
101+
{
102+
Value = value
103+
};
104+
}
105+
106+
public override void Write(Utf8JsonWriter writer, OneOrManyItem<T> value, JsonSerializerOptions options)
107+
{
108+
if (value is ListItem<T> list)
109+
{
110+
JsonSerializer.Serialize(writer, list.Value, options);
111+
}
112+
else if (value is SingleItem<T> item)
113+
{
114+
JsonSerializer.Serialize(writer, item.Value, options);
115+
}
116+
}
117+
}

‎dotnet/typeagent/src/common/StringExtensions.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
namespace TypeAgent.Common;
88

9-
public static partial class StringExtensions
9+
public static partial class StringExtensions
1010
{
1111
/// <summary>
1212
/// Splits an enumerable of strings into chunks, each chunk containing up to maxChunkLength strings and

‎dotnet/typeagent/src/knowpro/Answer/AnswerContexSchema.cs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace TypeAgent.KnowPro.Answer;
5+
6+
public class RelevantKnowledge
7+
{
8+
// Entity or entities who mentioned the knowledge
9+
[JsonPropertyName("origin")]
10+
public OneOrManyItem<string>? Origin { get; set; }
11+
12+
// Entity or entities who received or consumed this knowledge
13+
[JsonPropertyName("audience")]
14+
public OneOrManyItem<string>? Audience { get; set; }
15+
16+
// Time period during which this knowledge was gathered
17+
[JsonPropertyName("timeRange")]
18+
public DateRange? TimeRange { get; set; }
19+
};
20+
21+
public class RelevantTopic : RelevantKnowledge
22+
{
23+
[JsonPropertyName("knowledge")]
24+
public string? Topic { get; set; }
25+
}
26+
27+
public class RelevantEntity : RelevantKnowledge
28+
{
29+
[JsonPropertyName("knowledge")]
30+
public ConcreteEntity? Entity { get; set; }
31+
}
32+
33+
public partial class RelevantMessage
34+
{
35+
[JsonPropertyName("from")]
36+
public OneOrManyItem<string>? From { get; set; }
37+
38+
[JsonPropertyName("to")]
39+
public OneOrManyItem<string>? To { get; set; }
40+
41+
[JsonPropertyName("timestamp")]
42+
public string? Timestamp { get; set; }
43+
44+
[JsonPropertyName("messageText")]
45+
public OneOrManyItem<string>? MessageText { get; set; }
46+
}
47+
48+
public partial class AnswerContext
49+
{
50+
// Relevant entities
51+
// Use the 'name' and 'type' properties of entities to PRECISELY identify those that answer the user question.
52+
public IList<RelevantEntity>? Entities { get; set; }
53+
54+
// Relevant topics
55+
public IList<RelevantTopic> Topics { get; set; }
56+
57+
// Relevant messages
58+
public IList<RelevantMessage>? Messages { get; set; }
59+
};

0 commit comments

Comments
 (0)