Skip to content

Commit d9a2d16

Browse files
umeshmaumeshma
authored
KnowPro.NET Language Search Update (#1754)
Natural Language Search - SearchQuerySchema + Deserialization - SearchQueryCompiler - Compile natural language into SelectExpressions - Lang Search options - Bug fixes
1 parent f5426b9 commit d9a2d16

21 files changed

+944
-49
lines changed

‎dotnet/typeagent/examples/examplesLib/CommandLine/ConsoleApp.cs‎

Lines changed: 27 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -129,12 +129,6 @@ public virtual async Task<int> ProcessCommandAsync(string[] cmdLine, Cancellatio
129129
return CommandResult.NotHandled;
130130
}
131131
var parseResult = _allCommands.Parse(cmdLine);
132-
if (parseResult.Errors.Count > 0)
133-
{
134-
WriteArgErrors(parseResult);
135-
return CommandResult.NotHandled;
136-
}
137-
138132
return await parseResult.InvokeAsync(null, cancellationToken).ConfigureAwait(false);
139133
}
140134

@@ -145,7 +139,7 @@ async Task<int> EvalInputAsync(string input, CancellationToken cancellationToken
145139
{
146140
try
147141
{
148-
return input.StartsWith(CommandPrefix)
142+
return (input.StartsWith(CommandPrefix) || IsHelp(input))
149143
? await EvalCommandAsync(input, cancellationToken).ConfigureAwait(false)
150144
: await EvalLineAsync(input, cancellationToken).ConfigureAwait(false);
151145
}
@@ -203,6 +197,30 @@ bool IsStop(string? line)
203197
return line is not null ? line.Trim() : line;
204198
}
205199

200+
public void AddModule(ICommandModule module)
201+
{
202+
_allCommands.AddModule(module);
203+
}
204+
205+
public void AddModules(params ICommandModule[] modules)
206+
{
207+
foreach (var module in modules)
208+
{
209+
AddModule(module);
210+
}
211+
}
212+
213+
public void SortCommands()
214+
{
215+
var commands = _allCommands.Subcommands.ToList();
216+
_allCommands.Subcommands.Clear();
217+
commands.Sort((x, y) => x.Name.CompareTo(y.Name));
218+
foreach (var cmd in commands)
219+
{
220+
_allCommands.Add(cmd);
221+
}
222+
}
223+
206224
protected virtual void OnException(string input, Exception ex)
207225
{
208226
Console.WriteLine("## Could not process request");
@@ -231,27 +249,8 @@ protected void WriteArgErrors(ParseResult parseResult)
231249
}
232250
}
233251

234-
public void AddModule(ICommandModule module)
235-
{
236-
_allCommands.AddModule(module);
237-
}
238-
239-
public void AddModules(params ICommandModule[] modules)
240-
{
241-
foreach (var module in modules)
242-
{
243-
AddModule(module);
244-
}
245-
}
246-
247-
public void SortCommands()
252+
private bool IsHelp(string value)
248253
{
249-
var commands = _allCommands.Subcommands.ToList();
250-
_allCommands.Subcommands.Clear();
251-
commands.Sort((x, y) => x.Name.CompareTo(y.Name));
252-
foreach (var cmd in commands)
253-
{
254-
_allCommands.Add(cmd);
255-
}
254+
return value == "--help" || value == "--?" || value == "-?";
256255
}
257256
}

‎dotnet/typeagent/src/common/Resource.cs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace TypeAgent.Common;
5+
6+
public static class Resource
7+
{
8+
// Resource name is: <FullNamespace>.<filename>
9+
public static string LoadResourceText(Assembly assembly, string resourceName)
10+
{
11+
ArgumentVerify.ThrowIfNull(assembly, nameof(assembly));
12+
13+
using var stream = Load(assembly, resourceName);
14+
using var reader = new StreamReader(stream);
15+
string schemaText = reader.ReadToEnd();
16+
17+
return schemaText;
18+
}
19+
20+
// Resource name is: <FullNamespace>.<filename>
21+
public static List<string> LoadResourceLines(Assembly assembly, string resourceName)
22+
{
23+
ArgumentVerify.ThrowIfNull(assembly, nameof(assembly));
24+
25+
using var stream = Load(assembly, resourceName);
26+
using var reader = new StreamReader(stream);
27+
28+
List<string> lines = [];
29+
string line;
30+
while ((line = reader.ReadLine()) is not null)
31+
{
32+
lines.Add(line);
33+
}
34+
35+
return lines;
36+
}
37+
38+
private static Stream Load(Assembly assembly, string resourceName)
39+
{
40+
return assembly.GetManifestResourceStream(resourceName)
41+
?? throw new FileNotFoundException($"Resource not found: {resourceName}");
42+
}
43+
}

‎dotnet/typeagent/src/common/SchemaLoader.cs‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,7 @@ public static class SchemaLoader
88
// Resource name is: <FullNamespace>.<filename>
99
public static string LoadResource(Assembly assembly, string resourceName)
1010
{
11-
ArgumentVerify.ThrowIfNull(assembly, nameof(assembly));
12-
13-
using var stream = assembly.GetManifestResourceStream(resourceName)
14-
?? throw new FileNotFoundException($"Resource not found: {resourceName}");
15-
using var reader = new StreamReader(stream);
16-
string schemaText = reader.ReadToEnd();
11+
string schemaText = Resource.LoadResourceText(assembly, resourceName);
1712
RemoveCopyright(schemaText);
1813
return schemaText;
1914
}

‎dotnet/typeagent/src/common/SetExtensions.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace TypeAgent.Common;
5+
6+
public static class SetExtensions
7+
{
8+
public static void LoadFromFile(this HashSet<string> set, string filePath)
9+
{
10+
foreach(var line in File.ReadLines(filePath))
11+
{
12+
var entry = line.Trim();
13+
if (!string.IsNullOrEmpty(entry))
14+
{
15+
set.Add(entry);
16+
}
17+
}
18+
}
19+
20+
public static void LoadFromResource(this HashSet<string> set, Assembly assembly, string resourceName)
21+
{
22+
foreach (var line in Resource.LoadResourceLines(assembly, resourceName))
23+
{
24+
var entry = line.Trim();
25+
if (!string.IsNullOrEmpty(entry))
26+
{
27+
set.Add(entry);
28+
}
29+
}
30+
}
31+
}

‎dotnet/typeagent/src/knowpro/DateTimeSchema.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public class TimeVal
2929
public int Seconds { get; set; }
3030
}
3131

32-
public class DateTime
32+
public partial class DateTime
3333
{
3434
[JsonPropertyName("date")]
3535
public DateVal Date { get; set; }
@@ -38,7 +38,7 @@ public class DateTime
3838
public TimeVal? Time { get; set; }
3939
}
4040

41-
public class DateTimeRange
41+
public partial class DateTimeRange
4242
{
4343
[JsonPropertyName("startDate")]
4444
public DateTime StartDate { get; set; }

‎dotnet/typeagent/src/knowpro/DateTimeSchemaImpl.cs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace TypeAgent.KnowPro;
5+
6+
public partial class DateTime
7+
{
8+
public DateTimeOffset ToDateTimeOffset()
9+
{
10+
return Time is not null
11+
? new DateTimeOffset(
12+
Date.Year,
13+
Date.Month,
14+
Date.Day,
15+
Time.Hour,
16+
Time.Minute,
17+
Time.Seconds,
18+
TimeSpan.Zero
19+
)
20+
: new DateTimeOffset(
21+
Date.Year,
22+
Date.Month,
23+
Date.Day,
24+
0,
25+
0,
26+
0,
27+
TimeSpan.Zero
28+
);
29+
}
30+
31+
// Instance version of former toStartDate(dateTime).
32+
public DateTimeOffset ToStartDate()
33+
{
34+
return Time is not null
35+
? ToDateTimeOffset()
36+
: new DateTimeOffset(
37+
Date.Year,
38+
Date.Month,
39+
Date.Day,
40+
0,
41+
0,
42+
0,
43+
TimeSpan.Zero
44+
);
45+
}
46+
47+
// Instance version of former toStopDate(dateTime).
48+
// If no time component, returns inclusive end-of-day (23:59:59.999).
49+
public DateTimeOffset ToStopDate()
50+
{
51+
return Time is not null
52+
? ToDateTimeOffset()
53+
: new DateTimeOffset(
54+
Date.Year,
55+
Date.Month,
56+
Date.Day,
57+
23,
58+
59,
59+
59,
60+
TimeSpan.Zero
61+
).AddMilliseconds(999);
62+
}
63+
}
64+
65+
public partial class DateTimeRange
66+
{
67+
public DateRange ToDateRange()
68+
{
69+
var start = StartDate.ToStartDate();
70+
var end = StopDate is not null ? StopDate.ToStopDate() : (DateTimeOffset?)null;
71+
72+
return new DateRange
73+
{
74+
Start = start,
75+
End = end
76+
};
77+
}
78+
}

‎dotnet/typeagent/src/knowpro/Lang/LangQueryCompilerSettings.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace TypeAgent.KnowPro.Lang;
5+
6+
public class LangQueryCompilerSettings
7+
{
8+
public LangQueryCompilerSettings()
9+
{
10+
ApplyScope = true;
11+
ExactScope = false;
12+
VerbScope = true;
13+
}
14+
/// <summary>
15+
/// Is fuzzy matching enabled when applying scope?
16+
/// </summary>
17+
public bool ExactScope { get; set; }
18+
/// <summary>
19+
/// Is fuzzy matching enabled when applying scope?
20+
/// </summary>
21+
public bool VerbScope { get; set; }
22+
/// <summary>
23+
/// Are scope constraints enabled
24+
/// </summary>
25+
public bool ApplyScope { get; set; }
26+
/// <summary>
27+
/// Use to ignore noise terms etc.
28+
/// </summary>
29+
public Func<string, bool>? TermFilter { get; set; }
30+
}

‎dotnet/typeagent/src/knowpro/Lang/LangSearchFilter.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace TypeAgent.KnowPro.Lang;
5+
6+
public class LangSearchFilter
7+
{
8+
public KnowledgeType? KnowledgeType { get; set; }
9+
10+
public string? ThreadDescription { get; set; }
11+
12+
public IList<string>? Tags { get; set; }
13+
14+
public SearchTermGroup? ScopeDefiningTerms { get; set; }
15+
}

‎dotnet/typeagent/src/knowpro/Lang/LangSearchOptions.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace TypeAgent.KnowPro.Lang;
5+
6+
public class LangSearchOptions : SearchOptions
7+
{
8+
public LangSearchOptions()
9+
: base()
10+
{
11+
CompilerSettings = new();
12+
}
13+
14+
public LangQueryCompilerSettings CompilerSettings { get; }
15+
16+
public LangSearchRagOptions? FallbackRagOptions { get; set; }
17+
18+
public IList<PromptSection>? ModelInstructions { get; set; }
19+
}

‎dotnet/typeagent/src/knowpro/Lang/LangSearchRagOptions.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
namespace TypeAgent.KnowPro.Lang;
5+
6+
public class LangSearchRagOptions
7+
{
8+
public int? MaxMessageMatches { get; set; }
9+
10+
public bool? ExactMatch { get; set; }
11+
/// <summary>
12+
/// The maximum # of total message characters to select
13+
/// The query processor will ensure that the cumulative character count of message matches
14+
/// is less than this number
15+
/// </summary>
16+
public int? MaxCharsInBudget { get; set; }
17+
18+
}

0 commit comments

Comments
 (0)