Skip to content

Drop implausibly small per-occupation salary estimates (#275) #276

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Monster0506 merged 1 commit into from
Sep 20, 2026
Merged

Drop implausibly small per-occupation salary estimates (#275) #276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 68 additions & 22 deletions HumanCapital/AiSuggestedOccupationMatchingService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,14 @@ public AiSuggestedOccupationMatchingService(ILlmService llmService)

public async Task<IReadOnlyList<OccupationMatch>> MatchAsync(
ProfessionalProfile profile,
CancellationToken cancellationToken = default)
CancellationToken cancellationToken = default
)
{
if (profile.Skills.Count == 0 && profile.Experience.Count == 0 && profile.Education.Count == 0)
if (
profile.Skills.Count == 0
&& profile.Experience.Count == 0
&& profile.Education.Count == 0
)
{
return [];
}
Expand All @@ -37,7 +42,9 @@ public async Task<IReadOnlyList<OccupationMatch>> MatchAsync(
catch (LlmServiceException ex)
{
throw new OccupationMatchingException(
"Failed to match occupations: the LLM provider call failed.", ex);
"Failed to match occupations: the LLM provider call failed.",
ex
);
}

var json = StripCodeFence(completion);
Expand All @@ -50,34 +57,45 @@ public async Task<IReadOnlyList<OccupationMatch>> MatchAsync(
catch (JsonException ex)
{
throw new OccupationMatchingException(
"The LLM returned a response that could not be parsed as occupation matches.", ex);
"The LLM returned a response that could not be parsed as occupation matches.",
ex
);
}

if (response?.Occupations is not { } occupations)
{
throw new OccupationMatchingException(
"The LLM returned an occupation response without a structured occupations list.");
"The LLM returned an occupation response without a structured occupations list."
);
}

var matches = new List<OccupationMatch>(occupations.Count);
foreach (var occupation in occupations)
{
if (occupation is null
if (
occupation is null
|| string.IsNullOrWhiteSpace(occupation.Title)
|| string.IsNullOrWhiteSpace(occupation.Explanation))
|| string.IsNullOrWhiteSpace(occupation.Explanation)
)
{
throw new OccupationMatchingException(
"The LLM returned an occupation match without a title or explanation.");
"The LLM returned an occupation match without a title or explanation."
);
}

if (occupation.TypicalMinUsd is < 0
if (
occupation.TypicalMinUsd is < 0
|| occupation.TypicalMaxUsd is < 0
|| (occupation.TypicalMinUsd is not null
|| (
occupation.TypicalMinUsd is not null
&& occupation.TypicalMaxUsd is not null
&& occupation.TypicalMinUsd > occupation.TypicalMaxUsd))
&& occupation.TypicalMinUsd > occupation.TypicalMaxUsd
)
)
{
throw new OccupationMatchingException(
"The LLM returned an invalid compensation range for an occupation match.");
"The LLM returned an invalid compensation range for an occupation match."
);
}

var keySkills = (occupation.KeySkills ?? [])
Expand All @@ -86,18 +104,42 @@ public async Task<IReadOnlyList<OccupationMatch>> MatchAsync(
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();

matches.Add(new OccupationMatch(
occupation.Title.Trim(),
occupation.Explanation.Trim(),
IsAiEstimated: true,
keySkills,
occupation.TypicalMinUsd,
occupation.TypicalMaxUsd));
// The LLM occasionally returns an abbreviated figure (e.g. 80,
// presumably meaning "$80k") instead of a full annual-salary
// number - a positive value, so the min<0/max<0/min>max checks
// above don't catch it, but clearly not a real US salary. Treat
// it the same as the prompt's own "omit both if you cannot give
// a reasonable estimate" instruction: drop it rather than show
// a nonsensical range like "$80 - $120".
var (typicalMinUsd, typicalMaxUsd) =
IsPlausibleAnnualSalary(occupation.TypicalMinUsd)
&& IsPlausibleAnnualSalary(occupation.TypicalMaxUsd)
? (occupation.TypicalMinUsd, occupation.TypicalMaxUsd)
: (null, null);

matches.Add(
new OccupationMatch(
occupation.Title.Trim(),
occupation.Explanation.Trim(),
IsAiEstimated: true,
keySkills,
typicalMinUsd,
typicalMaxUsd
)
);
}

return matches;
}

// No real full-time US annual salary is this low; a value below this
// is almost certainly the model returning an abbreviated figure (e.g.
// 80 instead of 80000) rather than a genuine estimate.
private const int MinPlausibleAnnualSalaryUsd = 1_000;

private static bool IsPlausibleAnnualSalary(int? value) =>
value is null || value >= MinPlausibleAnnualSalaryUsd;

private static string BuildPrompt(ProfessionalProfile profile)
{
var summary = new StringBuilder();
Expand All @@ -112,7 +154,9 @@ private static string BuildPrompt(ProfessionalProfile profile)
summary.AppendLine("Experience:");
foreach (var exp in profile.Experience)
{
var description = exp.Description is not null ? $": {exp.Description}" : string.Empty;
var description = exp.Description is not null
? $": {exp.Description}"
: string.Empty;
summary.AppendLine($"- {exp.Title ?? "Role"} at {exp.Organization}{description}");
}
}
Expand Down Expand Up @@ -150,7 +194,8 @@ the profile above that justify the match (one sentence). Only suggest
occupation (not limited to skills already in the profile above).

"typicalMinUsd"/"typicalMaxUsd" should be a rough typical US annual
salary range for that specific occupation (not the whole profile).
salary range for that specific occupation (not the whole profile),
as a full number in dollars - e.g. 85000, never 85 or "85k".
Omit both if you cannot give a reasonable estimate.
""";
}
Expand Down Expand Up @@ -185,5 +230,6 @@ private record OccupationEntry(
string? Explanation,
IReadOnlyList<string?>? KeySkills,
int? TypicalMinUsd,
int? TypicalMaxUsd);
int? TypicalMaxUsd
);
}
55 changes: 46 additions & 9 deletions tests/MoneyMirror.Tests/HumanCapital/AiSuggestedOccupationMatchingServiceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,18 @@ public class AiSuggestedOccupationMatchingServiceTests
{
private class FakeLlmService(string response) : ILlmService
{
public Task<string> CompleteAsync(string prompt, CancellationToken cancellationToken = default) =>
Task.FromResult(response);
public Task<string> CompleteAsync(
string prompt,
CancellationToken cancellationToken = default
) => Task.FromResult(response);
}

private class ThrowingLlmService : ILlmService
{
public Task<string> CompleteAsync(string prompt, CancellationToken cancellationToken = default) =>
throw new LlmServiceException("provider unavailable");
public Task<string> CompleteAsync(
string prompt,
CancellationToken cancellationToken = default
) => throw new LlmServiceException("provider unavailable");
}

private static readonly ProfessionalProfile SampleProfile = ProfessionalProfile.Empty with
Expand Down Expand Up @@ -80,7 +84,9 @@ public async Task MatchAsync_MissingOccupationsList_ThrowsOccupationMatchingExce
{
var service = new AiSuggestedOccupationMatchingService(new FakeLlmService("{}"));

await Assert.ThrowsAsync<OccupationMatchingException>(() => service.MatchAsync(SampleProfile));
await Assert.ThrowsAsync<OccupationMatchingException>(() =>
service.MatchAsync(SampleProfile)
);
}

[Fact]
Expand All @@ -89,7 +95,9 @@ public async Task MatchAsync_MatchMissingTitleOrExplanation_ThrowsOccupationMatc
const string json = "{ \"occupations\": [{ \"title\": \"Software Engineer\" }] }";
var service = new AiSuggestedOccupationMatchingService(new FakeLlmService(json));

await Assert.ThrowsAsync<OccupationMatchingException>(() => service.MatchAsync(SampleProfile));
await Assert.ThrowsAsync<OccupationMatchingException>(() =>
service.MatchAsync(SampleProfile)
);
}

[Fact]
Expand All @@ -104,7 +112,32 @@ public async Task MatchAsync_InvalidCompensationRange_ThrowsOccupationMatchingEx
""";
var service = new AiSuggestedOccupationMatchingService(new FakeLlmService(json));

await Assert.ThrowsAsync<OccupationMatchingException>(() => service.MatchAsync(SampleProfile));
await Assert.ThrowsAsync<OccupationMatchingException>(() =>
service.MatchAsync(SampleProfile)
);
}

[Fact]
public async Task MatchAsync_ImplausiblySmallCompensationRange_NullsOutBothValuesRatherThanKeepingThem()
{
// The model sometimes returns an abbreviated figure (e.g. 80,
// presumably meaning "$80k") instead of a real annual-salary
// number. It's a valid, correctly-ordered positive range, so the
// negative/min>max checks don't catch it - it should still be
// dropped rather than shown as a nonsensical "$80 - $120".
const string json = """
{
"occupations": [
{"title": "Software Engineer", "explanation": "Strong C# background.", "typicalMinUsd": 80, "typicalMaxUsd": 120}
]
}
""";
var service = new AiSuggestedOccupationMatchingService(new FakeLlmService(json));

var match = Assert.Single(await service.MatchAsync(SampleProfile));

Assert.Null(match.TypicalMinUsd);
Assert.Null(match.TypicalMaxUsd);
}

[Fact]
Expand Down Expand Up @@ -150,14 +183,18 @@ public async Task MatchAsync_MalformedJson_ThrowsOccupationMatchingException()
{
var service = new AiSuggestedOccupationMatchingService(new FakeLlmService("not json"));

await Assert.ThrowsAsync<OccupationMatchingException>(() => service.MatchAsync(SampleProfile));
await Assert.ThrowsAsync<OccupationMatchingException>(() =>
service.MatchAsync(SampleProfile)
);
}

[Fact]
public async Task MatchAsync_LlmServiceFails_ThrowsOccupationMatchingException()
{
var service = new AiSuggestedOccupationMatchingService(new ThrowingLlmService());

await Assert.ThrowsAsync<OccupationMatchingException>(() => service.MatchAsync(SampleProfile));
await Assert.ThrowsAsync<OccupationMatchingException>(() =>
service.MatchAsync(SampleProfile)
);
}
}
Loading