From fbce0226f22323e7d5ec471f32d3c49745cbbe79 Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 07:01:06 -0400 Subject: [PATCH 1/4] Add eBay comparable-based asset valuations --- .env.example | 2 + Features/PhysicalAssets/InventoryList.razor | 4 +- Features/PhysicalAssets/PhysicalAssets.razor | 4 +- PhysicalAssets/AssetValuation.cs | 6 +- PhysicalAssets/Configuration/EbayOptions.cs | 12 + PhysicalAssets/EbayMarketDataException.cs | 8 + PhysicalAssets/EbayMarketDataService.cs | 240 ++++++++++++++++++ .../EvidenceBasedAssetValuationService.cs | 40 +++ PhysicalAssets/IAssetValuationService.cs | 9 +- PhysicalAssets/IEbayMarketDataService.cs | 13 + Program.cs | 7 +- appsettings.json | 6 + docker-compose.yml | 2 + docs/development.md | 20 +- .../EbayMarketDataServiceTests.cs | 128 ++++++++++ ...EvidenceBasedAssetValuationServiceTests.cs | 95 +++++++ 16 files changed, 575 insertions(+), 21 deletions(-) create mode 100644 PhysicalAssets/Configuration/EbayOptions.cs create mode 100644 PhysicalAssets/EbayMarketDataException.cs create mode 100644 PhysicalAssets/EbayMarketDataService.cs create mode 100644 PhysicalAssets/EvidenceBasedAssetValuationService.cs create mode 100644 PhysicalAssets/IEbayMarketDataService.cs create mode 100644 tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs create mode 100644 tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs diff --git a/.env.example b/.env.example index f73d292..0ab8727 100644 --- a/.env.example +++ b/.env.example @@ -7,3 +7,5 @@ POSTGRES_PASSWORD=postgres NEMOTRON_API_KEY= VISION_API_KEY= BLS_API_KEY= +EBAY_CLIENT_ID= +EBAY_CLIENT_SECRET= diff --git a/Features/PhysicalAssets/InventoryList.razor b/Features/PhysicalAssets/InventoryList.razor index 9c821d0..67c535e 100644 --- a/Features/PhysicalAssets/InventoryList.razor +++ b/Features/PhysicalAssets/InventoryList.razor @@ -370,8 +370,8 @@ else } } - // PA10.1/PA10.2: re-runs the (currently AI-estimated, see #138) valuation - // pipeline for this item's existing identification and appends the + // PA10.1/PA10.2: re-runs the market valuation pipeline for this item's + // existing identification and appends the // result as a new history entry - it never overwrites prior valuations. private async Task RevalueAsync() { diff --git a/Features/PhysicalAssets/PhysicalAssets.razor b/Features/PhysicalAssets/PhysicalAssets.razor index c6c4cfa..3657144 100644 --- a/Features/PhysicalAssets/PhysicalAssets.razor +++ b/Features/PhysicalAssets/PhysicalAssets.razor @@ -575,8 +575,8 @@ /// /// In-memory review state for one DetectedAsset: whether the user is - /// keeping it, any brand/model correction, an optional AI-estimated - /// valuation (#138), and (#72-74) whether/where it's been saved to the + /// keeping it, any brand/model correction, an optional market valuation, + /// and (#72-74) whether/where it's been saved to the /// inventory. This is the seam between PA3's detection output and /// PA7's save step. /// diff --git a/PhysicalAssets/AssetValuation.cs b/PhysicalAssets/AssetValuation.cs index b8cf731..94a0f9d 100644 --- a/PhysicalAssets/AssetValuation.cs +++ b/PhysicalAssets/AssetValuation.cs @@ -4,11 +4,7 @@ namespace MoneyMirror.PhysicalAssets; /// An estimated resale value for a physical asset. /// /// -/// True when the value came directly from the LLM's -/// judgment, not from real market comps. Never present this to the user -/// (or persist it) as if it were evidence-based - #138 is a deliberate, -/// temporary stand-in for the real PA5 (#14) -> PA6 (#68-71) pipeline, -/// which aggregates real comps deterministically and sets this false. +/// True when the value came from an LLM estimate rather than market listings. /// public record AssetValuation( decimal? EstimatedValueUsd, diff --git a/PhysicalAssets/Configuration/EbayOptions.cs b/PhysicalAssets/Configuration/EbayOptions.cs new file mode 100644 index 0000000..f8a90e0 --- /dev/null +++ b/PhysicalAssets/Configuration/EbayOptions.cs @@ -0,0 +1,12 @@ +namespace MoneyMirror.PhysicalAssets.Configuration; + +/// Credentials and endpoint configuration for eBay Browse API. +public sealed class EbayOptions +{ + public const string SectionName = "Ebay"; + + public string ClientId { get; set; } = string.Empty; + public string ClientSecret { get; set; } = string.Empty; + public string BaseUrl { get; set; } = "https://api.ebay.com"; + public string MarketplaceId { get; set; } = "EBAY_US"; +} diff --git a/PhysicalAssets/EbayMarketDataException.cs b/PhysicalAssets/EbayMarketDataException.cs new file mode 100644 index 0000000..5b66e07 --- /dev/null +++ b/PhysicalAssets/EbayMarketDataException.cs @@ -0,0 +1,8 @@ +namespace MoneyMirror.PhysicalAssets; + +/// Thrown when eBay Browse API cannot return comparable listings. +public sealed class EbayMarketDataException : Exception +{ + public EbayMarketDataException(string message, Exception? innerException = null) + : base(message, innerException) { } +} diff --git a/PhysicalAssets/EbayMarketDataService.cs b/PhysicalAssets/EbayMarketDataService.cs new file mode 100644 index 0000000..0b01bc0 --- /dev/null +++ b/PhysicalAssets/EbayMarketDataService.cs @@ -0,0 +1,240 @@ +using System.Globalization; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Options; +using MoneyMirror.PhysicalAssets.Configuration; + +namespace MoneyMirror.PhysicalAssets; + +/// Searches eBay's Browse API for used, USD-priced comparable listings. +public sealed class EbayMarketDataService : IEbayMarketDataService +{ + private const int ResultLimit = 20; + private const string BrowseScope = "https://api.ebay.com/oauth/api_scope"; + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + private readonly HttpClient _httpClient; + private readonly EbayOptions _options; + private readonly TimeProvider _timeProvider; + private readonly SemaphoreSlim _tokenLock = new(1, 1); + private CachedAccessToken? _cachedToken; + + public EbayMarketDataService( + HttpClient httpClient, + IOptions options, + TimeProvider? timeProvider = null + ) + { + _httpClient = httpClient; + _options = options.Value; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task> SearchUsedListingsAsync( + string label, + string? brand, + string? model, + CancellationToken cancellationToken = default + ) + { + ArgumentException.ThrowIfNullOrWhiteSpace(label); + EnsureCredentials(); + + var query = string.Join( + " ", + new[] { label, brand, model } + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value!.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + ); + var url = $"{_options.BaseUrl.TrimEnd('/')}/buy/browse/v1/item_summary/search" + + $"?q={Uri.EscapeDataString(query)}" + + $"&filter={Uri.EscapeDataString("conditions:{USED}")}" + + $"&limit={ResultLimit}"; + + using var request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.Authorization = new AuthenticationHeaderValue( + "Bearer", + await GetAccessTokenAsync(cancellationToken) + ); + request.Headers.Add("X-EBAY-C-MARKETPLACE-ID", _options.MarketplaceId); + + using var response = await SendAsync(request, "eBay Browse API", cancellationToken); + var result = await ReadResponseAsync(response, "eBay Browse API", cancellationToken); + + return result?.ItemSummaries? + .Select(ToEvidence) + .Where(evidence => evidence is not null) + .Select(evidence => evidence!) + .ToList() ?? []; + } + + private void EnsureCredentials() + { + if (string.IsNullOrWhiteSpace(_options.ClientId) || string.IsNullOrWhiteSpace(_options.ClientSecret)) + { + throw new EbayMarketDataException( + "eBay Browse API credentials are not configured. Set Ebay:ClientId and Ebay:ClientSecret." + ); + } + } + + private async Task GetAccessTokenAsync(CancellationToken cancellationToken) + { + var now = _timeProvider.GetUtcNow(); + if (_cachedToken is { ExpiresAt: var expiry } cached && expiry > now.AddMinutes(1)) + { + return cached.Value; + } + + await _tokenLock.WaitAsync(cancellationToken); + try + { + now = _timeProvider.GetUtcNow(); + if (_cachedToken is { ExpiresAt: var cachedExpiry } current && cachedExpiry > now.AddMinutes(1)) + { + return current.Value; + } + + using var request = new HttpRequestMessage( + HttpMethod.Post, + $"{_options.BaseUrl.TrimEnd('/')}/identity/v1/oauth2/token" + ); + var credentials = Convert.ToBase64String( + Encoding.UTF8.GetBytes($"{_options.ClientId}:{_options.ClientSecret}") + ); + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials); + request.Content = new FormUrlEncodedContent( + [ + new KeyValuePair("grant_type", "client_credentials"), + new KeyValuePair("scope", BrowseScope), + ]); + + using var response = await SendAsync(request, "eBay OAuth token service", cancellationToken); + var token = await ReadResponseAsync(response, "eBay OAuth token service", cancellationToken); + if ( + token is null + || string.IsNullOrWhiteSpace(token.AccessToken) + || token.ExpiresIn <= 0 + ) + { + throw new EbayMarketDataException("eBay OAuth token service returned an incomplete access token."); + } + + _cachedToken = new CachedAccessToken( + token.AccessToken, + _timeProvider.GetUtcNow().AddSeconds(token.ExpiresIn) + ); + return token.AccessToken; + } + finally + { + _tokenLock.Release(); + } + } + + private async Task SendAsync( + HttpRequestMessage request, + string serviceName, + CancellationToken cancellationToken + ) + { + try + { + return await _httpClient.SendAsync(request, cancellationToken); + } + catch (HttpRequestException ex) + { + throw new EbayMarketDataException($"Failed to reach the {serviceName}.", ex); + } + catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested) + { + throw new EbayMarketDataException($"The {serviceName} request timed out.", ex); + } + } + + private static async Task ReadResponseAsync( + HttpResponseMessage response, + string serviceName, + CancellationToken cancellationToken + ) + { + if (!response.IsSuccessStatusCode) + { + var errorBody = await response.Content.ReadAsStringAsync(cancellationToken); + throw new EbayMarketDataException( + $"{serviceName} returned {(int)response.StatusCode} {response.StatusCode}: {errorBody}" + ); + } + + try + { + return await response.Content.ReadFromJsonAsync(JsonOptions, cancellationToken); + } + catch (JsonException ex) + { + throw new EbayMarketDataException($"Failed to parse the {serviceName} response.", ex); + } + } + + private static AssetValuationEvidence? ToEvidence(ItemSummary item) + { + if ( + item.Price is null + || !string.Equals(item.Price.Currency, "USD", StringComparison.OrdinalIgnoreCase) + || !decimal.TryParse( + item.Price.Value, + NumberStyles.Number, + CultureInfo.InvariantCulture, + out var price + ) + || price <= 0 + ) + { + return null; + } + + return new AssetValuationEvidence(price, "eBay", item.Title, item.Condition); + } + + private sealed class SearchResponse + { + [JsonPropertyName("itemSummaries")] + public IReadOnlyList? ItemSummaries { get; init; } + } + + private sealed class ItemSummary + { + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("condition")] + public string? Condition { get; init; } + + [JsonPropertyName("price")] + public ItemPrice? Price { get; init; } + } + + private sealed class ItemPrice + { + [JsonPropertyName("value")] + public string? Value { get; init; } + + [JsonPropertyName("currency")] + public string? Currency { get; init; } + } + + private sealed class TokenResponse + { + [JsonPropertyName("access_token")] + public string? AccessToken { get; init; } + + [JsonPropertyName("expires_in")] + public int ExpiresIn { get; init; } + } + + private sealed record CachedAccessToken(string Value, DateTimeOffset ExpiresAt); +} diff --git a/PhysicalAssets/EvidenceBasedAssetValuationService.cs b/PhysicalAssets/EvidenceBasedAssetValuationService.cs new file mode 100644 index 0000000..bb9d8bc --- /dev/null +++ b/PhysicalAssets/EvidenceBasedAssetValuationService.cs @@ -0,0 +1,40 @@ +namespace MoneyMirror.PhysicalAssets; + +/// Builds an asset valuation from structured market listings, without an LLM. +public sealed class EvidenceBasedAssetValuationService : IAssetValuationService +{ + private readonly IEbayMarketDataService _marketDataService; + private readonly TimeProvider _timeProvider; + + public EvidenceBasedAssetValuationService( + IEbayMarketDataService marketDataService, + TimeProvider? timeProvider = null + ) + { + _marketDataService = marketDataService; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task EstimateAsync( + string label, + string? brand, + string? model, + CancellationToken cancellationToken = default + ) + { + try + { + var evidence = await _marketDataService.SearchUsedListingsAsync( + label, + brand, + model, + cancellationToken + ); + return MarketValuationCalculator.Calculate(evidence, _timeProvider.GetUtcNow()); + } + catch (EbayMarketDataException ex) + { + throw new AssetValuationException("Failed to retrieve comparable market listings.", ex); + } + } +} diff --git a/PhysicalAssets/IAssetValuationService.cs b/PhysicalAssets/IAssetValuationService.cs index e6df0fc..039ea5b 100644 --- a/PhysicalAssets/IAssetValuationService.cs +++ b/PhysicalAssets/IAssetValuationService.cs @@ -1,19 +1,14 @@ namespace MoneyMirror.PhysicalAssets; /// -/// Estimates a physical asset's resale value. The current implementation -/// (#138) is an explicit MVP placeholder that asks the LLM to guess a -/// plausible value - it is not grounded in real market comps. It exists so -/// the app has an end-to-end demo path before #14 (PA5, real market -/// evidence) and #68-71 (PA6, deterministic aggregation of that evidence) -/// are built; see #138 for the swap-out plan. +/// Estimates a physical asset's resale value from comparable-market evidence. /// public interface IAssetValuationService { /// A valuation whose EstimatedValueUsd is null when no market value is available. /// Callers must preserve that distinction instead of treating it as zero. /// - /// The LLM call failed, or its response couldn't be parsed. + /// The market-data provider failed or its response could not be parsed. /// Task EstimateAsync( string label, diff --git a/PhysicalAssets/IEbayMarketDataService.cs b/PhysicalAssets/IEbayMarketDataService.cs new file mode 100644 index 0000000..0442888 --- /dev/null +++ b/PhysicalAssets/IEbayMarketDataService.cs @@ -0,0 +1,13 @@ +namespace MoneyMirror.PhysicalAssets; + +/// Searches a market data source for real listings comparable to an identified item. +public interface IEbayMarketDataService +{ + /// Finds up to twenty used USD listings matching the confirmed item's label, brand and model. + Task> SearchUsedListingsAsync( + string label, + string? brand, + string? model, + CancellationToken cancellationToken = default + ); +} diff --git a/Program.cs b/Program.cs index bc72c5e..fca0960 100644 --- a/Program.cs +++ b/Program.cs @@ -8,6 +8,7 @@ using MoneyMirror.HumanCapital; using MoneyMirror.HumanCapital.Configuration; using MoneyMirror.PhysicalAssets; +using MoneyMirror.PhysicalAssets.Configuration; // Containers start with no LANG/LC_ALL, so .NET falls back to the invariant culture // and renders currency as "¤" instead of "$". Pin the formatting culture so money @@ -29,6 +30,7 @@ builder.Configuration.GetSection(NemotronOptions.SectionName) ); builder.Services.Configure(builder.Configuration.GetSection(BlsOptions.SectionName)); +builder.Services.Configure(builder.Configuration.GetSection(EbayOptions.SectionName)); builder.Services.Configure( builder.Configuration.GetSection(VisionModelOptions.SectionName) ); @@ -65,6 +67,9 @@ builder.Services.AddHttpClient(client => client.Timeout = TimeSpan.FromSeconds(15) ); +builder.Services.AddHttpClient(client => + client.Timeout = TimeSpan.FromSeconds(15) +); builder.Services.AddScoped< IMarketPotentialExplanationService, NemotronMarketPotentialExplanationService @@ -83,7 +88,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); -builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/appsettings.json b/appsettings.json index d8ca2ae..8b6ef71 100644 --- a/appsettings.json +++ b/appsettings.json @@ -24,5 +24,11 @@ "Bls": { "ApiKey": "", "BaseUrl": "https://api.bls.gov/publicAPI/v2/" + }, + "Ebay": { + "ClientId": "", + "ClientSecret": "", + "BaseUrl": "https://api.ebay.com", + "MarketplaceId": "EBAY_US" } } diff --git a/docker-compose.yml b/docker-compose.yml index 17172a2..2cf4da3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,8 @@ services: Ai__Nemotron__ApiKey: ${NEMOTRON_API_KEY:-} Ai__VisionModel__ApiKey: ${VISION_API_KEY:-} Bls__ApiKey: ${BLS_API_KEY:-} + Ebay__ClientId: ${EBAY_CLIENT_ID:-} + Ebay__ClientSecret: ${EBAY_CLIENT_SECRET:-} ports: - "${APP_PORT:-8000}:8080" volumes: diff --git a/docs/development.md b/docs/development.md index c666f4d..b815865 100644 --- a/docs/development.md +++ b/docs/development.md @@ -6,17 +6,27 @@ monolith, one database, one implicit user — no auth or multi-tenancy. ## Configuration -AI and BLS settings live under `Ai:Nemotron`, `Ai:VisionModel`, and `Bls` -(`BaseUrl`, `Model` / `ApiKey` as applicable). `appsettings.json` ships empty -`ApiKey` placeholders — never commit real keys. Set them locally: +AI, BLS, and eBay Browse API settings live under `Ai:Nemotron`, +`Ai:VisionModel`, `Bls`, and `Ebay` (`ClientId`, `ClientSecret`, `BaseUrl`, and +`MarketplaceId`). `appsettings.json` ships empty secret placeholders — never +commit real keys. Set them locally: ```sh dotnet user-secrets set "Ai:Nemotron:ApiKey" "" dotnet user-secrets set "Ai:VisionModel:ApiKey" "" dotnet user-secrets set "Bls:ApiKey" "" +dotnet user-secrets set "Ebay:ClientId" "" +dotnet user-secrets set "Ebay:ClientSecret" "" ``` -Or use env vars: `Ai__Nemotron__ApiKey`, `Ai__VisionModel__ApiKey`, `Bls__ApiKey`. +Or use env vars: `Ai__Nemotron__ApiKey`, `Ai__VisionModel__ApiKey`, +`Bls__ApiKey`, `Ebay__ClientId`, `Ebay__ClientSecret`. + +Physical asset valuations query up to 20 used eBay US listings and calculate a +median from usable USD prices. No usable listings produce a null value and an +explicit low-confidence explanation. The valuation returns each comparable's +price, source, title, and condition in `AssetValuation.Evidence`; that evidence +is persisted and displayed alongside each valuation in inventory history. A free BLS v2 key from [data.bls.gov/registrationEngine](https://data.bls.gov/registrationEngine/) raises rate limits; the app works without one at the unregistered limit. @@ -37,6 +47,8 @@ on Postgres volumes and host-local `dotnet run`: [backend/postgresql_setup.md](. | `NEMOTRON_API_KEY` | `Ai:Nemotron:ApiKey` | | `VISION_API_KEY` | `Ai:VisionModel:ApiKey` | | `BLS_API_KEY` | `Bls:ApiKey` | +| `EBAY_CLIENT_ID` | `Ebay:ClientId` | +| `EBAY_CLIENT_SECRET` | `Ebay:ClientSecret` | `docker compose down` stops the stack; add `-v` to wipe DB and image volumes. diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs new file mode 100644 index 0000000..2b94552 --- /dev/null +++ b/tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs @@ -0,0 +1,128 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Options; +using MoneyMirror.PhysicalAssets; +using MoneyMirror.PhysicalAssets.Configuration; + +namespace MoneyMirror.Tests.PhysicalAssets; + +public class EbayMarketDataServiceTests +{ + [Fact] + public async Task SearchUsedListingsAsync_RequestsUsedUsdListingsAndMapsEvidence() + { + var requests = new List(); + var handler = new CallbackHandler(async (request, cancellationToken) => + { + requests.Add(await RequestSnapshot.CaptureAsync(request, cancellationToken)); + return request.Method == HttpMethod.Post + ? JsonResponse("""{"access_token":"app-token","expires_in":7200}""") + : JsonResponse( + """{"itemSummaries":[{"title":"Fender CD-60S guitar","condition":"Used","price":{"value":"120.50","currency":"USD"}},{"title":"Foreign listing","condition":"Used","price":{"value":"80","currency":"CAD"}},{"title":"Missing price","condition":"Used"}]}""" + ); + }); + var service = CreateService(handler); + + var listings = await service.SearchUsedListingsAsync("acoustic guitar", "Fender", "CD-60S"); + + var listing = Assert.Single(listings); + Assert.Equal(120.50m, listing.PriceUsd); + Assert.Equal("eBay", listing.Source); + Assert.Equal("Fender CD-60S guitar", listing.ListingTitle); + Assert.Equal("Used", listing.Condition); + + var tokenRequest = requests[0]; + Assert.Equal(HttpMethod.Post, tokenRequest.Method); + Assert.EndsWith("/identity/v1/oauth2/token", tokenRequest.Uri.AbsoluteUri); + Assert.Equal( + "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes("test-client:test-secret")), + tokenRequest.Authorization + ); + Assert.Contains("grant_type=client_credentials", tokenRequest.Body); + Assert.Contains(Uri.EscapeDataString("https://api.ebay.com/oauth/api_scope"), tokenRequest.Body); + + var searchRequest = requests[1]; + Assert.Equal(HttpMethod.Get, searchRequest.Method); + Assert.Equal("Bearer app-token", searchRequest.Authorization); + Assert.Equal("EBAY_US", searchRequest.Marketplace); + Assert.Contains("q=acoustic%20guitar%20Fender%20CD-60S", searchRequest.Uri.Query); + Assert.Contains("conditions%3A%7BUSED%7D", searchRequest.Uri.Query); + Assert.Contains("limit=20", searchRequest.Uri.Query); + } + + [Fact] + public async Task SearchUsedListingsAsync_EmptyResponseReturnsNoListings() + { + var handler = new CallbackHandler((request, _) => Task.FromResult( + request.Method == HttpMethod.Post + ? JsonResponse("""{"access_token":"app-token","expires_in":7200}""") + : JsonResponse("""{"itemSummaries":[]}""") + )); + var service = CreateService(handler); + + var listings = await service.SearchUsedListingsAsync("lamp", null, null); + + Assert.Empty(listings); + } + + [Fact] + public async Task SearchUsedListingsAsync_MissingCredentialsFailsWithoutCallingProvider() + { + var handler = new CallbackHandler((_, _) => throw new InvalidOperationException("Unexpected request.")); + var service = new EbayMarketDataService( + new HttpClient(handler), + Options.Create(new EbayOptions()) + ); + + var exception = await Assert.ThrowsAsync( + () => service.SearchUsedListingsAsync("lamp", null, null) + ); + + Assert.Contains("credentials are not configured", exception.Message); + } + + private static EbayMarketDataService CreateService(HttpMessageHandler handler) => new( + new HttpClient(handler), + Options.Create(new EbayOptions { ClientId = "test-client", ClientSecret = "test-secret" }) + ); + + private static HttpResponseMessage JsonResponse(string body) => new(HttpStatusCode.OK) + { + Content = new StringContent(body, Encoding.UTF8, "application/json") + }; + + private sealed class CallbackHandler( + Func> callback + ) : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken + ) => callback(request, cancellationToken); + } + + private sealed record RequestSnapshot( + HttpMethod Method, + Uri Uri, + string? Authorization, + string? Marketplace, + string Body + ) + { + public static async Task CaptureAsync( + HttpRequestMessage request, + CancellationToken cancellationToken + ) => new( + request.Method, + request.RequestUri!, + request.Headers.Authorization?.ToString(), + request.Headers.TryGetValues("X-EBAY-C-MARKETPLACE-ID", out var marketplace) + ? marketplace.Single() + : null, + request.Content is null + ? string.Empty + : await request.Content.ReadAsStringAsync(cancellationToken) + ); + } +} diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs new file mode 100644 index 0000000..33fe593 --- /dev/null +++ b/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs @@ -0,0 +1,95 @@ +using MoneyMirror.PhysicalAssets; + +namespace MoneyMirror.Tests.PhysicalAssets; + +public class EvidenceBasedAssetValuationServiceTests +{ + private static readonly DateTimeOffset ValuationDate = new(2026, 9, 20, 12, 0, 0, TimeSpan.Zero); + + [Fact] + public async Task EstimateAsync_UsesMarketListingsAndReturnsStructuredEvidence() + { + var source = new FakeMarketDataService( + [ + new(80m, "eBay", "Used item A", "Used"), + new(100m, "eBay", "Used item B", "Very Good"), + new(140m, "eBay", "Used item C", "Used"), + ]); + var service = new EvidenceBasedAssetValuationService(source, new FixedTimeProvider(ValuationDate)); + + var valuation = await service.EstimateAsync("guitar", "Fender", "CD-60S"); + + Assert.Equal(100m, valuation.EstimatedValueUsd); + Assert.Equal(ValuationDate, valuation.ValuationDate); + Assert.False(valuation.IsAiEstimated); + Assert.False(valuation.IsLowConfidence); + Assert.Equal(3, valuation.Evidence.Count); + Assert.Equal("guitar", source.LastLabel); + Assert.Equal("Fender", source.LastBrand); + Assert.Equal("CD-60S", source.LastModel); + } + + [Fact] + public async Task EstimateAsync_NoListingsReturnsExplicitUnavailableLowConfidenceResult() + { + var source = new FakeMarketDataService([]); + var service = new EvidenceBasedAssetValuationService(source, new FixedTimeProvider(ValuationDate)); + + var valuation = await service.EstimateAsync("rare item", null, null); + + Assert.Null(valuation.EstimatedValueUsd); + Assert.Empty(valuation.Evidence); + Assert.False(valuation.IsAiEstimated); + Assert.True(valuation.IsLowConfidence); + Assert.Contains("no usable comparable listings", valuation.Reasoning); + } + + [Fact] + public async Task EstimateAsync_ProviderFailureBecomesDomainException() + { + var source = new FakeMarketDataService(new EbayMarketDataException("provider unavailable")); + var service = new EvidenceBasedAssetValuationService(source, new FixedTimeProvider(ValuationDate)); + + var exception = await Assert.ThrowsAsync( + () => service.EstimateAsync("guitar", null, null) + ); + + Assert.Equal("Failed to retrieve comparable market listings.", exception.Message); + Assert.IsType(exception.InnerException); + } + + private sealed class FakeMarketDataService(IReadOnlyList result) + : IEbayMarketDataService + { + private readonly Exception? _exception = null; + + public FakeMarketDataService(Exception exception) : this([]) + { + _exception = exception; + } + + public string? LastLabel { get; private set; } + public string? LastBrand { get; private set; } + public string? LastModel { get; private set; } + + public Task> SearchUsedListingsAsync( + string label, + string? brand, + string? model, + CancellationToken cancellationToken = default + ) + { + LastLabel = label; + LastBrand = brand; + LastModel = model; + return _exception is null + ? Task.FromResult(result) + : Task.FromException>(_exception); + } + } + + private sealed class FixedTimeProvider(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + } +} From 63369df47e5496fde05caf27ae2ef6b429d6be59 Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 08:15:50 -0400 Subject: [PATCH 2/4] Replace eBay comps with SerpApi Google Search --- .env.example | 3 +- HumanCapital/IMarketPotentialPipeline.cs | 6 +- PhysicalAssets/Configuration/EbayOptions.cs | 12 - .../Configuration/SerpApiOptions.cs | 12 + PhysicalAssets/EbayMarketDataException.cs | 8 - PhysicalAssets/EbayMarketDataService.cs | 240 -------------- .../EvidenceBasedAssetValuationService.cs | 6 +- PhysicalAssets/IEbayMarketDataService.cs | 13 - PhysicalAssets/ISerpApiMarketDataService.cs | 13 + PhysicalAssets/SerpApiMarketDataException.cs | 8 + PhysicalAssets/SerpApiMarketDataService.cs | 292 ++++++++++++++++++ Program.cs | 9 +- appsettings.json | 10 +- docker-compose.yml | 3 +- docs/development.md | 20 +- .../EbayMarketDataServiceTests.cs | 128 -------- .../EfPhysicalAssetRepositoryTests.cs | 101 ++++-- ...EvidenceBasedAssetValuationServiceTests.cs | 49 ++- .../SerpApiMarketDataServiceTests.cs | 137 ++++++++ 19 files changed, 608 insertions(+), 462 deletions(-) delete mode 100644 PhysicalAssets/Configuration/EbayOptions.cs create mode 100644 PhysicalAssets/Configuration/SerpApiOptions.cs delete mode 100644 PhysicalAssets/EbayMarketDataException.cs delete mode 100644 PhysicalAssets/EbayMarketDataService.cs delete mode 100644 PhysicalAssets/IEbayMarketDataService.cs create mode 100644 PhysicalAssets/ISerpApiMarketDataService.cs create mode 100644 PhysicalAssets/SerpApiMarketDataException.cs create mode 100644 PhysicalAssets/SerpApiMarketDataService.cs delete mode 100644 tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs create mode 100644 tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs diff --git a/.env.example b/.env.example index 0ab8727..d3375fd 100644 --- a/.env.example +++ b/.env.example @@ -7,5 +7,4 @@ POSTGRES_PASSWORD=postgres NEMOTRON_API_KEY= VISION_API_KEY= BLS_API_KEY= -EBAY_CLIENT_ID= -EBAY_CLIENT_SECRET= +SERPAPI_API_KEY= diff --git a/HumanCapital/IMarketPotentialPipeline.cs b/HumanCapital/IMarketPotentialPipeline.cs index 16d9809..8ba5f99 100644 --- a/HumanCapital/IMarketPotentialPipeline.cs +++ b/HumanCapital/IMarketPotentialPipeline.cs @@ -14,9 +14,9 @@ namespace MoneyMirror.HumanCapital; /// (#148), whose UI explicitly labels its output "AI estimate - not based /// on live wage data". This isn't an oversight: this method needs a BLS /// series ID per matched occupation, which was meant to come from O*NET -/// occupation data (#93/#94, HC4) - but eBay and O*NET were both dropped -/// from project scope (see #177) before that occupation-to-series mapping -/// was built. Wiring this in requires deciding how to get that mapping +/// occupation data (#93/#94, HC4) - but O*NET was dropped from project scope +/// (see #177) before that occupation-to-series mapping was built. Wiring this +/// in requires deciding how to get that mapping /// without O*NET (a fixed lookup table for common titles? a second LLM /// call to guess a series ID, no longer "no AI in choosing the number"?) /// before this pipeline can replace the AI-guess path in the UI. diff --git a/PhysicalAssets/Configuration/EbayOptions.cs b/PhysicalAssets/Configuration/EbayOptions.cs deleted file mode 100644 index f8a90e0..0000000 --- a/PhysicalAssets/Configuration/EbayOptions.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace MoneyMirror.PhysicalAssets.Configuration; - -/// Credentials and endpoint configuration for eBay Browse API. -public sealed class EbayOptions -{ - public const string SectionName = "Ebay"; - - public string ClientId { get; set; } = string.Empty; - public string ClientSecret { get; set; } = string.Empty; - public string BaseUrl { get; set; } = "https://api.ebay.com"; - public string MarketplaceId { get; set; } = "EBAY_US"; -} diff --git a/PhysicalAssets/Configuration/SerpApiOptions.cs b/PhysicalAssets/Configuration/SerpApiOptions.cs new file mode 100644 index 0000000..79481c3 --- /dev/null +++ b/PhysicalAssets/Configuration/SerpApiOptions.cs @@ -0,0 +1,12 @@ +namespace MoneyMirror.PhysicalAssets.Configuration; + +/// API key and Google Search endpoint configuration for SerpApi. +public sealed class SerpApiOptions +{ + public const string SectionName = "SerpApi"; + + public string ApiKey { get; set; } = string.Empty; + public string BaseUrl { get; set; } = "https://serpapi.com/search.json"; + public string CountryCode { get; set; } = "us"; + public string Language { get; set; } = "en"; +} diff --git a/PhysicalAssets/EbayMarketDataException.cs b/PhysicalAssets/EbayMarketDataException.cs deleted file mode 100644 index 5b66e07..0000000 --- a/PhysicalAssets/EbayMarketDataException.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace MoneyMirror.PhysicalAssets; - -/// Thrown when eBay Browse API cannot return comparable listings. -public sealed class EbayMarketDataException : Exception -{ - public EbayMarketDataException(string message, Exception? innerException = null) - : base(message, innerException) { } -} diff --git a/PhysicalAssets/EbayMarketDataService.cs b/PhysicalAssets/EbayMarketDataService.cs deleted file mode 100644 index 0b01bc0..0000000 --- a/PhysicalAssets/EbayMarketDataService.cs +++ /dev/null @@ -1,240 +0,0 @@ -using System.Globalization; -using System.Net.Http.Headers; -using System.Net.Http.Json; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.Extensions.Options; -using MoneyMirror.PhysicalAssets.Configuration; - -namespace MoneyMirror.PhysicalAssets; - -/// Searches eBay's Browse API for used, USD-priced comparable listings. -public sealed class EbayMarketDataService : IEbayMarketDataService -{ - private const int ResultLimit = 20; - private const string BrowseScope = "https://api.ebay.com/oauth/api_scope"; - private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); - - private readonly HttpClient _httpClient; - private readonly EbayOptions _options; - private readonly TimeProvider _timeProvider; - private readonly SemaphoreSlim _tokenLock = new(1, 1); - private CachedAccessToken? _cachedToken; - - public EbayMarketDataService( - HttpClient httpClient, - IOptions options, - TimeProvider? timeProvider = null - ) - { - _httpClient = httpClient; - _options = options.Value; - _timeProvider = timeProvider ?? TimeProvider.System; - } - - public async Task> SearchUsedListingsAsync( - string label, - string? brand, - string? model, - CancellationToken cancellationToken = default - ) - { - ArgumentException.ThrowIfNullOrWhiteSpace(label); - EnsureCredentials(); - - var query = string.Join( - " ", - new[] { label, brand, model } - .Where(value => !string.IsNullOrWhiteSpace(value)) - .Select(value => value!.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - ); - var url = $"{_options.BaseUrl.TrimEnd('/')}/buy/browse/v1/item_summary/search" - + $"?q={Uri.EscapeDataString(query)}" - + $"&filter={Uri.EscapeDataString("conditions:{USED}")}" - + $"&limit={ResultLimit}"; - - using var request = new HttpRequestMessage(HttpMethod.Get, url); - request.Headers.Authorization = new AuthenticationHeaderValue( - "Bearer", - await GetAccessTokenAsync(cancellationToken) - ); - request.Headers.Add("X-EBAY-C-MARKETPLACE-ID", _options.MarketplaceId); - - using var response = await SendAsync(request, "eBay Browse API", cancellationToken); - var result = await ReadResponseAsync(response, "eBay Browse API", cancellationToken); - - return result?.ItemSummaries? - .Select(ToEvidence) - .Where(evidence => evidence is not null) - .Select(evidence => evidence!) - .ToList() ?? []; - } - - private void EnsureCredentials() - { - if (string.IsNullOrWhiteSpace(_options.ClientId) || string.IsNullOrWhiteSpace(_options.ClientSecret)) - { - throw new EbayMarketDataException( - "eBay Browse API credentials are not configured. Set Ebay:ClientId and Ebay:ClientSecret." - ); - } - } - - private async Task GetAccessTokenAsync(CancellationToken cancellationToken) - { - var now = _timeProvider.GetUtcNow(); - if (_cachedToken is { ExpiresAt: var expiry } cached && expiry > now.AddMinutes(1)) - { - return cached.Value; - } - - await _tokenLock.WaitAsync(cancellationToken); - try - { - now = _timeProvider.GetUtcNow(); - if (_cachedToken is { ExpiresAt: var cachedExpiry } current && cachedExpiry > now.AddMinutes(1)) - { - return current.Value; - } - - using var request = new HttpRequestMessage( - HttpMethod.Post, - $"{_options.BaseUrl.TrimEnd('/')}/identity/v1/oauth2/token" - ); - var credentials = Convert.ToBase64String( - Encoding.UTF8.GetBytes($"{_options.ClientId}:{_options.ClientSecret}") - ); - request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials); - request.Content = new FormUrlEncodedContent( - [ - new KeyValuePair("grant_type", "client_credentials"), - new KeyValuePair("scope", BrowseScope), - ]); - - using var response = await SendAsync(request, "eBay OAuth token service", cancellationToken); - var token = await ReadResponseAsync(response, "eBay OAuth token service", cancellationToken); - if ( - token is null - || string.IsNullOrWhiteSpace(token.AccessToken) - || token.ExpiresIn <= 0 - ) - { - throw new EbayMarketDataException("eBay OAuth token service returned an incomplete access token."); - } - - _cachedToken = new CachedAccessToken( - token.AccessToken, - _timeProvider.GetUtcNow().AddSeconds(token.ExpiresIn) - ); - return token.AccessToken; - } - finally - { - _tokenLock.Release(); - } - } - - private async Task SendAsync( - HttpRequestMessage request, - string serviceName, - CancellationToken cancellationToken - ) - { - try - { - return await _httpClient.SendAsync(request, cancellationToken); - } - catch (HttpRequestException ex) - { - throw new EbayMarketDataException($"Failed to reach the {serviceName}.", ex); - } - catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested) - { - throw new EbayMarketDataException($"The {serviceName} request timed out.", ex); - } - } - - private static async Task ReadResponseAsync( - HttpResponseMessage response, - string serviceName, - CancellationToken cancellationToken - ) - { - if (!response.IsSuccessStatusCode) - { - var errorBody = await response.Content.ReadAsStringAsync(cancellationToken); - throw new EbayMarketDataException( - $"{serviceName} returned {(int)response.StatusCode} {response.StatusCode}: {errorBody}" - ); - } - - try - { - return await response.Content.ReadFromJsonAsync(JsonOptions, cancellationToken); - } - catch (JsonException ex) - { - throw new EbayMarketDataException($"Failed to parse the {serviceName} response.", ex); - } - } - - private static AssetValuationEvidence? ToEvidence(ItemSummary item) - { - if ( - item.Price is null - || !string.Equals(item.Price.Currency, "USD", StringComparison.OrdinalIgnoreCase) - || !decimal.TryParse( - item.Price.Value, - NumberStyles.Number, - CultureInfo.InvariantCulture, - out var price - ) - || price <= 0 - ) - { - return null; - } - - return new AssetValuationEvidence(price, "eBay", item.Title, item.Condition); - } - - private sealed class SearchResponse - { - [JsonPropertyName("itemSummaries")] - public IReadOnlyList? ItemSummaries { get; init; } - } - - private sealed class ItemSummary - { - [JsonPropertyName("title")] - public string? Title { get; init; } - - [JsonPropertyName("condition")] - public string? Condition { get; init; } - - [JsonPropertyName("price")] - public ItemPrice? Price { get; init; } - } - - private sealed class ItemPrice - { - [JsonPropertyName("value")] - public string? Value { get; init; } - - [JsonPropertyName("currency")] - public string? Currency { get; init; } - } - - private sealed class TokenResponse - { - [JsonPropertyName("access_token")] - public string? AccessToken { get; init; } - - [JsonPropertyName("expires_in")] - public int ExpiresIn { get; init; } - } - - private sealed record CachedAccessToken(string Value, DateTimeOffset ExpiresAt); -} diff --git a/PhysicalAssets/EvidenceBasedAssetValuationService.cs b/PhysicalAssets/EvidenceBasedAssetValuationService.cs index bb9d8bc..99f9d40 100644 --- a/PhysicalAssets/EvidenceBasedAssetValuationService.cs +++ b/PhysicalAssets/EvidenceBasedAssetValuationService.cs @@ -3,11 +3,11 @@ namespace MoneyMirror.PhysicalAssets; /// Builds an asset valuation from structured market listings, without an LLM. public sealed class EvidenceBasedAssetValuationService : IAssetValuationService { - private readonly IEbayMarketDataService _marketDataService; + private readonly ISerpApiMarketDataService _marketDataService; private readonly TimeProvider _timeProvider; public EvidenceBasedAssetValuationService( - IEbayMarketDataService marketDataService, + ISerpApiMarketDataService marketDataService, TimeProvider? timeProvider = null ) { @@ -32,7 +32,7 @@ public async Task EstimateAsync( ); return MarketValuationCalculator.Calculate(evidence, _timeProvider.GetUtcNow()); } - catch (EbayMarketDataException ex) + catch (SerpApiMarketDataException ex) { throw new AssetValuationException("Failed to retrieve comparable market listings.", ex); } diff --git a/PhysicalAssets/IEbayMarketDataService.cs b/PhysicalAssets/IEbayMarketDataService.cs deleted file mode 100644 index 0442888..0000000 --- a/PhysicalAssets/IEbayMarketDataService.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace MoneyMirror.PhysicalAssets; - -/// Searches a market data source for real listings comparable to an identified item. -public interface IEbayMarketDataService -{ - /// Finds up to twenty used USD listings matching the confirmed item's label, brand and model. - Task> SearchUsedListingsAsync( - string label, - string? brand, - string? model, - CancellationToken cancellationToken = default - ); -} diff --git a/PhysicalAssets/ISerpApiMarketDataService.cs b/PhysicalAssets/ISerpApiMarketDataService.cs new file mode 100644 index 0000000..29d5c97 --- /dev/null +++ b/PhysicalAssets/ISerpApiMarketDataService.cs @@ -0,0 +1,13 @@ +namespace MoneyMirror.PhysicalAssets; + +/// Searches Google results through SerpApi for priced used-item comparables. +public interface ISerpApiMarketDataService +{ + /// Returns up to twenty used USD listings matching an item's label, brand and model. + Task> SearchUsedListingsAsync( + string label, + string? brand, + string? model, + CancellationToken cancellationToken = default + ); +} diff --git a/PhysicalAssets/SerpApiMarketDataException.cs b/PhysicalAssets/SerpApiMarketDataException.cs new file mode 100644 index 0000000..58cb742 --- /dev/null +++ b/PhysicalAssets/SerpApiMarketDataException.cs @@ -0,0 +1,8 @@ +namespace MoneyMirror.PhysicalAssets; + +/// Thrown when SerpApi cannot return comparable Google search results. +public sealed class SerpApiMarketDataException : Exception +{ + public SerpApiMarketDataException(string message, Exception? innerException = null) + : base(message, innerException) { } +} diff --git a/PhysicalAssets/SerpApiMarketDataService.cs b/PhysicalAssets/SerpApiMarketDataService.cs new file mode 100644 index 0000000..f31dd58 --- /dev/null +++ b/PhysicalAssets/SerpApiMarketDataService.cs @@ -0,0 +1,292 @@ +using System.Globalization; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Options; +using MoneyMirror.PhysicalAssets.Configuration; + +namespace MoneyMirror.PhysicalAssets; + +/// Searches SerpApi's Google Search results for used, USD-priced listings. +public sealed class SerpApiMarketDataService : ISerpApiMarketDataService +{ + private const int ResultLimit = 20; + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + private static readonly Regex UsdPricePattern = new( + @"(?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d{1,2})?)(?![\d,])", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant + ); + private static readonly Regex UsedConditionPattern = new( + @"\b(?:used|pre[ -]?owned|second[ -]?hand|secondhand)\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant + ); + private static readonly Regex RefurbishedConditionPattern = new( + @"\b(?:refurbished|renewed|reconditioned)\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant + ); + + private readonly HttpClient _httpClient; + private readonly SerpApiOptions _options; + + public SerpApiMarketDataService(HttpClient httpClient, IOptions options) + { + _httpClient = httpClient; + _options = options.Value; + } + + public async Task> SearchUsedListingsAsync( + string label, + string? brand, + string? model, + CancellationToken cancellationToken = default + ) + { + ArgumentException.ThrowIfNullOrWhiteSpace(label); + EnsureCredentials(); + + var query = + string.Join( + " ", + new[] { label, brand, model } + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value!.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + ) + " used for sale price"; + + var uri = BuildSearchUri(query); + using var request = new HttpRequestMessage(HttpMethod.Get, uri); + using var response = await SendAsync(request, cancellationToken); + var result = await ReadResponseAsync(response, cancellationToken); + + if (!string.IsNullOrWhiteSpace(result.Error)) + { + throw new SerpApiMarketDataException("SerpApi Google Search returned an API error."); + } + + return MapEvidence(result); + } + + private void EnsureCredentials() + { + if (string.IsNullOrWhiteSpace(_options.ApiKey)) + { + throw new SerpApiMarketDataException( + "SerpApi credentials are not configured. Set SerpApi:ApiKey." + ); + } + } + + private Uri BuildSearchUri(string query) + { + var parameters = new Dictionary + { + ["engine"] = "google", + ["q"] = query, + ["gl"] = _options.CountryCode, + ["hl"] = _options.Language, + ["api_key"] = _options.ApiKey, + }; + var queryString = string.Join( + "&", + parameters.Select(pair => + $"{Uri.EscapeDataString(pair.Key)}={Uri.EscapeDataString(pair.Value)}" + ) + ); + + return new Uri($"{_options.BaseUrl.TrimEnd('?', '&')}?{queryString}", UriKind.Absolute); + } + + private static IReadOnlyList MapEvidence(SearchResponse result) + { + var evidence = new List(); + + foreach (var listing in result.ShoppingResults ?? []) + { + var condition = NormalizeCondition(listing.SecondHandCondition); + if (condition is null) + { + condition = NormalizeCondition($"{listing.Title} {listing.Snippet}"); + } + + var price = ParseSingleUsdPrice(listing.Price); + AddEvidence(evidence, listing.Title, listing.Source, listing.Link, condition, price); + } + + foreach (var listing in result.OrganicResults ?? []) + { + var text = $"{listing.Title} {listing.Snippet}"; + var condition = NormalizeCondition(text); + var price = ParseSingleUsdPrice(text); + AddEvidence(evidence, listing.Title, listing.Source, listing.Link, condition, price); + } + + return evidence.Distinct().Take(ResultLimit).ToArray(); + } + + private static void AddEvidence( + ICollection evidence, + string? title, + string? source, + string? link, + string? condition, + decimal? price + ) + { + if ( + string.IsNullOrWhiteSpace(title) + || string.IsNullOrWhiteSpace(condition) + || price is not > 0 + ) + { + return; + } + + evidence.Add( + new AssetValuationEvidence( + price.Value, + ResolveSource(source, link), + title.Trim(), + condition + ) + ); + } + + private static string? NormalizeCondition(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + if (RefurbishedConditionPattern.IsMatch(text)) + { + return "Refurbished"; + } + + return UsedConditionPattern.IsMatch(text) ? "Used" : null; + } + + private static decimal? ParseSingleUsdPrice(string? text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + var matches = UsdPricePattern.Matches(text); + if (matches.Count != 1) + { + return null; + } + + return + decimal.TryParse( + matches[0].Groups["amount"].Value, + NumberStyles.Number, + CultureInfo.InvariantCulture, + out var price + ) + && price > 0 + ? price + : null; + } + + private static string ResolveSource(string? source, string? link) + { + if (!string.IsNullOrWhiteSpace(source)) + { + return source.Trim(); + } + + return Uri.TryCreate(link, UriKind.Absolute, out var uri) + ? uri.Host.StartsWith("www.", StringComparison.OrdinalIgnoreCase) + ? uri.Host[4..] + : uri.Host + : "Google Search"; + } + + private async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken + ) + { + try + { + return await _httpClient.SendAsync(request, cancellationToken); + } + catch (HttpRequestException ex) + { + throw new SerpApiMarketDataException("Failed to reach SerpApi Google Search.", ex); + } + catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested) + { + throw new SerpApiMarketDataException("SerpApi Google Search request timed out.", ex); + } + } + + private static async Task ReadResponseAsync( + HttpResponseMessage response, + CancellationToken cancellationToken + ) + { + if (!response.IsSuccessStatusCode) + { + throw new SerpApiMarketDataException( + $"SerpApi Google Search returned {(int)response.StatusCode} {response.StatusCode}." + ); + } + + try + { + var result = await response.Content.ReadFromJsonAsync( + JsonOptions, + cancellationToken + ); + return result + ?? throw new SerpApiMarketDataException( + "SerpApi Google Search returned an empty response." + ); + } + catch (JsonException ex) + { + throw new SerpApiMarketDataException( + "Failed to parse the SerpApi Google Search response.", + ex + ); + } + } + + private sealed class SearchResponse + { + [JsonPropertyName("error")] + public string? Error { get; init; } + + [JsonPropertyName("shopping_results")] + public IReadOnlyList? ShoppingResults { get; init; } + + [JsonPropertyName("organic_results")] + public IReadOnlyList? OrganicResults { get; init; } + } + + private sealed class SearchListing + { + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("source")] + public string? Source { get; init; } + + [JsonPropertyName("link")] + public string? Link { get; init; } + + [JsonPropertyName("snippet")] + public string? Snippet { get; init; } + + [JsonPropertyName("price")] + public string? Price { get; init; } + + [JsonPropertyName("second_hand_condition")] + public string? SecondHandCondition { get; init; } + } +} diff --git a/Program.cs b/Program.cs index fca0960..0f5c399 100644 --- a/Program.cs +++ b/Program.cs @@ -30,7 +30,9 @@ builder.Configuration.GetSection(NemotronOptions.SectionName) ); builder.Services.Configure(builder.Configuration.GetSection(BlsOptions.SectionName)); -builder.Services.Configure(builder.Configuration.GetSection(EbayOptions.SectionName)); +builder.Services.Configure( + builder.Configuration.GetSection(SerpApiOptions.SectionName) +); builder.Services.Configure( builder.Configuration.GetSection(VisionModelOptions.SectionName) ); @@ -67,9 +69,12 @@ builder.Services.AddHttpClient(client => client.Timeout = TimeSpan.FromSeconds(15) ); -builder.Services.AddHttpClient(client => +builder.Services.AddHttpClient(client => client.Timeout = TimeSpan.FromSeconds(15) ); + +// SerpApi requires its key in the query string. Keep routine HTTP logs from recording it. +builder.Logging.AddFilter("System.Net.Http.HttpClient.ISerpApiMarketDataService", LogLevel.Warning); builder.Services.AddScoped< IMarketPotentialExplanationService, NemotronMarketPotentialExplanationService diff --git a/appsettings.json b/appsettings.json index 8b6ef71..8437182 100644 --- a/appsettings.json +++ b/appsettings.json @@ -25,10 +25,10 @@ "ApiKey": "", "BaseUrl": "https://api.bls.gov/publicAPI/v2/" }, - "Ebay": { - "ClientId": "", - "ClientSecret": "", - "BaseUrl": "https://api.ebay.com", - "MarketplaceId": "EBAY_US" + "SerpApi": { + "ApiKey": "", + "BaseUrl": "https://serpapi.com/search.json", + "CountryCode": "us", + "Language": "en" } } diff --git a/docker-compose.yml b/docker-compose.yml index 2cf4da3..2c5eaff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,8 +16,7 @@ services: Ai__Nemotron__ApiKey: ${NEMOTRON_API_KEY:-} Ai__VisionModel__ApiKey: ${VISION_API_KEY:-} Bls__ApiKey: ${BLS_API_KEY:-} - Ebay__ClientId: ${EBAY_CLIENT_ID:-} - Ebay__ClientSecret: ${EBAY_CLIENT_SECRET:-} + SerpApi__ApiKey: ${SERPAPI_API_KEY:-} ports: - "${APP_PORT:-8000}:8080" volumes: diff --git a/docs/development.md b/docs/development.md index b815865..cb1317d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -6,24 +6,25 @@ monolith, one database, one implicit user — no auth or multi-tenancy. ## Configuration -AI, BLS, and eBay Browse API settings live under `Ai:Nemotron`, -`Ai:VisionModel`, `Bls`, and `Ebay` (`ClientId`, `ClientSecret`, `BaseUrl`, and -`MarketplaceId`). `appsettings.json` ships empty secret placeholders — never +AI, BLS, and SerpApi Google Search settings live under `Ai:Nemotron`, +`Ai:VisionModel`, `Bls`, and `SerpApi` (`ApiKey`, `BaseUrl`, `CountryCode`, and +`Language`). `appsettings.json` ships empty secret placeholders — never commit real keys. Set them locally: ```sh dotnet user-secrets set "Ai:Nemotron:ApiKey" "" dotnet user-secrets set "Ai:VisionModel:ApiKey" "" dotnet user-secrets set "Bls:ApiKey" "" -dotnet user-secrets set "Ebay:ClientId" "" -dotnet user-secrets set "Ebay:ClientSecret" "" +dotnet user-secrets set "SerpApi:ApiKey" "" ``` Or use env vars: `Ai__Nemotron__ApiKey`, `Ai__VisionModel__ApiKey`, -`Bls__ApiKey`, `Ebay__ClientId`, `Ebay__ClientSecret`. +`Bls__ApiKey`, `SerpApi__ApiKey`. -Physical asset valuations query up to 20 used eBay US listings and calculate a -median from usable USD prices. No usable listings produce a null value and an +Physical asset valuations query SerpApi's Google Search API for up to 20 used +listings and calculate a median from unambiguous USD prices. Results without +clear used/refurbished condition or a single USD price are ignored. No usable +listings produce a null value and an explicit low-confidence explanation. The valuation returns each comparable's price, source, title, and condition in `AssetValuation.Evidence`; that evidence is persisted and displayed alongside each valuation in inventory history. @@ -47,8 +48,7 @@ on Postgres volumes and host-local `dotnet run`: [backend/postgresql_setup.md](. | `NEMOTRON_API_KEY` | `Ai:Nemotron:ApiKey` | | `VISION_API_KEY` | `Ai:VisionModel:ApiKey` | | `BLS_API_KEY` | `Bls:ApiKey` | -| `EBAY_CLIENT_ID` | `Ebay:ClientId` | -| `EBAY_CLIENT_SECRET` | `Ebay:ClientSecret` | +| `SERPAPI_API_KEY` | `SerpApi:ApiKey` | `docker compose down` stops the stack; add `-v` to wipe DB and image volumes. diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs deleted file mode 100644 index 2b94552..0000000 --- a/tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs +++ /dev/null @@ -1,128 +0,0 @@ -using System.Net; -using System.Net.Http.Headers; -using System.Text; -using Microsoft.Extensions.Options; -using MoneyMirror.PhysicalAssets; -using MoneyMirror.PhysicalAssets.Configuration; - -namespace MoneyMirror.Tests.PhysicalAssets; - -public class EbayMarketDataServiceTests -{ - [Fact] - public async Task SearchUsedListingsAsync_RequestsUsedUsdListingsAndMapsEvidence() - { - var requests = new List(); - var handler = new CallbackHandler(async (request, cancellationToken) => - { - requests.Add(await RequestSnapshot.CaptureAsync(request, cancellationToken)); - return request.Method == HttpMethod.Post - ? JsonResponse("""{"access_token":"app-token","expires_in":7200}""") - : JsonResponse( - """{"itemSummaries":[{"title":"Fender CD-60S guitar","condition":"Used","price":{"value":"120.50","currency":"USD"}},{"title":"Foreign listing","condition":"Used","price":{"value":"80","currency":"CAD"}},{"title":"Missing price","condition":"Used"}]}""" - ); - }); - var service = CreateService(handler); - - var listings = await service.SearchUsedListingsAsync("acoustic guitar", "Fender", "CD-60S"); - - var listing = Assert.Single(listings); - Assert.Equal(120.50m, listing.PriceUsd); - Assert.Equal("eBay", listing.Source); - Assert.Equal("Fender CD-60S guitar", listing.ListingTitle); - Assert.Equal("Used", listing.Condition); - - var tokenRequest = requests[0]; - Assert.Equal(HttpMethod.Post, tokenRequest.Method); - Assert.EndsWith("/identity/v1/oauth2/token", tokenRequest.Uri.AbsoluteUri); - Assert.Equal( - "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes("test-client:test-secret")), - tokenRequest.Authorization - ); - Assert.Contains("grant_type=client_credentials", tokenRequest.Body); - Assert.Contains(Uri.EscapeDataString("https://api.ebay.com/oauth/api_scope"), tokenRequest.Body); - - var searchRequest = requests[1]; - Assert.Equal(HttpMethod.Get, searchRequest.Method); - Assert.Equal("Bearer app-token", searchRequest.Authorization); - Assert.Equal("EBAY_US", searchRequest.Marketplace); - Assert.Contains("q=acoustic%20guitar%20Fender%20CD-60S", searchRequest.Uri.Query); - Assert.Contains("conditions%3A%7BUSED%7D", searchRequest.Uri.Query); - Assert.Contains("limit=20", searchRequest.Uri.Query); - } - - [Fact] - public async Task SearchUsedListingsAsync_EmptyResponseReturnsNoListings() - { - var handler = new CallbackHandler((request, _) => Task.FromResult( - request.Method == HttpMethod.Post - ? JsonResponse("""{"access_token":"app-token","expires_in":7200}""") - : JsonResponse("""{"itemSummaries":[]}""") - )); - var service = CreateService(handler); - - var listings = await service.SearchUsedListingsAsync("lamp", null, null); - - Assert.Empty(listings); - } - - [Fact] - public async Task SearchUsedListingsAsync_MissingCredentialsFailsWithoutCallingProvider() - { - var handler = new CallbackHandler((_, _) => throw new InvalidOperationException("Unexpected request.")); - var service = new EbayMarketDataService( - new HttpClient(handler), - Options.Create(new EbayOptions()) - ); - - var exception = await Assert.ThrowsAsync( - () => service.SearchUsedListingsAsync("lamp", null, null) - ); - - Assert.Contains("credentials are not configured", exception.Message); - } - - private static EbayMarketDataService CreateService(HttpMessageHandler handler) => new( - new HttpClient(handler), - Options.Create(new EbayOptions { ClientId = "test-client", ClientSecret = "test-secret" }) - ); - - private static HttpResponseMessage JsonResponse(string body) => new(HttpStatusCode.OK) - { - Content = new StringContent(body, Encoding.UTF8, "application/json") - }; - - private sealed class CallbackHandler( - Func> callback - ) : HttpMessageHandler - { - protected override Task SendAsync( - HttpRequestMessage request, - CancellationToken cancellationToken - ) => callback(request, cancellationToken); - } - - private sealed record RequestSnapshot( - HttpMethod Method, - Uri Uri, - string? Authorization, - string? Marketplace, - string Body - ) - { - public static async Task CaptureAsync( - HttpRequestMessage request, - CancellationToken cancellationToken - ) => new( - request.Method, - request.RequestUri!, - request.Headers.Authorization?.ToString(), - request.Headers.TryGetValues("X-EBAY-C-MARKETPLACE-ID", out var marketplace) - ? marketplace.Single() - : null, - request.Content is null - ? string.Empty - : await request.Content.ReadAsStringAsync(cancellationToken) - ); - } -} diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/EfPhysicalAssetRepositoryTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/EfPhysicalAssetRepositoryTests.cs index 5fa1b17..6a7d725 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/EfPhysicalAssetRepositoryTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/EfPhysicalAssetRepositoryTests.cs @@ -21,7 +21,9 @@ public EfPhysicalAssetRepositoryTests() _connection = new SqliteConnection("Filename=:memory:"); _connection.Open(); - var options = new DbContextOptionsBuilder().UseSqlite(_connection).Options; + var options = new DbContextOptionsBuilder() + .UseSqlite(_connection) + .Options; _db = new MoneyMirrorDbContext(options); _db.Database.EnsureCreated(); @@ -37,7 +39,9 @@ public void Dispose() [Fact] public async Task AddAsync_WithInitialValuation_IsReturnedByGetAllWithThatValuation() { - await _repository.AddAsync(new PhysicalAssetInput("Guitar", "Music", "Fender CD-60S", 120m)); + await _repository.AddAsync( + new PhysicalAssetInput("Guitar", "Music", "Fender CD-60S", 120m) + ); var all = await _repository.GetAllAsync(); @@ -62,9 +66,14 @@ public async Task AddAsync_WithoutValuation_HasNullCurrentValuation() [Fact] public async Task UpdateAsync_ExistingItem_ChangesItsFields() { - var id = await _repository.AddAsync(new PhysicalAssetInput("Old Name", "Old Category", null, null)); + var id = await _repository.AddAsync( + new PhysicalAssetInput("Old Name", "Old Category", null, null) + ); - var updated = await _repository.UpdateAsync(id, new PhysicalAssetInput("New Name", "New Category", "Model X", null)); + var updated = await _repository.UpdateAsync( + id, + new PhysicalAssetInput("New Name", "New Category", "Model X", null) + ); Assert.True(updated); var asset = Assert.Single(await _repository.GetAllAsync()); @@ -87,7 +96,10 @@ public async Task UpdateAsync_WithNewValuation_UpdatesCurrentValuation() [Fact] public async Task UpdateAsync_UnknownId_ReturnsFalse() { - var updated = await _repository.UpdateAsync(Guid.NewGuid(), new PhysicalAssetInput("X", null, null, null)); + var updated = await _repository.UpdateAsync( + Guid.NewGuid(), + new PhysicalAssetInput("X", null, null, null) + ); Assert.False(updated); } @@ -129,7 +141,15 @@ public async Task GetAllAsync_MultipleItems_SumsToCorrectTotalValue() public async Task AddFromScanAsync_CreatesScannedItemWithValuationAndEvidence() { var id = await _repository.AddFromScanAsync( - new ScannedAssetInput("Guitar", "Music", "Fender CD-60S", "abc123.jpg", 120m, "Typical used price.")); + new ScannedAssetInput( + "Guitar", + "Music", + "Fender CD-60S", + "abc123.jpg", + 120m, + "Typical used price." + ) + ); var summary = Assert.Single(await _repository.GetAllAsync()); Assert.Equal(id, summary.Id); @@ -172,7 +192,12 @@ public async Task AddValuationAsync_ExistingItem_AppendsToHistoryWithoutRemoving { var id = await _repository.AddAsync(new PhysicalAssetInput("Guitar", null, null, 100m)); - var updated = await _repository.AddValuationAsync(id, 175m, "AI estimate (revalue)", "Prices went up."); + var updated = await _repository.AddValuationAsync( + id, + 175m, + "AI estimate (revalue)", + "Prices went up." + ); Assert.True(updated); var detail = await _repository.GetByIdAsync(id); @@ -189,7 +214,12 @@ public async Task AddValuationAsync_ExistingItem_AppendsToHistoryWithoutRemoving [Fact] public async Task AddValuationAsync_UnknownId_ReturnsFalse() { - var updated = await _repository.AddValuationAsync(Guid.NewGuid(), 100m, "AI estimate", null); + var updated = await _repository.AddValuationAsync( + Guid.NewGuid(), + 100m, + "AI estimate", + null + ); Assert.False(updated); } @@ -201,12 +231,22 @@ public async Task AddFromScanAsync_WithComparableListings_PersistsAndReloadsEvid { IReadOnlyList comps = [ - new(115m, "eBay", "Fender CD-60S, used", "Good"), - new(130m, "eBay", "Fender CD-60S dreadnought", null), + new(115m, "Reverb", "Fender CD-60S, used", "Good"), + new(130m, "Marketplace", "Fender CD-60S dreadnought", null), ]; var id = await _repository.AddFromScanAsync( - new ScannedAssetInput("Guitar", "Music", "Fender CD-60S", "abc123.jpg", 122.5m, "Median of 2 listings.", "Market evidence", comps)); + new ScannedAssetInput( + "Guitar", + "Music", + "Fender CD-60S", + "abc123.jpg", + 122.5m, + "Median of 2 listings.", + "Market evidence", + comps + ) + ); var detail = await _repository.GetByIdAsync(id); @@ -214,7 +254,7 @@ public async Task AddFromScanAsync_WithComparableListings_PersistsAndReloadsEvid var entry = Assert.Single(detail!.ValuationHistory); Assert.Equal(2, entry.ComparableListings.Count); Assert.Equal(115m, entry.ComparableListings[0].PriceUsd); - Assert.Equal("eBay", entry.ComparableListings[0].Source); + Assert.Equal("Reverb", entry.ComparableListings[0].Source); Assert.Equal("Fender CD-60S, used", entry.ComparableListings[0].ListingTitle); Assert.Equal("Good", entry.ComparableListings[0].Condition); Assert.Null(entry.ComparableListings[1].Condition); @@ -223,11 +263,25 @@ public async Task AddFromScanAsync_WithComparableListings_PersistsAndReloadsEvid [Fact] public async Task AddValuationAsync_RevalueWithNewEvidence_DoesNotAffectPriorEntrysEvidence() { - IReadOnlyList firstComps = [new(100m, "eBay", null, null)]; - IReadOnlyList secondComps = [new(140m, "eBay", null, null), new(160m, "Craigslist", null, null)]; + IReadOnlyList firstComps = [new(100m, "Reverb", null, null)]; + IReadOnlyList secondComps = + [ + new(140m, "Marketplace", null, null), + new(160m, "Craigslist", null, null), + ]; var id = await _repository.AddFromScanAsync( - new ScannedAssetInput("Guitar", null, null, null, 100m, "First estimate.", "Market evidence", firstComps)); + new ScannedAssetInput( + "Guitar", + null, + null, + null, + 100m, + "First estimate.", + "Market evidence", + firstComps + ) + ); await _repository.AddValuationAsync(id, 150m, "Market evidence", "Revalued.", secondComps); var detail = await _repository.GetByIdAsync(id); @@ -257,9 +311,14 @@ public async Task AddValuationAsync_WithoutComparableListings_LeavesEvidenceEmpt [Fact] public async Task FindPossibleDuplicatesAsync_MatchingProductModel_ReturnsExistingItem() { - await _repository.AddAsync(new PhysicalAssetInput("Guitar", "Music", "Fender CD-60S", 120m)); + await _repository.AddAsync( + new PhysicalAssetInput("Guitar", "Music", "Fender CD-60S", 120m) + ); - var duplicates = await _repository.FindPossibleDuplicatesAsync("Acoustic guitar", "fender cd-60s"); + var duplicates = await _repository.FindPossibleDuplicatesAsync( + "Acoustic guitar", + "fender cd-60s" + ); var duplicate = Assert.Single(duplicates); Assert.Equal("Guitar", duplicate.Name); @@ -278,7 +337,9 @@ public async Task FindPossibleDuplicatesAsync_MatchingNameOnly_ReturnsExistingIt [Fact] public async Task FindPossibleDuplicatesAsync_NoMatch_ReturnsEmpty() { - await _repository.AddAsync(new PhysicalAssetInput("Guitar", "Music", "Fender CD-60S", 120m)); + await _repository.AddAsync( + new PhysicalAssetInput("Guitar", "Music", "Fender CD-60S", 120m) + ); var duplicates = await _repository.FindPossibleDuplicatesAsync("Lamp", "IKEA Foto"); @@ -288,7 +349,9 @@ public async Task FindPossibleDuplicatesAsync_NoMatch_ReturnsEmpty() [Fact] public async Task FindPossibleDuplicatesAsync_DifferentProductModelButSameName_StillMatchesOnName() { - await _repository.AddAsync(new PhysicalAssetInput("Guitar", "Music", "Fender CD-60S", 120m)); + await _repository.AddAsync( + new PhysicalAssetInput("Guitar", "Music", "Fender CD-60S", 120m) + ); var duplicates = await _repository.FindPossibleDuplicatesAsync("Guitar", "Gibson Les Paul"); diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs index 33fe593..329eb4e 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs @@ -4,18 +4,28 @@ namespace MoneyMirror.Tests.PhysicalAssets; public class EvidenceBasedAssetValuationServiceTests { - private static readonly DateTimeOffset ValuationDate = new(2026, 9, 20, 12, 0, 0, TimeSpan.Zero); + private static readonly DateTimeOffset ValuationDate = new( + 2026, + 9, + 20, + 12, + 0, + 0, + TimeSpan.Zero + ); [Fact] public async Task EstimateAsync_UsesMarketListingsAndReturnsStructuredEvidence() { - var source = new FakeMarketDataService( - [ - new(80m, "eBay", "Used item A", "Used"), - new(100m, "eBay", "Used item B", "Very Good"), - new(140m, "eBay", "Used item C", "Used"), + var source = new FakeMarketDataService([ + new(80m, "Reverb", "Used item A", "Used"), + new(100m, "Craigslist", "Used item B", "Used"), + new(140m, "Marketplace", "Used item C", "Used"), ]); - var service = new EvidenceBasedAssetValuationService(source, new FixedTimeProvider(ValuationDate)); + var service = new EvidenceBasedAssetValuationService( + source, + new FixedTimeProvider(ValuationDate) + ); var valuation = await service.EstimateAsync("guitar", "Fender", "CD-60S"); @@ -33,7 +43,10 @@ public async Task EstimateAsync_UsesMarketListingsAndReturnsStructuredEvidence() public async Task EstimateAsync_NoListingsReturnsExplicitUnavailableLowConfidenceResult() { var source = new FakeMarketDataService([]); - var service = new EvidenceBasedAssetValuationService(source, new FixedTimeProvider(ValuationDate)); + var service = new EvidenceBasedAssetValuationService( + source, + new FixedTimeProvider(ValuationDate) + ); var valuation = await service.EstimateAsync("rare item", null, null); @@ -47,23 +60,29 @@ public async Task EstimateAsync_NoListingsReturnsExplicitUnavailableLowConfidenc [Fact] public async Task EstimateAsync_ProviderFailureBecomesDomainException() { - var source = new FakeMarketDataService(new EbayMarketDataException("provider unavailable")); - var service = new EvidenceBasedAssetValuationService(source, new FixedTimeProvider(ValuationDate)); + var source = new FakeMarketDataService( + new SerpApiMarketDataException("provider unavailable") + ); + var service = new EvidenceBasedAssetValuationService( + source, + new FixedTimeProvider(ValuationDate) + ); - var exception = await Assert.ThrowsAsync( - () => service.EstimateAsync("guitar", null, null) + var exception = await Assert.ThrowsAsync(() => + service.EstimateAsync("guitar", null, null) ); Assert.Equal("Failed to retrieve comparable market listings.", exception.Message); - Assert.IsType(exception.InnerException); + Assert.IsType(exception.InnerException); } private sealed class FakeMarketDataService(IReadOnlyList result) - : IEbayMarketDataService + : ISerpApiMarketDataService { private readonly Exception? _exception = null; - public FakeMarketDataService(Exception exception) : this([]) + public FakeMarketDataService(Exception exception) + : this([]) { _exception = exception; } diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs new file mode 100644 index 0000000..e13bcb6 --- /dev/null +++ b/tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs @@ -0,0 +1,137 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Options; +using MoneyMirror.PhysicalAssets; +using MoneyMirror.PhysicalAssets.Configuration; + +namespace MoneyMirror.Tests.PhysicalAssets; + +public class SerpApiMarketDataServiceTests +{ + [Fact] + public async Task SearchUsedListingsAsync_QueriesGoogleAndMapsOnlyPricedUsedResults() + { + RequestSnapshot? requestSnapshot = null; + var handler = new CallbackHandler( + async (request, cancellationToken) => + { + requestSnapshot = await RequestSnapshot.CaptureAsync(request, cancellationToken); + return JsonResponse( + """{"shopping_results":[{"title":"Used Fender CD-60S Acoustic Guitar","source":"Reverb","price":"$130.00","second_hand_condition":"Used","link":"https://reverb.com/item/123"},{"title":"New Fender CD-60S Acoustic Guitar","source":"Retailer","price":"$300.00","second_hand_condition":"New","link":"https://store.example/guitar"},{"title":"Refurbished Fender CD-60S","source":"Reseller","price":"$145.00","second_hand_condition":"Refurbished","link":"https://reseller.example/guitar"}],"organic_results":[{"title":"Used Fender acoustic guitar listing","source":"Craigslist","snippet":"Clean condition, asking $110.00.","link":"https://pittsburgh.craigslist.org/item"},{"title":"Vintage guitar price guide","source":"Blog","snippet":"The new model costs $450.00."},{"title":"Used guitar price guide","source":"Blog","snippet":"Examples range from $80.00 to $100.00."}]}""" + ); + } + ); + var service = CreateService(handler); + + var listings = await service.SearchUsedListingsAsync("acoustic guitar", "Fender", "CD-60S"); + + Assert.Equal(3, listings.Count); + Assert.Contains( + listings, + item => item.PriceUsd == 130m && item.Source == "Reverb" && item.Condition == "Used" + ); + Assert.Contains(listings, item => item.PriceUsd == 145m && item.Condition == "Refurbished"); + Assert.Contains(listings, item => item.PriceUsd == 110m && item.Source == "Craigslist"); + Assert.All(listings, item => Assert.NotNull(item.ListingTitle)); + + Assert.NotNull(requestSnapshot); + Assert.Equal(HttpMethod.Get, requestSnapshot.Method); + Assert.Contains("engine=google", requestSnapshot.Uri.Query); + Assert.Contains( + "q=acoustic%20guitar%20Fender%20CD-60S%20used%20for%20sale%20price", + requestSnapshot.Uri.Query + ); + Assert.Contains("gl=us", requestSnapshot.Uri.Query); + Assert.Contains("hl=en", requestSnapshot.Uri.Query); + Assert.Contains("api_key=test-api-key", requestSnapshot.Uri.Query); + Assert.Null(requestSnapshot.Authorization); + } + + [Fact] + public async Task SearchUsedListingsAsync_EmptyResponseReturnsNoListings() + { + var service = CreateService( + new CallbackHandler((_, _) => Task.FromResult(JsonResponse("{}"))) + ); + + var listings = await service.SearchUsedListingsAsync("lamp", null, null); + + Assert.Empty(listings); + } + + [Fact] + public async Task SearchUsedListingsAsync_ApiErrorThrowsWithoutExposingApiKey() + { + var service = CreateService( + new CallbackHandler( + (_, _) => Task.FromResult(JsonResponse("""{"error":"Invalid API key"}""")) + ) + ); + + var exception = await Assert.ThrowsAsync(() => + service.SearchUsedListingsAsync("lamp", null, null) + ); + + Assert.DoesNotContain("test-api-key", exception.ToString()); + } + + [Fact] + public async Task SearchUsedListingsAsync_MissingCredentialsFailsWithoutCallingProvider() + { + var handler = new CallbackHandler( + (_, _) => throw new InvalidOperationException("Unexpected request.") + ); + var service = new SerpApiMarketDataService( + new HttpClient(handler), + Options.Create(new SerpApiOptions()) + ); + + var exception = await Assert.ThrowsAsync(() => + service.SearchUsedListingsAsync("lamp", null, null) + ); + + Assert.Contains("credentials are not configured", exception.Message); + } + + private static SerpApiMarketDataService CreateService(HttpMessageHandler handler) => + new( + new HttpClient(handler), + Options.Create(new SerpApiOptions { ApiKey = "test-api-key" }) + ); + + private static HttpResponseMessage JsonResponse(string body) => + new(HttpStatusCode.OK) + { + Content = new StringContent(body, Encoding.UTF8, "application/json"), + }; + + private sealed class CallbackHandler( + Func> callback + ) : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken + ) => callback(request, cancellationToken); + } + + private sealed record RequestSnapshot(HttpMethod Method, Uri Uri, string? Authorization) + { + public static async Task CaptureAsync( + HttpRequestMessage request, + CancellationToken cancellationToken + ) + { + if (request.Content is not null) + { + await request.Content.ReadAsStringAsync(cancellationToken); + } + + return new RequestSnapshot( + request.Method, + request.RequestUri!, + request.Headers.Authorization?.ToString() + ); + } + } +} From 9b5e1ad0c6eeb7637fba7742ed73b5c8ac9df153 Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 08:29:02 -0400 Subject: [PATCH 3/4] Cache SerpApi searches for 30 days --- .../Configuration/SerpApiOptions.cs | 2 + PhysicalAssets/SerpApiMarketDataService.cs | 48 +++++- PhysicalAssets/SerpApiSearchCache.cs | 154 ++++++++++++++++++ Program.cs | 9 + appsettings.json | 4 +- docs/development.md | 9 +- .../SerpApiMarketDataServiceTests.cs | 139 +++++++++++++++- 7 files changed, 350 insertions(+), 15 deletions(-) create mode 100644 PhysicalAssets/SerpApiSearchCache.cs diff --git a/PhysicalAssets/Configuration/SerpApiOptions.cs b/PhysicalAssets/Configuration/SerpApiOptions.cs index 79481c3..59ecbbb 100644 --- a/PhysicalAssets/Configuration/SerpApiOptions.cs +++ b/PhysicalAssets/Configuration/SerpApiOptions.cs @@ -9,4 +9,6 @@ public sealed class SerpApiOptions public string BaseUrl { get; set; } = "https://serpapi.com/search.json"; public string CountryCode { get; set; } = "us"; public string Language { get; set; } = "en"; + public int CacheDurationHours { get; set; } = 720; + public string CacheDirectory { get; set; } = "App_Data/serpapi-search-cache"; } diff --git a/PhysicalAssets/SerpApiMarketDataService.cs b/PhysicalAssets/SerpApiMarketDataService.cs index f31dd58..9e1b22b 100644 --- a/PhysicalAssets/SerpApiMarketDataService.cs +++ b/PhysicalAssets/SerpApiMarketDataService.cs @@ -1,5 +1,7 @@ using System.Globalization; using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; @@ -28,11 +30,17 @@ public sealed class SerpApiMarketDataService : ISerpApiMarketDataService private readonly HttpClient _httpClient; private readonly SerpApiOptions _options; + private readonly SerpApiSearchCache _searchCache; - public SerpApiMarketDataService(HttpClient httpClient, IOptions options) + public SerpApiMarketDataService( + HttpClient httpClient, + IOptions options, + SerpApiSearchCache searchCache + ) { _httpClient = httpClient; _options = options.Value; + _searchCache = searchCache; } public async Task> SearchUsedListingsAsync( @@ -43,7 +51,6 @@ public async Task> SearchUsedListingsAsync ) { ArgumentException.ThrowIfNullOrWhiteSpace(label); - EnsureCredentials(); var query = string.Join( @@ -54,6 +61,24 @@ public async Task> SearchUsedListingsAsync .Distinct(StringComparer.OrdinalIgnoreCase) ) + " used for sale price"; + var key = BuildCacheKey(query); + var cacheLifetime = TimeSpan.FromHours( + Math.Clamp(_options.CacheDurationHours, 1, 24 * 365) + ); + return await _searchCache.GetOrCreateAsync( + key, + cacheLifetime, + () => SearchProviderAsync(query, cancellationToken), + cancellationToken + ); + } + + private async Task> SearchProviderAsync( + string query, + CancellationToken cancellationToken + ) + { + EnsureCredentials(); var uri = BuildSearchUri(query); using var request = new HttpRequestMessage(HttpMethod.Get, uri); using var response = await SendAsync(request, cancellationToken); @@ -67,6 +92,25 @@ public async Task> SearchUsedListingsAsync return MapEvidence(result); } + private string BuildCacheKey(string query) + { + var normalizedQuery = Regex + .Replace(query.Normalize(NormalizationForm.FormKC), @"\s+", " ") + .Trim() + .ToLowerInvariant(); + var keyMaterial = string.Join( + "\n", + _options.BaseUrl.Trim(), + _options.CountryCode.Trim().ToLowerInvariant(), + _options.Language.Trim().ToLowerInvariant(), + _options.CacheDurationHours.ToString(CultureInfo.InvariantCulture), + normalizedQuery + ); + return Convert + .ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(keyMaterial))) + .ToLowerInvariant(); + } + private void EnsureCredentials() { if (string.IsNullOrWhiteSpace(_options.ApiKey)) diff --git a/PhysicalAssets/SerpApiSearchCache.cs b/PhysicalAssets/SerpApiSearchCache.cs new file mode 100644 index 0000000..ab1e789 --- /dev/null +++ b/PhysicalAssets/SerpApiSearchCache.cs @@ -0,0 +1,154 @@ +using System.Collections.Concurrent; +using System.Text.Json; + +namespace MoneyMirror.PhysicalAssets; + +/// Caches successful SerpApi searches on disk so app restarts do not spend the same search again. +public sealed class SerpApiSearchCache(string cacheDirectory) +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + private readonly ConcurrentDictionary< + string, + Lazy>> + > _inFlight = new(); + + public async Task> GetOrCreateAsync( + string key, + TimeSpan lifetime, + Func>> factory, + CancellationToken cancellationToken + ) + { + cancellationToken.ThrowIfCancellationRequested(); + + var pending = _inFlight.GetOrAdd( + key, + _ => new Lazy>>( + () => LoadOrCreateAsync(key, lifetime, factory), + LazyThreadSafetyMode.ExecutionAndPublication + ) + ); + var task = pending.Value; + _ = task.ContinueWith( + _ => + ( + (ICollection< + KeyValuePair>>> + >) + _inFlight + ).Remove( + new KeyValuePair>>>( + key, + pending + ) + ), + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default + ); + + return await task.WaitAsync(cancellationToken); + } + + private async Task> LoadOrCreateAsync( + string key, + TimeSpan lifetime, + Func>> factory + ) + { + try + { + Directory.CreateDirectory(cacheDirectory); + } + catch (IOException) + { + return await factory(); + } + catch (UnauthorizedAccessException) + { + return await factory(); + } + + var cachePath = Path.Combine(cacheDirectory, $"{key}.json"); + var cached = await ReadAsync(cachePath); + if ( + cached is { Evidence: not null, ExpiresAtUtc: var expiresAt } + && expiresAt > DateTimeOffset.UtcNow + ) + { + return cached.Evidence; + } + + var evidence = await factory(); + await WriteAsync(cachePath, new CacheEntry(DateTimeOffset.UtcNow.Add(lifetime), evidence)); + return evidence; + } + + private static async Task ReadAsync(string path) + { + try + { + if (!File.Exists(path)) + { + return null; + } + + await using var stream = File.OpenRead(path); + return await JsonSerializer.DeserializeAsync(stream, JsonOptions); + } + catch (IOException) + { + return null; + } + catch (UnauthorizedAccessException) + { + return null; + } + catch (JsonException) + { + return null; + } + } + + private static async Task WriteAsync(string path, CacheEntry entry) + { + var temporaryPath = $"{path}.{Guid.NewGuid():N}.tmp"; + try + { + await using (var stream = File.Create(temporaryPath)) + { + await JsonSerializer.SerializeAsync(stream, entry, JsonOptions); + } + + File.Move(temporaryPath, path, overwrite: true); + } + catch (IOException) + { + // The cache is an optimization; a storage hiccup must not fail a valuation. + } + catch (UnauthorizedAccessException) + { + // The cache is an optimization; a storage hiccup must not fail a valuation. + } + finally + { + try + { + File.Delete(temporaryPath); + } + catch (IOException) + { + // Ignore temporary-file cleanup failures. + } + catch (UnauthorizedAccessException) + { + // Ignore temporary-file cleanup failures. + } + } + } + + private sealed record CacheEntry( + DateTimeOffset ExpiresAtUtc, + IReadOnlyList Evidence + ); +} diff --git a/Program.cs b/Program.cs index 0f5c399..dbed91b 100644 --- a/Program.cs +++ b/Program.cs @@ -45,6 +45,15 @@ builder.Services.AddTransient(); builder.Services.AddMemoryCache(); +builder.Services.AddSingleton(sp => +{ + var options = + sp.GetRequiredService>().Value; + var cacheDirectory = Path.IsPathRooted(options.CacheDirectory) + ? options.CacheDirectory + : Path.Combine(builder.Environment.ContentRootPath, options.CacheDirectory); + return new SerpApiSearchCache(cacheDirectory); +}); // Both AI clients share NVIDIA's endpoint, which sheds load with a 503 when its // workers are saturated - see TransientFaultRetryHandler. HttpClient.Timeout diff --git a/appsettings.json b/appsettings.json index 8437182..ecaf003 100644 --- a/appsettings.json +++ b/appsettings.json @@ -29,6 +29,8 @@ "ApiKey": "", "BaseUrl": "https://serpapi.com/search.json", "CountryCode": "us", - "Language": "en" + "Language": "en", + "CacheDurationHours": 720, + "CacheDirectory": "App_Data/serpapi-search-cache" } } diff --git a/docs/development.md b/docs/development.md index cb1317d..47f674f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -7,9 +7,9 @@ monolith, one database, one implicit user — no auth or multi-tenancy. ## Configuration AI, BLS, and SerpApi Google Search settings live under `Ai:Nemotron`, -`Ai:VisionModel`, `Bls`, and `SerpApi` (`ApiKey`, `BaseUrl`, `CountryCode`, and -`Language`). `appsettings.json` ships empty secret placeholders — never -commit real keys. Set them locally: +`Ai:VisionModel`, `Bls`, and `SerpApi` (`ApiKey`, `BaseUrl`, `CountryCode`, +`Language`, and `CacheDurationHours`). `appsettings.json` ships empty secret +placeholders — never commit real keys. Set them locally: ```sh dotnet user-secrets set "Ai:Nemotron:ApiKey" "" @@ -28,6 +28,9 @@ listings produce a null value and an explicit low-confidence explanation. The valuation returns each comparable's price, source, title, and condition in `AssetValuation.Evidence`; that evidence is persisted and displayed alongside each valuation in inventory history. +Successful results, including empty results, are cached for 30 days in +`App_Data/serpapi-search-cache`, which is on the persistent Docker app-data +volume. Repeated or concurrent searches for the same item reuse that result. A free BLS v2 key from [data.bls.gov/registrationEngine](https://data.bls.gov/registrationEngine/) raises rate limits; the app works without one at the unregistered limit. diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs index e13bcb6..fc3a58a 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs @@ -11,6 +11,7 @@ public class SerpApiMarketDataServiceTests [Fact] public async Task SearchUsedListingsAsync_QueriesGoogleAndMapsOnlyPricedUsedResults() { + using var cache = new TemporarySearchCache(); RequestSnapshot? requestSnapshot = null; var handler = new CallbackHandler( async (request, cancellationToken) => @@ -21,7 +22,7 @@ public async Task SearchUsedListingsAsync_QueriesGoogleAndMapsOnlyPricedUsedResu ); } ); - var service = CreateService(handler); + var service = CreateService(handler, cache.Create()); var listings = await service.SearchUsedListingsAsync("acoustic guitar", "Fender", "CD-60S"); @@ -50,22 +51,36 @@ public async Task SearchUsedListingsAsync_QueriesGoogleAndMapsOnlyPricedUsedResu [Fact] public async Task SearchUsedListingsAsync_EmptyResponseReturnsNoListings() { + using var cache = new TemporarySearchCache(); + var requestCount = 0; var service = CreateService( - new CallbackHandler((_, _) => Task.FromResult(JsonResponse("{}"))) + new CallbackHandler( + (_, _) => + { + requestCount++; + return Task.FromResult(JsonResponse("{}")); + } + ), + cache.Create() ); var listings = await service.SearchUsedListingsAsync("lamp", null, null); + var repeatedListings = await service.SearchUsedListingsAsync("lamp", null, null); Assert.Empty(listings); + Assert.Empty(repeatedListings); + Assert.Equal(1, requestCount); } [Fact] public async Task SearchUsedListingsAsync_ApiErrorThrowsWithoutExposingApiKey() { + using var cache = new TemporarySearchCache(); var service = CreateService( new CallbackHandler( (_, _) => Task.FromResult(JsonResponse("""{"error":"Invalid API key"}""")) - ) + ), + cache.Create() ); var exception = await Assert.ThrowsAsync(() => @@ -78,13 +93,11 @@ public async Task SearchUsedListingsAsync_ApiErrorThrowsWithoutExposingApiKey() [Fact] public async Task SearchUsedListingsAsync_MissingCredentialsFailsWithoutCallingProvider() { + using var cache = new TemporarySearchCache(); var handler = new CallbackHandler( (_, _) => throw new InvalidOperationException("Unexpected request.") ); - var service = new SerpApiMarketDataService( - new HttpClient(handler), - Options.Create(new SerpApiOptions()) - ); + var service = CreateService(handler, cache.Create(), new SerpApiOptions()); var exception = await Assert.ThrowsAsync(() => service.SearchUsedListingsAsync("lamp", null, null) @@ -93,10 +106,100 @@ public async Task SearchUsedListingsAsync_MissingCredentialsFailsWithoutCallingP Assert.Contains("credentials are not configured", exception.Message); } - private static SerpApiMarketDataService CreateService(HttpMessageHandler handler) => + [Fact] + public async Task SearchUsedListingsAsync_ReusesDiskCacheForEquivalentQueriesAcrossServiceInstances() + { + using var cache = new TemporarySearchCache(); + var requestCount = 0; + var firstService = CreateService( + new CallbackHandler( + (_, _) => + { + requestCount++; + return Task.FromResult( + JsonResponse( + """{"shopping_results":[{"title":"Used Fender CD-60S guitar","source":"Reverb","price":"$130.00","second_hand_condition":"Used"}]}""" + ) + ); + } + ), + cache.Create() + ); + var first = await firstService.SearchUsedListingsAsync( + "acoustic guitar", + "Fender", + "CD-60S" + ); + + // A new cache object simulates an app restart while reading the same persistent directory. + var secondService = CreateService( + new CallbackHandler( + (_, _) => + throw new InvalidOperationException("A cached search should not call SerpApi.") + ), + cache.Create() + ); + var second = await secondService.SearchUsedListingsAsync( + " ACOUSTIC GUITAR ", + "fender", + "cd-60s" + ); + + Assert.Equal(1, requestCount); + Assert.Equal(first, second); + } + + [Fact] + public async Task SearchUsedListingsAsync_CoalescesConcurrentIdenticalQueries() + { + using var cache = new TemporarySearchCache(); + var sharedSearchCache = cache.Create(); + var requestCount = 0; + var requestStarted = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously + ); + var releaseRequest = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously + ); + var firstService = CreateService( + new CallbackHandler( + async (_, cancellationToken) => + { + Interlocked.Increment(ref requestCount); + requestStarted.SetResult(); + await releaseRequest.Task.WaitAsync(cancellationToken); + return JsonResponse("{}"); + } + ), + sharedSearchCache + ); + var secondService = CreateService( + new CallbackHandler( + (_, _) => throw new InvalidOperationException("A duplicate search was sent.") + ), + sharedSearchCache + ); + + var firstTask = firstService.SearchUsedListingsAsync("lamp", null, null); + await requestStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var secondTask = secondService.SearchUsedListingsAsync("lamp", null, null); + releaseRequest.SetResult(); + var results = await Task.WhenAll(firstTask, secondTask); + + Assert.Equal(1, requestCount); + Assert.Empty(results[0]); + Assert.Empty(results[1]); + } + + private static SerpApiMarketDataService CreateService( + HttpMessageHandler handler, + SerpApiSearchCache cache, + SerpApiOptions? options = null + ) => new( new HttpClient(handler), - Options.Create(new SerpApiOptions { ApiKey = "test-api-key" }) + Options.Create(options ?? new SerpApiOptions { ApiKey = "test-api-key" }), + cache ); private static HttpResponseMessage JsonResponse(string body) => @@ -115,6 +218,24 @@ CancellationToken cancellationToken ) => callback(request, cancellationToken); } + private sealed class TemporarySearchCache : IDisposable + { + private readonly string _directory = Path.Combine( + Path.GetTempPath(), + $"MoneyMirror-SerpApiCacheTests-{Guid.NewGuid():N}" + ); + + public SerpApiSearchCache Create() => new(_directory); + + public void Dispose() + { + if (Directory.Exists(_directory)) + { + Directory.Delete(_directory, recursive: true); + } + } + } + private sealed record RequestSnapshot(HttpMethod Method, Uri Uri, string? Authorization) { public static async Task CaptureAsync( From 670ab2913434c992f55ac9db1c98063acabba884 Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 09:09:35 -0400 Subject: [PATCH 4/4] Restore eBay Browse API with AI-guess fallback --- .env.example | 3 +- PhysicalAssets/Configuration/EbayOptions.cs | 19 + .../Configuration/SerpApiOptions.cs | 14 - PhysicalAssets/EbayMarketDataException.cs | 8 + PhysicalAssets/EbayMarketDataService.cs | 298 +++++++++++++++ PhysicalAssets/EbayTokenCache.cs | 45 +++ .../EvidenceBasedAssetValuationService.cs | 16 +- PhysicalAssets/IMarketDataService.cs | 13 + PhysicalAssets/ISerpApiMarketDataService.cs | 13 - ...earchCache.cs => MarketDataSearchCache.cs} | 5 +- PhysicalAssets/SerpApiMarketDataException.cs | 8 - PhysicalAssets/SerpApiMarketDataService.cs | 336 ----------------- Program.cs | 21 +- appsettings.json | 13 +- docker-compose.yml | 3 +- docs/development.md | 53 +-- .../EbayMarketDataServiceTests.cs | 353 ++++++++++++++++++ ...EvidenceBasedAssetValuationServiceTests.cs | 62 ++- .../SerpApiMarketDataServiceTests.cs | 258 ------------- 19 files changed, 861 insertions(+), 680 deletions(-) create mode 100644 PhysicalAssets/Configuration/EbayOptions.cs delete mode 100644 PhysicalAssets/Configuration/SerpApiOptions.cs create mode 100644 PhysicalAssets/EbayMarketDataException.cs create mode 100644 PhysicalAssets/EbayMarketDataService.cs create mode 100644 PhysicalAssets/EbayTokenCache.cs create mode 100644 PhysicalAssets/IMarketDataService.cs delete mode 100644 PhysicalAssets/ISerpApiMarketDataService.cs rename PhysicalAssets/{SerpApiSearchCache.cs => MarketDataSearchCache.cs} (94%) delete mode 100644 PhysicalAssets/SerpApiMarketDataException.cs delete mode 100644 PhysicalAssets/SerpApiMarketDataService.cs create mode 100644 tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs delete mode 100644 tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs diff --git a/.env.example b/.env.example index d3375fd..0ab8727 100644 --- a/.env.example +++ b/.env.example @@ -7,4 +7,5 @@ POSTGRES_PASSWORD=postgres NEMOTRON_API_KEY= VISION_API_KEY= BLS_API_KEY= -SERPAPI_API_KEY= +EBAY_CLIENT_ID= +EBAY_CLIENT_SECRET= diff --git a/PhysicalAssets/Configuration/EbayOptions.cs b/PhysicalAssets/Configuration/EbayOptions.cs new file mode 100644 index 0000000..563d316 --- /dev/null +++ b/PhysicalAssets/Configuration/EbayOptions.cs @@ -0,0 +1,19 @@ +namespace MoneyMirror.PhysicalAssets.Configuration; + +/// Client-credentials app keys and Browse API endpoint configuration for eBay. +/// Defaults point at eBay's sandbox environment; swap both URLs to the production hosts +/// once the app has production-approved keys (sandbox listings are seeded test data, not +/// a real catalog). +public sealed class EbayOptions +{ + public const string SectionName = "Ebay"; + + public string ClientId { get; set; } = string.Empty; + public string ClientSecret { get; set; } = string.Empty; + public string AuthUrl { get; set; } = "https://api.sandbox.ebay.com/identity/v1/oauth2/token"; + public string SearchUrl { get; set; } = + "https://api.sandbox.ebay.com/buy/browse/v1/item_summary/search"; + public string MarketplaceId { get; set; } = "EBAY_US"; + public int CacheDurationHours { get; set; } = 720; + public string CacheDirectory { get; set; } = "App_Data/ebay-search-cache"; +} diff --git a/PhysicalAssets/Configuration/SerpApiOptions.cs b/PhysicalAssets/Configuration/SerpApiOptions.cs deleted file mode 100644 index 59ecbbb..0000000 --- a/PhysicalAssets/Configuration/SerpApiOptions.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace MoneyMirror.PhysicalAssets.Configuration; - -/// API key and Google Search endpoint configuration for SerpApi. -public sealed class SerpApiOptions -{ - public const string SectionName = "SerpApi"; - - public string ApiKey { get; set; } = string.Empty; - public string BaseUrl { get; set; } = "https://serpapi.com/search.json"; - public string CountryCode { get; set; } = "us"; - public string Language { get; set; } = "en"; - public int CacheDurationHours { get; set; } = 720; - public string CacheDirectory { get; set; } = "App_Data/serpapi-search-cache"; -} diff --git a/PhysicalAssets/EbayMarketDataException.cs b/PhysicalAssets/EbayMarketDataException.cs new file mode 100644 index 0000000..a1bd216 --- /dev/null +++ b/PhysicalAssets/EbayMarketDataException.cs @@ -0,0 +1,8 @@ +namespace MoneyMirror.PhysicalAssets; + +/// Thrown when eBay's Browse API cannot return comparable listings. +public sealed class EbayMarketDataException : Exception +{ + public EbayMarketDataException(string message, Exception? innerException = null) + : base(message, innerException) { } +} diff --git a/PhysicalAssets/EbayMarketDataService.cs b/PhysicalAssets/EbayMarketDataService.cs new file mode 100644 index 0000000..ceedacd --- /dev/null +++ b/PhysicalAssets/EbayMarketDataService.cs @@ -0,0 +1,298 @@ +using System.Globalization; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Options; +using MoneyMirror.PhysicalAssets.Configuration; + +namespace MoneyMirror.PhysicalAssets; + +/// Searches eBay's Browse API for priced used/refurbished listings via a +/// client-credentials app token. Defaults to eBay's sandbox, whose inventory is seeded +/// test data - it answers real requests but rarely has comps for a real product name, and +/// its own condition filter does not reliably exclude new items, so results are always +/// re-checked against 's USD/condition requirements +/// here rather than trusted as returned. +public sealed class EbayMarketDataService : IMarketDataService +{ + private const int ResultLimit = 50; + private const string ConditionFilter = "conditions:{USED|CERTIFIED_REFURBISHED|SELLER_REFURBISHED}"; + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + private readonly HttpClient _httpClient; + private readonly EbayOptions _options; + private readonly MarketDataSearchCache _searchCache; + private readonly EbayTokenCache _tokenCache; + + public EbayMarketDataService( + HttpClient httpClient, + IOptions options, + MarketDataSearchCache searchCache, + EbayTokenCache tokenCache + ) + { + _httpClient = httpClient; + _options = options.Value; + _searchCache = searchCache; + _tokenCache = tokenCache; + } + + public async Task> SearchUsedListingsAsync( + string label, + string? brand, + string? model, + CancellationToken cancellationToken = default + ) + { + ArgumentException.ThrowIfNullOrWhiteSpace(label); + + var query = string.Join( + " ", + new[] { label, brand, model } + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value!.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + ); + + var key = BuildCacheKey(query); + var cacheLifetime = TimeSpan.FromHours(Math.Clamp(_options.CacheDurationHours, 1, 24 * 365)); + return await _searchCache.GetOrCreateAsync( + key, + cacheLifetime, + () => SearchProviderAsync(query, cancellationToken), + cancellationToken + ); + } + + private async Task> SearchProviderAsync( + string query, + CancellationToken cancellationToken + ) + { + EnsureCredentials(); + var token = await _tokenCache.GetOrCreateAsync(ct => FetchTokenAsync(ct), cancellationToken); + + using var request = new HttpRequestMessage(HttpMethod.Get, BuildSearchUri(query)); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); + request.Headers.Add("X-EBAY-C-MARKETPLACE-ID", _options.MarketplaceId); + + using var response = await SendAsync(request, cancellationToken); + var result = await ReadResponseAsync( + response, + "eBay Browse API search", + cancellationToken + ); + + return MapEvidence(result); + } + + private async Task<(string Token, TimeSpan ExpiresIn)> FetchTokenAsync( + CancellationToken cancellationToken + ) + { + using var request = new HttpRequestMessage(HttpMethod.Post, _options.AuthUrl) + { + Content = new FormUrlEncodedContent( + new Dictionary + { + ["grant_type"] = "client_credentials", + ["scope"] = "https://api.ebay.com/oauth/api_scope", + } + ), + }; + var credentials = Convert.ToBase64String( + Encoding.UTF8.GetBytes($"{_options.ClientId}:{_options.ClientSecret}") + ); + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials); + + using var response = await SendAsync(request, cancellationToken); + var result = await ReadResponseAsync( + response, + "eBay OAuth token request", + cancellationToken + ); + + if (string.IsNullOrWhiteSpace(result.AccessToken)) + { + throw new EbayMarketDataException("eBay OAuth token response did not include a token."); + } + + return (result.AccessToken, TimeSpan.FromSeconds(Math.Max(result.ExpiresIn, 60))); + } + + private string BuildCacheKey(string query) + { + var normalizedQuery = System.Text.RegularExpressions.Regex + .Replace(query.Normalize(NormalizationForm.FormKC), @"\s+", " ") + .Trim() + .ToLowerInvariant(); + var keyMaterial = string.Join( + "\n", + _options.SearchUrl.Trim(), + _options.MarketplaceId.Trim().ToUpperInvariant(), + _options.CacheDurationHours.ToString(CultureInfo.InvariantCulture), + normalizedQuery + ); + return Convert + .ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(keyMaterial))) + .ToLowerInvariant(); + } + + private void EnsureCredentials() + { + if (string.IsNullOrWhiteSpace(_options.ClientId) || string.IsNullOrWhiteSpace(_options.ClientSecret)) + { + throw new EbayMarketDataException( + "eBay credentials are not configured. Set Ebay:ClientId and Ebay:ClientSecret." + ); + } + } + + private Uri BuildSearchUri(string query) + { + var parameters = new Dictionary + { + ["q"] = query, + ["filter"] = ConditionFilter, + ["limit"] = ResultLimit.ToString(CultureInfo.InvariantCulture), + }; + var queryString = string.Join( + "&", + parameters.Select(pair => + $"{Uri.EscapeDataString(pair.Key)}={Uri.EscapeDataString(pair.Value)}" + ) + ); + + return new Uri($"{_options.SearchUrl.TrimEnd('?', '&')}?{queryString}", UriKind.Absolute); + } + + private static IReadOnlyList MapEvidence(SearchResponse result) + { + var evidence = new List(); + + foreach (var item in result.ItemSummaries ?? []) + { + var condition = NormalizeCondition(item.ConditionId); + if ( + condition is null + || string.IsNullOrWhiteSpace(item.Title) + || item.Price is not { Currency: "USD" } price + || !decimal.TryParse( + price.Value, + NumberStyles.Number, + CultureInfo.InvariantCulture, + out var priceUsd + ) + || priceUsd <= 0 + ) + { + continue; + } + + evidence.Add(new AssetValuationEvidence(priceUsd, "eBay", item.Title.Trim(), condition)); + } + + return evidence.Distinct().Take(ResultLimit).ToArray(); + } + + // eBay's documented conditionId taxonomy: 1000-1999 covers New/New-with-defects, 2000-2999 + // is the Refurbished band (certified/seller/excellent/very-good/good), 3000-6999 is Used at + // every grade (Used, Very Good, Good, Acceptable), and 7000 is "for parts or not working" - + // not representative of a working item's resale value, so it is excluded like New. + private static string? NormalizeCondition(string? conditionId) + { + if (!int.TryParse(conditionId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var id)) + { + return null; + } + + return id switch + { + >= 2000 and < 3000 => "Refurbished", + >= 3000 and < 7000 => "Used", + _ => null, + }; + } + + private async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken + ) + { + try + { + return await _httpClient.SendAsync(request, cancellationToken); + } + catch (HttpRequestException ex) + { + throw new EbayMarketDataException("Failed to reach eBay.", ex); + } + catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested) + { + throw new EbayMarketDataException("eBay request timed out.", ex); + } + } + + private static async Task ReadResponseAsync( + HttpResponseMessage response, + string what, + CancellationToken cancellationToken + ) + { + if (!response.IsSuccessStatusCode) + { + throw new EbayMarketDataException( + $"{what} returned {(int)response.StatusCode} {response.StatusCode}." + ); + } + + try + { + var result = await response.Content.ReadFromJsonAsync(JsonOptions, cancellationToken); + return result ?? throw new EbayMarketDataException($"{what} returned an empty response."); + } + catch (JsonException ex) + { + throw new EbayMarketDataException($"Failed to parse the {what} response.", ex); + } + } + + private sealed class TokenResponse + { + [JsonPropertyName("access_token")] + public string? AccessToken { get; init; } + + [JsonPropertyName("expires_in")] + public int ExpiresIn { get; init; } + } + + private sealed class SearchResponse + { + [JsonPropertyName("itemSummaries")] + public IReadOnlyList? ItemSummaries { get; init; } + } + + private sealed class ItemSummary + { + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("price")] + public Money? Price { get; init; } + + [JsonPropertyName("conditionId")] + public string? ConditionId { get; init; } + } + + private sealed class Money + { + [JsonPropertyName("value")] + public string? Value { get; init; } + + [JsonPropertyName("currency")] + public string? Currency { get; init; } + } +} diff --git a/PhysicalAssets/EbayTokenCache.cs b/PhysicalAssets/EbayTokenCache.cs new file mode 100644 index 0000000..2791c6c --- /dev/null +++ b/PhysicalAssets/EbayTokenCache.cs @@ -0,0 +1,45 @@ +namespace MoneyMirror.PhysicalAssets; + +/// Caches eBay's client-credentials OAuth token in memory so every search does not +/// spend a round trip re-authenticating. Registered as a singleton: +/// is a typed HttpClient, which is transient, so a per-instance cache would never be reused. +public sealed class EbayTokenCache(TimeProvider? timeProvider = null) +{ + // A token is refreshed a minute before it actually expires, so a request in flight + // never races an expiring token. + private static readonly TimeSpan ExpiryMargin = TimeSpan.FromMinutes(1); + + private readonly TimeProvider _timeProvider = timeProvider ?? TimeProvider.System; + private readonly SemaphoreSlim _lock = new(1, 1); + private string? _token; + private DateTimeOffset _expiresAtUtc = DateTimeOffset.MinValue; + + public async Task GetOrCreateAsync( + Func> factory, + CancellationToken cancellationToken + ) + { + if (_token is { } cached && _timeProvider.GetUtcNow() < _expiresAtUtc) + { + return cached; + } + + await _lock.WaitAsync(cancellationToken); + try + { + if (_token is { } stillCached && _timeProvider.GetUtcNow() < _expiresAtUtc) + { + return stillCached; + } + + var (token, expiresIn) = await factory(cancellationToken); + _token = token; + _expiresAtUtc = _timeProvider.GetUtcNow() + expiresIn - ExpiryMargin; + return token; + } + finally + { + _lock.Release(); + } + } +} diff --git a/PhysicalAssets/EvidenceBasedAssetValuationService.cs b/PhysicalAssets/EvidenceBasedAssetValuationService.cs index 99f9d40..7938aed 100644 --- a/PhysicalAssets/EvidenceBasedAssetValuationService.cs +++ b/PhysicalAssets/EvidenceBasedAssetValuationService.cs @@ -1,17 +1,23 @@ namespace MoneyMirror.PhysicalAssets; -/// Builds an asset valuation from structured market listings, without an LLM. +/// Builds an asset valuation from structured market listings, without an LLM. +/// Falls back to 's price guess when the market-data +/// provider itself fails (credentials, network, rate limit) - not when it succeeds with zero +/// usable listings, which is already a valid, explicit "no market value" result. public sealed class EvidenceBasedAssetValuationService : IAssetValuationService { - private readonly ISerpApiMarketDataService _marketDataService; + private readonly IMarketDataService _marketDataService; + private readonly AiEstimatedValuationService _fallback; private readonly TimeProvider _timeProvider; public EvidenceBasedAssetValuationService( - ISerpApiMarketDataService marketDataService, + IMarketDataService marketDataService, + AiEstimatedValuationService fallback, TimeProvider? timeProvider = null ) { _marketDataService = marketDataService; + _fallback = fallback; _timeProvider = timeProvider ?? TimeProvider.System; } @@ -32,9 +38,9 @@ public async Task EstimateAsync( ); return MarketValuationCalculator.Calculate(evidence, _timeProvider.GetUtcNow()); } - catch (SerpApiMarketDataException ex) + catch (EbayMarketDataException) { - throw new AssetValuationException("Failed to retrieve comparable market listings.", ex); + return await _fallback.EstimateAsync(label, brand, model, cancellationToken); } } } diff --git a/PhysicalAssets/IMarketDataService.cs b/PhysicalAssets/IMarketDataService.cs new file mode 100644 index 0000000..232b7a9 --- /dev/null +++ b/PhysicalAssets/IMarketDataService.cs @@ -0,0 +1,13 @@ +namespace MoneyMirror.PhysicalAssets; + +/// Searches an external market-data provider for priced used-item comparables. +public interface IMarketDataService +{ + /// Returns priced used/refurbished listings matching an item's label, brand and model. + Task> SearchUsedListingsAsync( + string label, + string? brand, + string? model, + CancellationToken cancellationToken = default + ); +} diff --git a/PhysicalAssets/ISerpApiMarketDataService.cs b/PhysicalAssets/ISerpApiMarketDataService.cs deleted file mode 100644 index 29d5c97..0000000 --- a/PhysicalAssets/ISerpApiMarketDataService.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace MoneyMirror.PhysicalAssets; - -/// Searches Google results through SerpApi for priced used-item comparables. -public interface ISerpApiMarketDataService -{ - /// Returns up to twenty used USD listings matching an item's label, brand and model. - Task> SearchUsedListingsAsync( - string label, - string? brand, - string? model, - CancellationToken cancellationToken = default - ); -} diff --git a/PhysicalAssets/SerpApiSearchCache.cs b/PhysicalAssets/MarketDataSearchCache.cs similarity index 94% rename from PhysicalAssets/SerpApiSearchCache.cs rename to PhysicalAssets/MarketDataSearchCache.cs index ab1e789..bd35d29 100644 --- a/PhysicalAssets/SerpApiSearchCache.cs +++ b/PhysicalAssets/MarketDataSearchCache.cs @@ -3,8 +3,9 @@ namespace MoneyMirror.PhysicalAssets; -/// Caches successful SerpApi searches on disk so app restarts do not spend the same search again. -public sealed class SerpApiSearchCache(string cacheDirectory) +/// Caches successful market-data provider searches on disk so app restarts do not +/// spend the same search again, and coalesces concurrent identical in-flight searches. +public sealed class MarketDataSearchCache(string cacheDirectory) { private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); private readonly ConcurrentDictionary< diff --git a/PhysicalAssets/SerpApiMarketDataException.cs b/PhysicalAssets/SerpApiMarketDataException.cs deleted file mode 100644 index 58cb742..0000000 --- a/PhysicalAssets/SerpApiMarketDataException.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace MoneyMirror.PhysicalAssets; - -/// Thrown when SerpApi cannot return comparable Google search results. -public sealed class SerpApiMarketDataException : Exception -{ - public SerpApiMarketDataException(string message, Exception? innerException = null) - : base(message, innerException) { } -} diff --git a/PhysicalAssets/SerpApiMarketDataService.cs b/PhysicalAssets/SerpApiMarketDataService.cs deleted file mode 100644 index 9e1b22b..0000000 --- a/PhysicalAssets/SerpApiMarketDataService.cs +++ /dev/null @@ -1,336 +0,0 @@ -using System.Globalization; -using System.Net.Http.Json; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Text.RegularExpressions; -using Microsoft.Extensions.Options; -using MoneyMirror.PhysicalAssets.Configuration; - -namespace MoneyMirror.PhysicalAssets; - -/// Searches SerpApi's Google Search results for used, USD-priced listings. -public sealed class SerpApiMarketDataService : ISerpApiMarketDataService -{ - private const int ResultLimit = 20; - private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); - private static readonly Regex UsdPricePattern = new( - @"(?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d{1,2})?)(?![\d,])", - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant - ); - private static readonly Regex UsedConditionPattern = new( - @"\b(?:used|pre[ -]?owned|second[ -]?hand|secondhand)\b", - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant - ); - private static readonly Regex RefurbishedConditionPattern = new( - @"\b(?:refurbished|renewed|reconditioned)\b", - RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant - ); - - private readonly HttpClient _httpClient; - private readonly SerpApiOptions _options; - private readonly SerpApiSearchCache _searchCache; - - public SerpApiMarketDataService( - HttpClient httpClient, - IOptions options, - SerpApiSearchCache searchCache - ) - { - _httpClient = httpClient; - _options = options.Value; - _searchCache = searchCache; - } - - public async Task> SearchUsedListingsAsync( - string label, - string? brand, - string? model, - CancellationToken cancellationToken = default - ) - { - ArgumentException.ThrowIfNullOrWhiteSpace(label); - - var query = - string.Join( - " ", - new[] { label, brand, model } - .Where(value => !string.IsNullOrWhiteSpace(value)) - .Select(value => value!.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - ) + " used for sale price"; - - var key = BuildCacheKey(query); - var cacheLifetime = TimeSpan.FromHours( - Math.Clamp(_options.CacheDurationHours, 1, 24 * 365) - ); - return await _searchCache.GetOrCreateAsync( - key, - cacheLifetime, - () => SearchProviderAsync(query, cancellationToken), - cancellationToken - ); - } - - private async Task> SearchProviderAsync( - string query, - CancellationToken cancellationToken - ) - { - EnsureCredentials(); - var uri = BuildSearchUri(query); - using var request = new HttpRequestMessage(HttpMethod.Get, uri); - using var response = await SendAsync(request, cancellationToken); - var result = await ReadResponseAsync(response, cancellationToken); - - if (!string.IsNullOrWhiteSpace(result.Error)) - { - throw new SerpApiMarketDataException("SerpApi Google Search returned an API error."); - } - - return MapEvidence(result); - } - - private string BuildCacheKey(string query) - { - var normalizedQuery = Regex - .Replace(query.Normalize(NormalizationForm.FormKC), @"\s+", " ") - .Trim() - .ToLowerInvariant(); - var keyMaterial = string.Join( - "\n", - _options.BaseUrl.Trim(), - _options.CountryCode.Trim().ToLowerInvariant(), - _options.Language.Trim().ToLowerInvariant(), - _options.CacheDurationHours.ToString(CultureInfo.InvariantCulture), - normalizedQuery - ); - return Convert - .ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(keyMaterial))) - .ToLowerInvariant(); - } - - private void EnsureCredentials() - { - if (string.IsNullOrWhiteSpace(_options.ApiKey)) - { - throw new SerpApiMarketDataException( - "SerpApi credentials are not configured. Set SerpApi:ApiKey." - ); - } - } - - private Uri BuildSearchUri(string query) - { - var parameters = new Dictionary - { - ["engine"] = "google", - ["q"] = query, - ["gl"] = _options.CountryCode, - ["hl"] = _options.Language, - ["api_key"] = _options.ApiKey, - }; - var queryString = string.Join( - "&", - parameters.Select(pair => - $"{Uri.EscapeDataString(pair.Key)}={Uri.EscapeDataString(pair.Value)}" - ) - ); - - return new Uri($"{_options.BaseUrl.TrimEnd('?', '&')}?{queryString}", UriKind.Absolute); - } - - private static IReadOnlyList MapEvidence(SearchResponse result) - { - var evidence = new List(); - - foreach (var listing in result.ShoppingResults ?? []) - { - var condition = NormalizeCondition(listing.SecondHandCondition); - if (condition is null) - { - condition = NormalizeCondition($"{listing.Title} {listing.Snippet}"); - } - - var price = ParseSingleUsdPrice(listing.Price); - AddEvidence(evidence, listing.Title, listing.Source, listing.Link, condition, price); - } - - foreach (var listing in result.OrganicResults ?? []) - { - var text = $"{listing.Title} {listing.Snippet}"; - var condition = NormalizeCondition(text); - var price = ParseSingleUsdPrice(text); - AddEvidence(evidence, listing.Title, listing.Source, listing.Link, condition, price); - } - - return evidence.Distinct().Take(ResultLimit).ToArray(); - } - - private static void AddEvidence( - ICollection evidence, - string? title, - string? source, - string? link, - string? condition, - decimal? price - ) - { - if ( - string.IsNullOrWhiteSpace(title) - || string.IsNullOrWhiteSpace(condition) - || price is not > 0 - ) - { - return; - } - - evidence.Add( - new AssetValuationEvidence( - price.Value, - ResolveSource(source, link), - title.Trim(), - condition - ) - ); - } - - private static string? NormalizeCondition(string? text) - { - if (string.IsNullOrWhiteSpace(text)) - { - return null; - } - - if (RefurbishedConditionPattern.IsMatch(text)) - { - return "Refurbished"; - } - - return UsedConditionPattern.IsMatch(text) ? "Used" : null; - } - - private static decimal? ParseSingleUsdPrice(string? text) - { - if (string.IsNullOrWhiteSpace(text)) - { - return null; - } - - var matches = UsdPricePattern.Matches(text); - if (matches.Count != 1) - { - return null; - } - - return - decimal.TryParse( - matches[0].Groups["amount"].Value, - NumberStyles.Number, - CultureInfo.InvariantCulture, - out var price - ) - && price > 0 - ? price - : null; - } - - private static string ResolveSource(string? source, string? link) - { - if (!string.IsNullOrWhiteSpace(source)) - { - return source.Trim(); - } - - return Uri.TryCreate(link, UriKind.Absolute, out var uri) - ? uri.Host.StartsWith("www.", StringComparison.OrdinalIgnoreCase) - ? uri.Host[4..] - : uri.Host - : "Google Search"; - } - - private async Task SendAsync( - HttpRequestMessage request, - CancellationToken cancellationToken - ) - { - try - { - return await _httpClient.SendAsync(request, cancellationToken); - } - catch (HttpRequestException ex) - { - throw new SerpApiMarketDataException("Failed to reach SerpApi Google Search.", ex); - } - catch (TaskCanceledException ex) when (!cancellationToken.IsCancellationRequested) - { - throw new SerpApiMarketDataException("SerpApi Google Search request timed out.", ex); - } - } - - private static async Task ReadResponseAsync( - HttpResponseMessage response, - CancellationToken cancellationToken - ) - { - if (!response.IsSuccessStatusCode) - { - throw new SerpApiMarketDataException( - $"SerpApi Google Search returned {(int)response.StatusCode} {response.StatusCode}." - ); - } - - try - { - var result = await response.Content.ReadFromJsonAsync( - JsonOptions, - cancellationToken - ); - return result - ?? throw new SerpApiMarketDataException( - "SerpApi Google Search returned an empty response." - ); - } - catch (JsonException ex) - { - throw new SerpApiMarketDataException( - "Failed to parse the SerpApi Google Search response.", - ex - ); - } - } - - private sealed class SearchResponse - { - [JsonPropertyName("error")] - public string? Error { get; init; } - - [JsonPropertyName("shopping_results")] - public IReadOnlyList? ShoppingResults { get; init; } - - [JsonPropertyName("organic_results")] - public IReadOnlyList? OrganicResults { get; init; } - } - - private sealed class SearchListing - { - [JsonPropertyName("title")] - public string? Title { get; init; } - - [JsonPropertyName("source")] - public string? Source { get; init; } - - [JsonPropertyName("link")] - public string? Link { get; init; } - - [JsonPropertyName("snippet")] - public string? Snippet { get; init; } - - [JsonPropertyName("price")] - public string? Price { get; init; } - - [JsonPropertyName("second_hand_condition")] - public string? SecondHandCondition { get; init; } - } -} diff --git a/Program.cs b/Program.cs index dbed91b..17a6527 100644 --- a/Program.cs +++ b/Program.cs @@ -30,8 +30,8 @@ builder.Configuration.GetSection(NemotronOptions.SectionName) ); builder.Services.Configure(builder.Configuration.GetSection(BlsOptions.SectionName)); -builder.Services.Configure( - builder.Configuration.GetSection(SerpApiOptions.SectionName) +builder.Services.Configure( + builder.Configuration.GetSection(EbayOptions.SectionName) ); builder.Services.Configure( builder.Configuration.GetSection(VisionModelOptions.SectionName) @@ -45,14 +45,14 @@ builder.Services.AddTransient(); builder.Services.AddMemoryCache(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => { - var options = - sp.GetRequiredService>().Value; + var options = sp.GetRequiredService>().Value; var cacheDirectory = Path.IsPathRooted(options.CacheDirectory) ? options.CacheDirectory : Path.Combine(builder.Environment.ContentRootPath, options.CacheDirectory); - return new SerpApiSearchCache(cacheDirectory); + return new MarketDataSearchCache(cacheDirectory); }); // Both AI clients share NVIDIA's endpoint, which sheds load with a 503 when its @@ -78,12 +78,13 @@ builder.Services.AddHttpClient(client => client.Timeout = TimeSpan.FromSeconds(15) ); -builder.Services.AddHttpClient(client => +builder.Services.AddHttpClient(client => client.Timeout = TimeSpan.FromSeconds(15) ); -// SerpApi requires its key in the query string. Keep routine HTTP logs from recording it. -builder.Logging.AddFilter("System.Net.Http.HttpClient.ISerpApiMarketDataService", LogLevel.Warning); +// eBay's OAuth token is a Bearer header, not a query-string secret, but keep routine +// HTTP logs at Warning so a future log-level bump cannot echo it either. +builder.Logging.AddFilter("System.Net.Http.HttpClient.IMarketDataService", LogLevel.Warning); builder.Services.AddScoped< IMarketPotentialExplanationService, NemotronMarketPotentialExplanationService @@ -102,6 +103,10 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +// Registered concretely (not just via IAssetValuationService) so +// EvidenceBasedAssetValuationService can take it as its fallback without a +// circular resolution through the interface both of them implement. +builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); diff --git a/appsettings.json b/appsettings.json index ecaf003..2868530 100644 --- a/appsettings.json +++ b/appsettings.json @@ -25,12 +25,13 @@ "ApiKey": "", "BaseUrl": "https://api.bls.gov/publicAPI/v2/" }, - "SerpApi": { - "ApiKey": "", - "BaseUrl": "https://serpapi.com/search.json", - "CountryCode": "us", - "Language": "en", + "Ebay": { + "ClientId": "", + "ClientSecret": "", + "AuthUrl": "https://api.sandbox.ebay.com/identity/v1/oauth2/token", + "SearchUrl": "https://api.sandbox.ebay.com/buy/browse/v1/item_summary/search", + "MarketplaceId": "EBAY_US", "CacheDurationHours": 720, - "CacheDirectory": "App_Data/serpapi-search-cache" + "CacheDirectory": "App_Data/ebay-search-cache" } } diff --git a/docker-compose.yml b/docker-compose.yml index 2c5eaff..2cf4da3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,8 @@ services: Ai__Nemotron__ApiKey: ${NEMOTRON_API_KEY:-} Ai__VisionModel__ApiKey: ${VISION_API_KEY:-} Bls__ApiKey: ${BLS_API_KEY:-} - SerpApi__ApiKey: ${SERPAPI_API_KEY:-} + Ebay__ClientId: ${EBAY_CLIENT_ID:-} + Ebay__ClientSecret: ${EBAY_CLIENT_SECRET:-} ports: - "${APP_PORT:-8000}:8080" volumes: diff --git a/docs/development.md b/docs/development.md index 47f674f..2333c63 100644 --- a/docs/development.md +++ b/docs/development.md @@ -6,34 +6,44 @@ monolith, one database, one implicit user — no auth or multi-tenancy. ## Configuration -AI, BLS, and SerpApi Google Search settings live under `Ai:Nemotron`, -`Ai:VisionModel`, `Bls`, and `SerpApi` (`ApiKey`, `BaseUrl`, `CountryCode`, -`Language`, and `CacheDurationHours`). `appsettings.json` ships empty secret -placeholders — never commit real keys. Set them locally: +AI, BLS, and eBay Browse API settings live under `Ai:Nemotron`, +`Ai:VisionModel`, `Bls`, and `Ebay` (`ClientId`, `ClientSecret`, `AuthUrl`, +`SearchUrl`, `MarketplaceId`, and `CacheDurationHours`). `appsettings.json` +ships empty secret placeholders — never commit real keys. Set them locally: ```sh dotnet user-secrets set "Ai:Nemotron:ApiKey" "" dotnet user-secrets set "Ai:VisionModel:ApiKey" "" dotnet user-secrets set "Bls:ApiKey" "" -dotnet user-secrets set "SerpApi:ApiKey" "" +dotnet user-secrets set "Ebay:ClientId" "" +dotnet user-secrets set "Ebay:ClientSecret" "" ``` Or use env vars: `Ai__Nemotron__ApiKey`, `Ai__VisionModel__ApiKey`, -`Bls__ApiKey`, `SerpApi__ApiKey`. - -Physical asset valuations query SerpApi's Google Search API for up to 20 used -listings and calculate a median from unambiguous USD prices. Results without -clear used/refurbished condition or a single USD price are ignored. No usable -listings produce a null value and an -explicit low-confidence explanation. The valuation returns each comparable's -price, source, title, and condition in `AssetValuation.Evidence`; that evidence -is persisted and displayed alongside each valuation in inventory history. -Successful results, including empty results, are cached for 30 days in -`App_Data/serpapi-search-cache`, which is on the persistent Docker app-data -volume. Repeated or concurrent searches for the same item reuse that result. - -A free BLS v2 key from [data.bls.gov/registrationEngine](https://data.bls.gov/registrationEngine/) -raises rate limits; the app works without one at the unregistered limit. +`Bls__ApiKey`, `Ebay__ClientId`, `Ebay__ClientSecret`. + +Physical asset valuations authenticate to eBay's Browse API with an +OAuth2 client-credentials app token (cached in memory for its ~2-hour +lifetime), then search for up to 50 listings filtered to used/refurbished +condition and calculate a median from USD-priced results. eBay's own +condition filter does not reliably exclude new items, so results are +re-checked client-side against each listing's `conditionId`. No usable +listings produce a null value and an explicit low-confidence explanation. +If eBay itself fails - missing credentials, network error, rate limit - +the valuation falls back to an LLM price guess (`AiEstimatedValuationService`) +rather than failing the request outright. The valuation returns each +comparable's price, source, title, and condition in `AssetValuation.Evidence`; +that evidence is persisted and displayed alongside each valuation in +inventory history. Successful results, including empty results, are cached +for 30 days in `App_Data/ebay-search-cache`, which is on the persistent +Docker app-data volume. Repeated or concurrent searches for the same item +reuse that result. + +The default `AuthUrl`/`SearchUrl` point at eBay's **sandbox** environment, +whose inventory is seeded test data - it answers real requests but rarely +has comps for a real product name. Swap both URLs to the production hosts +(`api.ebay.com` instead of `api.sandbox.ebay.com`) once the app has +production-approved keys from the [eBay Developer Program](https://developer.ebay.com/). ## Run with Docker @@ -51,7 +61,8 @@ on Postgres volumes and host-local `dotnet run`: [backend/postgresql_setup.md](. | `NEMOTRON_API_KEY` | `Ai:Nemotron:ApiKey` | | `VISION_API_KEY` | `Ai:VisionModel:ApiKey` | | `BLS_API_KEY` | `Bls:ApiKey` | -| `SERPAPI_API_KEY` | `SerpApi:ApiKey` | +| `EBAY_CLIENT_ID` | `Ebay:ClientId` | +| `EBAY_CLIENT_SECRET` | `Ebay:ClientSecret` | `docker compose down` stops the stack; add `-v` to wipe DB and image volumes. diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs new file mode 100644 index 0000000..0e8f604 --- /dev/null +++ b/tests/MoneyMirror.Tests/PhysicalAssets/EbayMarketDataServiceTests.cs @@ -0,0 +1,353 @@ +using System.Net; +using System.Text; +using Microsoft.Extensions.Options; +using MoneyMirror.PhysicalAssets; +using MoneyMirror.PhysicalAssets.Configuration; + +namespace MoneyMirror.Tests.PhysicalAssets; + +public class EbayMarketDataServiceTests +{ + private const string TokenResponse = + """{"access_token":"test-access-token","expires_in":7200,"token_type":"Application Access Token"}"""; + + [Fact] + public async Task SearchUsedListingsAsync_AuthenticatesThenSearchesAndMapsOnlyUsedOrRefurbishedUsdResults() + { + using var cache = new TemporarySearchCache(); + RequestSnapshot? tokenRequest = null; + RequestSnapshot? searchRequest = null; + var handler = new CallbackHandler(async (request, cancellationToken) => + { + if (request.RequestUri!.AbsolutePath.Contains("oauth2/token")) + { + tokenRequest = await RequestSnapshot.CaptureAsync(request, cancellationToken); + return JsonResponse(TokenResponse); + } + + searchRequest = await RequestSnapshot.CaptureAsync(request, cancellationToken); + return JsonResponse( + """ + {"itemSummaries":[ + {"title":"Used Fender CD-60S Acoustic Guitar","price":{"value":"130.00","currency":"USD"},"conditionId":"3000"}, + {"title":"New Fender CD-60S Acoustic Guitar","price":{"value":"300.00","currency":"USD"},"conditionId":"1000"}, + {"title":"Certified Refurbished Fender CD-60S","price":{"value":"145.00","currency":"USD"},"conditionId":"2010"}, + {"title":"For parts Fender CD-60S","price":{"value":"40.00","currency":"USD"},"conditionId":"7000"}, + {"title":"Used guitar, foreign listing","price":{"value":"110.00","currency":"GBP"},"conditionId":"3000"} + ]} + """ + ); + }); + var service = CreateService(handler, cache.Create(), new EbayTokenCache()); + + var listings = await service.SearchUsedListingsAsync("acoustic guitar", "Fender", "CD-60S"); + + Assert.Equal(2, listings.Count); + Assert.Contains( + listings, + item => item.PriceUsd == 130m && item.Source == "eBay" && item.Condition == "Used" + ); + Assert.Contains(listings, item => item.PriceUsd == 145m && item.Condition == "Refurbished"); + Assert.All(listings, item => Assert.NotNull(item.ListingTitle)); + + Assert.NotNull(tokenRequest); + Assert.Equal(HttpMethod.Post, tokenRequest.Method); + Assert.StartsWith("Basic ", tokenRequest.Authorization); + Assert.Contains("grant_type=client_credentials", tokenRequest.Body); + + Assert.NotNull(searchRequest); + Assert.Equal(HttpMethod.Get, searchRequest.Method); + Assert.Equal("Bearer test-access-token", searchRequest.Authorization); + Assert.Equal("EBAY_US", searchRequest.MarketplaceId); + Assert.Contains("q=acoustic%20guitar%20Fender%20CD-60S", searchRequest.Uri.Query); + Assert.Contains( + "filter=conditions%3A%7BUSED%7CCERTIFIED_REFURBISHED%7CSELLER_REFURBISHED%7D", + searchRequest.Uri.Query + ); + } + + [Fact] + public async Task SearchUsedListingsAsync_ReusesTheCachedTokenAcrossSearches() + { + using var cache = new TemporarySearchCache(); + var tokenRequests = 0; + var handler = new CallbackHandler((request, _) => + { + if (request.RequestUri!.AbsolutePath.Contains("oauth2/token")) + { + tokenRequests++; + return Task.FromResult(JsonResponse(TokenResponse)); + } + + return Task.FromResult(JsonResponse("""{"itemSummaries":[]}""")); + }); + var tokenCache = new EbayTokenCache(); + var service = CreateService(handler, cache.Create(), tokenCache); + + await service.SearchUsedListingsAsync("lamp", null, null); + await service.SearchUsedListingsAsync("chair", null, null); + + Assert.Equal(1, tokenRequests); + } + + [Fact] + public async Task SearchUsedListingsAsync_EmptyResponseReturnsNoListings() + { + using var cache = new TemporarySearchCache(); + var service = CreateService( + new CallbackHandler((request, _) => + Task.FromResult( + JsonResponse( + request.RequestUri!.AbsolutePath.Contains("oauth2/token") + ? TokenResponse + : "{}" + ) + ) + ), + cache.Create(), + new EbayTokenCache() + ); + + var listings = await service.SearchUsedListingsAsync("lamp", null, null); + + Assert.Empty(listings); + } + + [Fact] + public async Task SearchUsedListingsAsync_SearchApiErrorThrows() + { + using var cache = new TemporarySearchCache(); + var service = CreateService( + new CallbackHandler((request, _) => + Task.FromResult( + request.RequestUri!.AbsolutePath.Contains("oauth2/token") + ? JsonResponse(TokenResponse) + : new HttpResponseMessage(HttpStatusCode.Unauthorized) + { + Content = new StringContent( + """{"errors":[{"message":"Invalid access token"}]}""", + Encoding.UTF8, + "application/json" + ), + } + ) + ), + cache.Create(), + new EbayTokenCache() + ); + + var exception = await Assert.ThrowsAsync(() => + service.SearchUsedListingsAsync("lamp", null, null) + ); + + Assert.Contains("401", exception.Message); + } + + [Fact] + public async Task SearchUsedListingsAsync_TokenRequestFailureThrowsWithoutExposingTheSecret() + { + using var cache = new TemporarySearchCache(); + var service = CreateService( + new CallbackHandler( + (_, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.Unauthorized)) + ), + cache.Create(), + new EbayTokenCache(), + new EbayOptions + { + ClientId = "test-client-id", + ClientSecret = "super-secret-value", + } + ); + + var exception = await Assert.ThrowsAsync(() => + service.SearchUsedListingsAsync("lamp", null, null) + ); + + Assert.DoesNotContain("super-secret-value", exception.ToString()); + } + + [Fact] + public async Task SearchUsedListingsAsync_MissingCredentialsFailsWithoutCallingProvider() + { + using var cache = new TemporarySearchCache(); + var handler = new CallbackHandler( + (_, _) => throw new InvalidOperationException("Unexpected request.") + ); + var service = CreateService( + handler, + cache.Create(), + new EbayTokenCache(), + new EbayOptions() + ); + + var exception = await Assert.ThrowsAsync(() => + service.SearchUsedListingsAsync("lamp", null, null) + ); + + Assert.Contains("credentials are not configured", exception.Message); + } + + [Fact] + public async Task SearchUsedListingsAsync_ReusesDiskCacheForEquivalentQueriesAcrossServiceInstances() + { + using var cache = new TemporarySearchCache(); + var requestCount = 0; + var firstService = CreateService( + new CallbackHandler((request, _) => + { + requestCount++; + return Task.FromResult( + JsonResponse( + request.RequestUri!.AbsolutePath.Contains("oauth2/token") + ? TokenResponse + : """{"itemSummaries":[{"title":"Used Fender CD-60S guitar","price":{"value":"130.00","currency":"USD"},"conditionId":"3000"}]}""" + ) + ); + }), + cache.Create(), + new EbayTokenCache() + ); + var first = await firstService.SearchUsedListingsAsync("acoustic guitar", "Fender", "CD-60S"); + + // A new cache object simulates an app restart while reading the same persistent + // directory; a fresh token cache too, since it is per-process in production. + var secondService = CreateService( + new CallbackHandler( + (_, _) => throw new InvalidOperationException("A cached search should not call eBay.") + ), + cache.Create(), + new EbayTokenCache() + ); + var second = await secondService.SearchUsedListingsAsync( + " ACOUSTIC GUITAR ", + "fender", + "cd-60s" + ); + + Assert.Equal(2, requestCount); // one token fetch + one search, both on the first service + Assert.Equal(first, second); + } + + [Fact] + public async Task SearchUsedListingsAsync_CoalescesConcurrentIdenticalQueries() + { + using var cache = new TemporarySearchCache(); + var sharedSearchCache = cache.Create(); + var sharedTokenCache = new EbayTokenCache(); + var searchRequests = 0; + var requestStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseRequest = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var firstService = CreateService( + new CallbackHandler(async (request, cancellationToken) => + { + if (request.RequestUri!.AbsolutePath.Contains("oauth2/token")) + { + return JsonResponse(TokenResponse); + } + + Interlocked.Increment(ref searchRequests); + requestStarted.SetResult(); + await releaseRequest.Task.WaitAsync(cancellationToken); + return JsonResponse("""{"itemSummaries":[]}"""); + }), + sharedSearchCache, + sharedTokenCache + ); + var secondService = CreateService( + new CallbackHandler((request, _) => + request.RequestUri!.AbsolutePath.Contains("oauth2/token") + ? Task.FromResult(JsonResponse(TokenResponse)) + : throw new InvalidOperationException("A duplicate search was sent.") + ), + sharedSearchCache, + sharedTokenCache + ); + + var firstTask = firstService.SearchUsedListingsAsync("lamp", null, null); + await requestStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var secondTask = secondService.SearchUsedListingsAsync("lamp", null, null); + releaseRequest.SetResult(); + var results = await Task.WhenAll(firstTask, secondTask); + + Assert.Equal(1, searchRequests); + Assert.Empty(results[0]); + Assert.Empty(results[1]); + } + + private static EbayMarketDataService CreateService( + HttpMessageHandler handler, + MarketDataSearchCache cache, + EbayTokenCache tokenCache, + EbayOptions? options = null + ) => + new( + new HttpClient(handler), + Options.Create( + options + ?? new EbayOptions + { + ClientId = "test-client-id", + ClientSecret = "test-client-secret", + } + ), + cache, + tokenCache + ); + + private static HttpResponseMessage JsonResponse(string body) => + new(HttpStatusCode.OK) { Content = new StringContent(body, Encoding.UTF8, "application/json") }; + + private sealed class CallbackHandler( + Func> callback + ) : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken + ) => callback(request, cancellationToken); + } + + private sealed class TemporarySearchCache : IDisposable + { + private readonly string _directory = Path.Combine( + Path.GetTempPath(), + $"MoneyMirror-EbaySearchCacheTests-{Guid.NewGuid():N}" + ); + + public MarketDataSearchCache Create() => new(_directory); + + public void Dispose() + { + if (Directory.Exists(_directory)) + { + Directory.Delete(_directory, recursive: true); + } + } + } + + private sealed record RequestSnapshot( + HttpMethod Method, + Uri Uri, + string? Authorization, + string? MarketplaceId, + string Body + ) + { + public static async Task CaptureAsync( + HttpRequestMessage request, + CancellationToken cancellationToken + ) => + new( + request.Method, + request.RequestUri!, + request.Headers.Authorization?.ToString(), + request.Headers.TryGetValues("X-EBAY-C-MARKETPLACE-ID", out var values) + ? values.FirstOrDefault() + : null, + request.Content is null + ? string.Empty + : await request.Content.ReadAsStringAsync(cancellationToken) + ); + } +} diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs index 329eb4e..998e590 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/EvidenceBasedAssetValuationServiceTests.cs @@ -1,3 +1,4 @@ +using MoneyMirror.Ai; using MoneyMirror.PhysicalAssets; namespace MoneyMirror.Tests.PhysicalAssets; @@ -24,6 +25,7 @@ public async Task EstimateAsync_UsesMarketListingsAndReturnsStructuredEvidence() ]); var service = new EvidenceBasedAssetValuationService( source, + UnusedFallback(), new FixedTimeProvider(ValuationDate) ); @@ -45,6 +47,7 @@ public async Task EstimateAsync_NoListingsReturnsExplicitUnavailableLowConfidenc var source = new FakeMarketDataService([]); var service = new EvidenceBasedAssetValuationService( source, + UnusedFallback(), new FixedTimeProvider(ValuationDate) ); @@ -58,26 +61,54 @@ public async Task EstimateAsync_NoListingsReturnsExplicitUnavailableLowConfidenc } [Fact] - public async Task EstimateAsync_ProviderFailureBecomesDomainException() + public async Task EstimateAsync_ProviderFailureFallsBackToAiEstimate() { var source = new FakeMarketDataService( - new SerpApiMarketDataException("provider unavailable") + new EbayMarketDataException("provider unavailable") + ); + var fallback = new AiEstimatedValuationService( + new FakeLlmService( + """{"estimatedValueUsd": 75.00, "reasoning": "A typical used guitar in this condition sells for about $75."}""" + ), + new FixedTimeProvider(ValuationDate) ); var service = new EvidenceBasedAssetValuationService( source, + fallback, new FixedTimeProvider(ValuationDate) ); - var exception = await Assert.ThrowsAsync(() => - service.EstimateAsync("guitar", null, null) + var valuation = await service.EstimateAsync("guitar", null, null); + + Assert.Equal(75.00m, valuation.EstimatedValueUsd); + Assert.True(valuation.IsAiEstimated); + Assert.Equal("AI estimate (low confidence; not based on live market data)", valuation.SourceLabel); + Assert.Empty(valuation.Evidence); + } + + [Fact] + public async Task EstimateAsync_ProviderFailureAndFallbackFailureThrowsTheFallbacksException() + { + var source = new FakeMarketDataService( + new EbayMarketDataException("provider unavailable") + ); + var fallback = new AiEstimatedValuationService( + new ThrowingLlmService(), + new FixedTimeProvider(ValuationDate) + ); + var service = new EvidenceBasedAssetValuationService( + source, + fallback, + new FixedTimeProvider(ValuationDate) ); - Assert.Equal("Failed to retrieve comparable market listings.", exception.Message); - Assert.IsType(exception.InnerException); + await Assert.ThrowsAsync(() => + service.EstimateAsync("guitar", null, null) + ); } private sealed class FakeMarketDataService(IReadOnlyList result) - : ISerpApiMarketDataService + : IMarketDataService { private readonly Exception? _exception = null; @@ -107,6 +138,23 @@ public Task> SearchUsedListingsAsync( } } + private sealed class FakeLlmService(string completion) : ILlmService + { + public Task CompleteAsync(string prompt, CancellationToken cancellationToken = default) => + Task.FromResult(completion); + } + + private sealed class ThrowingLlmService : ILlmService + { + public Task CompleteAsync(string prompt, CancellationToken cancellationToken = default) => + Task.FromException(new LlmServiceException("The LLM is unavailable.")); + } + + // A fallback that must never actually be called - used by tests where the market + // data provider succeeds, so the fallback path should never execute. + private static AiEstimatedValuationService UnusedFallback() => + new(new ThrowingLlmService()); + private sealed class FixedTimeProvider(DateTimeOffset now) : TimeProvider { public override DateTimeOffset GetUtcNow() => now; diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs deleted file mode 100644 index fc3a58a..0000000 --- a/tests/MoneyMirror.Tests/PhysicalAssets/SerpApiMarketDataServiceTests.cs +++ /dev/null @@ -1,258 +0,0 @@ -using System.Net; -using System.Text; -using Microsoft.Extensions.Options; -using MoneyMirror.PhysicalAssets; -using MoneyMirror.PhysicalAssets.Configuration; - -namespace MoneyMirror.Tests.PhysicalAssets; - -public class SerpApiMarketDataServiceTests -{ - [Fact] - public async Task SearchUsedListingsAsync_QueriesGoogleAndMapsOnlyPricedUsedResults() - { - using var cache = new TemporarySearchCache(); - RequestSnapshot? requestSnapshot = null; - var handler = new CallbackHandler( - async (request, cancellationToken) => - { - requestSnapshot = await RequestSnapshot.CaptureAsync(request, cancellationToken); - return JsonResponse( - """{"shopping_results":[{"title":"Used Fender CD-60S Acoustic Guitar","source":"Reverb","price":"$130.00","second_hand_condition":"Used","link":"https://reverb.com/item/123"},{"title":"New Fender CD-60S Acoustic Guitar","source":"Retailer","price":"$300.00","second_hand_condition":"New","link":"https://store.example/guitar"},{"title":"Refurbished Fender CD-60S","source":"Reseller","price":"$145.00","second_hand_condition":"Refurbished","link":"https://reseller.example/guitar"}],"organic_results":[{"title":"Used Fender acoustic guitar listing","source":"Craigslist","snippet":"Clean condition, asking $110.00.","link":"https://pittsburgh.craigslist.org/item"},{"title":"Vintage guitar price guide","source":"Blog","snippet":"The new model costs $450.00."},{"title":"Used guitar price guide","source":"Blog","snippet":"Examples range from $80.00 to $100.00."}]}""" - ); - } - ); - var service = CreateService(handler, cache.Create()); - - var listings = await service.SearchUsedListingsAsync("acoustic guitar", "Fender", "CD-60S"); - - Assert.Equal(3, listings.Count); - Assert.Contains( - listings, - item => item.PriceUsd == 130m && item.Source == "Reverb" && item.Condition == "Used" - ); - Assert.Contains(listings, item => item.PriceUsd == 145m && item.Condition == "Refurbished"); - Assert.Contains(listings, item => item.PriceUsd == 110m && item.Source == "Craigslist"); - Assert.All(listings, item => Assert.NotNull(item.ListingTitle)); - - Assert.NotNull(requestSnapshot); - Assert.Equal(HttpMethod.Get, requestSnapshot.Method); - Assert.Contains("engine=google", requestSnapshot.Uri.Query); - Assert.Contains( - "q=acoustic%20guitar%20Fender%20CD-60S%20used%20for%20sale%20price", - requestSnapshot.Uri.Query - ); - Assert.Contains("gl=us", requestSnapshot.Uri.Query); - Assert.Contains("hl=en", requestSnapshot.Uri.Query); - Assert.Contains("api_key=test-api-key", requestSnapshot.Uri.Query); - Assert.Null(requestSnapshot.Authorization); - } - - [Fact] - public async Task SearchUsedListingsAsync_EmptyResponseReturnsNoListings() - { - using var cache = new TemporarySearchCache(); - var requestCount = 0; - var service = CreateService( - new CallbackHandler( - (_, _) => - { - requestCount++; - return Task.FromResult(JsonResponse("{}")); - } - ), - cache.Create() - ); - - var listings = await service.SearchUsedListingsAsync("lamp", null, null); - var repeatedListings = await service.SearchUsedListingsAsync("lamp", null, null); - - Assert.Empty(listings); - Assert.Empty(repeatedListings); - Assert.Equal(1, requestCount); - } - - [Fact] - public async Task SearchUsedListingsAsync_ApiErrorThrowsWithoutExposingApiKey() - { - using var cache = new TemporarySearchCache(); - var service = CreateService( - new CallbackHandler( - (_, _) => Task.FromResult(JsonResponse("""{"error":"Invalid API key"}""")) - ), - cache.Create() - ); - - var exception = await Assert.ThrowsAsync(() => - service.SearchUsedListingsAsync("lamp", null, null) - ); - - Assert.DoesNotContain("test-api-key", exception.ToString()); - } - - [Fact] - public async Task SearchUsedListingsAsync_MissingCredentialsFailsWithoutCallingProvider() - { - using var cache = new TemporarySearchCache(); - var handler = new CallbackHandler( - (_, _) => throw new InvalidOperationException("Unexpected request.") - ); - var service = CreateService(handler, cache.Create(), new SerpApiOptions()); - - var exception = await Assert.ThrowsAsync(() => - service.SearchUsedListingsAsync("lamp", null, null) - ); - - Assert.Contains("credentials are not configured", exception.Message); - } - - [Fact] - public async Task SearchUsedListingsAsync_ReusesDiskCacheForEquivalentQueriesAcrossServiceInstances() - { - using var cache = new TemporarySearchCache(); - var requestCount = 0; - var firstService = CreateService( - new CallbackHandler( - (_, _) => - { - requestCount++; - return Task.FromResult( - JsonResponse( - """{"shopping_results":[{"title":"Used Fender CD-60S guitar","source":"Reverb","price":"$130.00","second_hand_condition":"Used"}]}""" - ) - ); - } - ), - cache.Create() - ); - var first = await firstService.SearchUsedListingsAsync( - "acoustic guitar", - "Fender", - "CD-60S" - ); - - // A new cache object simulates an app restart while reading the same persistent directory. - var secondService = CreateService( - new CallbackHandler( - (_, _) => - throw new InvalidOperationException("A cached search should not call SerpApi.") - ), - cache.Create() - ); - var second = await secondService.SearchUsedListingsAsync( - " ACOUSTIC GUITAR ", - "fender", - "cd-60s" - ); - - Assert.Equal(1, requestCount); - Assert.Equal(first, second); - } - - [Fact] - public async Task SearchUsedListingsAsync_CoalescesConcurrentIdenticalQueries() - { - using var cache = new TemporarySearchCache(); - var sharedSearchCache = cache.Create(); - var requestCount = 0; - var requestStarted = new TaskCompletionSource( - TaskCreationOptions.RunContinuationsAsynchronously - ); - var releaseRequest = new TaskCompletionSource( - TaskCreationOptions.RunContinuationsAsynchronously - ); - var firstService = CreateService( - new CallbackHandler( - async (_, cancellationToken) => - { - Interlocked.Increment(ref requestCount); - requestStarted.SetResult(); - await releaseRequest.Task.WaitAsync(cancellationToken); - return JsonResponse("{}"); - } - ), - sharedSearchCache - ); - var secondService = CreateService( - new CallbackHandler( - (_, _) => throw new InvalidOperationException("A duplicate search was sent.") - ), - sharedSearchCache - ); - - var firstTask = firstService.SearchUsedListingsAsync("lamp", null, null); - await requestStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); - var secondTask = secondService.SearchUsedListingsAsync("lamp", null, null); - releaseRequest.SetResult(); - var results = await Task.WhenAll(firstTask, secondTask); - - Assert.Equal(1, requestCount); - Assert.Empty(results[0]); - Assert.Empty(results[1]); - } - - private static SerpApiMarketDataService CreateService( - HttpMessageHandler handler, - SerpApiSearchCache cache, - SerpApiOptions? options = null - ) => - new( - new HttpClient(handler), - Options.Create(options ?? new SerpApiOptions { ApiKey = "test-api-key" }), - cache - ); - - private static HttpResponseMessage JsonResponse(string body) => - new(HttpStatusCode.OK) - { - Content = new StringContent(body, Encoding.UTF8, "application/json"), - }; - - private sealed class CallbackHandler( - Func> callback - ) : HttpMessageHandler - { - protected override Task SendAsync( - HttpRequestMessage request, - CancellationToken cancellationToken - ) => callback(request, cancellationToken); - } - - private sealed class TemporarySearchCache : IDisposable - { - private readonly string _directory = Path.Combine( - Path.GetTempPath(), - $"MoneyMirror-SerpApiCacheTests-{Guid.NewGuid():N}" - ); - - public SerpApiSearchCache Create() => new(_directory); - - public void Dispose() - { - if (Directory.Exists(_directory)) - { - Directory.Delete(_directory, recursive: true); - } - } - } - - private sealed record RequestSnapshot(HttpMethod Method, Uri Uri, string? Authorization) - { - public static async Task CaptureAsync( - HttpRequestMessage request, - CancellationToken cancellationToken - ) - { - if (request.Content is not null) - { - await request.Content.ReadAsStringAsync(cancellationToken); - } - - return new RequestSnapshot( - request.Method, - request.RequestUri!, - request.Headers.Authorization?.ToString() - ); - } - } -}