diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor b/Features/PhysicalAssets/LiveCameraScanner.razor index 78f4095..b1a4014 100644 --- a/Features/PhysicalAssets/LiveCameraScanner.razor +++ b/Features/PhysicalAssets/LiveCameraScanner.razor @@ -341,10 +341,29 @@ try { - await using var streamRef = await _cameraModule.InvokeAsync("getCanvasBlob", _canvas); + var jsMask = polygon.Select(p => new { x = p.X, y = p.Y }).ToArray(); + var jsBounds = new + { + x = _lastResult.Bounds.X, + y = _lastResult.Bounds.Y, + width = _lastResult.Bounds.Width, + height = _lastResult.Bounds.Height, + }; + + await using var streamRef = await _cameraModule.InvokeAsync( + "getCutoutBlob", + _canvas, + jsMask, + jsBounds + ); + + using var ms = new MemoryStream(); await using var stream = await streamRef.OpenReadStreamAsync(10 * 1024 * 1024); + await stream.CopyToAsync(ms); + var cutoutBytes = ms.ToArray(); - var imageReference = await ImageStorage.SaveAsync(stream, "live-scan.jpg"); + ms.Position = 0; + var imageReference = await ImageStorage.SaveAsync(ms, "live-scan-cutout.png"); var item = new LiveScannedItem( polygon, @@ -352,7 +371,8 @@ _lastResult.Confidence, _lastResult.MaskWidth, _lastResult.MaskHeight, - imageReference + imageReference, + cutoutBytes ); await OnItemAccepted.InvokeAsync(item); diff --git a/Features/PhysicalAssets/PhysicalAssets.razor b/Features/PhysicalAssets/PhysicalAssets.razor index 1b30050..bb181fe 100644 --- a/Features/PhysicalAssets/PhysicalAssets.razor +++ b/Features/PhysicalAssets/PhysicalAssets.razor @@ -8,6 +8,7 @@ @inject IAssetValuationService ValuationService @inject IPhysicalAssetRepository AssetRepository @inject IOptions UploadOptions +@inject ILogger Logger Physical Assets @@ -249,26 +250,65 @@ _sortOrder = "confidence-desc"; _detections = items.Select(item => new ReviewableDetection(item.Detection) { ImageReference = item.ImageReference }).ToList(); } - private void ReviewLiveScannedItem(LiveScannedItem item) + private async Task ReviewLiveScannedItem(LiveScannedItem item) { ClearError(); _imageReference = null; _categoryFilter = _confidenceFilter = "all"; _sortOrder = "confidence-desc"; - var detection = new DetectedAsset( - "Scanned item", - item.Confidence, - item.Region, - null, - [], - item.Mask - ); + DetectedAsset detection; - _detections = new List + if (item.CutoutBytes is { Length: > 0 } bytes) { - new(detection) { ImageReference = item.ImageReference, Included = true } + _isProcessing = true; + _processingMessage = "Identifying item with AI..."; + StateHasChanged(); + + try + { + var identified = await DetectionService.IdentifyCutoutAsync(bytes, "image/png"); + detection = identified with { Mask = item.Mask, Region = item.Region }; + } + catch (Exception ex) + { + Logger.LogError(ex, "Single-item identification failed. Falling back to generic label."); + detection = new DetectedAsset( + "Scanned item", + item.Confidence, + item.Region, + null, + [], + item.Mask + ); + } + finally + { + _isProcessing = false; + _processingMessage = null; + } + } + else + { + detection = new DetectedAsset( + "Scanned item", + item.Confidence, + item.Region, + null, + [], + item.Mask + ); + } + + var reviewable = new ReviewableDetection(detection) + { + ImageReference = item.ImageReference, + Included = true, + Brand = detection.Identification?.Brand ?? "", + Model = detection.Identification?.Model ?? "" }; + + _detections = new List { reviewable }; } private IReadOnlyList CategoryOptions => _detections is null diff --git a/PhysicalAssets/IPhysicalAssetDetectionService.cs b/PhysicalAssets/IPhysicalAssetDetectionService.cs index bc6d08f..a6ac8f8 100644 --- a/PhysicalAssets/IPhysicalAssetDetectionService.cs +++ b/PhysicalAssets/IPhysicalAssetDetectionService.cs @@ -19,6 +19,16 @@ public interface IPhysicalAssetDetectionService Task> DetectAsync( byte[] imageBytes, string mediaType, - CancellationToken cancellationToken = default); -} + CancellationToken cancellationToken = default + ); + /// + /// Identifies a single physical possession from an isolated cutout image (background stripped). + /// Focuses on brand, model/flavor/edition, condition, and category tags with high precision. + /// + Task IdentifyCutoutAsync( + byte[] imageBytes, + string mediaType, + CancellationToken cancellationToken = default + ); +} diff --git a/PhysicalAssets/NvidiaAssetDetectionService.cs b/PhysicalAssets/NvidiaAssetDetectionService.cs index cbee504..2e7ec80 100644 --- a/PhysicalAssets/NvidiaAssetDetectionService.cs +++ b/PhysicalAssets/NvidiaAssetDetectionService.cs @@ -26,7 +26,8 @@ public NvidiaAssetDetectionService(IVisionService visionService) public async Task> DetectAsync( byte[] imageBytes, string mediaType, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default + ) { // Observed live against meta/llama-3.2-11b-vision-instruct: for the same // photo it honours "respond with ONLY a single JSON object" about four @@ -54,27 +55,77 @@ public async Task> DetectAsync( : new AssetDetectionException(message, parseError) { RawResponse = completion }; } + public async Task IdentifyCutoutAsync( + byte[] imageBytes, + string mediaType, + CancellationToken cancellationToken = default + ) + { + var completion = await RequestAsync( + imageBytes, + mediaType, + SingleItemPrompt, + cancellationToken + ); + if (TryParseSingleItem(completion, out var asset, out _)) + { + return asset; + } + + completion = await RequestAsync( + imageBytes, + mediaType, + SingleItemRetryPrompt, + cancellationToken + ); + if (TryParseSingleItem(completion, out asset, out var parseError)) + { + return asset; + } + + var general = await DetectAsync(imageBytes, mediaType, cancellationToken); + if (general.Count > 0) + { + return general[0]; + } + + const string message = + "The vision model returned a response that could not be parsed as a detected item."; + throw parseError is null + ? new AssetDetectionException(message) { RawResponse = completion } + : new AssetDetectionException(message, parseError) { RawResponse = completion }; + } + private async Task RequestAsync( byte[] imageBytes, string mediaType, string prompt, - CancellationToken cancellationToken) + CancellationToken cancellationToken + ) { try { - return await _visionService.DetectAsync(imageBytes, mediaType, prompt, cancellationToken); + return await _visionService.DetectAsync( + imageBytes, + mediaType, + prompt, + cancellationToken + ); } catch (VisionServiceException ex) { throw new AssetDetectionException( - "Failed to detect objects: the vision provider call failed.", ex); + "Failed to detect objects: the vision provider call failed.", + ex + ); } } private static bool TryParseDetections( string completion, [NotNullWhen(true)] out IReadOnlyList? detections, - out JsonException? parseError) + out JsonException? parseError + ) { parseError = null; @@ -88,8 +139,10 @@ private static bool TryParseDetections( { try { - if (JsonSerializer.Deserialize(candidate, JsonOptions) - is { Objects: { } objects }) + if ( + JsonSerializer.Deserialize(candidate, JsonOptions) is + { Objects: { } objects } + ) { detections = Normalize(objects); return true; @@ -107,8 +160,10 @@ private static bool TryParseDetections( { try { - if (JsonSerializer.Deserialize(repaired, JsonOptions) - is { Objects: { } repairedObjects }) + if ( + JsonSerializer.Deserialize(repaired, JsonOptions) is + { Objects: { } repairedObjects } + ) { detections = Normalize(repairedObjects); return true; @@ -124,6 +179,48 @@ private static bool TryParseDetections( return false; } + private static bool TryParseSingleItem( + string completion, + [NotNullWhen(true)] out DetectedAsset? asset, + out JsonException? parseError + ) + { + parseError = null; + asset = null; + + foreach (var candidate in JsonObjectCandidates(completion)) + { + try + { + var single = JsonSerializer.Deserialize(candidate, JsonOptions); + if (single is not null && !string.IsNullOrWhiteSpace(single.Label)) + { + asset = single.Tags is null ? single with { Tags = [] } : single; + return true; + } + + if ( + JsonSerializer.Deserialize(candidate, JsonOptions) is + { Objects: { Count: > 0 } objects } + ) + { + var first = objects[0]; + if (first is not null && !string.IsNullOrWhiteSpace(first.Label)) + { + asset = first.Tags is null ? first with { Tags = [] } : first; + return true; + } + } + } + catch (JsonException ex) + { + parseError = ex; + } + } + + return false; + } + /// /// Rebuilds the response's bracket structure from the first { onwards, /// closing each open bracket with the closer it actually needs. Observed live, @@ -268,6 +365,32 @@ the last must be a closing brace. private const string RetryPrompt = Prompt + RetrySuffix; + private const string SingleItemPrompt = """ + You identify a single physical possession from an isolated cutout photo (background removed). + Carefully examine all visible logos, brand text, packaging labels, model badges, materials, and form factors. + Respond with ONLY a single JSON object - no markdown code fences, no commentary - matching exactly this shape: + + { + "label": string, + "confidence": number between 0 and 1, + "identification": { + "brand": string|null, + "model": string|null, + "confidence": number between 0 and 1 + }, + "tags": [string] + } + + Rules: + - "label" is a concise item category/name (for example: "Potato Chips", "Computer Monitor", "Running Shoes"). + - "identification.brand" is the visible or inferred brand/manufacturer (for example: "Miss Vickie's", "Dell", "Nike"). If unknown, set null. + - "identification.model" is the specific product name, flavor, or model (for example: "Jalapeño", "UltraSharp U2720Q", "Air Force 1"). If unknown, set null. + - "tags" contains visible descriptors, such as color, flavor, material, or packaging (for example: ["green", "bag", "snack", "jalapeno"]). + - Never hallucinate brands or models not supported by visual evidence. + """; + + private const string SingleItemRetryPrompt = SingleItemPrompt + RetrySuffix; + /// /// Yields every balanced {...} span in , outermost /// first. The model regularly ignores the "JSON only" instruction and precedes @@ -284,7 +407,6 @@ private static IEnumerable JsonObjectCandidates(string text) yield return candidate; } } - } /// @@ -349,9 +471,11 @@ private static IEnumerable JsonObjectCandidates(string text) // there kills the Blazor circuit, which leaves the scan stuck on its // spinner, so normalize here and let every caller trust the shape. private static IReadOnlyList Normalize(IReadOnlyList objects) => - [.. objects - .Where(detected => detected is not null) - .Select(detected => detected.Tags is null ? detected with { Tags = [] } : detected)]; + [ + .. objects + .Where(detected => detected is not null) + .Select(detected => detected.Tags is null ? detected with { Tags = [] } : detected), + ]; private record DetectionResponse(IReadOnlyList Objects); } diff --git a/PhysicalAssets/SamSegmentation.cs b/PhysicalAssets/SamSegmentation.cs index 807426c..0d261fb 100644 --- a/PhysicalAssets/SamSegmentation.cs +++ b/PhysicalAssets/SamSegmentation.cs @@ -38,7 +38,8 @@ public record LiveScannedItem( double Confidence, int Width, int Height, - string? ImageReference = null + string? ImageReference = null, + byte[]? CutoutBytes = null ); public interface ISamSegmentationEngine : IAsyncDisposable diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs index 27d0098..6304e03 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs @@ -30,6 +30,7 @@ public LiveCameraScannerTests() _module .Setup("normalizePoint", _ => true) .SetResult(new NormalizedPoint(0.5, 0.5)); + _module.Setup("getCutoutBlob", _ => true).SetResult(new FakeStream()); _module.Setup("getCanvasBlob", _ => true).SetResult(new FakeStream()); } @@ -147,6 +148,8 @@ await cut.Find(".viewfinder-stage") Assert.NotNull(acceptedItem); Assert.True(acceptedItem.Mask.Count >= 3); + Assert.NotNull(acceptedItem.CutoutBytes); + Assert.NotEmpty(acceptedItem.CutoutBytes); Assert.Equal(1, _storage.Calls); } diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/NvidiaAssetDetectionServiceTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/NvidiaAssetDetectionServiceTests.cs index 996590b..f2850fe 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/NvidiaAssetDetectionServiceTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/NvidiaAssetDetectionServiceTests.cs @@ -507,4 +507,75 @@ Hope this helps! var asset = Assert.Single(results); Assert.Equal("keyboard", asset.Label); } + + [Fact] + public async Task IdentifyCutoutAsync_ValidSingleItemJson_ReturnsIdentifiedAsset() + { + var response = """ + { + "label": "Potato Chips", + "confidence": 0.98, + "identification": { + "brand": "Miss Vickie's", + "model": "Jalapeño", + "confidence": 0.95 + }, + "tags": ["green", "bag", "snack"] + } + """; + var service = new NvidiaAssetDetectionService(new FakeVisionService(response)); + + var asset = await service.IdentifyCutoutAsync(Image, "image/png"); + + Assert.Equal("Potato Chips", asset.Label); + Assert.Equal(0.98, asset.Confidence); + Assert.NotNull(asset.Identification); + Assert.Equal("Miss Vickie's", asset.Identification.Brand); + Assert.Equal("Jalapeño", asset.Identification.Model); + Assert.Equal(3, asset.Tags.Count); + } + + [Fact] + public async Task IdentifyCutoutAsync_ArrayWrappedJson_FallsBackAndParsesFirstItem() + { + var response = """ + { + "objects": [ + { + "label": "Monitor", + "confidence": 0.99, + "identification": { + "brand": "Dell", + "model": "UltraSharp", + "confidence": 0.92 + }, + "tags": ["black", "screen"] + } + ] + } + """; + var service = new NvidiaAssetDetectionService(new FakeVisionService(response)); + + var asset = await service.IdentifyCutoutAsync(Image, "image/png"); + + Assert.Equal("Monitor", asset.Label); + Assert.Equal("Dell", asset.Identification?.Brand); + Assert.Equal("UltraSharp", asset.Identification?.Model); + } + + [Fact] + public async Task IdentifyCutoutAsync_RetriesOnFirstParseFailure_AndRecovers() + { + var vision = new SequencedVisionService( + "This is a photo of a Miss Vickie's chips bag.", + """{"label": "Chips", "confidence": 0.9, "identification": {"brand": "Miss Vickie's", "model": null, "confidence": 0.9}, "tags": ["snack"]}""" + ); + var service = new NvidiaAssetDetectionService(vision); + + var asset = await service.IdentifyCutoutAsync(Image, "image/png"); + + Assert.Equal(2, vision.Prompts.Count); + Assert.Equal("Chips", asset.Label); + Assert.Equal("Miss Vickie's", asset.Identification?.Brand); + } } diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs index 1cdf9d8..5106274 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs @@ -438,6 +438,21 @@ public Task> DetectAsync( string mediaType, CancellationToken cancellationToken = default ) => Detect(++_calls, cancellationToken); + + public Task IdentifyCutoutAsync( + byte[] imageBytes, + string mediaType, + CancellationToken cancellationToken = default + ) => + Task.FromResult( + new DetectedAsset( + "Chair", + 0.95, + new BoundingBox(0.1, 0.1, 0.3, 0.3), + new AssetIdentification("Brand", "Model", 0.95), + [] + ) + ); } private sealed class FakeStorage : IPossessionImageStorage diff --git a/tests/js/camera-scanner.test.mjs b/tests/js/camera-scanner.test.mjs index 6b84eb2..758397e 100644 --- a/tests/js/camera-scanner.test.mjs +++ b/tests/js/camera-scanner.test.mjs @@ -176,3 +176,46 @@ test("normalizePoint converts viewport coordinates to clamped [0, 1] range", () assert.equal(outside.x, 1); assert.equal(outside.y, 1); }); + +test("extractCutout creates cropped canvas with polygon clipping and returns blob", async () => { + let clipped = false; + let drawn = false; + const paths = []; + + globalThis.document = { + createElement: () => ({ + width: 0, + height: 0, + getContext: () => ({ + beginPath: () => {}, + moveTo: (x, y) => paths.push({ op: "move", x, y }), + lineTo: (x, y) => paths.push({ op: "line", x, y }), + closePath: () => {}, + clip: () => { + clipped = true; + }, + drawImage: () => { + drawn = true; + }, + }), + toBlob: (cb) => cb(new Blob(["cutout-png"], { type: "image/png" })), + }), + }; + + const sourceCanvas = { width: 1000, height: 1000 }; + const mask = [ + { x: 0.2, y: 0.2 }, + { x: 0.6, y: 0.2 }, + { x: 0.6, y: 0.6 }, + { x: 0.2, y: 0.6 }, + ]; + const bounds = { x: 0.2, y: 0.2, width: 0.4, height: 0.4 }; + + const blob = await camera.extractCutout(sourceCanvas, mask, bounds); + + assert.ok(blob instanceof Blob); + assert.equal(blob.type, "image/png"); + assert.ok(clipped); + assert.ok(drawn); + assert.equal(paths.length, 4); +}); diff --git a/wwwroot/js/camera-scanner.js b/wwwroot/js/camera-scanner.js index 28d2a35..6c31eb3 100644 --- a/wwwroot/js/camera-scanner.js +++ b/wwwroot/js/camera-scanner.js @@ -156,7 +156,9 @@ export function normalizePoint(clientX, clientY, containerElement, videoElement) } const video = - videoElement || containerElement.querySelector?.("video") || containerElement.querySelector?.("canvas"); + videoElement || + containerElement.querySelector?.("video") || + containerElement.querySelector?.("canvas"); const videoWidth = video?.videoWidth || video?.width || 0; const videoHeight = video?.videoHeight || video?.height || 0; @@ -221,3 +223,64 @@ export async function captureBlob(canvas) { export async function getCanvasBlob(canvas) { return await defaultCapture.toBlob(canvas); } + +export async function extractCutout(canvas, mask, bounds) { + if (!canvas) { + throw new Error("Canvas is required for cutout extraction."); + } + + let effectiveBounds = bounds; + if (!effectiveBounds && mask && mask.length >= 3) { + const xs = mask.map((p) => p.x); + const ys = mask.map((p) => p.y); + const minX = Math.max(0, Math.min(...xs)); + const minY = Math.max(0, Math.min(...ys)); + const maxX = Math.min(1, Math.max(...xs)); + const maxY = Math.min(1, Math.max(...ys)); + effectiveBounds = { x: minX, y: minY, width: maxX - minX, height: maxY - minY }; + } + + const srcW = canvas.width; + const srcH = canvas.height; + + const cropX = Math.floor((effectiveBounds?.x || 0) * srcW); + const cropY = Math.floor((effectiveBounds?.y || 0) * srcH); + const cropW = Math.max( + 1, + Math.min(srcW - cropX, Math.ceil((effectiveBounds?.width || 1) * srcW)), + ); + const cropH = Math.max( + 1, + Math.min(srcH - cropY, Math.ceil((effectiveBounds?.height || 1) * srcH)), + ); + + const cropCanvas = document.createElement("canvas"); + cropCanvas.width = cropW; + cropCanvas.height = cropH; + const ctx = cropCanvas.getContext("2d"); + + if (mask && mask.length >= 3) { + ctx.beginPath(); + const startX = mask[0].x * srcW - cropX; + const startY = mask[0].y * srcH - cropY; + ctx.moveTo(startX, startY); + for (let i = 1; i < mask.length; i++) { + ctx.lineTo(mask[i].x * srcW - cropX, mask[i].y * srcH - cropY); + } + ctx.closePath(); + ctx.clip(); + } + + ctx.drawImage(canvas, cropX, cropY, cropW, cropH, 0, 0, cropW, cropH); + + return new Promise((resolve, reject) => { + cropCanvas.toBlob( + (blob) => (blob ? resolve(blob) : reject(new Error("Could not create cutout blob."))), + "image/png", + ); + }); +} + +export async function getCutoutBlob(canvas, mask, bounds) { + return await extractCutout(canvas, mask, bounds); +}