Targeted cutout extraction and single-item cloud identification

Features/PhysicalAssets/LiveCameraScanner.razor

@@ -341,18 +341,38 @@
 
         try
         {
-            await using var streamRef = await _cameraModule.InvokeAsync<IJSStreamReference>("getCanvasBlob", _canvas);
+            var jsMask = polygon.Select(p => new { x = p.X, y = p.Y }).ToArray();
+            var jsBounds = new
+            {
+                x = _lastResult.Bounds.X,
+                y = _lastResult.Bounds.Y,
+                width = _lastResult.Bounds.Width,
+                height = _lastResult.Bounds.Height,
+            };
+
+            await using var streamRef = await _cameraModule.InvokeAsync<IJSStreamReference>(
+                "getCutoutBlob",
+                _canvas,
+                jsMask,
+                jsBounds
+            );
+
+            using var ms = new MemoryStream();
             await using var stream = await streamRef.OpenReadStreamAsync(10 * 1024 * 1024);
+            await stream.CopyToAsync(ms);
+            var cutoutBytes = ms.ToArray();
 
-            var imageReference = await ImageStorage.SaveAsync(stream, "live-scan.jpg");
+            ms.Position = 0;
+            var imageReference = await ImageStorage.SaveAsync(ms, "live-scan-cutout.png");
 
             var item = new LiveScannedItem(
                 polygon,
                 _lastResult.Bounds,
                 _lastResult.Confidence,
                 _lastResult.MaskWidth,
                 _lastResult.MaskHeight,
-                imageReference
+                imageReference,
+                cutoutBytes
             );
 
             await OnItemAccepted.InvokeAsync(item);

Features/PhysicalAssets/PhysicalAssets.razor

@@ -8,6 +8,7 @@
 @inject IAssetValuationService ValuationService
 @inject IPhysicalAssetRepository AssetRepository
 @inject IOptions<ImageUploadOptions> UploadOptions
+@inject ILogger<PhysicalAssets> Logger
 
 <PageTitle>Physical Assets</PageTitle>
 
@@ -249,26 +250,65 @@
         _sortOrder = "confidence-desc";
         _detections = items.Select(item => new ReviewableDetection(item.Detection) { ImageReference = item.ImageReference }).ToList();
     }
-    private void ReviewLiveScannedItem(LiveScannedItem item)
+    private async Task ReviewLiveScannedItem(LiveScannedItem item)
     {
         ClearError();
         _imageReference = null;
         _categoryFilter = _confidenceFilter = "all";
         _sortOrder = "confidence-desc";
 
-        var detection = new DetectedAsset(
-            "Scanned item",
-            item.Confidence,
-            item.Region,
-            null,
-            [],
-            item.Mask
-        );
+        DetectedAsset detection;
 
-        _detections = new List<ReviewableDetection>
+        if (item.CutoutBytes is { Length: > 0 } bytes)
         {
-            new(detection) { ImageReference = item.ImageReference, Included = true }
+            _isProcessing = true;
+            _processingMessage = "Identifying item with AI...";
+            StateHasChanged();
+
+            try
+            {
+                var identified = await DetectionService.IdentifyCutoutAsync(bytes, "image/png");
+                detection = identified with { Mask = item.Mask, Region = item.Region };
+            }
+            catch (Exception ex)
+            {
+                Logger.LogError(ex, "Single-item identification failed. Falling back to generic label.");
+                detection = new DetectedAsset(
+                    "Scanned item",
+                    item.Confidence,
+                    item.Region,
+                    null,
+                    [],
+                    item.Mask
+                );
+            }
+            finally
+            {
+                _isProcessing = false;
+                _processingMessage = null;
+            }
+        }
+        else
+        {
+            detection = new DetectedAsset(
+                "Scanned item",
+                item.Confidence,
+                item.Region,
+                null,
+                [],
+                item.Mask
+            );
+        }
+
+        var reviewable = new ReviewableDetection(detection)
+        {
+            ImageReference = item.ImageReference,
+            Included = true,
+            Brand = detection.Identification?.Brand ?? "",
+            Model = detection.Identification?.Model ?? ""
         };
+
+        _detections = new List<ReviewableDetection> { reviewable };
     }
 
     private IReadOnlyList<string> CategoryOptions => _detections is null

PhysicalAssets/IPhysicalAssetDetectionService.cs

@@ -19,6 +19,16 @@ public interface IPhysicalAssetDetectionService
     Task<IReadOnlyList<DetectedAsset>> DetectAsync(
         byte[] imageBytes,
         string mediaType,
-        CancellationToken cancellationToken = default);
-}
+        CancellationToken cancellationToken = default
+    );
 
+    /// <summary>
+    /// Identifies a single physical possession from an isolated cutout image (background stripped).
+    /// Focuses on brand, model/flavor/edition, condition, and category tags with high precision.
+    /// </summary>
+    Task<DetectedAsset> IdentifyCutoutAsync(
+        byte[] imageBytes,
+        string mediaType,
+        CancellationToken cancellationToken = default
+    );
+}

PhysicalAssets/NvidiaAssetDetectionService.cs

@@ -26,7 +26,8 @@ public NvidiaAssetDetectionService(IVisionService visionService)
     public async Task<IReadOnlyList<DetectedAsset>> DetectAsync(
         byte[] imageBytes,
         string mediaType,
-        CancellationToken cancellationToken = default)
+        CancellationToken cancellationToken = default
+    )
     {
         // Observed live against meta/llama-3.2-11b-vision-instruct: for the same
         // photo it honours "respond with ONLY a single JSON object" about four
@@ -54,27 +55,77 @@ public async Task<IReadOnlyList<DetectedAsset>> DetectAsync(
             : new AssetDetectionException(message, parseError) { RawResponse = completion };
     }
 
+    public async Task<DetectedAsset> IdentifyCutoutAsync(
+        byte[] imageBytes,
+        string mediaType,
+        CancellationToken cancellationToken = default
+    )
+    {
+        var completion = await RequestAsync(
+            imageBytes,
+            mediaType,
+            SingleItemPrompt,
+            cancellationToken
+        );
+        if (TryParseSingleItem(completion, out var asset, out _))
+        {
+            return asset;
+        }
+
+        completion = await RequestAsync(
+            imageBytes,
+            mediaType,
+            SingleItemRetryPrompt,
+            cancellationToken
+        );
+        if (TryParseSingleItem(completion, out asset, out var parseError))
+        {
+            return asset;
+        }
+
+        var general = await DetectAsync(imageBytes, mediaType, cancellationToken);
+        if (general.Count > 0)
+        {
+            return general[0];
+        }
+
+        const string message =
+            "The vision model returned a response that could not be parsed as a detected item.";
+        throw parseError is null
+            ? new AssetDetectionException(message) { RawResponse = completion }
+            : new AssetDetectionException(message, parseError) { RawResponse = completion };
+    }
+
     private async Task<string> RequestAsync(
         byte[] imageBytes,
         string mediaType,
         string prompt,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken
+    )
     {
         try
         {
-            return await _visionService.DetectAsync(imageBytes, mediaType, prompt, cancellationToken);
+            return await _visionService.DetectAsync(
+                imageBytes,
+                mediaType,
+                prompt,
+                cancellationToken
+            );
         }
         catch (VisionServiceException ex)
         {
             throw new AssetDetectionException(
-                "Failed to detect objects: the vision provider call failed.", ex);
+                "Failed to detect objects: the vision provider call failed.",
+                ex
+            );
         }
     }
 
     private static bool TryParseDetections(
         string completion,
         [NotNullWhen(true)] out IReadOnlyList<DetectedAsset>? detections,
-        out JsonException? parseError)
+        out JsonException? parseError
+    )
     {
         parseError = null;
 
@@ -88,8 +139,10 @@ private static bool TryParseDetections(
         {
             try
             {
-                if (JsonSerializer.Deserialize<DetectionResponse>(candidate, JsonOptions)
-                    is { Objects: { } objects })
+                if (
+                    JsonSerializer.Deserialize<DetectionResponse>(candidate, JsonOptions) is
+                    { Objects: { } objects }
+                )
                 {
                     detections = Normalize(objects);
                     return true;
@@ -107,8 +160,10 @@ private static bool TryParseDetections(
         {
             try
             {
-                if (JsonSerializer.Deserialize<DetectionResponse>(repaired, JsonOptions)
-                    is { Objects: { } repairedObjects })
+                if (
+                    JsonSerializer.Deserialize<DetectionResponse>(repaired, JsonOptions) is
+                    { Objects: { } repairedObjects }
+                )
                 {
                     detections = Normalize(repairedObjects);
                     return true;
@@ -124,6 +179,48 @@ private static bool TryParseDetections(
         return false;
     }
 
+    private static bool TryParseSingleItem(
+        string completion,
+        [NotNullWhen(true)] out DetectedAsset? asset,
+        out JsonException? parseError
+    )
+    {
+        parseError = null;
+        asset = null;
+
+        foreach (var candidate in JsonObjectCandidates(completion))
+        {
+            try
+            {
+                var single = JsonSerializer.Deserialize<DetectedAsset>(candidate, JsonOptions);
+                if (single is not null && !string.IsNullOrWhiteSpace(single.Label))
+                {
+                    asset = single.Tags is null ? single with { Tags = [] } : single;
+                    return true;
+                }
+
+                if (
+                    JsonSerializer.Deserialize<DetectionResponse>(candidate, JsonOptions) is
+                    { Objects: { Count: > 0 } objects }
+                )
+                {
+                    var first = objects[0];
+                    if (first is not null && !string.IsNullOrWhiteSpace(first.Label))
+                    {
+                        asset = first.Tags is null ? first with { Tags = [] } : first;
+                        return true;
+                    }
+                }
+            }
+            catch (JsonException ex)
+            {
+                parseError = ex;
+            }
+        }
+
+        return false;
+    }
+
     /// <summary>
     /// Rebuilds the response's bracket structure from the first <c>{</c> onwards,
     /// closing each open bracket with the closer it actually needs. Observed live,
@@ -268,6 +365,32 @@ the last must be a closing brace.
 
     private const string RetryPrompt = Prompt + RetrySuffix;
 
+    private const string SingleItemPrompt = """
+        You identify a single physical possession from an isolated cutout photo (background removed).
+        Carefully examine all visible logos, brand text, packaging labels, model badges, materials, and form factors.
+        Respond with ONLY a single JSON object - no markdown code fences, no commentary - matching exactly this shape:
+
+        {
+          "label": string,
+          "confidence": number between 0 and 1,
+          "identification": {
+            "brand": string|null,
+            "model": string|null,
+            "confidence": number between 0 and 1
+          },
+          "tags": [string]
+        }
+
+        Rules:
+        - "label" is a concise item category/name (for example: "Potato Chips", "Computer Monitor", "Running Shoes").
+        - "identification.brand" is the visible or inferred brand/manufacturer (for example: "Miss Vickie's", "Dell", "Nike"). If unknown, set null.
+        - "identification.model" is the specific product name, flavor, or model (for example: "Jalapeño", "UltraSharp U2720Q", "Air Force 1"). If unknown, set null.
+        - "tags" contains visible descriptors, such as color, flavor, material, or packaging (for example: ["green", "bag", "snack", "jalapeno"]).
+        - Never hallucinate brands or models not supported by visual evidence.
+        """;
+
+    private const string SingleItemRetryPrompt = SingleItemPrompt + RetrySuffix;
+
     /// <summary>
     /// Yields every balanced <c>{...}</c> span in <paramref name="text"/>, outermost
     /// first. The model regularly ignores the "JSON only" instruction and precedes
@@ -284,7 +407,6 @@ private static IEnumerable<string> JsonObjectCandidates(string text)
                 yield return candidate;
             }
         }
-
     }
 
     /// <summary>
@@ -349,9 +471,11 @@ private static IEnumerable<string> JsonObjectCandidates(string text)
     // there kills the Blazor circuit, which leaves the scan stuck on its
     // spinner, so normalize here and let every caller trust the shape.
     private static IReadOnlyList<DetectedAsset> Normalize(IReadOnlyList<DetectedAsset> objects) =>
-        [.. objects
-            .Where(detected => detected is not null)
-            .Select(detected => detected.Tags is null ? detected with { Tags = [] } : detected)];
+        [
+            .. objects
+                .Where(detected => detected is not null)
+                .Select(detected => detected.Tags is null ? detected with { Tags = [] } : detected),
+        ];
 
     private record DetectionResponse(IReadOnlyList<DetectedAsset> Objects);
 }

PhysicalAssets/SamSegmentation.cs

@@ -38,7 +38,8 @@ public record LiveScannedItem(
     double Confidence,
     int Width,
     int Height,
-    string? ImageReference = null
+    string? ImageReference = null,
+    byte[]? CutoutBytes = null
 );
 
 public interface ISamSegmentationEngine : IAsyncDisposable

tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs

@@ -30,6 +30,7 @@ public LiveCameraScannerTests()
         _module
             .Setup<NormalizedPoint>("normalizePoint", _ => true)
             .SetResult(new NormalizedPoint(0.5, 0.5));
+        _module.Setup<IJSStreamReference>("getCutoutBlob", _ => true).SetResult(new FakeStream());
         _module.Setup<IJSStreamReference>("getCanvasBlob", _ => true).SetResult(new FakeStream());
     }
 
@@ -147,6 +148,8 @@ await cut.Find(".viewfinder-stage")
 
         Assert.NotNull(acceptedItem);
         Assert.True(acceptedItem.Mask.Count >= 3);
+        Assert.NotNull(acceptedItem.CutoutBytes);
+        Assert.NotEmpty(acceptedItem.CutoutBytes);
         Assert.Equal(1, _storage.Calls);
     }