From 8fd4800fc2fa1cf24364123e9d3f6c89a27cacd1 Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 03:46:24 -0400 Subject: [PATCH 1/9] Add live camera viewfinder with tap to segment --- .../PhysicalAssets/LiveCameraScanner.razor | 342 ++++++++++++++++++ .../LiveCameraScanner.razor.css | 153 ++++++++ Features/PhysicalAssets/PhysicalAssets.razor | 27 +- PhysicalAssets/SamSegmentation.cs | 9 + package.json | 3 +- .../PhysicalAssets/FakeSamEngine.cs | 75 ++++ .../PhysicalAssets/LiveCameraScannerTests.cs | 192 ++++++++++ .../PhysicalAssets/VideoScanPickerTests.cs | 257 ++++++++++--- tests/js/camera-scanner.test.mjs | 178 +++++++++ tests/js/mobile-sam.test.mjs | 4 +- wwwroot/js/camera-scanner.js | 189 ++++++++++ wwwroot/js/mobile-sam.js | 28 +- 12 files changed, 1372 insertions(+), 85 deletions(-) create mode 100644 Features/PhysicalAssets/LiveCameraScanner.razor create mode 100644 Features/PhysicalAssets/LiveCameraScanner.razor.css create mode 100644 tests/MoneyMirror.Tests/PhysicalAssets/FakeSamEngine.cs create mode 100644 tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs create mode 100644 tests/js/camera-scanner.test.mjs create mode 100644 wwwroot/js/camera-scanner.js diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor b/Features/PhysicalAssets/LiveCameraScanner.razor new file mode 100644 index 0000000..4bdc102 --- /dev/null +++ b/Features/PhysicalAssets/LiveCameraScanner.razor @@ -0,0 +1,342 @@ +@using MoneyMirror.PhysicalAssets +@using System.Globalization +@using System.Text +@implements IAsyncDisposable +@inject IJSRuntime JS +@inject ISamSegmentationEngine SamEngine +@inject IPossessionImageStorage ImageStorage +@inject ILogger Logger + +
+
+

Live camera scanner

+ @if (_isStreaming) + { + Camera active + } +
+ +

Point your camera at an item and tap it to snap an outline in real time.

+ +
+ @if (!_isStreaming && !_isFrozen) + { +
+

Camera is inactive.

+ +
+ } + + + + + @if (_lastResult?.Polygon is { Count: >= 3 } polygon) + { + + } + + @if (_isStreaming && !_isFrozen) + { +
Tap any item to segment
+ } + else if (_isFrozen && _isSegmenting) + { +
Segmenting...
+ } + else if (_isFrozen) + { +
Tap to refine (+/-) or accept item below
+ } +
+ + @if (_errorMessage is not null) + { + + } + +
+ @if (_isStreaming || _isFrozen) + { + + } + + @if (_isFrozen) + { +
+ + +
+ + + + + } + else if (_isStreaming) + { + + } +
+
+ +@code { + private const string CameraModulePath = "./js/camera-scanner.js"; + private const string FacingEnvironment = "environment"; + private const string FacingUser = "user"; + + [Parameter] public bool Disabled { get; set; } + [Parameter] public EventCallback OnItemAccepted { get; set; } + [Parameter] public EventCallback BusyChanged { get; set; } + + private ElementReference _viewfinder; + private ElementReference _video; + private ElementReference _canvas; + + private IJSObjectReference? _cameraModule; + private bool _isStreaming; + private bool _isFrozen; + private bool _isSegmenting; + private bool _isWorking; + private string _cameraFacing = FacingEnvironment; + private SamPromptType _currentPromptMode = SamPromptType.Positive; + private string? _errorMessage; + private SamSegmentationResult? _lastResult; + private readonly List _prompts = new(); + + public async Task StartCameraAsync() + { + _errorMessage = null; + + try + { + _cameraModule ??= await JS.InvokeAsync("import", CameraModulePath); + await _cameraModule.InvokeVoidAsync("startCamera", _video, _cameraFacing); + _isStreaming = true; + _isFrozen = false; + + _ = SamEngine.InitializeAsync(); + } + catch (Exception ex) + { + _errorMessage = ex.Message; + Logger.LogError(ex, "Failed to start camera stream."); + } + } + + public async Task StopCameraAsync() + { + if (_cameraModule is not null) + { + await _cameraModule.InvokeVoidAsync("stopCamera", _video); + } + + _isStreaming = false; + _isFrozen = false; + } + + public async Task SwitchCameraAsync() + { + _errorMessage = null; + + if (_cameraModule is null) + { + return; + } + + try + { + var nextFacing = _cameraFacing == FacingEnvironment ? FacingUser : FacingEnvironment; + await _cameraModule.InvokeVoidAsync("switchCamera", _video, _cameraFacing); + _cameraFacing = nextFacing; + } + catch (Exception ex) + { + _errorMessage = ex.Message; + } + } + + public async Task HandleTapAsync(MouseEventArgs e) + { + if (!_isStreaming && !_isFrozen) + { + return; + } + + if (_isWorking || _isSegmenting || _cameraModule is null) + { + return; + } + + var coords = await _cameraModule.InvokeAsync( + "normalizePoint", + e.ClientX, + e.ClientY, + _viewfinder + ); + + if (coords.X < 0 || coords.X > 1 || coords.Y < 0 || coords.Y > 1) + { + return; + } + + _isSegmenting = true; + await BusyChanged.InvokeAsync(true); + + try + { + if (!_isFrozen) + { + await _cameraModule.InvokeVoidAsync("freezeFrame", _video, _canvas); + _isFrozen = true; + + await SamEngine.EncodeFrameAsync(_canvas); + _prompts.Clear(); + + var prompt = new SamPointPrompt(coords.X, coords.Y, SamPromptType.Positive); + _prompts.Add(prompt); + + _lastResult = await SamEngine.DecodePointAsync(prompt.X, prompt.Y, prompt.Type); + } + else + { + var prompt = new SamPointPrompt(coords.X, coords.Y, _currentPromptMode); + _prompts.Add(prompt); + + _lastResult = await SamEngine.DecodePointsAsync(_prompts); + } + } + catch (Exception ex) + { + _errorMessage = ex.Message; + Logger.LogError(ex, "Tap segmentation failed."); + } + finally + { + _isSegmenting = false; + await BusyChanged.InvokeAsync(false); + } + } + + public void SetPromptMode(SamPromptType mode) + { + _currentPromptMode = mode; + } + + public async Task RetapAsync() + { + _prompts.Clear(); + _lastResult = null; + _isFrozen = false; + _errorMessage = null; + + await SamEngine.ResetPriorAsync(); + + if (_cameraModule is not null) + { + await _cameraModule.InvokeVoidAsync("unfreezeFrame", _video); + } + } + + public async Task AcceptItemAsync() + { + if (_lastResult?.Polygon is not { Count: >= 3 } polygon || _cameraModule is null) + { + return; + } + + _isWorking = true; + await BusyChanged.InvokeAsync(true); + + try + { + await using var streamRef = await _cameraModule.InvokeAsync("getCanvasBlob", _canvas); + await using var stream = await streamRef.OpenReadStreamAsync(10 * 1024 * 1024); + + var imageReference = await ImageStorage.SaveAsync(stream, "live-scan.jpg"); + + var item = new LiveScannedItem( + polygon, + _lastResult.Bounds, + _lastResult.Confidence, + _lastResult.MaskWidth, + _lastResult.MaskHeight, + imageReference + ); + + await OnItemAccepted.InvokeAsync(item); + await RetapAsync(); + } + catch (Exception ex) + { + _errorMessage = ex.Message; + Logger.LogError(ex, "Failed to accept segmented item."); + } + finally + { + _isWorking = false; + await BusyChanged.InvokeAsync(false); + } + } + + private string PolygonPoints(IReadOnlyList points) + { + var builder = new StringBuilder(); + + for (var i = 0; i < points.Count; i++) + { + if (i > 0) + { + builder.Append(' '); + } + + builder.Append((points[i].X * 100).ToString("F2", CultureInfo.InvariantCulture)); + builder.Append(','); + builder.Append((points[i].Y * 100).ToString("F2", CultureInfo.InvariantCulture)); + } + + return builder.ToString(); + } + + public async ValueTask DisposeAsync() + { + if (_cameraModule is not null) + { + try + { + await _cameraModule.InvokeVoidAsync("stopCamera", _video); + await _cameraModule.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + finally + { + _cameraModule = null; + } + } + } +} diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor.css b/Features/PhysicalAssets/LiveCameraScanner.razor.css new file mode 100644 index 0000000..26f7fd3 --- /dev/null +++ b/Features/PhysicalAssets/LiveCameraScanner.razor.css @@ -0,0 +1,153 @@ +.camera-scanner { + position: relative; + width: 100%; + margin-bottom: 1.5rem; +} + +.viewfinder-stage { + position: relative; + width: 100%; + max-width: 720px; + margin: 0 auto; + aspect-ratio: 4 / 3; + background-color: #0b0f19; + border-radius: 0.75rem; + overflow: hidden; + touch-action: none; + user-select: none; + -webkit-user-select: none; + overscroll-behavior: none; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.4); + cursor: crosshair; +} + +.camera-video, +.freeze-canvas { + width: 100%; + height: 100%; + object-fit: cover; + display: block; +} + +.freeze-canvas { + position: absolute; + top: 0; + left: 0; + z-index: 1; +} + +.contour-overlay { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + z-index: 2; + pointer-events: none; +} + +.glowing-contour { + fill: rgba(0, 220, 255, 0.25); + stroke: #00dcff; + stroke-width: 0.8; + stroke-linejoin: round; + stroke-linecap: round; + filter: drop-shadow(0 0 6px rgba(0, 220, 255, 0.85)) + drop-shadow(0 0 12px rgba(0, 220, 255, 0.45)); + animation: contour-pulse 2s ease-in-out infinite alternate; +} + +@keyframes contour-pulse { + 0% { + fill: rgba(0, 220, 255, 0.2); + stroke: #00dcff; + filter: drop-shadow(0 0 4px rgba(0, 220, 255, 0.7)); + } + 100% { + fill: rgba(0, 220, 255, 0.35); + stroke: #33e6ff; + filter: drop-shadow(0 0 8px rgba(0, 220, 255, 1)) + drop-shadow(0 0 16px rgba(0, 220, 255, 0.6)); + } +} + +.prompt-marker { + stroke-width: 0.4; + transition: transform 0.15s ease-out; +} + +.prompt-marker.positive { + fill: #10b981; + stroke: #ffffff; + filter: drop-shadow(0 0 3px rgba(16, 185, 129, 0.9)); +} + +.prompt-marker.negative { + fill: #ef4444; + stroke: #ffffff; + filter: drop-shadow(0 0 3px rgba(239, 68, 68, 0.9)); +} + +.viewfinder-hint { + position: absolute; + bottom: 12px; + left: 50%; + transform: translateX(-50%); + z-index: 3; + background-color: rgba(15, 23, 42, 0.75); + backdrop-filter: blur(4px); + color: #f8fafc; + padding: 0.35rem 0.75rem; + border-radius: 9999px; + font-size: 0.85rem; + pointer-events: none; + white-space: nowrap; +} + +.viewfinder-placeholder { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + width: 100%; + height: 100%; + color: #94a3b8; + padding: 2rem; + text-align: center; +} + +.action-bar { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + align-items: center; + margin-top: 1rem; +} + +.mode-toggle { + display: inline-flex; + border-radius: 0.375rem; + overflow: hidden; + border: 1px solid #cbd5e1; +} + +.mode-btn { + padding: 0.25rem 0.75rem; + font-size: 0.875rem; + border: none; + background-color: #f8fafc; + color: #334155; + cursor: pointer; +} + +.mode-btn.active.positive { + background-color: #10b981; + color: #ffffff; + font-weight: 600; +} + +.mode-btn.active.negative { + background-color: #ef4444; + color: #ffffff; + font-weight: 600; +} diff --git a/Features/PhysicalAssets/PhysicalAssets.razor b/Features/PhysicalAssets/PhysicalAssets.razor index c9b8043..de069b4 100644 --- a/Features/PhysicalAssets/PhysicalAssets.razor +++ b/Features/PhysicalAssets/PhysicalAssets.razor @@ -26,6 +26,8 @@ + + @if (_isProcessing) {
@@ -220,7 +222,8 @@ private bool _isProcessing; private bool _videoBusy; - private bool IsWorking => _isProcessing || (_detections?.Any(d => d.IsSaving || d.IsEstimating) ?? false); + private bool _cameraBusy; + private bool IsWorking => _isProcessing || _videoBusy || _cameraBusy || (_detections?.Any(d => d.IsSaving || d.IsEstimating) ?? false); private string? _processingMessage; private string? _errorMessage; private string? _errorRawResponse; @@ -240,6 +243,28 @@ _detections = items.Select(item => new ReviewableDetection(item.Detection) { ImageReference = item.ImageReference }).ToList(); } + private void ReviewLiveScannedItem(LiveScannedItem item) + { + ClearError(); + _imageReference = null; + _categoryFilter = _confidenceFilter = "all"; + _sortOrder = "confidence-desc"; + + var detection = new DetectedAsset( + "Scanned item", + item.Confidence, + item.Region, + null, + [], + item.Mask + ); + + _detections = new List + { + new(detection) { ImageReference = item.ImageReference, Included = true } + }; + } + private IReadOnlyList CategoryOptions => _detections is null ? Array.Empty() : _detections diff --git a/PhysicalAssets/SamSegmentation.cs b/PhysicalAssets/SamSegmentation.cs index 4b031e0..23e5f23 100644 --- a/PhysicalAssets/SamSegmentation.cs +++ b/PhysicalAssets/SamSegmentation.cs @@ -32,6 +32,15 @@ int Height public record SamEncodeResult(double ElapsedMs, string? Device, int Width, int Height); +public record LiveScannedItem( + IReadOnlyList Mask, + BoundingBox Region, + double Confidence, + int Width, + int Height, + string? ImageReference = null +); + public interface ISamSegmentationEngine : IAsyncDisposable { ValueTask InitializeAsync(CancellationToken cancellationToken = default); diff --git a/package.json b/package.json index eb99efd..c99714e 100644 --- a/package.json +++ b/package.json @@ -5,9 +5,10 @@ "description": "Dev tooling for Money Mirror (formatters only — the app itself is ASP.NET Core).", "license": "UNLICENSED", "scripts": { - "test": "npm run test:video && npm run test:sam", + "test": "npm run test:video && npm run test:sam && npm run test:camera", "test:video": "node --test tests/js/video-scan.test.mjs", "test:sam": "node --test tests/js/mobile-sam.test.mjs", + "test:camera": "node --test tests/js/camera-scanner.test.mjs", "format": "npm run format:cs && npm run format:css", "format:cs": "dotnet csharpier format .", "format:css": "prettier --write \"**/*.css\"", diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/FakeSamEngine.cs b/tests/MoneyMirror.Tests/PhysicalAssets/FakeSamEngine.cs new file mode 100644 index 0000000..b294ab5 --- /dev/null +++ b/tests/MoneyMirror.Tests/PhysicalAssets/FakeSamEngine.cs @@ -0,0 +1,75 @@ +using Microsoft.AspNetCore.Components; +using MoneyMirror.PhysicalAssets; + +namespace MoneyMirror.Tests.PhysicalAssets; + +public sealed class FakeSamEngine : ISamSegmentationEngine +{ + public int InitCalls { get; private set; } + public int EncodeCalls { get; private set; } + public int DecodePointCalls { get; private set; } + public int DecodePointsCalls { get; private set; } + public int ResetCalls { get; private set; } + public int DisposeCalls { get; private set; } + + public SamSegmentationResult Result { get; set; } = + new( + [new(0.2, 0.2), new(0.8, 0.2), new(0.8, 0.8), new(0.2, 0.8)], + new(0.2, 0.2, 0.6, 0.6), + 0.96, + 100, + 100, + 12.5 + ); + + public ValueTask InitializeAsync(CancellationToken cancellationToken = default) + { + InitCalls++; + return ValueTask.FromResult(new SamEngineStatus("ready", "webgpu", false, 0, 0)); + } + + public ValueTask GetStatusAsync( + CancellationToken cancellationToken = default + ) => ValueTask.FromResult(new SamEngineStatus("ready", "webgpu", true, 1024, 768)); + + public ValueTask EncodeFrameAsync( + ElementReference frameSource, + CancellationToken cancellationToken = default + ) + { + EncodeCalls++; + return ValueTask.FromResult(new SamEncodeResult(25.0, "webgpu", 1280, 720)); + } + + public ValueTask DecodePointAsync( + double x, + double y, + SamPromptType type = SamPromptType.Positive, + CancellationToken cancellationToken = default + ) + { + DecodePointCalls++; + return ValueTask.FromResult(Result); + } + + public ValueTask DecodePointsAsync( + IReadOnlyList points, + CancellationToken cancellationToken = default + ) + { + DecodePointsCalls++; + return ValueTask.FromResult(Result); + } + + public ValueTask ResetPriorAsync(CancellationToken cancellationToken = default) + { + ResetCalls++; + return ValueTask.CompletedTask; + } + + public ValueTask DisposeAsync() + { + DisposeCalls++; + return ValueTask.CompletedTask; + } +} diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs new file mode 100644 index 0000000..27d0098 --- /dev/null +++ b/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs @@ -0,0 +1,192 @@ +using Bunit; +using Microsoft.AspNetCore.Components; +using Microsoft.AspNetCore.Components.Web; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.JSInterop; +using MoneyMirror.Features.PhysicalAssets; +using MoneyMirror.PhysicalAssets; + +namespace MoneyMirror.Tests.PhysicalAssets; + +public sealed class LiveCameraScannerTests : IDisposable +{ + private readonly BunitContext _context = new(); + private readonly BunitJSModuleInterop _module; + private readonly FakeSamEngine _samEngine = new(); + private readonly FakeStorage _storage = new(); + + public LiveCameraScannerTests() + { + _context.Services.AddLogging(); + _context.Services.AddSingleton(_samEngine); + _context.Services.AddSingleton(_storage); + + _module = _context.JSInterop.SetupModule("./js/camera-scanner.js"); + _module.SetupVoid("startCamera", _ => true).SetVoidResult(); + _module.SetupVoid("stopCamera", _ => true).SetVoidResult(); + _module.SetupVoid("switchCamera", _ => true).SetVoidResult(); + _module.SetupVoid("freezeFrame", _ => true).SetVoidResult(); + _module.SetupVoid("unfreezeFrame", _ => true).SetVoidResult(); + _module + .Setup("normalizePoint", _ => true) + .SetResult(new NormalizedPoint(0.5, 0.5)); + _module.Setup("getCanvasBlob", _ => true).SetResult(new FakeStream()); + } + + [Fact] + public void RendersInactivePlaceholderInitially() + { + var cut = _context.Render(); + + Assert.Contains("Start live camera", cut.Markup); + Assert.DoesNotContain("glowing-contour", cut.Markup); + } + + [Fact] + public async Task StartCamera_InitializesEngineAndStream() + { + var cut = _context.Render(); + + await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); + + _module.VerifyInvoke("startCamera", 1); + Assert.True(_samEngine.InitCalls >= 1); + Assert.Contains("Camera active", cut.Markup); + } + + [Fact] + public async Task Tap_FreezesAndSegmentsFrame() + { + var cut = _context.Render(); + await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); + + await cut.Find(".viewfinder-stage") + .ClickAsync(new MouseEventArgs { ClientX = 200, ClientY = 150 }); + + _module.VerifyInvoke("freezeFrame", 1); + Assert.Equal(1, _samEngine.EncodeCalls); + Assert.Equal(1, _samEngine.DecodePointCalls); + Assert.Contains("glowing-contour", cut.Markup); + Assert.Contains("prompt-marker", cut.Markup); + } + + [Fact] + public async Task RefinementTap_CallsDecodePoints() + { + var cut = _context.Render(); + await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); + + await cut.Find(".viewfinder-stage") + .ClickAsync(new MouseEventArgs { ClientX = 200, ClientY = 150 }); + await cut.Find(".viewfinder-stage") + .ClickAsync(new MouseEventArgs { ClientX = 220, ClientY = 170 }); + + Assert.Equal(1, _samEngine.DecodePointsCalls); + } + + [Fact] + public async Task SwitchPromptMode_TogglesMode() + { + var cut = _context.Render(); + await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); + await cut.Find(".viewfinder-stage") + .ClickAsync(new MouseEventArgs { ClientX = 200, ClientY = 150 }); + + var excludeButton = cut.FindAll("button.mode-btn") + .Single(b => b.TextContent.Contains("Exclude")); + await excludeButton.ClickAsync(new MouseEventArgs()); + + var updatedButton = cut.FindAll("button.mode-btn") + .Single(b => b.TextContent.Contains("Exclude")); + Assert.Contains("active negative", updatedButton.ClassName); + } + + [Fact] + public async Task Retap_ResetsFrameAndPrior() + { + var cut = _context.Render(); + await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); + await cut.Find(".viewfinder-stage") + .ClickAsync(new MouseEventArgs { ClientX = 200, ClientY = 150 }); + + var retapButton = cut.FindAll("button").Single(b => b.TextContent.Contains("Retap")); + await retapButton.ClickAsync(new MouseEventArgs()); + + _module.VerifyInvoke("unfreezeFrame", 1); + Assert.Equal(1, _samEngine.ResetCalls); + Assert.DoesNotContain("glowing-contour", cut.Markup); + } + + [Fact] + public async Task SwitchCamera_CallsJsSwitchCamera() + { + var cut = _context.Render(); + await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); + + var switchButton = cut.FindAll("button") + .Single(b => b.TextContent.Contains("Switch camera")); + await switchButton.ClickAsync(new MouseEventArgs()); + + _module.VerifyInvoke("switchCamera", 1); + } + + [Fact] + public async Task AcceptItem_SavesImageAndEmitsEvent() + { + LiveScannedItem? acceptedItem = null; + var cut = _context.Render(parameters => + parameters.Add(p => p.OnItemAccepted, (LiveScannedItem item) => acceptedItem = item) + ); + + await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); + await cut.Find(".viewfinder-stage") + .ClickAsync(new MouseEventArgs { ClientX = 200, ClientY = 150 }); + + var acceptButton = cut.FindAll("button").Single(b => b.TextContent.Contains("Accept item")); + await acceptButton.ClickAsync(new MouseEventArgs()); + + Assert.NotNull(acceptedItem); + Assert.True(acceptedItem.Mask.Count >= 3); + Assert.Equal(1, _storage.Calls); + } + + public void Dispose() + { + _context.Dispose(); + } + + private sealed class FakeStream : IJSStreamReference + { + public long Length => 3; + + public ValueTask OpenReadStreamAsync( + long maxAllowedSize = 512000, + CancellationToken cancellationToken = default + ) => ValueTask.FromResult(new MemoryStream([1, 2, 3])); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class FakeStorage : IPossessionImageStorage + { + public List References { get; } = []; + public int Calls { get; private set; } + + public Task SaveAsync( + Stream content, + string fileName, + CancellationToken cancellationToken = default + ) + { + Calls++; + var reference = $"live-scan-{Calls}.jpg"; + References.Add(reference); + return Task.FromResult(reference); + } + + public Task OpenReadAsync( + string reference, + CancellationToken cancellationToken = default + ) => Task.FromResult(null); + } +} diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs index 9036594..47786f1 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs @@ -26,7 +26,9 @@ public sealed class VideoScanPickerTests : IDisposable public VideoScanPickerTests() { _connection.Open(); - _db = new MoneyMirrorDbContext(new DbContextOptionsBuilder().UseSqlite(_connection).Options); + _db = new MoneyMirrorDbContext( + new DbContextOptionsBuilder().UseSqlite(_connection).Options + ); _db.Database.EnsureCreated(); _repository = new EfPhysicalAssetRepository(_db); _context.Services.AddLogging(); @@ -36,6 +38,7 @@ public VideoScanPickerTests() _context.Services.AddSingleton(_storage); _context.Services.AddSingleton(_valuation); _context.Services.AddSingleton(_repository); + _context.Services.AddSingleton(new FakeSamEngine()); _module = _context.JSInterop.SetupModule("./js/video-scan.js"); _module.Setup("prepare", _ => true).SetResult([0.5, 1.5]); _module.Setup("frameUrl", _ => true).SetResult("blob:test-frame"); @@ -49,34 +52,60 @@ public async Task SelectedItemsOnly_GetSeparateCrops_AndPersistThroughExistingRe { var page = _context.Render(); page.Find("#room-video").Change("room.mp4"); - await page.FindAll("button").Single(b => b.TextContent == "Find items in video").ClickAsync(new MouseEventArgs()); + await page.FindAll("button") + .Single(b => b.TextContent == "Find items in video") + .ClickAsync(new MouseEventArgs()); page.WaitForAssertion(() => Assert.Equal(2, page.FindAll(".item-region").Count)); Assert.Empty(_storage.References); Assert.Empty(await _repository.GetAllAsync()); // Two adjacent sightings of each item share selection. No items start selected. - Assert.All(page.FindAll(".item-region"), b => Assert.Equal("false", b.GetAttribute("aria-pressed"))); + Assert.All( + page.FindAll(".item-region"), + b => Assert.Equal("false", b.GetAttribute("aria-pressed")) + ); page.FindAll(".item-region")[0].Click(); page.FindAll(".item-region")[1].Click(); page.FindAll("button").Single(b => b.TextContent.Trim() == "1.5 s").Click(); - Assert.All(page.FindAll(".item-region"), b => Assert.Equal("true", b.GetAttribute("aria-pressed"))); - await page.FindAll("button").Single(b => b.TextContent.Trim() == "Review selected items").ClickAsync(new MouseEventArgs()); - page.WaitForAssertion(() => Assert.Equal(2, page.FindAll("img[src^='/api/possession-images/']").Count)); + Assert.All( + page.FindAll(".item-region"), + b => Assert.Equal("true", b.GetAttribute("aria-pressed")) + ); + await page.FindAll("button") + .Single(b => b.TextContent.Trim() == "Review selected items") + .ClickAsync(new MouseEventArgs()); + page.WaitForAssertion(() => + Assert.Equal(2, page.FindAll("img[src^='/api/possession-images/']").Count) + ); Assert.Equal(2, _storage.References.Count); Assert.Equal(2, _module.Invocations.Count(i => i.Identifier == "cropStream")); Assert.Empty(await _repository.GetAllAsync()); // Reviewing is not inventory consent. for (var i = 0; i < 2; i++) { - await page.FindAll("button").Where(b => b.TextContent.Trim() == "Estimate value").ElementAt(i).ClickAsync(new MouseEventArgs()); - page.WaitForAssertion(() => Assert.Single(page.FindAll("button"), b => b.TextContent.Trim() == "Save as new item")); - var saveButton = page.FindAll("button").Single(b => b.TextContent.Trim() == "Save as new item"); + await page.FindAll("button") + .Where(b => b.TextContent.Trim() == "Estimate value") + .ElementAt(i) + .ClickAsync(new MouseEventArgs()); + page.WaitForAssertion(() => + Assert.Single( + page.FindAll("button"), + b => b.TextContent.Trim() == "Save as new item" + ) + ); + var saveButton = page.FindAll("button") + .Single(b => b.TextContent.Trim() == "Save as new item"); Assert.False(saveButton.HasAttribute("disabled"), page.Markup); await saveButton.ClickAsync(new MouseEventArgs()); Assert.DoesNotContain("Failed to save to inventory:", page.Markup); Assert.Equal(i + 1, (await _repository.GetAllAsync()).Count); - page.WaitForAssertion(() => Assert.Equal(i + 1, - page.FindAll(".alert-success").Count(a => a.TextContent.Contains("Saved to inventory.")))); + page.WaitForAssertion(() => + Assert.Equal( + i + 1, + page.FindAll(".alert-success") + .Count(a => a.TextContent.Contains("Saved to inventory.")) + ) + ); } var saved = await _repository.GetAllAsync(); @@ -89,12 +118,17 @@ public async Task SelectedItemsOnly_GetSeparateCrops_AndPersistThroughExistingRe public void UnselectedItems_AreNotCroppedOrPassedToReview() { IReadOnlyList? reviewed = null; - var picker = _context.Render(p => p.Add(c => c.OnSelected, items => reviewed = items)); + var picker = _context.Render(p => + p.Add(c => c.OnSelected, items => reviewed = items) + ); picker.Find("#room-video").Change("room.mp4"); picker.Find("button").Click(); picker.WaitForAssertion(() => Assert.Equal(2, picker.FindAll(".item-region").Count)); picker.FindAll(".item-region")[1].Click(); - picker.FindAll("button").Single(b => b.TextContent.Trim() == "Review selected items").Click(); + picker + .FindAll("button") + .Single(b => b.TextContent.Trim() == "Review selected items") + .Click(); picker.WaitForAssertion(() => Assert.NotNull(reviewed)); Assert.Equal("Lamp", Assert.Single(reviewed!).Detection.Label); Assert.Single(_storage.References); @@ -105,16 +139,21 @@ public void UnselectedItems_AreNotCroppedOrPassedToReview() [Fact] public void SelectHighConfidence_SelectsItemsAboveThreshold_AndDeselectAllClears() { - _detector.Detect = (_, _) => Task.FromResult>([ - new("Chair", 0.9, new(0.1, 0.1, 0.3, 0.3), null, []), - new("Pillow", 0.5, new(0.6, 0.1, 0.2, 0.4), null, [])]); + _detector.Detect = (_, _) => + Task.FromResult>([ + new("Chair", 0.9, new(0.1, 0.1, 0.3, 0.3), null, []), + new("Pillow", 0.5, new(0.6, 0.1, 0.2, 0.4), null, []), + ]); var picker = _context.Render(); picker.Find("#room-video").Change("room.mp4"); picker.Find("button").Click(); picker.WaitForAssertion(() => Assert.Equal(2, picker.FindAll(".item-region").Count)); - picker.FindAll("button").Single(b => b.TextContent.Trim() == "Select high-confidence items").Click(); + picker + .FindAll("button") + .Single(b => b.TextContent.Trim() == "Select high-confidence items") + .Click(); Assert.Contains(picker.FindAll(".item-region")[0].ClassList, c => c == "selected"); Assert.DoesNotContain(picker.FindAll(".item-region")[1].ClassList, c => c == "selected"); @@ -126,9 +165,17 @@ public void SelectHighConfidence_SelectsItemsAboveThreshold_AndDeselectAllClears [Fact] public void SegmentedItem_RendersPolygonOverlay_AndPassesMaskToCropStream() { - IReadOnlyList mask = [new(0.1, 0.1), new(0.4, 0.1), new(0.4, 0.4), new(0.1, 0.4)]; - _detector.Detect = (_, _) => Task.FromResult>([ - new("Chair", 0.9, new(0.1, 0.1, 0.3, 0.3), null, [], mask)]); + IReadOnlyList mask = + [ + new(0.1, 0.1), + new(0.4, 0.1), + new(0.4, 0.4), + new(0.1, 0.4), + ]; + _detector.Detect = (_, _) => + Task.FromResult>([ + new("Chair", 0.9, new(0.1, 0.1, 0.3, 0.3), null, [], mask), + ]); var picker = _context.Render(); picker.Find("#room-video").Change("room.mp4"); @@ -142,7 +189,10 @@ public void SegmentedItem_RendersPolygonOverlay_AndPassesMaskToCropStream() polygon.Click(); Assert.Contains("selected", polygon.ClassList); - picker.FindAll("button").Single(b => b.TextContent.Trim() == "Review selected items").Click(); + picker + .FindAll("button") + .Single(b => b.TextContent.Trim() == "Review selected items") + .Click(); picker.WaitForAssertion(() => Assert.Single(_storage.References)); var crop = Assert.Single(_module.Invocations, i => i.Identifier == "cropStream"); @@ -152,20 +202,32 @@ public void SegmentedItem_RendersPolygonOverlay_AndPassesMaskToCropStream() [Fact] public void MissingBoxes_UseCheckbox_AndFailedFramesDoNotDiscardSuccessfulOnes() { - _detector.Detect = (call, _) => call == 1 - ? Task.FromException>(new AssetDetectionException("Provider unavailable")) - : Task.FromResult>([new("Chair", 0.9, null, null, [])]); + _detector.Detect = (call, _) => + call == 1 + ? Task.FromException>( + new AssetDetectionException("Provider unavailable") + ) + : Task.FromResult>([ + new("Chair", 0.9, null, null, []), + ]); var picker = _context.Render(); picker.Find("#room-video").Change("room.mp4"); picker.Find("button").Click(); - picker.WaitForAssertion(() => Assert.Contains("1 frame(s) could not be analyzed", picker.Markup)); + picker.WaitForAssertion(() => + Assert.Contains("1 frame(s) could not be analyzed", picker.Markup) + ); picker.FindAll("button").Single(b => b.TextContent.Trim() == "1.5 s").Click(); Assert.Empty(picker.FindAll(".item-region")); Assert.Contains("saves the whole frame", picker.Markup); picker.Find("input[type=checkbox]").Change(true); - picker.FindAll("button").Single(b => b.TextContent.Trim() == "Review selected items").Click(); + picker + .FindAll("button") + .Single(b => b.TextContent.Trim() == "Review selected items") + .Click(); picker.WaitForAssertion(() => Assert.Single(_storage.References)); - Assert.Null(Assert.Single(_module.Invocations, i => i.Identifier == "cropStream").Arguments[2]); + Assert.Null( + Assert.Single(_module.Invocations, i => i.Identifier == "cropStream").Arguments[2] + ); } [Fact] @@ -179,7 +241,9 @@ public async Task CancelDetection_ReleasesBusyState_WithoutSavingImages() var picker = _context.Render(); picker.Find("#room-video").Change("room.mp4"); var scan = picker.Find("button").ClickAsync(new MouseEventArgs()); - picker.WaitForAssertion(() => Assert.True(picker.Find("#room-video").HasAttribute("disabled"))); + picker.WaitForAssertion(() => + Assert.True(picker.Find("#room-video").HasAttribute("disabled")) + ); picker.FindAll("button").Single(b => b.TextContent.Trim() == "Cancel").Click(); await scan; picker.WaitForAssertion(() => Assert.Contains("Scan cancelled", picker.Markup)); @@ -197,11 +261,19 @@ public void SaveFailure_PreservesSelection_AndRetryReusesAlreadySavedCrop() picker.WaitForAssertion(() => Assert.Equal(2, picker.FindAll(".item-region").Count)); picker.FindAll(".item-region")[0].Click(); picker.FindAll(".item-region")[1].Click(); - picker.FindAll("button").Single(b => b.TextContent.Trim() == "Review selected items").Click(); + picker + .FindAll("button") + .Single(b => b.TextContent.Trim() == "Review selected items") + .Click(); picker.WaitForAssertion(() => Assert.Contains("selection is preserved", picker.Markup)); Assert.Single(_storage.References); - picker.FindAll("button").Single(b => b.TextContent.Trim() == "Review selected items").Click(); - picker.WaitForAssertion(() => Assert.Contains("Selected items are ready below", picker.Markup)); + picker + .FindAll("button") + .Single(b => b.TextContent.Trim() == "Review selected items") + .Click(); + picker.WaitForAssertion(() => + Assert.Contains("Selected items are ready below", picker.Markup) + ); Assert.Equal(2, _storage.References.Count); Assert.Equal(3, _storage.Calls); } @@ -215,26 +287,41 @@ public async Task CancelSaving_PreservesSelection_ForRetry() picker.Find("button").Click(); picker.WaitForAssertion(() => Assert.Equal(2, picker.FindAll(".item-region").Count)); picker.FindAll(".item-region")[0].Click(); - var save = picker.FindAll("button").Single(b => b.TextContent.Trim() == "Review selected items").ClickAsync(new MouseEventArgs()); - picker.WaitForAssertion(() => Assert.Contains("Saving selected item images", picker.Markup)); + var save = picker + .FindAll("button") + .Single(b => b.TextContent.Trim() == "Review selected items") + .ClickAsync(new MouseEventArgs()); + picker.WaitForAssertion(() => + Assert.Contains("Saving selected item images", picker.Markup) + ); picker.FindAll("button").Single(b => b.TextContent.Trim() == "Cancel").Click(); await save; picker.WaitForAssertion(() => Assert.Contains("Saving cancelled", picker.Markup)); Assert.Empty(_storage.References); Assert.Equal("true", picker.FindAll(".item-region")[0].GetAttribute("aria-pressed")); _storage.WaitForCancellation = false; - picker.FindAll("button").Single(b => b.TextContent.Trim() == "Review selected items").Click(); + picker + .FindAll("button") + .Single(b => b.TextContent.Trim() == "Review selected items") + .Click(); picker.WaitForAssertion(() => Assert.Single(_storage.References)); } [Fact] public void DecodeFailure_ShowsActionableError_AndAllowsRetry() { - _module.Setup("prepare", _ => true).SetException(new JSException("Choose a video up to 30 seconds long.\nstack trace")); + _module + .Setup("prepare", _ => true) + .SetException(new JSException("Choose a video up to 30 seconds long.\nstack trace")); var picker = _context.Render(); picker.Find("#room-video").Change("room.mp4"); picker.Find("button").Click(); - picker.WaitForAssertion(() => Assert.Contains("Choose a video up to 30 seconds long.", picker.Find("[role=alert]").TextContent)); + picker.WaitForAssertion(() => + Assert.Contains( + "Choose a video up to 30 seconds long.", + picker.Find("[role=alert]").TextContent + ) + ); Assert.DoesNotContain("stack trace", picker.Markup); Assert.False(picker.Find("#room-video").HasAttribute("disabled")); Assert.Empty(_storage.References); @@ -247,10 +334,16 @@ public async Task NoMarketEvidence_ShowsUnavailable_WithoutSaveOrMergeActions() _valuation.Result = MarketValuationCalculator.Calculate([], DateTimeOffset.UtcNow); var page = _context.Render(); page.Find("#room-video").Change("room.mp4"); - await page.FindAll("button").Single(b => b.TextContent.Trim() == "Find items in video").ClickAsync(new MouseEventArgs()); + await page.FindAll("button") + .Single(b => b.TextContent.Trim() == "Find items in video") + .ClickAsync(new MouseEventArgs()); page.FindAll(".item-region")[0].Click(); - await page.FindAll("button").Single(b => b.TextContent.Trim() == "Review selected items").ClickAsync(new MouseEventArgs()); - await page.FindAll("button").Single(b => b.TextContent.Trim() == "Estimate value").ClickAsync(new MouseEventArgs()); + await page.FindAll("button") + .Single(b => b.TextContent.Trim() == "Review selected items") + .ClickAsync(new MouseEventArgs()); + await page.FindAll("button") + .Single(b => b.TextContent.Trim() == "Estimate value") + .ClickAsync(new MouseEventArgs()); Assert.Contains("No market value available", page.Markup); Assert.Contains("Market evidence (low confidence)", page.Markup); @@ -263,15 +356,26 @@ public async Task NoMarketEvidence_ShowsUnavailable_WithoutSaveOrMergeActions() [Fact] public async Task SparseMarketEvidence_PreservesConfidenceLabelWhenSaved() { - _valuation.Result = MarketValuationCalculator.Calculate([new(25, "Market", "Chair", "Used")], DateTimeOffset.UtcNow); + _valuation.Result = MarketValuationCalculator.Calculate( + [new(25, "Market", "Chair", "Used")], + DateTimeOffset.UtcNow + ); var page = _context.Render(); page.Find("#room-video").Change("room.mp4"); - await page.FindAll("button").Single(b => b.TextContent.Trim() == "Find items in video").ClickAsync(new MouseEventArgs()); + await page.FindAll("button") + .Single(b => b.TextContent.Trim() == "Find items in video") + .ClickAsync(new MouseEventArgs()); page.FindAll(".item-region")[0].Click(); - await page.FindAll("button").Single(b => b.TextContent.Trim() == "Review selected items").ClickAsync(new MouseEventArgs()); - await page.FindAll("button").Single(b => b.TextContent.Trim() == "Estimate value").ClickAsync(new MouseEventArgs()); + await page.FindAll("button") + .Single(b => b.TextContent.Trim() == "Review selected items") + .ClickAsync(new MouseEventArgs()); + await page.FindAll("button") + .Single(b => b.TextContent.Trim() == "Estimate value") + .ClickAsync(new MouseEventArgs()); Assert.Contains("Market evidence (low confidence)", page.Markup); - await page.FindAll("button").Single(b => b.TextContent.Trim() == "Save as new item").ClickAsync(new MouseEventArgs()); + await page.FindAll("button") + .Single(b => b.TextContent.Trim() == "Save as new item") + .ClickAsync(new MouseEventArgs()); var record = Assert.Single(_db.AssetValuationRecords); Assert.Equal(25m, record.EstimatedValue); @@ -286,7 +390,9 @@ public async Task NoMarketEvidence_OnRevalue_PreservesPreviousValuation() _valuation.Result = MarketValuationCalculator.Calculate([], DateTimeOffset.UtcNow); var page = _context.Render(); await page.Find(".asset-row-summary").ClickAsync(new MouseEventArgs()); - await page.FindAll("button").Single(b => b.TextContent.Trim() == "Revalue").ClickAsync(new MouseEventArgs()); + await page.FindAll("button") + .Single(b => b.TextContent.Trim() == "Revalue") + .ClickAsync(new MouseEventArgs()); Assert.Contains("no usable comparable listings", page.Find("[role=alert]").TextContent); var detail = await _repository.GetByIdAsync(id); @@ -303,20 +409,34 @@ public void Dispose() private sealed class FakeStream : IJSStreamReference { public long Length => 3; - public ValueTask OpenReadStreamAsync(long maxAllowedSize = 512000, CancellationToken cancellationToken = default) => - ValueTask.FromResult(new MemoryStream([1, 2, 3])); + + public ValueTask OpenReadStreamAsync( + long maxAllowedSize = 512000, + CancellationToken cancellationToken = default + ) => ValueTask.FromResult(new MemoryStream([1, 2, 3])); + public ValueTask DisposeAsync() => ValueTask.CompletedTask; } private sealed class FakeDetector : IPhysicalAssetDetectionService { private int _calls; - public Func>> Detect { get; set; } = (_, _) => - Task.FromResult>([ - new("Chair", 0.9, new(0.1, 0.1, 0.3, 0.3), null, []), - new("Lamp", 0.8, new(0.6, 0.1, 0.2, 0.4), null, [])]); - public Task> DetectAsync(byte[] imageBytes, string mediaType, CancellationToken cancellationToken = default) => - Detect(++_calls, cancellationToken); + public Func< + int, + CancellationToken, + Task> + > Detect { get; set; } = + (_, _) => + Task.FromResult>([ + new("Chair", 0.9, new(0.1, 0.1, 0.3, 0.3), null, []), + new("Lamp", 0.8, new(0.6, 0.1, 0.2, 0.4), null, []), + ]); + + public Task> DetectAsync( + byte[] imageBytes, + string mediaType, + CancellationToken cancellationToken = default + ) => Detect(++_calls, cancellationToken); } private sealed class FakeStorage : IPossessionImageStorage @@ -325,21 +445,38 @@ private sealed class FakeStorage : IPossessionImageStorage public int Calls { get; private set; } public int FailOnCall { get; set; } public bool WaitForCancellation { get; set; } - public async Task SaveAsync(Stream content, string fileName, CancellationToken cancellationToken = default) + + public async Task SaveAsync( + Stream content, + string fileName, + CancellationToken cancellationToken = default + ) { - if (WaitForCancellation) await Task.Delay(Timeout.Infinite, cancellationToken); - if (++Calls == FailOnCall) throw new PossessionImageStorageException("Test disk failure"); + if (WaitForCancellation) + await Task.Delay(Timeout.Infinite, cancellationToken); + if (++Calls == FailOnCall) + throw new PossessionImageStorageException("Test disk failure"); var reference = $"item-{Calls}.jpg"; References.Add(reference); return reference; } - public Task OpenReadAsync(string reference, CancellationToken cancellationToken = default) => Task.FromResult(null); + + public Task OpenReadAsync( + string reference, + CancellationToken cancellationToken = default + ) => Task.FromResult(null); } private sealed class FakeValuation : IAssetValuationService { - public AssetValuation Result { get; set; } = new(100, "Test estimate", DateTimeOffset.UtcNow, true); - public Task EstimateAsync(string label, string? brand, string? model, CancellationToken cancellationToken = default) => - Task.FromResult(Result); + public AssetValuation Result { get; set; } = + new(100, "Test estimate", DateTimeOffset.UtcNow, true); + + public Task EstimateAsync( + string label, + string? brand, + string? model, + CancellationToken cancellationToken = default + ) => Task.FromResult(Result); } } diff --git a/tests/js/camera-scanner.test.mjs b/tests/js/camera-scanner.test.mjs new file mode 100644 index 0000000..6b84eb2 --- /dev/null +++ b/tests/js/camera-scanner.test.mjs @@ -0,0 +1,178 @@ +import { afterEach, beforeEach, test, mock } from "node:test"; +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; + +const source = await readFile(new URL("../../wwwroot/js/camera-scanner.js", import.meta.url)); +const camera = await import(`data:text/javascript;base64,${source.toString("base64")}`); + +const originalDescriptor = Object.getOwnPropertyDescriptor(globalThis, "navigator"); + +function setMockNavigator(mediaDevices) { + Object.defineProperty(globalThis, "navigator", { + value: { mediaDevices }, + configurable: true, + writable: true, + }); +} + +function createMockTrack() { + return { + stop: mock.fn(), + }; +} + +function createMockStream() { + const tracks = [createMockTrack(), createMockTrack()]; + return { + getTracks: () => tracks, + }; +} + +function createMockVideo() { + return { + videoWidth: 1280, + videoHeight: 720, + srcObject: null, + setAttribute: mock.fn(), + play: mock.fn(async () => {}), + pause: mock.fn(), + }; +} + +function createMockCanvas() { + return { + width: 0, + height: 0, + getContext: () => ({ + drawImage: mock.fn(), + }), + toBlob: (cb) => cb(new Blob(["test-image"], { type: "image/jpeg" })), + }; +} + +afterEach(() => { + if (originalDescriptor) { + Object.defineProperty(globalThis, "navigator", originalDescriptor); + } +}); + +test("CameraDriver starts video stream with environment facing mode constraints", async () => { + const mockStream = createMockStream(); + let requestedConstraints = null; + + setMockNavigator({ + getUserMedia: mock.fn(async (constraints) => { + requestedConstraints = constraints; + return mockStream; + }), + }); + + const video = createMockVideo(); + const driver = new camera.CameraDriver(); + + const result = await driver.start(video, camera.CameraFacing.Environment); + + assert.equal(result.facing, camera.CameraFacing.Environment); + assert.equal(result.width, 1280); + assert.equal(result.height, 720); + assert.equal(video.srcObject, mockStream); + assert.equal(video.setAttribute.mock.callCount(), 2); + assert.equal(video.play.mock.callCount(), 1); + assert.equal(requestedConstraints.video.facingMode.ideal, "environment"); +}); + +test("CameraDriver switches facing mode between environment and user", async () => { + const mockStream = createMockStream(); + const calls = []; + + setMockNavigator({ + getUserMedia: mock.fn(async (constraints) => { + calls.push(constraints.video.facingMode.ideal); + return mockStream; + }), + }); + + const video = createMockVideo(); + const driver = new camera.CameraDriver(); + + await driver.start(video, camera.CameraFacing.Environment); + const switched = await driver.switchFacing(video, camera.CameraFacing.Environment); + + assert.equal(switched.facing, camera.CameraFacing.User); + assert.deepEqual(calls, ["environment", "user"]); +}); + +test("CameraDriver maps permission denial to clear user guidance", async () => { + const permissionError = new Error("Permission denied"); + permissionError.name = "NotAllowedError"; + + setMockNavigator({ + getUserMedia: mock.fn(async () => { + throw permissionError; + }), + }); + + const video = createMockVideo(); + const driver = new camera.CameraDriver(); + + await assert.rejects(async () => { + await driver.start(video); + }, /Camera access was denied/); +}); + +test("CameraDriver stops all stream tracks on stop", async () => { + const mockStream = createMockStream(); + + setMockNavigator({ + getUserMedia: mock.fn(async () => mockStream), + }); + + const video = createMockVideo(); + const driver = new camera.CameraDriver(); + + await driver.start(video); + driver.stop(video); + + for (const track of mockStream.getTracks()) { + assert.equal(track.stop.mock.callCount(), 1); + } + assert.equal(video.srcObject, null); + assert.equal(video.pause.mock.callCount(), 1); +}); + +test("FrameCapture draws video to canvas and pauses video on freeze", () => { + const video = createMockVideo(); + const canvas = createMockCanvas(); + const capture = new camera.FrameCapture(); + + const dimensions = capture.freeze(video, canvas); + + assert.equal(dimensions.width, 1280); + assert.equal(dimensions.height, 720); + assert.equal(canvas.width, 1280); + assert.equal(canvas.height, 720); + assert.equal(video.pause.mock.callCount(), 1); +}); + +test("normalizePoint converts viewport coordinates to clamped [0, 1] range", () => { + const container = { + getBoundingClientRect: () => ({ + left: 100, + top: 200, + width: 400, + height: 300, + }), + }; + + const center = camera.normalizePoint(300, 350, container); + assert.equal(center.x, 0.5); + assert.equal(center.y, 0.5); + + const topLeft = camera.normalizePoint(100, 200, container); + assert.equal(topLeft.x, 0); + assert.equal(topLeft.y, 0); + + const outside = camera.normalizePoint(600, 600, container); + assert.equal(outside.x, 1); + assert.equal(outside.y, 1); +}); diff --git a/tests/js/mobile-sam.test.mjs b/tests/js/mobile-sam.test.mjs index 16fc941..1fdbecf 100644 --- a/tests/js/mobile-sam.test.mjs +++ b/tests/js/mobile-sam.test.mjs @@ -186,8 +186,8 @@ test("ImageProcessor resizes input preserving aspect ratio and normalizes RGB ch assert.equal(result.origHeight, 1080); assert.equal(result.scaledWidth, 1024); assert.equal(result.scaledHeight, 576); - assert.deepEqual(result.tensor.dims, [1, 3, 1024, 1024]); - assert.equal(result.tensor.data.length, 3 * 1024 * 1024); + assert.deepEqual(result.tensor.dims, [576, 1024, 3]); + assert.equal(result.tensor.data.length, 576 * 1024 * 3); }); test("ContourExtractor thresholds logits and extracts normalized polygon coordinates", () => { diff --git a/wwwroot/js/camera-scanner.js b/wwwroot/js/camera-scanner.js new file mode 100644 index 0000000..7554cfb --- /dev/null +++ b/wwwroot/js/camera-scanner.js @@ -0,0 +1,189 @@ +export const CameraFacing = Object.freeze({ + Environment: "environment", + User: "user", +}); + +const DEFAULT_IDEAL_WIDTH = 1280; +const DEFAULT_IDEAL_HEIGHT = 720; +const JPEG_QUALITY = 0.9; + +const activeStreams = new WeakMap(); + +export class CameraDriver { + async start(video, facing = CameraFacing.Environment) { + if (!navigator?.mediaDevices?.getUserMedia) { + throw new Error("Camera streaming is not supported by this browser."); + } + + this.stop(video); + + const constraints = { + audio: false, + video: { + facingMode: { ideal: facing }, + width: { ideal: DEFAULT_IDEAL_WIDTH }, + height: { ideal: DEFAULT_IDEAL_HEIGHT }, + }, + }; + + try { + const stream = await navigator.mediaDevices.getUserMedia(constraints); + activeStreams.set(video, stream); + + video.srcObject = stream; + video.setAttribute("playsinline", "true"); + video.setAttribute("autoplay", "true"); + video.muted = true; + + await video.play(); + + return { + facing, + width: video.videoWidth || DEFAULT_IDEAL_WIDTH, + height: video.videoHeight || DEFAULT_IDEAL_HEIGHT, + }; + } catch (error) { + throw new Error(this.#mapErrorMessage(error)); + } + } + + #mapErrorMessage(error) { + const name = error?.name || ""; + + if (name === "NotAllowedError" || name === "PermissionDeniedError") { + return "Camera access was denied. Grant camera permission in browser settings."; + } + + if (name === "NotFoundError" || name === "DevicesNotFoundError") { + return "No camera device found on this system."; + } + + if (name === "NotReadableError" || name === "TrackStartError") { + return "Camera is in use by another application or tab."; + } + + if (name === "OverconstrainedError") { + return "Requested camera settings are not supported on this device."; + } + + return error?.message || "Failed to start camera feed."; + } + + async switchFacing(video, currentFacing) { + const nextFacing = + currentFacing === CameraFacing.Environment + ? CameraFacing.User + : CameraFacing.Environment; + + const result = await this.start(video, nextFacing); + return result; + } + + stop(video) { + const stream = activeStreams.get(video); + if (!stream) { + return; + } + + for (const track of stream.getTracks()) { + track.stop(); + } + + activeStreams.delete(video); + + if (video) { + video.pause(); + video.srcObject = null; + } + } +} + +export class FrameCapture { + freeze(video, canvas) { + if (!video) { + throw new Error("Video element is required to freeze frame."); + } + + const width = video.videoWidth || video.clientWidth || DEFAULT_IDEAL_WIDTH; + const height = video.videoHeight || video.clientHeight || DEFAULT_IDEAL_HEIGHT; + + canvas.width = width; + canvas.height = height; + + const ctx = canvas.getContext("2d"); + ctx.drawImage(video, 0, 0, width, height); + + video.pause(); + + return { width, height }; + } + + async unfreeze(video) { + if (video && video.srcObject) { + await video.play(); + } + } + + async toBlob(canvas, quality = JPEG_QUALITY) { + return new Promise((resolve, reject) => { + canvas.toBlob( + (blob) => { + if (blob) { + resolve(blob); + } else { + reject(new Error("Failed to extract image blob from canvas.")); + } + }, + "image/jpeg", + quality, + ); + }); + } +} + +export function normalizePoint(clientX, clientY, containerElement) { + if (!containerElement) { + return { x: 0, y: 0 }; + } + + const rect = containerElement.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) { + return { x: 0, y: 0 }; + } + + const x = Math.max(0, Math.min(1, (clientX - rect.left) / rect.width)); + const y = Math.max(0, Math.min(1, (clientY - rect.top) / rect.height)); + + return { x, y }; +} + +const defaultDriver = new CameraDriver(); +const defaultCapture = new FrameCapture(); + +export async function startCamera(video, facing = CameraFacing.Environment) { + return await defaultDriver.start(video, facing); +} + +export async function switchCamera(video, currentFacing) { + return await defaultDriver.switchFacing(video, currentFacing); +} + +export function stopCamera(video) { + defaultDriver.stop(video); +} + +export function freezeFrame(video, canvas) { + return defaultCapture.freeze(video, canvas); +} + +export async function unfreezeFrame(video) { + await defaultCapture.unfreeze(video); +} + +export async function captureBlob(canvas) { + return await defaultCapture.toBlob(canvas); +} + +export async function getCanvasBlob(canvas) { + return await defaultCapture.toBlob(canvas); +} diff --git a/wwwroot/js/mobile-sam.js b/wwwroot/js/mobile-sam.js index ffb7e59..4710afd 100644 --- a/wwwroot/js/mobile-sam.js +++ b/wwwroot/js/mobile-sam.js @@ -229,30 +229,16 @@ export class ImageProcessor { const imageData = ctx.getImageData(0, 0, scaledWidth, scaledHeight); const pixels = imageData.data; - const planeSize = TARGET_SIZE * TARGET_SIZE; - const tensorData = new Float32Array(CHANNELS * planeSize); + const totalPixels = scaledWidth * scaledHeight; + const tensorData = new Float32Array(totalPixels * CHANNELS); - for (let y = 0; y < scaledHeight; y++) { - const rowOffset = y * TARGET_SIZE; - const srcRowOffset = y * scaledWidth * 4; - - for (let x = 0; x < scaledWidth; x++) { - const srcIdx = srcRowOffset + x * 4; - const dstIdx = rowOffset + x; - - tensorData[dstIdx] = (pixels[srcIdx] - PIXEL_MEAN_R) / PIXEL_STD_R; - tensorData[planeSize + dstIdx] = (pixels[srcIdx + 1] - PIXEL_MEAN_G) / PIXEL_STD_G; - tensorData[2 * planeSize + dstIdx] = - (pixels[srcIdx + 2] - PIXEL_MEAN_B) / PIXEL_STD_B; - } + for (let i = 0, j = 0; i < pixels.length; i += 4, j += 3) { + tensorData[j] = pixels[i]; + tensorData[j + 1] = pixels[i + 1]; + tensorData[j + 2] = pixels[i + 2]; } - const tensor = new ort.Tensor("float32", tensorData, [ - 1, - CHANNELS, - TARGET_SIZE, - TARGET_SIZE, - ]); + const tensor = new ort.Tensor("float32", tensorData, [scaledHeight, scaledWidth, CHANNELS]); return { tensor, origWidth, From d9c073665f7d20327d927fb57ae09a55d9bba5ff Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 03:49:00 -0400 Subject: [PATCH 2/9] Fix test whitespace for dotnet format --- tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs index 47786f1..4b732a0 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs @@ -425,7 +425,8 @@ public Func< int, CancellationToken, Task> - > Detect { get; set; } = + > Detect + { get; set; } = (_, _) => Task.FromResult>([ new("Chair", 0.9, new(0.1, 0.1, 0.3, 0.3), null, []), From 9cf5974197e24aed9718fb8aa75d48ca3d405c7b Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 03:54:12 -0400 Subject: [PATCH 3/9] Pull video scan input and fix segmentation stall --- .../PhysicalAssets/LiveCameraScanner.razor | 46 +++++++++++--- Features/PhysicalAssets/PhysicalAssets.razor | 34 ++++++----- .../PhysicalAssets/VideoScanPickerTests.cs | 6 +- wwwroot/js/mobile-sam.js | 61 +++++++++++++++---- 4 files changed, 112 insertions(+), 35 deletions(-) diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor b/Features/PhysicalAssets/LiveCameraScanner.razor index 4bdc102..7f4df2f 100644 --- a/Features/PhysicalAssets/LiveCameraScanner.razor +++ b/Features/PhysicalAssets/LiveCameraScanner.razor @@ -46,10 +46,14 @@ } - @if (_isStreaming && !_isFrozen) + @if (_isStreaming && !_isFrozen && _statusHint is null) {
Tap any item to segment
} + else if (_statusHint is not null) + { +
@_statusHint
+ } else if (_isFrozen && _isSegmenting) {
Segmenting...
@@ -121,10 +125,11 @@ private bool _isFrozen; private bool _isSegmenting; private bool _isWorking; - private string _cameraFacing = FacingEnvironment; - private SamPromptType _currentPromptMode = SamPromptType.Positive; private string? _errorMessage; + private string? _statusHint; private SamSegmentationResult? _lastResult; + private string _cameraFacing = FacingEnvironment; + private SamPromptType _currentPromptMode = SamPromptType.Positive; private readonly List _prompts = new(); public async Task StartCameraAsync() @@ -205,38 +210,65 @@ _isSegmenting = true; await BusyChanged.InvokeAsync(true); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); try { if (!_isFrozen) { await _cameraModule.InvokeVoidAsync("freezeFrame", _video, _canvas); + + _statusHint = "Encoding frame..."; + StateHasChanged(); + + await SamEngine.EncodeFrameAsync(_canvas, cts.Token); _isFrozen = true; - await SamEngine.EncodeFrameAsync(_canvas); _prompts.Clear(); - var prompt = new SamPointPrompt(coords.X, coords.Y, SamPromptType.Positive); _prompts.Add(prompt); - _lastResult = await SamEngine.DecodePointAsync(prompt.X, prompt.Y, prompt.Type); + _statusHint = "Snapping outline..."; + StateHasChanged(); + + _lastResult = await SamEngine.DecodePointAsync(prompt.X, prompt.Y, prompt.Type, cts.Token); + _statusHint = null; } else { + _statusHint = "Refining outline..."; + StateHasChanged(); + var prompt = new SamPointPrompt(coords.X, coords.Y, _currentPromptMode); _prompts.Add(prompt); - _lastResult = await SamEngine.DecodePointsAsync(_prompts); + _lastResult = await SamEngine.DecodePointsAsync(_prompts, cts.Token); + _statusHint = null; + } + } + catch (OperationCanceledException) + { + _errorMessage = "Segmentation timed out. Please try tapping again."; + _statusHint = null; + if (_cameraModule is not null) + { + await _cameraModule.InvokeVoidAsync("unfreezeFrame", _video); } } catch (Exception ex) { _errorMessage = ex.Message; + _statusHint = null; + if (_cameraModule is not null) + { + await _cameraModule.InvokeVoidAsync("unfreezeFrame", _video); + } Logger.LogError(ex, "Tap segmentation failed."); } finally { _isSegmenting = false; + _statusHint = null; await BusyChanged.InvokeAsync(false); } } diff --git a/Features/PhysicalAssets/PhysicalAssets.razor b/Features/PhysicalAssets/PhysicalAssets.razor index de069b4..1b30050 100644 --- a/Features/PhysicalAssets/PhysicalAssets.razor +++ b/Features/PhysicalAssets/PhysicalAssets.razor @@ -15,19 +15,22 @@

Physical Assets

-

Scan a photo or short video to identify and value your possessions.

+

Scan a photo or point your camera at possessions to identify and value them in real time.

Scan a possession

- -
- - + +
- @@ -184,7 +187,7 @@ type="button" class="btn btn-outline-warning btn-sm me-2 mb-1" @onclick="() => SaveToInventoryAsync(detection, mergeIntoId: duplicate.Id)" - disabled="@(IsWorking || _videoBusy)"> + disabled="@IsWorking"> Merge into "@duplicate.Name" } @@ -194,7 +197,7 @@ @@ -219,11 +222,13 @@ @code { private const double LowConfidenceThreshold = 0.6; private const double HighConfidenceThreshold = 0.8; + [Parameter] + public bool ShowVideoScan { get; set; } private bool _isProcessing; - private bool _videoBusy; private bool _cameraBusy; - private bool IsWorking => _isProcessing || _videoBusy || _cameraBusy || (_detections?.Any(d => d.IsSaving || d.IsEstimating) ?? false); + private bool _videoBusy; + private bool IsWorking => _isProcessing || _cameraBusy || (_videoBusy && ShowVideoScan) || (_detections?.Any(d => d.IsSaving || d.IsEstimating) ?? false); private string? _processingMessage; private string? _errorMessage; private string? _errorRawResponse; @@ -234,6 +239,8 @@ private string _confidenceFilter = "all"; private string _sortOrder = "confidence-desc"; + + private void ReviewVideoItems(IReadOnlyList items) { ClearError(); @@ -242,7 +249,6 @@ _sortOrder = "confidence-desc"; _detections = items.Select(item => new ReviewableDetection(item.Detection) { ImageReference = item.ImageReference }).ToList(); } - private void ReviewLiveScannedItem(LiveScannedItem item) { ClearError(); @@ -340,7 +346,7 @@ private async Task OnFileSelectedAsync(InputFileChangeEventArgs e) { - if (IsWorking || _videoBusy) return; + if (IsWorking) return; ClearError(); _imageReference = null; _detections = null; @@ -404,7 +410,7 @@ private async Task EstimateValueAsync(ReviewableDetection detection) { - if (IsWorking || _videoBusy || detection.SavedItemId is not null) return; + if (IsWorking || detection.SavedItemId is not null) return; detection.ValuationError = null; detection.ValuationRawResponse = null; detection.IsEstimating = true; @@ -444,7 +450,7 @@ // were found. private async Task SaveToInventoryAsync(ReviewableDetection detection, Guid? mergeIntoId) { - if (detection.Valuation?.EstimatedValueUsd is not { } value || IsWorking || _videoBusy || detection.SavedItemId is not null) + if (detection.Valuation?.EstimatedValueUsd is not { } value || IsWorking || detection.SavedItemId is not null) { return; } diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs index 4b732a0..1cdf9d8 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs @@ -50,7 +50,7 @@ public VideoScanPickerTests() [Fact] public async Task SelectedItemsOnly_GetSeparateCrops_AndPersistThroughExistingReview() { - var page = _context.Render(); + var page = _context.Render(p => p.Add(x => x.ShowVideoScan, true)); page.Find("#room-video").Change("room.mp4"); await page.FindAll("button") .Single(b => b.TextContent == "Find items in video") @@ -332,7 +332,7 @@ public async Task NoMarketEvidence_ShowsUnavailable_WithoutSaveOrMergeActions() { await _repository.AddAsync(new PhysicalAssetInput("Chair", null, null, 50)); _valuation.Result = MarketValuationCalculator.Calculate([], DateTimeOffset.UtcNow); - var page = _context.Render(); + var page = _context.Render(p => p.Add(x => x.ShowVideoScan, true)); page.Find("#room-video").Change("room.mp4"); await page.FindAll("button") .Single(b => b.TextContent.Trim() == "Find items in video") @@ -360,7 +360,7 @@ public async Task SparseMarketEvidence_PreservesConfidenceLabelWhenSaved() [new(25, "Market", "Chair", "Used")], DateTimeOffset.UtcNow ); - var page = _context.Render(); + var page = _context.Render(p => p.Add(x => x.ShowVideoScan, true)); page.Find("#room-video").Change("room.mp4"); await page.FindAll("button") .Single(b => b.TextContent.Trim() == "Find items in video") diff --git a/wwwroot/js/mobile-sam.js b/wwwroot/js/mobile-sam.js index 4710afd..c460451 100644 --- a/wwwroot/js/mobile-sam.js +++ b/wwwroot/js/mobile-sam.js @@ -84,23 +84,31 @@ export class ModelStorage { throw new Error(`Failed to download model asset from ${url}: HTTP ${response.status}`); } + const totalBytes = Number(response.headers.get("content-length")) || 0; + let buffer; + + if (response.body && onProgress && totalBytes > 0) { + buffer = await this.#readStreamWithProgress(response.body, totalBytes, onProgress); + } else { + buffer = await response.arrayBuffer(); + } + if ("caches" in globalThis) { try { const cache = await globalThis.caches.open(this.#cacheName); - await cache.put(url, response.clone()); + const cacheResponse = new Response(buffer.slice(0), { + headers: { + "content-type": "application/octet-stream", + }, + }); + await cache.put(url, cacheResponse); } catch { // Storage quota exceeded or private mode restriction } } - const totalBytes = Number(response.headers.get("content-length")) || 0; - if (!response.body || !onProgress || totalBytes <= 0) { - return await response.arrayBuffer(); - } - - return await this.#readStreamWithProgress(response.body, totalBytes, onProgress); + return buffer; } - async #readStreamWithProgress(body, totalBytes, onProgress) { const reader = body.getReader(); const chunks = []; @@ -471,6 +479,7 @@ export class MobileSamEngine { #processor; #extractor; #state = EngineState.Uninitialized; + #initPromise = null; #ort = null; #encoderSession = null; @@ -480,7 +489,6 @@ export class MobileSamEngine { #currentEmbeddings = null; #currentMeta = null; #currentMaskPrior = null; - constructor(options = {}) { this.#storage = options.storage || new ModelStorage(); this.#driver = options.driver || new OrtDriver(options); @@ -501,8 +509,34 @@ export class MobileSamEngine { return this.getStatus(); } + if (this.#state === EngineState.Loading && this.#initPromise) { + return await this.#initPromise; + } + this.#state = EngineState.Loading; + this.#initPromise = this.#doInit(options); + + try { + return await this.#initPromise; + } finally { + this.#initPromise = null; + } + } + + async waitForReady() { + if (this.#state === EngineState.Ready) { + return; + } + if (this.#initPromise) { + await this.#initPromise; + return; + } + + await this.init(); + } + + async #doInit(options) { try { this.#ort = await this.#driver.getOrt(); @@ -707,8 +741,13 @@ export async function initEngine(options = {}) { } export async function encodeFrame(source) { - if (!defaultEngine) { - throw new Error("MobileSAM engine is not initialized. Call initEngine first."); + if (!defaultEngine || defaultEngine.state === EngineState.Disposed) { + defaultEngine = new MobileSamEngine(); + await defaultEngine.init(); + } else if (defaultEngine.state === EngineState.Loading) { + await defaultEngine.waitForReady(); + } else if (defaultEngine.state === EngineState.Uninitialized) { + await defaultEngine.init(); } return await defaultEngine.encodeImage(source); } From 1239583a67fc14e486a7de051c903708f37d6a52 Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 04:07:21 -0400 Subject: [PATCH 4/9] Add model readiness check and extend timeout --- .../PhysicalAssets/LiveCameraScanner.razor | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor b/Features/PhysicalAssets/LiveCameraScanner.razor index 7f4df2f..adcf1fa 100644 --- a/Features/PhysicalAssets/LiveCameraScanner.razor +++ b/Features/PhysicalAssets/LiveCameraScanner.razor @@ -12,7 +12,15 @@

Live camera scanner

@if (_isStreaming) { - Camera active +
+ Camera active + @if (_isModelReady) + { + + @(_engineDevice == "webgpu" ? "WebGPU" : "WASM fallback") + + } +
} @@ -105,6 +113,13 @@ } + + @if (_engineDevice == "wasm") + { +

+ Note: Running on CPU WASM. WebGPU requires HTTPS or localhost (127.0.0.1). +

+ } @code { @@ -125,13 +140,14 @@ private bool _isFrozen; private bool _isSegmenting; private bool _isWorking; + private bool _isModelReady; + private string? _engineDevice; private string? _errorMessage; private string? _statusHint; private SamSegmentationResult? _lastResult; private string _cameraFacing = FacingEnvironment; private SamPromptType _currentPromptMode = SamPromptType.Positive; private readonly List _prompts = new(); - public async Task StartCameraAsync() { _errorMessage = null; @@ -143,11 +159,18 @@ _isStreaming = true; _isFrozen = false; - _ = SamEngine.InitializeAsync(); + _statusHint = "Downloading MobileSAM model (~40 MB)..."; + StateHasChanged(); + + var status = await SamEngine.InitializeAsync(); + _engineDevice = status.Device; + _isModelReady = true; + _statusHint = null; } catch (Exception ex) { _errorMessage = ex.Message; + _statusHint = null; Logger.LogError(ex, "Failed to start camera stream."); } } @@ -190,12 +213,16 @@ { return; } - if (_isWorking || _isSegmenting || _cameraModule is null) { return; } + if (!_isModelReady) + { + _errorMessage = "MobileSAM model is downloading (~40 MB). Please wait a moment."; + return; + } var coords = await _cameraModule.InvokeAsync( "normalizePoint", e.ClientX, @@ -210,7 +237,7 @@ _isSegmenting = true; await BusyChanged.InvokeAsync(true); - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120)); try { @@ -218,12 +245,13 @@ { await _cameraModule.InvokeVoidAsync("freezeFrame", _video, _canvas); - _statusHint = "Encoding frame..."; + _statusHint = _engineDevice == "webgpu" + ? "Encoding frame (WebGPU)..." + : "Encoding frame (CPU WASM fallback)..."; StateHasChanged(); await SamEngine.EncodeFrameAsync(_canvas, cts.Token); _isFrozen = true; - _prompts.Clear(); var prompt = new SamPointPrompt(coords.X, coords.Y, SamPromptType.Positive); _prompts.Add(prompt); From ee5e9e206fdba56774cb9f7aad4bd863f2487879 Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 04:23:59 -0400 Subject: [PATCH 5/9] Optimize contour extraction and polygon decimation --- wwwroot/js/mobile-sam.js | 150 +++++++++++++++++++++------------------ 1 file changed, 82 insertions(+), 68 deletions(-) diff --git a/wwwroot/js/mobile-sam.js b/wwwroot/js/mobile-sam.js index c460451..a49ce3d 100644 --- a/wwwroot/js/mobile-sam.js +++ b/wwwroot/js/mobile-sam.js @@ -298,34 +298,40 @@ export class ImageProcessor { export class ContourExtractor { processMask(maskData, maskWidth, maskHeight, confidence = 1.0) { - const totalPixels = maskWidth * maskHeight; - const binaryMask = new Uint8Array(totalPixels); + const maxDim = 256; + const step = Math.max(1, Math.floor(Math.max(maskWidth, maskHeight) / maxDim)); + const gridW = Math.ceil(maskWidth / step); + const gridH = Math.ceil(maskHeight / step); + const grid = new Uint8Array(gridW * gridH); - let minX = maskWidth; + let minX = gridW; let maxX = -1; - let minY = maskHeight; + let minY = gridH; let maxY = -1; let foregroundCount = 0; - for (let y = 0; y < maskHeight; y++) { - const rowOffset = y * maskWidth; - for (let x = 0; x < maskWidth; x++) { - const idx = rowOffset + x; - if (maskData[idx] > MASK_THRESHOLD) { - binaryMask[idx] = 1; + for (let gy = 0; gy < gridH; gy++) { + const my = Math.min(maskHeight - 1, gy * step); + const maskRow = my * maskWidth; + const gridRow = gy * gridW; + + for (let gx = 0; gx < gridW; gx++) { + const mx = Math.min(maskWidth - 1, gx * step); + if (maskData[maskRow + mx] > MASK_THRESHOLD) { + grid[gridRow + gx] = 1; foregroundCount++; - if (x < minX) { - minX = x; + if (gx < minX) { + minX = gx; } - if (x > maxX) { - maxX = x; + if (gx > maxX) { + maxX = gx; } - if (y < minY) { - minY = y; + if (gy < minY) { + minY = gy; } - if (y > maxY) { - maxY = y; + if (gy > maxY) { + maxY = gy; } } } @@ -336,44 +342,44 @@ export class ContourExtractor { polygon: [], bounds: { x: 0, y: 0, width: 0, height: 0 }, confidence, - binaryMask, - maskWidth, - maskHeight, + binaryMask: grid, + maskWidth: gridW, + maskHeight: gridH, }; } - const rawContour = this.#traceBorder(binaryMask, maskWidth, maskHeight, minX, minY); + const rawContour = this.#traceBorder(grid, gridW, gridH, minX, minY); const simplified = this.#simplifyRdp(rawContour, SIMPLIFICATION_EPSILON); const polygon = simplified.map((p) => ({ - x: Math.max(0, Math.min(1, p.x / maskWidth)), - y: Math.max(0, Math.min(1, p.y / maskHeight)), + x: Math.max(0, Math.min(1, (p.x * step) / maskWidth)), + y: Math.max(0, Math.min(1, (p.y * step) / maskHeight)), })); const bounds = { - x: minX / maskWidth, - y: minY / maskHeight, - width: (maxX - minX + 1) / maskWidth, - height: (maxY - minY + 1) / maskHeight, + x: Math.max(0, (minX * step) / maskWidth), + y: Math.max(0, (minY * step) / maskHeight), + width: Math.min(1, ((maxX - minX + 1) * step) / maskWidth), + height: Math.min(1, ((maxY - minY + 1) * step) / maskHeight), }; return { polygon, bounds, confidence, - binaryMask, - maskWidth, - maskHeight, + binaryMask: grid, + maskWidth: gridW, + maskHeight: gridH, }; } - #traceBorder(binaryMask, width, height, startX, startY) { + #traceBorder(grid, width, height, startX, startY) { let firstX = -1; let firstY = -1; for (let y = startY; y < height; y++) { for (let x = 0; x < width; x++) { - if (binaryMask[y * width + x] === 1) { + if (grid[y * width + x] === 1) { firstX = x; firstY = y; break; @@ -391,17 +397,16 @@ export class ContourExtractor { const contour = []; let currX = firstX; let currY = firstY; - let enterDir = 0; - const maxSteps = width * height * 2; + let checkDir = 7; + const maxSteps = 1000; let steps = 0; do { contour.push({ x: currX, y: currY }); let foundNext = false; - const checkStart = (enterDir + 4 + 1) % 8; for (let i = 0; i < 8; i++) { - const dir = (checkStart + i) % 8; + const dir = (checkDir + i) % 8; const nextX = currX + NEIGHBOR_DX[dir]; const nextY = currY + NEIGHBOR_DY[dir]; @@ -409,10 +414,10 @@ export class ContourExtractor { continue; } - if (binaryMask[nextY * width + nextX] === 1) { + if (grid[nextY * width + nextX] === 1) { currX = nextX; currY = nextY; - enterDir = dir; + checkDir = (dir + 5) % 8; foundNext = true; break; } @@ -436,40 +441,49 @@ export class ContourExtractor { return points; } - let maxDist = 0; - let maxIdx = 0; - const last = points.length - 1; + const keep = new Uint8Array(points.length); + keep[0] = 1; + keep[points.length - 1] = 1; + + const stack = [[0, points.length - 1]]; + + while (stack.length > 0) { + const [start, end] = stack.pop(); + let maxDist = 0; + let maxIdx = 0; + + const sx = points[start].x; + const sy = points[start].y; + const ex = points[end].x; + const ey = points[end].y; + const dx = ex - sx; + const dy = ey - sy; + const lenSq = dx * dx + dy * dy; + + for (let i = start + 1; i < end; i++) { + let dist; + if (lenSq === 0) { + dist = Math.hypot(points[i].x - sx, points[i].y - sy); + } else { + dist = + Math.abs(dy * points[i].x - dx * points[i].y + ex * sy - ey * sx) / + Math.sqrt(lenSq); + } - for (let i = 1; i < last; i++) { - const dist = this.#pointToLineDist(points[i], points[0], points[last]); - if (dist > maxDist) { - maxDist = dist; - maxIdx = i; + if (dist > maxDist) { + maxDist = dist; + maxIdx = i; + } } - } - if (maxDist > epsilon) { - const left = this.#simplifyRdp(points.slice(0, maxIdx + 1), epsilon); - const right = this.#simplifyRdp(points.slice(maxIdx), epsilon); - return left.slice(0, -1).concat(right); - } - - return [points[0], points[last]]; - } - - #pointToLineDist(point, start, end) { - const dx = end.x - start.x; - const dy = end.y - start.y; - const lenSq = dx * dx + dy * dy; - - if (lenSq === 0) { - const px = point.x - start.x; - const py = point.y - start.y; - return Math.sqrt(px * px + py * py); + if (maxDist > epsilon) { + keep[maxIdx] = 1; + stack.push([start, maxIdx]); + stack.push([maxIdx, end]); + } } - const numerator = Math.abs(dy * point.x - dx * point.y + end.x * start.y - end.y * start.x); - return numerator / Math.sqrt(lenSq); + return points.filter((_, i) => keep[i] === 1); } } From 3acd9099f28422ce88f7cecd12d1e4ee394fec73 Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 04:32:49 -0400 Subject: [PATCH 6/9] Clamp confidence score and add inline SVG styling --- Features/PhysicalAssets/LiveCameraScanner.razor | 14 +++++++++++--- wwwroot/js/mobile-sam.js | 3 ++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor b/Features/PhysicalAssets/LiveCameraScanner.razor index adcf1fa..4cef7e8 100644 --- a/Features/PhysicalAssets/LiveCameraScanner.razor +++ b/Features/PhysicalAssets/LiveCameraScanner.razor @@ -43,17 +43,25 @@ @if (_lastResult?.Polygon is { Count: >= 3 } polygon) { } - @if (_isStreaming && !_isFrozen && _statusHint is null) {
Tap any item to segment
diff --git a/wwwroot/js/mobile-sam.js b/wwwroot/js/mobile-sam.js index a49ce3d..c8b2bd1 100644 --- a/wwwroot/js/mobile-sam.js +++ b/wwwroot/js/mobile-sam.js @@ -682,7 +682,8 @@ export class MobileSamEngine { const masksTensor = results.masks; const iouTensor = results.iou_predictions; - const confidence = iouTensor ? Number(iouTensor.data[0]) : 1.0; + const rawConfidence = iouTensor ? Number(iouTensor.data[0]) : 1.0; + const confidence = Math.max(0.0, Math.min(1.0, rawConfidence)); const maskDims = masksTensor.dims; const maskHeight = maskDims[maskDims.length - 2]; From 84111b0763720018cae7b7eecaf9712a132bd6be Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 04:37:19 -0400 Subject: [PATCH 7/9] Strip mask from interop and raise message limit --- Program.cs | 4 ++++ wwwroot/js/mobile-sam.js | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Program.cs b/Program.cs index 7cb7e89..4bd1040 100644 --- a/Program.cs +++ b/Program.cs @@ -20,6 +20,10 @@ // Add services to the container. builder.Services.AddRazorComponents().AddInteractiveServerComponents(); +builder.Services.Configure(options => +{ + options.MaximumReceiveMessageSize = 512 * 1024; +}); builder.Services.Configure( builder.Configuration.GetSection(NemotronOptions.SectionName) diff --git a/wwwroot/js/mobile-sam.js b/wwwroot/js/mobile-sam.js index c8b2bd1..c607808 100644 --- a/wwwroot/js/mobile-sam.js +++ b/wwwroot/js/mobile-sam.js @@ -342,7 +342,6 @@ export class ContourExtractor { polygon: [], bounds: { x: 0, y: 0, width: 0, height: 0 }, confidence, - binaryMask: grid, maskWidth: gridW, maskHeight: gridH, }; @@ -367,7 +366,6 @@ export class ContourExtractor { polygon, bounds, confidence, - binaryMask: grid, maskWidth: gridW, maskHeight: gridH, }; @@ -696,7 +694,11 @@ export class MobileSamEngine { confidence, ); return { - ...processed, + polygon: processed.polygon, + bounds: processed.bounds, + confidence: processed.confidence, + maskWidth: processed.maskWidth, + maskHeight: processed.maskHeight, elapsedMs, }; } From 6d069478aba397fc0060eee8d96e11f3bff603d1 Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 04:44:56 -0400 Subject: [PATCH 8/9] Fix unpacked multi point prompts and missing origX --- PhysicalAssets/SamSegmentation.cs | 2 +- wwwroot/js/mobile-sam.js | 27 ++++++++++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/PhysicalAssets/SamSegmentation.cs b/PhysicalAssets/SamSegmentation.cs index 23e5f23..807426c 100644 --- a/PhysicalAssets/SamSegmentation.cs +++ b/PhysicalAssets/SamSegmentation.cs @@ -141,7 +141,7 @@ public async ValueTask DecodePointsAsync( return await module.InvokeAsync( "decodeMultiPoints", cancellationToken, - jsPoints + new object[] { jsPoints } ); } diff --git a/wwwroot/js/mobile-sam.js b/wwwroot/js/mobile-sam.js index c607808..d769f07 100644 --- a/wwwroot/js/mobile-sam.js +++ b/wwwroot/js/mobile-sam.js @@ -618,22 +618,27 @@ export class MobileSamEngine { throw new Error("No image embeddings available. Call encodeImage first."); } - if (!points || points.length === 0) { + const pointList = Array.isArray(points) + ? points + : points && typeof points.x === "number" + ? [points] + : []; + + if (pointList.length === 0) { throw new Error("At least one point prompt is required."); } const startTime = performance.now(); - const pointCount = points.length; + const pointCount = pointList.length; const coordData = new Float32Array(pointCount * 2); const labelData = new Float32Array(pointCount); const { scale, origWidth, origHeight } = this.#currentMeta; for (let i = 0; i < pointCount; i++) { - const pt = points[i]; + const pt = pointList[i]; const origX = pt.x <= 1.0 ? pt.x * origWidth : pt.x; const origY = pt.y <= 1.0 ? pt.y * origHeight : pt.y; - coordData[i * 2] = origX * scale; coordData[i * 2 + 1] = origY * scale; labelData[i] = typeof pt.type === "number" ? pt.type : PromptType.Positive; @@ -773,10 +778,22 @@ export async function decodePoint(x, y, type = PromptType.Positive) { return await decodeMultiPoints([{ x, y, type }]); } -export async function decodeMultiPoints(points, options = {}) { +export async function decodeMultiPoints(...args) { if (!defaultEngine) { throw new Error("MobileSAM engine is not initialized. Call initEngine first."); } + + let points = []; + let options = {}; + + if (Array.isArray(args[0])) { + points = args[0]; + options = args[1] || {}; + } else { + points = args.filter((a) => a && typeof a.x === "number"); + options = args.find((a) => a && typeof a.x !== "number") || {}; + } + return await defaultEngine.decodePoints(points, options); } From 1f0a8ed47ff8aaef6902809a9bcf5348a9cc9cc6 Mon Sep 17 00:00:00 2001 From: Monster0506 Date: Sun, 20 Sep 2026 04:48:25 -0400 Subject: [PATCH 9/9] Project tap coordinates to uncropped video frame --- .../PhysicalAssets/LiveCameraScanner.razor | 4 +-- .../LiveCameraScanner.razor.css | 3 +- wwwroot/js/camera-scanner.js | 36 ++++++++++++++++++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor b/Features/PhysicalAssets/LiveCameraScanner.razor index 4cef7e8..78f4095 100644 --- a/Features/PhysicalAssets/LiveCameraScanner.razor +++ b/Features/PhysicalAssets/LiveCameraScanner.razor @@ -235,9 +235,9 @@ "normalizePoint", e.ClientX, e.ClientY, - _viewfinder + _viewfinder, + _video ); - if (coords.X < 0 || coords.X > 1 || coords.Y < 0 || coords.Y > 1) { return; diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor.css b/Features/PhysicalAssets/LiveCameraScanner.razor.css index 26f7fd3..120c315 100644 --- a/Features/PhysicalAssets/LiveCameraScanner.razor.css +++ b/Features/PhysicalAssets/LiveCameraScanner.razor.css @@ -9,7 +9,6 @@ width: 100%; max-width: 720px; margin: 0 auto; - aspect-ratio: 4 / 3; background-color: #0b0f19; border-radius: 0.75rem; overflow: hidden; @@ -25,7 +24,7 @@ .freeze-canvas { width: 100%; height: 100%; - object-fit: cover; + object-fit: contain; display: block; } diff --git a/wwwroot/js/camera-scanner.js b/wwwroot/js/camera-scanner.js index 7554cfb..28d2a35 100644 --- a/wwwroot/js/camera-scanner.js +++ b/wwwroot/js/camera-scanner.js @@ -37,6 +37,10 @@ export class CameraDriver { await video.play(); + if (video.parentElement && video.videoWidth > 0 && video.videoHeight > 0) { + video.parentElement.style.aspectRatio = `${video.videoWidth} / ${video.videoHeight}`; + } + return { facing, width: video.videoWidth || DEFAULT_IDEAL_WIDTH, @@ -141,7 +145,7 @@ export class FrameCapture { } } -export function normalizePoint(clientX, clientY, containerElement) { +export function normalizePoint(clientX, clientY, containerElement, videoElement) { if (!containerElement) { return { x: 0, y: 0 }; } @@ -151,6 +155,36 @@ export function normalizePoint(clientX, clientY, containerElement) { return { x: 0, y: 0 }; } + const video = + videoElement || containerElement.querySelector?.("video") || containerElement.querySelector?.("canvas"); + const videoWidth = video?.videoWidth || video?.width || 0; + const videoHeight = video?.videoHeight || video?.height || 0; + + if (videoWidth > 0 && videoHeight > 0) { + const containerAspect = rect.width / rect.height; + const videoAspect = videoWidth / videoHeight; + + let visibleW = rect.width; + let visibleH = rect.height; + let offsetX = 0; + let offsetY = 0; + + if (videoAspect > containerAspect) { + visibleH = rect.width / videoAspect; + offsetY = (rect.height - visibleH) / 2; + } else { + visibleW = rect.height * videoAspect; + offsetX = (rect.width - visibleW) / 2; + } + + const relX = clientX - rect.left - offsetX; + const relY = clientY - rect.top - offsetY; + + const x = Math.max(0, Math.min(1, relX / visibleW)); + const y = Math.max(0, Math.min(1, relY / visibleH)); + return { x, y }; + } + const x = Math.max(0, Math.min(1, (clientX - rect.left) / rect.width)); const y = Math.max(0, Math.min(1, (clientY - rect.top) / rect.height));