From 476bbedbd0e11fd6a7b7f54228af4c8d1eb7ffd2 Mon Sep 17 00:00:00 2001 From: Andrew Roddy Date: Sun, 20 Sep 2026 10:12:51 -0400 Subject: [PATCH 1/2] Cut the segmentation memory peak so iOS stops killing the tab Tapping an item on an iPhone reloaded the page. Nothing was caught and nothing was logged, because nothing threw: iOS gives a tab a few hundred megabytes and kills it outright when it goes over. The tap is where the encoder runs, so the tap is where the budget ran out. Four things were spending it before the model ever did any work: - The ORT bundle was ort.all, which carries WebGPU, WebGL, WebNN and the training runtime, and loads the 23 MB JSEP WebAssembly binary whether or not the device has a GPU. The plain WASM build is 12 MB. Pick the bundle from navigator.gpu, and skip the doomed WebGPU session attempt where the runtime was never loaded. - numThreads was set to min(4, cores) unconditionally. Threaded WASM runs on SharedArrayBuffer, which needs COOP/COEP headers this app does not send, so those threads could never exist. Ask for one. - Both models were loaded and instantiated in parallel, holding two ONNX buffers and the runtime's copy of each at once. Sequential, releasing each buffer before reaching for the next, halves that high water mark. The download is the slow part and it is cached after the first run. - Caching a downloaded model sliced it first, so a 40 MB encoder existed three times over: ours, the slice, and the Response body. Response copies what it is handed, so the slice bought nothing. The scratch canvas in prepareTensor is also given back explicitly now rather than waiting for a collection Safari is in no hurry to run. --- tests/js/mobile-sam.test.mjs | 130 ++++++++++++++++++++++++++++++++++- wwwroot/js/mobile-sam.js | 106 ++++++++++++++++++++++------ 2 files changed, 214 insertions(+), 22 deletions(-) diff --git a/tests/js/mobile-sam.test.mjs b/tests/js/mobile-sam.test.mjs index 1fdbecf..c726b0f 100644 --- a/tests/js/mobile-sam.test.mjs +++ b/tests/js/mobile-sam.test.mjs @@ -100,6 +100,8 @@ afterEach(() => { globalThis.caches = originalCaches; globalThis.fetch = originalFetch; globalThis.document = originalDocument; + delete globalThis.crossOriginIsolated; + delete globalThis.ort; sam.disposeEngine(); }); @@ -160,15 +162,49 @@ test("OrtDriver attempts WebGPU and falls back to WASM if WebGPU is unsupported" }, }; + // A device that advertises a GPU, so the attempt is worth making here. + const originalNavigator = Object.getOwnPropertyDescriptor(globalThis, "navigator"); + Object.defineProperty(globalThis, "navigator", { + value: { gpu: {} }, + configurable: true, + writable: true, + }); + + try { + const driver = new sam.OrtDriver({ ort: mockOrt }); + const sessionResult = await driver.createSession( + mockOrt, + new ArrayBuffer(8), + sam.ExecutionDevice.Auto, + ); + + assert.equal(sessionResult.device, sam.ExecutionDevice.Wasm); + assert.equal(callCount, 2); + } finally { + if (originalNavigator) { + Object.defineProperty(globalThis, "navigator", originalNavigator); + } + } +}); + +test("OrtDriver does not attempt WebGPU on a device that has no GPU", async () => { + const mockOrt = createMockOrt(); const driver = new sam.OrtDriver({ ort: mockOrt }); + const sessionResult = await driver.createSession( mockOrt, new ArrayBuffer(8), sam.ExecutionDevice.Auto, ); + // The WebGPU runtime was never loaded here, so a session attempt could only + // allocate and then fail. assert.equal(sessionResult.device, sam.ExecutionDevice.Wasm); - assert.equal(callCount, 2); + assert.equal(mockOrt.InferenceSession.create.mock.callCount(), 1); + assert.deepEqual( + mockOrt.InferenceSession.create.mock.calls[0].arguments[1].executionProviders, + ["wasm"], + ); }); test("ImageProcessor resizes input preserving aspect ratio and normalizes RGB channels", () => { @@ -365,3 +401,95 @@ test("Module helpers provide singleton access for Blazor interop", async () => { const afterDispose = sam.getEngineStatus(); assert.equal(afterDispose.state, sam.EngineState.Uninitialized); }); + +test("OrtDriver loads the WebGPU runtime only where there is a GPU to use it", () => { + const withoutGpu = new sam.OrtDriver(); + // ort.all drags in a 23 MB WebAssembly binary; the plain WASM build is 12 MB, + // and on a phone that gap is the difference between running and being killed. + assert.match(withoutGpu.resolveCdnUrl(), /ort\.wasm\.bundle\.min\.mjs$/); + + const originalNavigator = Object.getOwnPropertyDescriptor(globalThis, "navigator"); + Object.defineProperty(globalThis, "navigator", { + value: { gpu: {} }, + configurable: true, + writable: true, + }); + + try { + assert.match(new sam.OrtDriver().resolveCdnUrl(), /ort\.webgpu\.bundle\.min\.mjs$/); + } finally { + if (originalNavigator) { + Object.defineProperty(globalThis, "navigator", originalNavigator); + } + } + + assert.equal( + new sam.OrtDriver({ cdnUrl: "https://example.com/ort.mjs" }).resolveCdnUrl(), + "https://example.com/ort.mjs", + ); +}); + +test("OrtDriver asks for a single WASM thread when the page is not cross-origin isolated", async () => { + const mockOrt = createMockOrt(); + mockOrt.env.wasm.numThreads = 4; + + globalThis.crossOriginIsolated = false; + + globalThis.ort = mockOrt; + + const driver = new sam.OrtDriver(); + await driver.getOrt(); + + // Threads need SharedArrayBuffer, which needs COOP/COEP headers the app does + // not send. Asking for four buys worker infrastructure and nothing else. + assert.equal(mockOrt.env.wasm.numThreads, 1); +}); + +test("MobileSamEngine releases each model before reaching for the next", async () => { + const mockOrt = createMockOrt(); + const order = []; + + const storage = { + loadBuffer: mock.fn(async (url) => { + order.push(`load:${url.includes("encoder") ? "encoder" : "decoder"}`); + return new ArrayBuffer(16); + }), + }; + const driver = { + getOrt: async () => mockOrt, + createSession: mock.fn(async () => { + order.push("create"); + return { + session: await mockOrt.InferenceSession.create(), + device: sam.ExecutionDevice.Wasm, + }; + }), + }; + + const engine = new sam.MobileSamEngine({ + storage, + driver, + processor: { prepareTensor: () => ({}) }, + }); + await engine.init({ + device: sam.ExecutionDevice.Wasm, + encoderUrl: "https://example.com/encoder.onnx", + decoderUrl: "https://example.com/decoder.onnx", + }); + + // Strictly one model in flight: loading both up front held two ONNX buffers + // and the runtime's copy of each at the same time, roughly doubling the peak. + assert.deepEqual(order, ["load:encoder", "create", "load:decoder", "create"]); +}); + +test("ImageProcessor hands the scratch canvas back once the pixels are copied out", () => { + const canvas = createMockCanvas(512, 288); + globalThis.document = { createElement: () => canvas }; + + new sam.ImageProcessor().prepareTensor({ width: 1920, height: 1080 }, createMockOrt()); + + // Safari keeps canvas backing stores alive well past the last reference to + // them, so this one is given up explicitly. + assert.equal(canvas.width, 0); + assert.equal(canvas.height, 0); +}); diff --git a/wwwroot/js/mobile-sam.js b/wwwroot/js/mobile-sam.js index d769f07..231e105 100644 --- a/wwwroot/js/mobile-sam.js +++ b/wwwroot/js/mobile-sam.js @@ -24,10 +24,18 @@ const DEFAULT_MODEL_BASE_URL = "https://huggingface.co/Acly/MobileSAM/resolve/ma const DEFAULT_ENCODER_PATH = "mobile_sam_image_encoder.onnx"; const DEFAULT_DECODER_PATH = "sam_mask_decoder_single.onnx"; const CACHE_NAME = "mobilesam-model-v1"; -const ORT_CDN_URL = - "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.21.0/dist/ort.all.bundle.min.mjs"; const ORT_WASM_PATH = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.21.0/dist/"; +// onnxruntime publishes one bundle per set of execution providers, and they are +// not close in size. ort.all pulls in WebGPU, WebGL, WebNN and the training +// runtime, and with them the 23 MB JSEP WebAssembly binary - which it loads +// whether or not the device has a GPU to point it at. The plain WASM build is +// 12 MB. That difference is the whole budget on a phone: iOS gives a tab a few +// hundred megabytes and kills it outright when it goes over, with no error and +// no way to catch it, so the page simply reloads itself mid-tap. +const ORT_WEBGPU_BUNDLE = "ort.webgpu.bundle.min.mjs"; +const ORT_WASM_BUNDLE = "ort.wasm.bundle.min.mjs"; + const TARGET_SIZE = 1024; const CHANNELS = 3; const LOW_RES_SIZE = 256; @@ -95,13 +103,18 @@ export class ModelStorage { if ("caches" in globalThis) { try { + // Response copies what it is handed, so slicing first meant three + // copies of a 40 MB model alive at once: ours, the slice, and the + // body. Hand it the buffer directly and there are two. const cache = await globalThis.caches.open(this.#cacheName); - const cacheResponse = new Response(buffer.slice(0), { - headers: { - "content-type": "application/octet-stream", - }, - }); - await cache.put(url, cacheResponse); + await cache.put( + url, + new Response(buffer, { + headers: { + "content-type": "application/octet-stream", + }, + }), + ); } catch { // Storage quota exceeded or private mode restriction } @@ -151,10 +164,21 @@ export class OrtDriver { constructor(options = {}) { this.#ort = options.ort || null; - this.#cdnUrl = options.cdnUrl || ORT_CDN_URL; + this.#cdnUrl = options.cdnUrl || null; this.#wasmPath = options.wasmPath || ORT_WASM_PATH; } + // Loading the WebGPU runtime on a device with no navigator.gpu costs ~11 MB + // of WebAssembly that can never run anything. + resolveCdnUrl() { + if (this.#cdnUrl) { + return this.#cdnUrl; + } + + const bundle = globalThis.navigator?.gpu ? ORT_WEBGPU_BUNDLE : ORT_WASM_BUNDLE; + return `${ORT_WASM_PATH}${bundle}`; + } + async getOrt() { if (this.#ort) { return this.#ort; @@ -166,7 +190,7 @@ export class OrtDriver { return this.#ort; } - const loaded = await import(/* webpackIgnore: true */ this.#cdnUrl); + const loaded = await import(/* webpackIgnore: true */ this.resolveCdnUrl()); this.#ort = loaded.default || loaded; this.#configureWasm(this.#ort); return this.#ort; @@ -181,8 +205,16 @@ export class OrtDriver { ort.env.wasm.wasmPaths = this.#wasmPath; } - const cores = navigator.hardwareConcurrency || 2; - ort.env.wasm.numThreads = Math.min(4, cores); + // Threaded WASM runs on SharedArrayBuffer, which the browser only exposes + // to a cross-origin-isolated page - COOP and COEP headers this app does + // not send. Asking for threads anyway has onnxruntime stand up worker + // and memory infrastructure it can never use. + const isolated = + globalThis.crossOriginIsolated === true && typeof SharedArrayBuffer !== "undefined"; + + ort.env.wasm.numThreads = isolated + ? Math.min(4, globalThis.navigator?.hardwareConcurrency || 2) + : 1; } async createSession(ort, bufferOrUrl, preferredDevice = ExecutionDevice.Auto) { @@ -198,6 +230,13 @@ export class OrtDriver { } async #tryWebGpuThenWasm(ort, bufferOrUrl, strictGpu) { + // resolveCdnUrl only loads the WebGPU runtime where the device reported a + // GPU, so without one this attempt cannot do anything except fail - after + // allocating its way there, which is the part worth avoiding. + if (!strictGpu && !globalThis.navigator?.gpu) { + return await this.#createWasmSession(ort, bufferOrUrl); + } + try { const session = await ort.InferenceSession.create(bufferOrUrl, { executionProviders: [ExecutionDevice.WebGpu], @@ -237,6 +276,13 @@ export class ImageProcessor { const imageData = ctx.getImageData(0, 0, scaledWidth, scaledHeight); const pixels = imageData.data; + // Safari holds on to canvas backing stores long after the element is + // unreachable, and this one exists only for the pixels just copied out + // of it. Zeroing the dimensions releases it now rather than whenever a + // collection happens to run. + canvas.width = 0; + canvas.height = 0; + const totalPixels = scaledWidth * scaledHeight; const tensorData = new Float32Array(totalPixels * CHANNELS); @@ -561,15 +607,33 @@ export class MobileSamEngine { new URL(options.decoderPath || DEFAULT_DECODER_PATH, baseUrl).href; const preferredDevice = options.device || ExecutionDevice.Auto; - const [encoderBuffer, decoderBuffer] = await Promise.all([ - this.#storage.loadBuffer(encoderUrl, options.onEncoderProgress), - this.#storage.loadBuffer(decoderUrl, options.onDecoderProgress), - ]); - - const [encoderResult, decoderResult] = await Promise.all([ - this.#driver.createSession(this.#ort, encoderBuffer, preferredDevice), - this.#driver.createSession(this.#ort, decoderBuffer, preferredDevice), - ]); + // One model at a time. Loading both in parallel held two ONNX buffers + // and the runtime's own copy of each in memory simultaneously, which + // roughly doubles the peak for no wall-clock gain worth having - the + // download is the slow part and it is cached after the first run. + // Releasing each buffer before reaching for the next keeps the high + // water mark at one model rather than two. + let encoderBuffer = await this.#storage.loadBuffer( + encoderUrl, + options.onEncoderProgress, + ); + const encoderResult = await this.#driver.createSession( + this.#ort, + encoderBuffer, + preferredDevice, + ); + encoderBuffer = null; + + let decoderBuffer = await this.#storage.loadBuffer( + decoderUrl, + options.onDecoderProgress, + ); + const decoderResult = await this.#driver.createSession( + this.#ort, + decoderBuffer, + preferredDevice, + ); + decoderBuffer = null; this.#encoderSession = encoderResult.session; this.#decoderSession = decoderResult.session; From a693e9849a3f58988e40bc06095d28236423be0d Mon Sep 17 00:00:00 2001 From: Andrew Roddy Date: Sun, 20 Sep 2026 10:14:55 -0400 Subject: [PATCH 2/2] Say why the camera is unavailable, and how to get it back Two failures looked identical from the viewfinder: a plain http:// origin, where Safari withholds navigator.mediaDevices entirely and there is no prompt to show, and a refusal, which Safari remembers per site and then repeats without ever prompting again. Both left the same inactive stage and a button that did nothing, and tapping it again could not help in either case. The browser is now asked what it supports before anything is offered, so an insecure origin says so instead of presenting a dead button. And openCamera reports its outcome rather than throwing it, because interop flattens a thrown error down to its message and the DOMException name is the part that separates a refusal the person can undo from a device that simply has no camera. A refusal now carries the steps back through Safari's own settings. Flipping the camera tears the old stream down before asking for the new one, so it reports the same way - a refusal there used to leave a blank stage with nothing said. --- .../PhysicalAssets/LiveCameraScanner.razor | 117 +++++++++++++++++- .../LiveCameraScanner.razor.css | 26 ++++ PhysicalAssets/CameraSupport.cs | 25 ++++ .../PhysicalAssets/LiveCameraScannerTests.cs | 73 ++++++++++- .../PhysicalAssets/VideoScanPickerTests.cs | 6 + tests/js/camera-scanner.test.mjs | 71 ++++++++++- wwwroot/js/camera-scanner.js | 60 ++++++++- 7 files changed, 366 insertions(+), 12 deletions(-) create mode 100644 PhysicalAssets/CameraSupport.cs diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor b/Features/PhysicalAssets/LiveCameraScanner.razor index b1a4014..48b4a6d 100644 --- a/Features/PhysicalAssets/LiveCameraScanner.razor +++ b/Features/PhysicalAssets/LiveCameraScanner.razor @@ -30,10 +30,29 @@ @if (!_isStreaming && !_isFrozen) {
-

Camera is inactive.

+ @if (_support is { HasMediaDevices: false }) + { + @* + Safari withholds the camera API entirely on a plain http:// + origin, so there is nothing here for a button to ask for. + Say what would make it work instead of offering a dead tap. + *@ +

This browser will not share a camera over an insecure connection.

+

+ Open Money Mirror over https://, or on localhost, + and the camera becomes available. +

+ } + else + { +

Camera is inactive.

+

+ Your browser will ask for permission — choose Allow to start scanning. +

+ }
} @@ -82,7 +101,32 @@ @if (_errorMessage is not null) { - + }
@@ -151,22 +195,67 @@ private bool _isModelReady; private string? _engineDevice; private string? _errorMessage; + private string? _errorCode; private string? _statusHint; + private CameraSupport? _support; private SamSegmentationResult? _lastResult; private string _cameraFacing = FacingEnvironment; private SamPromptType _currentPromptMode = SamPromptType.Positive; private readonly List _prompts = new(); + /// + /// Asks the browser what it is willing to do before the person taps anything, + /// so an origin with no camera API can say so rather than offer a button that + /// cannot work. + /// + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (!firstRender) + { + return; + } + + _cameraModule ??= await JS.InvokeAsync("import", CameraModulePath); + _support = await _cameraModule.InvokeAsync("describeSupport"); + StateHasChanged(); + } + public async Task StartCameraAsync() { _errorMessage = null; + _errorCode = null; try { _cameraModule ??= await JS.InvokeAsync("import", CameraModulePath); - await _cameraModule.InvokeVoidAsync("startCamera", _video, _cameraFacing); + var result = await _cameraModule.InvokeAsync( + "openCamera", + _video, + _cameraFacing + ); + + if (!result.Ok) + { + _errorCode = result.Code; + _errorMessage = result.Message; + Logger.LogWarning( + "Camera could not be started ({Code}): {Message}", + result.Code, + result.Message + ); + return; + } + + _cameraFacing = result.Facing ?? _cameraFacing; _isStreaming = true; _isFrozen = false; + // The model is ~40 MB and has nothing to do with which camera is + // running, so a second start should not fetch it again. + if (_isModelReady) + { + return; + } + _statusHint = "Downloading MobileSAM model (~40 MB)..."; StateHasChanged(); @@ -197,6 +286,7 @@ public async Task SwitchCameraAsync() { _errorMessage = null; + _errorCode = null; if (_cameraModule is null) { @@ -206,8 +296,25 @@ try { var nextFacing = _cameraFacing == FacingEnvironment ? FacingUser : FacingEnvironment; - await _cameraModule.InvokeVoidAsync("switchCamera", _video, _cameraFacing); - _cameraFacing = nextFacing; + var result = await _cameraModule.InvokeAsync( + "flipCamera", + _video, + _cameraFacing + ); + + if (!result.Ok) + { + // Flipping tears the old stream down before asking for the new + // one, so a refusal here leaves no feed at all - worth explaining + // rather than leaving as a blank stage. + _errorCode = result.Code; + _errorMessage = result.Message; + _isStreaming = false; + _isFrozen = false; + return; + } + + _cameraFacing = result.Facing ?? nextFacing; } catch (Exception ex) { diff --git a/Features/PhysicalAssets/LiveCameraScanner.razor.css b/Features/PhysicalAssets/LiveCameraScanner.razor.css index f60d1c8..81a54b9 100644 --- a/Features/PhysicalAssets/LiveCameraScanner.razor.css +++ b/Features/PhysicalAssets/LiveCameraScanner.razor.css @@ -117,6 +117,32 @@ text-align: center; } +/* Sits on the dark viewfinder stage, so it borrows the placeholder's colour + and steps back from it rather than choosing one of its own. */ +.permission-help { + max-width: 24rem; + font-size: 0.875rem; + line-height: 1.5; + opacity: 0.85; +} + +.permission-help code { + font-size: 0.9em; +} + +/* Recovery steps live inside the denial alert, which is a light surface - these + inherit its colour, not the stage's. */ +.permission-steps { + margin: 0.5rem 0 0; + padding-left: 1.25rem; + font-size: 0.875rem; + line-height: 1.5; +} + +.permission-steps li + li { + margin-top: 0.25rem; +} + .action-bar { display: flex; flex-wrap: wrap; diff --git a/PhysicalAssets/CameraSupport.cs b/PhysicalAssets/CameraSupport.cs new file mode 100644 index 0000000..95798dd --- /dev/null +++ b/PhysicalAssets/CameraSupport.cs @@ -0,0 +1,25 @@ +namespace MoneyMirror.PhysicalAssets; + +/// +/// What the visitor's browser will allow before they tap anything, as reported +/// by describeSupport in camera-scanner.js. +/// +/// +/// False on a plain http:// origin. Safari withholds the camera API entirely +/// there, so the failure needs explaining rather than retrying. +/// +/// Whether navigator.mediaDevices.getUserMedia exists at all. +/// Whether to offer iOS-shaped recovery steps when permission is refused. +public record CameraSupport(bool SecureContext, bool HasMediaDevices, bool IsIos); + +/// +/// The outcome of asking the browser for a camera, as reported by +/// openCamera in camera-scanner.js. +/// +/// +/// The DOMException name - NotAllowedError for a refusal, +/// NotFoundError for a device with no camera, and so on. Interop would +/// drop this if the browser threw instead of reporting, and a refusal is the one +/// failure the person can actually do something about. +/// +public record CameraStartResult(bool Ok, string? Facing, string? Code, string? Message); diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs index 6304e03..94bcb66 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/LiveCameraScannerTests.cs @@ -22,9 +22,16 @@ public LiveCameraScannerTests() _context.Services.AddSingleton(_storage); _module = _context.JSInterop.SetupModule("./js/camera-scanner.js"); - _module.SetupVoid("startCamera", _ => true).SetVoidResult(); + _module + .Setup("describeSupport", _ => true) + .SetResult(new CameraSupport(SecureContext: true, HasMediaDevices: true, IsIos: true)); + _module + .Setup("openCamera", _ => true) + .SetResult(new CameraStartResult(Ok: true, Facing: "environment", null, null)); + _module + .Setup("flipCamera", _ => true) + .SetResult(new CameraStartResult(Ok: true, Facing: "user", null, null)); _module.SetupVoid("stopCamera", _ => true).SetVoidResult(); - _module.SetupVoid("switchCamera", _ => true).SetVoidResult(); _module.SetupVoid("freezeFrame", _ => true).SetVoidResult(); _module.SetupVoid("unfreezeFrame", _ => true).SetVoidResult(); _module @@ -50,7 +57,7 @@ public async Task StartCamera_InitializesEngineAndStream() await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); - _module.VerifyInvoke("startCamera", 1); + _module.VerifyInvoke("openCamera", 1); Assert.True(_samEngine.InitCalls >= 1); Assert.Contains("Camera active", cut.Markup); } @@ -128,7 +135,65 @@ public async Task SwitchCamera_CallsJsSwitchCamera() .Single(b => b.TextContent.Contains("Switch camera")); await switchButton.ClickAsync(new MouseEventArgs()); - _module.VerifyInvoke("switchCamera", 1); + _module.VerifyInvoke("flipCamera", 1); + } + + [Fact] + public async Task CameraRefused_ExplainsHowToGrantItBack() + { + using var context = new BunitContext(); + context.Services.AddLogging(); + context.Services.AddSingleton(new FakeSamEngine()); + context.Services.AddSingleton(new FakeStorage()); + + var module = context.JSInterop.SetupModule("./js/camera-scanner.js"); + module.SetupVoid("stopCamera", _ => true).SetVoidResult(); + module + .Setup("describeSupport", _ => true) + .SetResult(new CameraSupport(SecureContext: true, HasMediaDevices: true, IsIos: true)); + module + .Setup("openCamera", _ => true) + .SetResult( + new CameraStartResult( + Ok: false, + Facing: null, + Code: "NotAllowedError", + Message: "Camera access was denied." + ) + ); + + var cut = context.Render(); + await cut.Find("button.btn-primary").ClickAsync(new MouseEventArgs()); + + // Safari refuses again without prompting once it has been told no, so a + // second tap cannot help and the way back has to be spelled out. + Assert.Contains("Camera access was denied.", cut.Markup); + Assert.Contains("Website Settings", cut.Markup); + Assert.DoesNotContain("Camera active", cut.Markup); + } + + [Fact] + public void InsecureOrigin_OffersNoCameraButtonAtAll() + { + using var context = new BunitContext(); + context.Services.AddLogging(); + context.Services.AddSingleton(new FakeSamEngine()); + context.Services.AddSingleton(new FakeStorage()); + + var module = context.JSInterop.SetupModule("./js/camera-scanner.js"); + module.SetupVoid("stopCamera", _ => true).SetVoidResult(); + module + .Setup("describeSupport", _ => true) + .SetResult( + new CameraSupport(SecureContext: false, HasMediaDevices: false, IsIos: true) + ); + + var cut = context.Render(); + + // There is no camera API on a plain http:// origin, so a button could do + // nothing but fail. + Assert.DoesNotContain("Start live camera", cut.Markup); + Assert.Contains("insecure connection", cut.Markup); } [Fact] diff --git a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs index 7e99aa0..671c1f4 100644 --- a/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs +++ b/tests/MoneyMirror.Tests/PhysicalAssets/VideoScanPickerTests.cs @@ -39,6 +39,12 @@ public VideoScanPickerTests() _context.Services.AddSingleton(_valuation); _context.Services.AddSingleton(_repository); _context.Services.AddSingleton(new FakeSamEngine()); + // The page also hosts LiveCameraScanner, which asks the browser what it + // supports on first render. + _context.JSInterop.SetupModule("./js/camera-scanner.js") + .Setup("describeSupport", _ => true) + .SetResult(new CameraSupport(SecureContext: true, HasMediaDevices: true, IsIos: false)); + _module = _context.JSInterop.SetupModule("./js/video-scan.js"); _module.Setup("prepare", _ => true).SetResult([0.5, 1.5]); _module.Setup("frameUrl", _ => true).SetResult("blob:test-frame"); diff --git a/tests/js/camera-scanner.test.mjs b/tests/js/camera-scanner.test.mjs index 758397e..7b3473c 100644 --- a/tests/js/camera-scanner.test.mjs +++ b/tests/js/camera-scanner.test.mjs @@ -7,9 +7,9 @@ const camera = await import(`data:text/javascript;base64,${source.toString("base const originalDescriptor = Object.getOwnPropertyDescriptor(globalThis, "navigator"); -function setMockNavigator(mediaDevices) { +function setMockNavigator(mediaDevices, userAgent = "", maxTouchPoints = 0) { Object.defineProperty(globalThis, "navigator", { - value: { mediaDevices }, + value: { mediaDevices, userAgent, maxTouchPoints }, configurable: true, writable: true, }); @@ -54,6 +54,7 @@ afterEach(() => { if (originalDescriptor) { Object.defineProperty(globalThis, "navigator", originalDescriptor); } + delete globalThis.isSecureContext; }); test("CameraDriver starts video stream with environment facing mode constraints", async () => { @@ -219,3 +220,69 @@ test("extractCutout creates cropped canvas with polygon clipping and returns blo assert.ok(drawn); assert.equal(paths.length, 4); }); + +test("openCamera reports a refusal by name instead of throwing it away", async () => { + const denial = new Error("Permission denied"); + denial.name = "NotAllowedError"; + + setMockNavigator({ + getUserMedia: mock.fn(async () => { + throw denial; + }), + }); + + const result = await camera.openCamera(createMockVideo()); + + // Interop flattens a thrown error down to its message, and the name is the + // part that says the person can grant this back. + assert.equal(result.ok, false); + assert.equal(result.code, "NotAllowedError"); + assert.match(result.message, /Camera access was denied/); +}); + +test("openCamera reports the facing mode it actually got", async () => { + setMockNavigator({ getUserMedia: mock.fn(async () => createMockStream()) }); + + const result = await camera.openCamera(createMockVideo(), camera.CameraFacing.User); + + assert.equal(result.ok, true); + assert.equal(result.facing, "user"); + assert.equal(result.code, null); +}); + +test("flipCamera reports a refusal rather than leaving a dead stage", async () => { + const failure = new Error("Camera is in use"); + failure.name = "NotReadableError"; + + setMockNavigator({ + getUserMedia: mock.fn(async () => { + throw failure; + }), + }); + + const result = await camera.flipCamera(createMockVideo(), camera.CameraFacing.Environment); + + assert.equal(result.ok, false); + assert.equal(result.code, "NotReadableError"); +}); + +test("describeSupport reports a withheld camera API instead of guessing", () => { + setMockNavigator(undefined); + globalThis.isSecureContext = false; + + const support = camera.describeSupport(); + + assert.equal(support.hasMediaDevices, false); + assert.equal(support.secureContext, false); +}); + +test("describeSupport recognises iOS, including an iPad claiming to be a Mac", () => { + setMockNavigator({ getUserMedia: () => {} }, "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0)"); + assert.equal(camera.describeSupport().isIos, true); + + setMockNavigator({ getUserMedia: () => {} }, "Mozilla/5.0 (Macintosh; Intel Mac OS X)", 5); + assert.equal(camera.describeSupport().isIos, true); + + setMockNavigator({ getUserMedia: () => {} }, "Mozilla/5.0 (Macintosh; Intel Mac OS X)", 0); + assert.equal(camera.describeSupport().isIos, false); +}); diff --git a/wwwroot/js/camera-scanner.js b/wwwroot/js/camera-scanner.js index 6c31eb3..f091d69 100644 --- a/wwwroot/js/camera-scanner.js +++ b/wwwroot/js/camera-scanner.js @@ -47,7 +47,12 @@ export class CameraDriver { height: video.videoHeight || DEFAULT_IDEAL_HEIGHT, }; } catch (error) { - throw new Error(this.#mapErrorMessage(error)); + // The name is what separates a refusal from a missing device, and the + // two want very different advice, so keep it rather than flattening + // everything into one sentence. + const wrapped = new Error(this.#mapErrorMessage(error)); + wrapped.code = error?.name || "UnknownError"; + throw wrapped; } } @@ -208,6 +213,59 @@ export function stopCamera(video) { defaultDriver.stop(video); } +/** + * What this browser is willing to do, before anyone taps anything. + * + * WebKit withholds navigator.mediaDevices entirely outside a secure context, so + * over plain http:// on a LAN address there is no camera API to refuse and no + * prompt to show - only an undefined property. Knowing that up front lets the + * page explain itself instead of offering a button that cannot work. + */ +export function describeSupport() { + const agent = navigator?.userAgent || ""; + + // iPadOS reports itself as a Mac; the touch points are what give it away. + const isIos = + /iPad|iPhone|iPod/.test(agent) || + (agent.includes("Macintosh") && (navigator?.maxTouchPoints || 0) > 1); + + return { + secureContext: typeof isSecureContext === "boolean" ? isSecureContext : true, + hasMediaDevices: !!navigator?.mediaDevices?.getUserMedia, + isIos, + }; +} + +/** + * Open the camera, reporting a refusal instead of throwing one. + * + * Interop turns a thrown error into a JSException carrying only its message, so + * the DOMException name - the part that says whether the person refused or the + * device simply has no camera - would be lost on the way across. Handing back a + * result keeps it. + */ +export async function openCamera(video, facing = CameraFacing.Environment) { + return await attemptCamera(() => defaultDriver.start(video, facing)); +} + +export async function flipCamera(video, currentFacing) { + return await attemptCamera(() => defaultDriver.switchFacing(video, currentFacing)); +} + +async function attemptCamera(run) { + try { + const result = await run(); + return { ok: true, facing: result.facing, code: null, message: null }; + } catch (error) { + return { + ok: false, + facing: null, + code: error?.code || "UnknownError", + message: error?.message || "Failed to start camera feed.", + }; + } +} + export function freezeFrame(video, canvas) { return defaultCapture.freeze(video, canvas); }