diff --git a/android/src/androidTest/java/io/github/webview_native_ui/engine/NativeUIContactCaptureTest.kt b/android/src/androidTest/java/io/github/webview_native_ui/engine/NativeUIContactCaptureTest.kt new file mode 100644 index 0000000..f864850 --- /dev/null +++ b/android/src/androidTest/java/io/github/webview_native_ui/engine/NativeUIContactCaptureTest.kt @@ -0,0 +1,241 @@ +package io.github.webview_native_ui.engine + +import android.content.Intent +import android.graphics.RectF +import android.os.SystemClock +import android.view.MotionEvent +import android.widget.Button +import android.widget.FrameLayout +import androidx.test.platform.app.InstrumentationRegistry +import java.io.File +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.* +import org.junit.Test + +class NativeUIContactCaptureTest { + private val instrumentation + get() = InstrumentationRegistry.getInstrumentation() + + private fun main(block: () -> Unit) = instrumentation.runOnMainSync(block) + + @Test fun visualInvalidationRetainsPendingContactAndNativeTap() = verify(false) + + @Test fun sourceInvalidationCancelsPendingContactWithoutDroppingNativeTap() = verify(true) + + @Test fun captureTimeoutPreservesNativeTap() = verify(false, true) + + @Test fun requestArmedDuringWebGestureSurvivesToNativeContact() = verify(false, false, true) + + @Test fun contactAcquisitionPreservesNativeLongPress() = verify(false, longPress = true) + + @Test fun releasedPixelsAreNotSelectedByLaterContact() = verify(false, release = true) + + private fun verify( + cancel: Boolean, + stall: Boolean = false, + duringGesture: Boolean = false, + longPress: Boolean = false, + release: Boolean = false, + ) { + val activity = + instrumentation.startActivitySync( + Intent(instrumentation.targetContext, EngineTestActivity::class.java) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + ) as EngineTestActivity + lateinit var snapshots: NativeUISnapshots + lateinit var button: Button + val done = CountDownLatch(1) + val clicked = CountDownLatch(1) + val clickedAfterRelease = CountDownLatch(1) + var tapCount = 0 + var longCount = 0 + val selected = mutableListOf() + var result = JSONObject() + var callbackBeforePress = false + try { + main { + if (stall) { + val previous = activity.web + activity.root.removeView(previous) + previous.destroy() + activity.web = + object : android.webkit.WebView(activity) { + override fun postVisualStateCallback( + id: Long, + callback: VisualStateCallback, + ) {} + } + activity.root.addView(activity.web, FrameLayout.LayoutParams(-1, -1)) + } + snapshots = + NativeUISnapshots(File(activity.cacheDir, "contact-${System.nanoTime()}")) + button = + Button(activity).apply { + text = "SDK button" + setOnClickListener { + tapCount++ + clicked.countDown() + if (tapCount == 2) clickedAfterRelease.countDown() + } + setOnLongClickListener { + longCount++ + clicked.countDown() + true + } + } + activity.root.addView( + button, + FrameLayout.LayoutParams(220, 90).apply { + leftMargin = 30 + topMargin = 180 + }, + ) + snapshots.observeInteractions(activity.window) { event -> + if (event.has("captureID")) { + selected.add(event) + callbackBeforePress = !button.isPressed + } + } + } + instrumentation.waitForIdleSync() + if (!stall) { + val painted = CountDownLatch(1) + main { + activity.web.postVisualStateCallback( + 0, + object : android.webkit.WebView.VisualStateCallback() { + override fun onComplete(id: Long) { + activity.web.postOnAnimation { + activity.web.postOnAnimation { + activity.web.post { painted.countDown() } + } + } + } + }, + ) + } + assertTrue("Fixture Web draw did not complete", painted.await(5, TimeUnit.SECONDS)) + } + val request = + JSONObject() + .put("action", "prepare") + .put("ids", JSONArray().put("sdk")) + .put("revision", 1) + .put("viewportWidth", activity.web.width) + .put("captureID", "contact") + .put("captureOwner", "owner") + .put("captureDelayMs", 10000) + .put("stopScrollBeforeCapture", true) + fun send(request: JSONObject, complete: (JSONObject) -> Unit = {}) { + snapshots.handle( + request, + activity.window, + activity.web, + frames = { _, _ -> + mapOf( + "sdk" to + RectF( + button.x, + button.y, + button.x + button.width, + button.y + button.height, + ) + ) + }, + views = { listOf(button) }, + ) { + complete(it.getOrThrow()) + } + } + main { + if (duringGesture) snapshots.interaction(true) + send(request) { + result = it + done.countDown() + } + repeat(5) { + send( + JSONObject() + .put("action", "change") + .put("captureOwner", "owner") + .put("retainPending", true) + ) + } + button.x = 60f + if (cancel) send(JSONObject().put("action", "change").put("captureOwner", "owner")) + if (duringGesture) snapshots.interaction(false) + val time = SystemClock.uptimeMillis() + val location = IntArray(2) + button.getLocationOnScreen(location) + val root = IntArray(2) + activity.window.decorView.getLocationOnScreen(root) + for ((action, at) in + listOf( + MotionEvent.ACTION_DOWN to time, + MotionEvent.ACTION_UP to + time + + (if (longPress) + android.view.ViewConfiguration.getLongPressTimeout() + 100 + else 40), + )) { + val event = + MotionEvent.obtain( + time, + at, + action, + location[0] - root[0] + 20f, + location[1] - root[1] + 20f, + 0, + ) + activity.window.callback.dispatchTouchEvent(event) + event.recycle() + } + } + assertTrue("Capture did not complete", done.await(10, TimeUnit.SECONDS)) + assertTrue("SDK tap was lost", clicked.await(3, TimeUnit.SECONDS)) + assertEquals(if (longPress) 0 else 1, tapCount) + assertEquals(if (longPress) 1 else 0, longCount) + if (cancel || stall) { + assertFalse(result.has("token")) + assertTrue(selected.isEmpty()) + } else { + assertTrue("No immutable pixels", result.has("token")) + assertEquals(1, selected.size) + assertEquals("contact", selected.single().getString("captureID")) + assertTrue("Captured SDK pressed state", callbackBeforePress) + assertEquals( + 48.0, + result.getJSONArray("parts").getJSONObject(0).getDouble("x"), + 1.0, + ) + } + if (release) { + main { + snapshots.release(result.getString("token")) + selected.clear() + val time = SystemClock.uptimeMillis() + for (action in listOf(MotionEvent.ACTION_DOWN, MotionEvent.ACTION_UP)) { + val event = + MotionEvent.obtain(time, time, action, button.x + 20, button.y + 20, 0) + activity.window.callback.dispatchTouchEvent(event) + event.recycle() + } + assertTrue("Released capture selected by later contact", selected.isEmpty()) + } + assertTrue( + "SDK tap lost after release", + clickedAfterRelease.await(3, TimeUnit.SECONDS), + ) + assertEquals(2, tapCount) + } + } finally { + main { + snapshots.close() + activity.finish() + } + } + } +} diff --git a/android/src/androidTest/java/io/github/webview_native_ui/engine/NativeUIProjectionContactTest.kt b/android/src/androidTest/java/io/github/webview_native_ui/engine/NativeUIProjectionContactTest.kt new file mode 100644 index 0000000..56c53f3 --- /dev/null +++ b/android/src/androidTest/java/io/github/webview_native_ui/engine/NativeUIProjectionContactTest.kt @@ -0,0 +1,341 @@ +package io.github.webview_native_ui.engine + +import android.content.Intent +import android.graphics.Canvas +import android.graphics.Color +import android.os.SystemClock +import android.view.MotionEvent +import android.view.View +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebView +import android.webkit.WebViewClient +import android.widget.Button +import android.widget.FrameLayout +import androidx.test.platform.app.InstrumentationRegistry +import java.io.File +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import org.junit.Assert.* +import org.junit.Test + +/** Exercise the public SDK registration, including its own transport and original native input. */ +class NativeUIProjectionContactTest { + private val instrumentation + get() = InstrumentationRegistry.getInstrumentation() + + private fun main(block: () -> Unit) = instrumentation.runOnMainSync(block) + + @Test + fun compositedSDKContactCapturesDuringVirtualChanges() = + verify(NativeUIProjection.CaptureMode.COMPOSITED) + + @Test + fun viewSDKContactCapturesDuringVirtualChanges() = verify(NativeUIProjection.CaptureMode.VIEW) + + @Test + fun hostedSDKStopsWebMotionBeforeAcquisitionAndOriginalInput() = + verify(NativeUIProjection.CaptureMode.VIEW, hosted = true) + + private fun verify(mode: NativeUIProjection.CaptureMode, hosted: Boolean = false) { + val activity = + instrumentation.startActivitySync( + Intent(instrumentation.targetContext, EngineTestActivity::class.java) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + ) as EngineTestActivity + lateinit var registration: NativeUIProjection + lateinit var button: Button + val loaded = CountDownLatch(1) + val clicked = CountDownLatch(1) + var clickCount = 0 + var trackingContact = false + var webDowns = 0 + var stopsAtAcquisition: Int? = null + fun js(expression: String): String { + val done = CountDownLatch(1) + var result = "" + main { + activity.web.evaluateJavascript(expression) { + result = it + done.countDown() + } + } + assertTrue("Web evaluation did not finish", done.await(3, TimeUnit.SECONDS)) + return result + } + fun diagnostics(): String = + js( + """ + JSON.stringify({events:nativeEvents.filter(e=>!e.capturePolicy).slice(-25), + transactions:nativeRequests.filter(e=>!['change','track'].includes(e.request.action)).slice(-30).map(e=>({...e,reply:nativeReplies.find(r=>r.key===e.key)})), + virtualChanges,attributes:[...document.querySelector('#slot').attributes].map(a=>[a.name,a.value]), + images:[...document.querySelectorAll('#slot img')].map(i=>({src:i.src,complete:i.complete,width:i.naturalWidth,opacity:getComputedStyle(i).opacity,display:getComputedStyle(i).display}))}) + """ + .trimIndent() + ) + fun waitFor(expression: String) { + val deadline = SystemClock.uptimeMillis() + 10000 + while (SystemClock.uptimeMillis() < deadline) { + if (js(expression) == "true") return + Thread.sleep(20) + } + fail("Timed out: $expression; state=" + diagnostics()) + } + fun waitNative(visibility: Int) { + val deadline = SystemClock.uptimeMillis() + 5000 + while (SystemClock.uptimeMillis() < deadline) { + var actual = -1 + main { actual = button.visibility } + if (actual == visibility) return + Thread.sleep(20) + } + fail("Native visibility did not reach $visibility; state=" + diagnostics()) + } + try { + main { + val density = activity.resources.displayMetrics.density + if (hosted) { + val previous = activity.web + activity.root.removeView(previous) + previous.destroy() + activity.web = + object : WebView(activity) { + override fun onTouchEvent(event: MotionEvent): Boolean { + if ( + trackingContact && + event.actionMasked == MotionEvent.ACTION_DOWN + ) + webDowns++ + return super.onTouchEvent(event) + } + } + .apply { settings.javaScriptEnabled = true } + activity.root.addView(activity.web, 0, FrameLayout.LayoutParams(-1, -1)) + } + button = + object : Button(activity) { + override fun draw(canvas: Canvas) { + if ( + trackingContact && + !canvas.isHardwareAccelerated && + stopsAtAcquisition == null + ) + stopsAtAcquisition = webDowns + super.draw(canvas) + } + } + .apply { + text = "SDK" + setBackgroundColor(Color.RED) + setOnClickListener { + clickCount++ + trackingContact = false + // SDK input changes the live pixels; capture must precede it. + setBackgroundColor(Color.GREEN) + registration.receive("moving") + // Capacitor forwards the same SDK event to its Web proxy. + activity.web.evaluateJavascript("testEvents.moving({})", null) + clicked.countDown() + } + } + activity.root.addView( + if (hosted) NativeUIScrollHosting(activity.web, button) else button, + FrameLayout.LayoutParams((100 * density).toInt(), (50 * density).toInt()) + .apply { + leftMargin = (20 * density).toInt() + topMargin = (100 * density).toInt() + }, + ) + activity.web.webViewClient = + object : WebViewClient() { + override fun onPageFinished(view: WebView, url: String) { + loaded.countDown() + } + + override fun shouldInterceptRequest( + view: WebView, + request: WebResourceRequest, + ): WebResourceResponse? { + val path = request.url.path ?: return null + if (!path.startsWith("/_capacitor_file_")) return null + val file = File(path.removePrefix("/_capacitor_file_")) + if ( + !file.canonicalPath.startsWith( + activity.cacheDir.canonicalPath + "/" + ) + ) + return null + return runCatching { + WebResourceResponse("image/png", null, file.inputStream()) + } + .getOrNull() + } + } + registration = + NativeUIProjection( + activity.window, + activity.web, + "SDK", + "#slot", + receiveListeners = + NativeUIProjection.ReceiveListeners( + projection = + NativeUIProjection.ReceiveListeners.Projection( + native = listOf("shown"), + web = listOf("moving"), + ) + ), + failedFallback = NativeUIProjection.FailedFallback.WEB, + captureMode = mode, + captureDelayMs = 1000, + view = { button }, + ) + activity.web.loadDataWithBaseURL( + "https://localhost", + """ + + + +
Web fallback
+
Recycled rows
+ + """ + .trimIndent(), + "text/html", + "utf-8", + null, + ) + } + assertTrue("Fixture navigation did not finish", loaded.await(10, TimeUnit.SECONDS)) + waitFor("typeof testEvents.shown==='function'") + js("testEvents.shown({})") + waitFor("document.querySelector('#slot').hasAttribute('data-native-ui-snapshot')") + waitFor( + "nativeEvents.some(event=>event.capturePolicy==='preinteraction'&&event.prepareDuringInteraction===true)" + ) + // Retire the settled image, then keep changing virtual rows. Native acquisition is + // delayed by 1s, so a fresh immutable image is available only through contact. + js( + """ + window.policiesBefore=nativeEvents.filter(e=>e.capturePolicy).length; + window.virtualChanges=0;window.keepChanging=true; + function recycle(){ + if(!keepChanging)return; + const rows=document.querySelector('#rows'); + rows.textContent='Recycled '+(++virtualChanges); + rows.style.transform='translateY('+(virtualChanges%7)+'px)'; + requestAnimationFrame(recycle); + } + recycle();dispatchEvent(new Event('resize')); + """ + .trimIndent() + ) + waitFor( + "nativeEvents.filter(e=>e.capturePolicy).length>policiesBefore&&virtualChanges>=3" + ) + assertEquals( + "Contact must not use a previously settled image", + "false", + js("document.querySelector('#slot').hasAttribute('data-native-ui-snapshot')"), + ) + main { + assertEquals(View.VISIBLE, button.visibility) + // Require fresh VIEW pixels, rather than reusing the initial settled raster. + if (hosted) button.alpha = 0.99f + trackingContact = hosted + val location = IntArray(2) + val origin = IntArray(2) + button.getLocationOnScreen(location) + activity.window.decorView.getLocationOnScreen(origin) + val now = SystemClock.uptimeMillis() + for ((action, time) in + listOf(MotionEvent.ACTION_DOWN to now, MotionEvent.ACTION_UP to now + 40)) { + val event = + MotionEvent.obtain( + now, + time, + action, + location[0] - origin[0] + button.width / 2f, + location[1] - origin[1] + button.height / 2f, + 0, + ) + activity.window.callback.dispatchTouchEvent(event) + event.recycle() + } + } + assertTrue("Original SDK tap was lost", clicked.await(4, TimeUnit.SECONDS)) + assertEquals(1, clickCount) + if (hosted) { + assertEquals( + "Web motion was not stopped before acquiring VIEW pixels", + 1, + stopsAtAcquisition ?: -1, + ) + } + waitFor( + "document.querySelector('#slot').hasAttribute('data-native-ui-snapshot-visible')" + ) + waitFor( + "[...document.querySelectorAll('#slot img[data-native-ui-projection-image]')].some(image=>image.complete&&image.naturalWidth>0&&getComputedStyle(image).opacity==='1')" + ) + waitFor( + "nativeEvents.some(event=>event.captureID&&event.captureOwner&&event.captureState==='processing')" + ) + // The SDK paints green as soon as its original click is dispatched. Red pixels prove + // this is the pre-contact image rather than a late screenshot of the SDK action. + assertEquals( + "SDK action was captured instead of pre-contact pixels", + "true", + js( + """ + (()=>{ + const image=document.querySelector('#slot img[data-native-ui-projection-image]'); + const canvas=document.createElement('canvas');canvas.width=image.naturalWidth;canvas.height=image.naturalHeight; + const context=canvas.getContext('2d');context.drawImage(image,0,0); + const pixels=context.getImageData(0,0,canvas.width,canvas.height).data; + let red=0,green=0; + for(let i=0;i200&&pixels[i+1]<80&&pixels[i+2]<80)red++; + if(pixels[i]<80&&pixels[i+1]>200&&pixels[i+2]<80)green++; + } + return red>canvas.width*canvas.height/4&&green===0; + })() + """ + .trimIndent() + ), + ) + waitNative(View.INVISIBLE) + main { registration.receive("shown") } + js("keepChanging=false;testEvents.shown({})") + waitFor("document.querySelector('#slot').dataset.nativeUiDestination==='native'") + waitNative(View.VISIBLE) + } catch (error: Throwable) { + throw AssertionError( + "SDK contact fixture failed ($mode, hosted=$hosted); state=" + + runCatching { diagnostics() } + .getOrElse { "diagnostics unavailable: ${it.message}" }, + error, + ) + } finally { + main { + registration.close() + activity.finish() + } + } + } +} diff --git a/android/src/main/java/io/github/webview_native_ui/engine/NativeUIProjection.kt b/android/src/main/java/io/github/webview_native_ui/engine/NativeUIProjection.kt index 12b9b0a..6032c5b 100644 --- a/android/src/main/java/io/github/webview_native_ui/engine/NativeUIProjection.kt +++ b/android/src/main/java/io/github/webview_native_ui/engine/NativeUIProjection.kt @@ -431,16 +431,19 @@ constructor( throw error } } else throw UnsupportedOperationException("WebView document-start scripts are required") - stopInteraction = + val stopCapture = snapshots.observeInteractions(window) { event -> reply(null, event) } + val stopHosting = NativeUIWindowInteraction.observe( window, drag = ::coverBeforeScroll, contact = ::restoreForContact, ) { active -> windowInteracting = active - snapshots.interaction(active) - reply(null, JSONObject().put("active", active)) } + stopInteraction = { + stopHosting() + stopCapture() + } } private fun frame(view: View): RectF { @@ -547,10 +550,15 @@ constructor( complete(JSONObject()) return } - snapshots.prepare( + // The original SDK DOWN is replayed after acquisition; stop source motion now. + request.put( + "stopScrollBeforeCapture", + selected.values.any { NativeUIScrollHosting.owns(it, webView) }, + ) + snapshots.prepareContact( + request, window, webView, - request.optDouble("viewportWidth"), { val current = views() if ( @@ -563,8 +571,8 @@ constructor( emptyMap() else selected.mapValues { frame(it.value) } }, + views = { selected.values.toList() }, sourceViews = if (captureMode == CaptureMode.VIEW) ({ selected }) else null, - contentRevision = request.optLong("revision"), ) { result -> result .optString("token") @@ -589,7 +597,25 @@ constructor( val ids = part.getJSONArray("ids") for (i in 0 until ids.length()) { val id = ids.getString(i) - selected[id]?.let { entries[id] = Anchor(frame(it), image) } + if (selected.containsKey(id)) { + val anchor = part.getJSONObject("anchorBounds") + entries[id] = + Anchor( + RectF( + x + anchor.getDouble("x").toFloat() * scale, + y + anchor.getDouble("y").toFloat() * scale, + x + + (anchor.getDouble("x") + + anchor.getDouble("width")) + .toFloat() * scale, + y + + (anchor.getDouble("y") + + anchor.getDouble("height")) + .toFloat() * scale, + ), + image, + ) + } } } anchors[token] = entries @@ -598,6 +624,13 @@ constructor( complete(result) } } + "change" -> { + snapshots.invalidateAcquisitions( + request.optString("captureOwner").takeIf { request.has("captureOwner") }, + request.optBoolean("retainPending"), + ) + complete(JSONObject()) + } "prepared" -> { awaitWebDraw( webView, diff --git a/android/src/main/java/io/github/webview_native_ui/engine/NativeUISnapshots.kt b/android/src/main/java/io/github/webview_native_ui/engine/NativeUISnapshots.kt index 6b66563..155986b 100644 --- a/android/src/main/java/io/github/webview_native_ui/engine/NativeUISnapshots.kt +++ b/android/src/main/java/io/github/webview_native_ui/engine/NativeUISnapshots.kt @@ -9,6 +9,8 @@ import android.net.Uri import android.os.Build import android.os.Handler import android.os.Looper +import android.os.SystemClock +import android.view.MotionEvent import android.view.PixelCopy import android.view.View import android.view.ViewGroup @@ -91,6 +93,114 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { private var touching = false private var pending = false private var closed = false + + private class Acquisition(val id: String, val owner: String, val epoch: Long) { + var run: (() -> Unit)? = null + var cancel: (() -> Unit)? = null + var hit: ((MotionEvent) -> Boolean)? = null + var stop: ((MotionEvent) -> Unit)? = null + var contact: (() -> Unit)? = null + var contactTimeout: Runnable? = null + var pixels = false + var token: String? = null + var pinned = false + var changedAt = SystemClock.uptimeMillis() + } + + private var epoch = 0L + private val acquisitions = linkedMapOf() + private var observedWindow: Window? = null + private var interactionListener: ((JSONObject) -> Unit)? = null + private var stopObserver: (() -> Unit)? = null + + /** Acquire immutable pixels before native input; encoding continues after input is replayed. */ + fun observeInteractions(window: Window, changed: (JSONObject) -> Unit): () -> Unit { + onMain() + stopObserver?.invoke() + if (!supported || closed) return {} + observedWindow = window + interactionListener = changed + val remove = + NativeUIWindowInteraction.observe( + window, + contact = { event, complete -> + val selected = + acquisitions.values.toList().filter { + it.pixels || it.hit?.invoke(event) == true + } + if (selected.isEmpty()) false + else { + val deferred = selected.any { !it.pixels } + var remaining = selected.size + selected.forEach { job -> + job.pinned = true + job.contact = { + if (job.pixels) + changed( + JSONObject() + .put("active", true) + .put("captureID", job.id) + .put("captureOwner", job.owner) + .put("captureState", "processing") + ) + if (--remaining == 0 && deferred) complete(true) + } + if (job.hit?.invoke(event) == true) job.stop?.invoke(event) + if (job.pixels) finishContact(job) + else { + // Optional acquisition must release SDK input before the window's + // mandatory-return timeout can cancel the original gesture. + val expired = Runnable { + job.cancel?.invoke() ?: finishContact(job) + } + job.contactTimeout = expired + main.postDelayed(expired, 750) + job.run?.invoke() ?: finishContact(job) + } + } + deferred + } + }, + ) { active -> + interaction(active) + changed(JSONObject().put("active", active)) + } + var listening = true + val stop = { + if (listening) { + listening = false + remove() + observedWindow = null + interactionListener = null + epoch++ + acquisitions.values.toList().forEach { it.cancel?.invoke() } + acquisitions.clear() + } + } + stopObserver = stop + return stop + } + + private fun finishContact(job: Acquisition) { + job.contactTimeout?.let { main.removeCallbacks(it) } + job.contactTimeout = null + val complete = job.contact + job.contact = null + complete?.invoke() + } + + internal fun invalidateAcquisitions(owner: String? = null, retainPending: Boolean = false) { + val selected = + owner?.let { listOfNotNull(acquisitions[it]) } ?: acquisitions.values.toList() + selected.forEach { job -> + if (retainPending && !job.pixels) job.changedAt = SystemClock.uptimeMillis() + else { + acquisitions.remove(job.owner) + if (!job.pinned) job.cancel?.invoke() + } + } + } + var coveringToken: String? = null private set @@ -147,23 +257,29 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { return } val expected = ids.toSet() - var initial: Map? = frames(expected, request.getLong("revision")) - if (initial!!.keys != expected) { + if (frames(expected, request.getLong("revision")).keys != expected) { respond(JSONObject()) return } - prepare( + prepareContact( + request, window, webView, - width, { - val measured = initial ?: frames(expected, request.getLong("revision")) - initial = null - measured.takeIf { it.keys == expected } ?: emptyMap() + frames(expected, request.getLong("revision")).takeIf { it.keys == expected } + ?: emptyMap() }, + { views(expected).orEmpty() }, complete = respond, ) } + "change" -> { + invalidateAcquisitions( + request.optString("captureOwner").takeIf { request.has("captureOwner") }, + request.optBoolean("retainPending"), + ) + respond(JSONObject()) + } "cover" -> { val ids = identifiers(token) val selected = if (ids.isEmpty()) null else views(ids) @@ -254,6 +370,7 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { fun changed() { onMain() generation++ + invalidateAcquisitions() } fun interaction(active: Boolean) { @@ -271,73 +388,220 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { sourceViews: (() -> Map)? = null, contentRevision: Long = 0, complete: (JSONObject) -> Unit, + ) = + prepareCapture( + window, + reference, + viewportWidth, + frames, + sourceViews, + contentRevision, + complete = complete, + ) + + internal fun prepareContact( + request: JSONObject, + window: Window, + reference: WebView, + frames: () -> Map, + views: () -> List, + sourceViews: (() -> Map)? = null, + complete: (JSONObject) -> Unit, + ) { + val job = + request + .optString("captureID") + .takeIf { it.isNotEmpty() && observedWindow === window } + ?.let { Acquisition(it, request.optString("captureOwner"), epoch) } + if (job != null) { + invalidateAcquisitions(job.owner) + acquisitions[job.owner] = job + job.hit = { event -> + views().any { view -> + val bounds = Rect() + val location = IntArray(2) + view.rootView.getLocationOnScreen(location) + val visible = view.isShown && view.getGlobalVisibleRect(bounds) + bounds.offset(location[0], location[1]) + visible && bounds.contains(event.rawX.toInt(), event.rawY.toInt()) + } + } + if (request.optBoolean("stopScrollBeforeCapture")) + job.stop = { event -> + val contact = MotionEvent.obtain(event) + val origin = IntArray(2) + reference.getLocationOnScreen(origin) + try { + contact.setLocation(event.rawX - origin[0], event.rawY - origin[1]) + reference.onTouchEvent(contact) + contact.action = MotionEvent.ACTION_CANCEL + reference.onTouchEvent(contact) + } finally { + contact.recycle() + } + } + interactionListener?.invoke( + JSONObject() + .put("active", touching) + .put("capturePolicy", "preinteraction") + .put("prepareDuringInteraction", true) + ) + } + prepareCapture( + window, + reference, + request.optDouble("viewportWidth"), + frames, + sourceViews = sourceViews, + contentRevision = request.optLong("revision"), + acquisition = job, + delay = request.optLong("captureDelayMs", 0).coerceAtLeast(0), + complete = complete, + ) + } + + private fun prepareCapture( + window: Window, + reference: View, + viewportWidth: Double, + frames: () -> Map, + sourceViews: (() -> Map)? = null, + contentRevision: Long = 0, + acquisition: Acquisition? = null, + delay: Long = 0, + complete: (JSONObject) -> Unit, ) { onMain() if ( !supported || closed || - pending || - touching || + (acquisition == null && (pending || touching)) || viewportWidth <= 0 || !viewportWidth.isFinite() || reference.width <= 0 ) { + if (acquisition != null && acquisitions[acquisition.owner] === acquisition) + acquisitions.remove(acquisition.owner) complete(JSONObject()) return } - pending = true + if (acquisition == null) pending = true val stamp = generation - val started = android.os.SystemClock.uptimeMillis() - fun fail() { - pending = false - complete(JSONObject()) - } - fun sample(previous: Map, stable: Int) { - if ( - closed || - generation != stamp || - touching || - android.os.SystemClock.uptimeMillis() - started >= 1000 - ) { - fail() - return + val started = SystemClock.uptimeMillis() + var finished = false + var attempt = 0 + fun current() = + !closed && + !finished && + (acquisition == null || acquisition.epoch == epoch) && + (acquisition?.pinned == true || + (!touching && (acquisition != null || generation == stamp))) + fun finish(result: JSONObject) { + if (finished) return + finished = true + if (acquisition == null) pending = false + acquisition?.run = null + acquisition?.cancel = null + acquisition?.let { job -> + job.token = result.optString("token").takeIf { it.isNotEmpty() } + if (!result.has("token") && acquisitions[job.owner] === job) + acquisitions.remove(job.owner) + finishContact(job) } - val current = - try { - frames().mapValues { RectF(it.value) } - } catch (_: Exception) { + complete(result) + } + fun fail() = finish(JSONObject()) + acquisition?.cancel = { fail() } + lateinit var sample: (Map, Int) -> Unit + fun acquire() { + val ticket = ++attempt + val valid = { current() && ticket == attempt } + fun readPixels() { + if (!valid()) return + val currentFrames = + try { + frames() + } catch (_: RuntimeException) { + fail() + return + } + if (currentFrames.isEmpty()) { fail() return } - if (current.isEmpty() || current.any { it.key.isEmpty() || !it.value.valid() }) { - fail() - return - } - val count = if (current == previous) stable + 1 else 0 - if (count < 3) { - main.postDelayed({ sample(current, count) }, 16) - return - } - capture( - window, - reference, - viewportWidth, - current, - stamp, - sourceViews?.invoke(), - contentRevision, - ) { - pending = false - complete(it) + capture( + window, + reference, + viewportWidth, + currentFrames, + valid, + sourceViews?.invoke(), + contentRevision, + acquired = { + acquisition?.pixels = true + acquisition?.run = null + acquisition?.let(::finishContact) + }, + ) { + if (ticket == attempt) { + if ( + !it.has("token") && + acquisition != null && + !acquisition.pinned && + touching && + !finished + ) + main.postDelayed({ sample(emptyMap(), 0) }, 16) + else finish(it) + } + } } + // A contact restarts an in-flight idle copy after Chromium has stopped. + if (acquisition != null && reference is WebView) + NativeUIWebReadiness.prepare(reference, valid) { + if (it) readPixels() else if (valid()) fail() + } + else readPixels() } + acquisition?.run = { acquire() } + sample = + fun(previous: Map, stable: Int) { + if (finished || acquisition?.pinned == true) return + if (acquisition != null && touching) { + main.postDelayed({ sample(previous, stable) }, 16) + return + } + if ( + !current() || SystemClock.uptimeMillis() - started >= maxOf(1000, delay + 1000) + ) { + fail() + return + } + val current = + try { + frames().mapValues { RectF(it.value) } + } catch (_: Exception) { + fail() + return + } + if (current.isEmpty() || current.any { it.key.isEmpty() || !it.value.valid() }) { + fail() + return + } + val count = if (current == previous) stable + 1 else 0 + if ( + count < 3 || + SystemClock.uptimeMillis() - (acquisition?.changedAt ?: started) < delay + ) { + main.postDelayed({ sample(current, count) }, 16) + return + } + acquire() + } // Stable native bounds do not imply that the Web background has painted. // Prepare the Web draw before measuring/capturing the composited window. - if (reference is WebView) { - NativeUIWebReadiness.prepare( - reference, - { !closed && generation == stamp && !touching }, - ) { ready -> + if (reference is WebView && acquisition == null) { + NativeUIWebReadiness.prepare(reference, { current() }) { ready -> if (ready) sample(emptyMap(), 0) else fail() } } else sample(emptyMap(), 0) @@ -348,9 +612,10 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { reference: View, viewportWidth: Double, frames: Map, - stamp: Long, + current: () -> Boolean, sourceViews: Map?, contentRevision: Long, + acquired: () -> Unit, complete: (JSONObject) -> Unit, ) { if (Build.VERSION.SDK_INT < 26) { @@ -379,6 +644,16 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { JSONObject() .put("ids", JSONArray(listOf(part.id))) .put("url", Uri.fromFile(part.file).toString()) + .put( + "anchorBounds", + JSONObject().apply { + val anchor = frames.getValue(part.id) + put("x", (anchor.left - part.frame.left) * ratio) + put("y", (anchor.top - part.frame.top) * ratio) + put("width", anchor.width() * ratio) + put("height", anchor.height() * ratio) + }, + ) .put("x", part.frame.left * ratio) .put("y", part.frame.top * ratio) .put("width", part.frame.width() * ratio) @@ -430,6 +705,7 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { } == true } ) { + acquired() deliver( cached.parts.map { it.copy(frame = RectF(frames.getValue(it.id))) }, reused = true, @@ -457,11 +733,12 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { } try { val acceptPixels: (Int) -> Unit = accept@{ result -> - if (result != PixelCopy.SUCCESS || closed || generation != stamp || touching) { + if (result != PixelCopy.SUCCESS || !current()) { pixels.recycle() complete(JSONObject()) return@accept } + acquired() val token = UUID.randomUUID().toString() encoder.execute { val parts = mutableListOf() @@ -537,9 +814,7 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { } main.post { if ( - closed || - generation != stamp || - touching || + !current() || root.width != width || root.height != height || origin(reference) != location || @@ -829,11 +1104,18 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { if (hidden?.first == token) restoreViews() if (coveringToken == token) coveringToken = null if (copy?.token == token) clearCopy() + acquisitions.values + .toList() + .filter { it.token == token } + .forEach { invalidateAcquisitions(it.owner) } captures.remove(token)?.parts?.forEach(::releasePixelsIfUnused) } fun reset() { onMain() + epoch++ + acquisitions.values.toList().forEach { it.cancel?.invoke() } + acquisitions.clear() generation++ touching = false val previous = viewCache @@ -846,6 +1128,8 @@ class NativeUISnapshots(private val directory: File) : AutoCloseable { override fun close() { onMain() closed = true + stopObserver?.invoke() + stopObserver = null reset() encoder.shutdown() } diff --git a/docs/advanced/preinteraction-capture.md b/docs/advanced/preinteraction-capture.md index 93c216d..a9b1b8b 100644 --- a/docs/advanced/preinteraction-capture.md +++ b/docs/advanced/preinteraction-capture.md @@ -18,7 +18,7 @@ NativeUICoveredSnapshots.automaticallyInterceptsEvents = false With interception disabled, Engine retains its existing idle capture behavior. -Custom transports should use `observeInteractions(_:changed:)`, forward the entire dictionary to the Web listener, and pass requests through `handle`. Android retains its existing acquisition behavior. +Custom transports should use `observeInteractions(_:changed:)`, forward the entire dictionary to the Web listener, and pass requests through `handle`. Android hosts use the [snapshot window integration](../reference/platforms.md#android-library) with asynchronous PixelCopy acquisition and preserved native input. Apple documents [Window event dispatch](https://developer.apple.com/documentation/uikit/uiwindow/sendevent%28_%3A%29) and [adding an inherited-method override](https://developer.apple.com/documentation/objectivec/class_addmethod%28_%3A_%3A_%3A_%3A%29). A [gesture recognizer receives touches before its attached view](https://developer.apple.com/documentation/uikit/uigesturerecognizer/touchesbegan%28_%3Awith%3A%29); that does not establish its ordering relative to other recognizers implementing press appearance, so it is not used as the pre-dispatch guarantee. diff --git a/docs/reference/platforms.md b/docs/reference/platforms.md index 71f1eee..e88b07f 100644 --- a/docs/reference/platforms.md +++ b/docs/reference/platforms.md @@ -56,6 +56,12 @@ android/gradlew -p android testDebugUnitTest assembleRelease connectedDebugAndro `NativeUISnapshots` prepares composited PixelCopy images and local PNG files, then covers/restores from that cache. Encoding runs off the UI thread; only one preparation runs per owner. Notify it of visual changes and interaction, and release tokens on teardown. All view/lifecycle calls run on the UI thread. The plugin retains revision, eligibility and fallback policy. +SDK registrations and custom hosts share the same contact acquisition. A Web fallback waits for the selected contact request to settle before hiding its native source; failures and explicit hide still use the normal fallback. + +Custom snapshot hosts can call `observeInteractions(window, changed)` and forward the complete event dictionary to their Web transport. The `prepareDuringInteraction` capability keeps a request armed during a Web gesture without acquiring idle pixels while input is active. This enables contact capture: visual changes retain an unacquired request, while renderer revision changes and reset retire obsolete work. Native contact remeasures current frames and acquires PixelCopy pixels before delivering the original SDK input. `stopScrollBeforeCapture` also cancels WebView momentum for that native contact; disabling the flag preserves scrolling. Completed pixels can be selected without another copy. PNG encoding and Web decoding remain asynchronous. + +PixelCopy is asynchronous, so the existing window input replay preserves the original event sequence and gesture duration while acquisition completes. An optional capture deadline releases input before the window return deadline; a failed capture uses the normal Web fallback and must not consume the SDK tap. Retain the returned cleanup function and remove the observation on teardown. Do not also forward `NativeUIClippedView.interaction` for the same window, since the window observer already reports that input. + After prepared Web pixels are shown or retired, `NativeUIWebReadiness.prepare` checks readiness for a subsequent WebView draw. Its `current` predicate must validate the owning session, source generation and image revision. The temporary native copy covers this asynchronous handoff; overlay startup itself does not wait. Neither bridge acknowledgement nor this callback proves physical display presentation. An opaque image can remain beneath an opaque native view between handoffs, but diff --git a/src/native/session.ts b/src/native/session.ts index 1ceedc0..3bf739f 100644 --- a/src/native/session.ts +++ b/src/native/session.ts @@ -325,7 +325,7 @@ export const startNativeSession = async ( // image. That intermediate state is not a failed projection or a new reveal. if (stopped || startingProjection) return; applyFailedFallback(); - if (destination === 'web' && manager && !manager.token && !manager.restoring) { + if (destination === 'web' && manager && !manager.token && !manager.restoring && !manager.capturingContact) { void bridge.send({ action: 'visibility', visible: false }).catch(() => {}); target?.removeAttribute(marker); return; @@ -463,7 +463,10 @@ export const startNativeSession = async ( nativeScroll = tracking?.tracked === true; manager.consider(config.captureMode === 'view' ? contentRevision : ++revision, sources()); } else { - await bridge.send({ action: 'visibility', visible: false }); + // Once a token exists, cover owns hiding native and preserves its entry copy until painted. + // A separate visibility:false would tear down that handoff; pending contact gets the same guard. + if (destination !== 'web' || (!manager.token && !manager.capturingContact)) + await bridge.send({ action: 'visibility', visible: false }); setVisibility(); } }; @@ -632,7 +635,7 @@ export const startNativeSession = async ( // A native show starts a new local request; the screen-wide Web request prevents restoration. if (force === 'native' && projectionRequest.isPaused()) manager?.end(); beginProjection(); - if (!manager?.token) void bridge.send({ action: 'visibility', visible: false }).catch(() => {}); + if (!manager?.token && !manager?.capturingContact) void bridge.send({ action: 'visibility', visible: false }).catch(() => {}); setVisibility(); } else if (destination === 'native') { manager?.end(); diff --git a/src/projection/web.ts b/src/projection/web.ts index dd5c903..64694d5 100644 --- a/src/projection/web.ts +++ b/src/projection/web.ts @@ -180,8 +180,9 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP let visualRevision = 0; let preparedVisualRevision = -1; let stopped = false; - let flight = 0; + const capturesInFlight = new Set(); let preinteraction = false; + let prepareDuringInteraction = false; let captureSerial = 0; let requestedGeneration = -1; let ownerGeneration = 0; @@ -253,8 +254,8 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP if ( stopped || destination !== 'native' || - (flight > 0 && (!preinteraction || requestedGeneration === generation)) || - touching || + (capturesInFlight.size > 0 && (!preinteraction || requestedGeneration === generation)) || + (touching && !prepareDuringInteraction) || handoff || callbacks.blocked() || !latest || @@ -269,9 +270,9 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP if (!selected.length) return; const capturedLayout = signature(selected); const capturedOwnership = selected.map((candidate) => signature([candidate], true)); - flight++; requestedGeneration = generation; const captureID = `${imageIdentity}-${++captureSerial}`; + capturesInFlight.add(captureID); const requestAt = performance.now(); const owner = ownerGeneration; const pinned = () => preinteraction && pinnedCaptureID === captureID && ownerGeneration === owner; @@ -418,7 +419,8 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP retireImage(image); }); if (result?.token && !adopted) void call({ action: 'release', token: result.token }).catch(() => {}); - flight--; + capturesInFlight.delete(captureID); + if (pinnedCaptureID === captureID) callbacks.wake(); if (latest && (latest.revision !== revision || visualRevision !== visual)) schedule(); } }; @@ -426,7 +428,7 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP // available while their replacement is being rendered and decoded. const schedule = () => { win.clearTimeout(timer); - if (!stopped && destination === 'native' && !handoff && !touching && !callbacks.blocked()) + if (!stopped && destination === 'native' && !handoff && (!touching || prepareDuringInteraction) && !callbacks.blocked()) timer = win.setTimeout(() => void prepare(), preinteraction ? 0 : captureDelayMs); }; let consideredLayout: string | undefined; @@ -481,31 +483,45 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP } visualRevision++; generation++; - if (preinteraction) - void call({ action: 'change', captureOwner: imageIdentity, ...(scrollOwner ? { retainPending: true } : {}) }).catch(() => {}); + // Virtual scrollers mutate rows and wrapper transforms as well as emitting + // scroll. Retire completed pixels, but keep an unacquired native lease: + // contact still reads the current renderer geometry and fresh window pixels. + // Ownership/revision changes continue to cancel it through consider/invalidate. + if (preinteraction) void call({ action: 'change', captureOwner: imageIdentity, retainPending: true }).catch(() => {}); schedule(); }; const listener = await boundedListener( - transport.listen(({ active: pressed, capturePolicy, captureID, captureOwner, captureState, mainThreadMs }) => { - if (captureOwner !== undefined && captureOwner !== imageIdentity) return; - if (capturePolicy === 'preinteraction') { - if (!preinteraction) { - preinteraction = true; - schedule(); + transport.listen( + ({ + active: pressed, + capturePolicy, + captureID, + captureOwner, + captureState, + mainThreadMs, + prepareDuringInteraction: armDuringInput, + }) => { + if (captureOwner !== undefined && captureOwner !== imageIdentity) return; + if (capturePolicy === 'preinteraction') { + prepareDuringInteraction = armDuringInput === true; + if (!preinteraction) { + preinteraction = true; + schedule(); + } + return; } - return; - } - if (pressed && captureID) { - pinnedCaptureID = captureID; - contactAt = performance.now(); - reportTiming({ captureID, stage: cached?.captureID === captureID ? 'ready' : captureState, mainThreadMs }); - } else if (pressed && !touching) pinnedCaptureID = preinteraction ? '' : undefined; - touching = pressed; - generation++; - visualRevision++; - win.clearTimeout(timer); - if (!pressed) schedule(); - }), + if (pressed && captureID) { + pinnedCaptureID = captureID; + contactAt = performance.now(); + reportTiming({ captureID, stage: cached?.captureID === captureID ? 'ready' : captureState, mainThreadMs }); + } else if (pressed && !touching) pinnedCaptureID = preinteraction ? '' : undefined; + touching = pressed; + generation++; + visualRevision++; + win.clearTimeout(timer); + if (!pressed || prepareDuringInteraction) schedule(); + }, + ), ); const projectionNode = (node: Node | null) => node?.nodeType === 1 && @@ -1060,6 +1076,9 @@ const createProjectionEngine = async (scope: WebProjectionScope, transport: WebP get handoffPending() { return handoffPending; }, + get capturingContact() { + return !!pinnedCaptureID && capturesInFlight.has(pinnedCaptureID); + }, retireRestoration, end, updated, @@ -1131,6 +1150,9 @@ export const createWebProjectionBatch = async ( get handoffPending() { return engine?.handoffPending ?? false; }, + get capturingContact() { + return engine?.capturingContact ?? false; + }, consider(revision, sources) { if (stopped) return; const doc = batchDocument ?? sources[0]?.scope.element.ownerDocument; diff --git a/src/types.ts b/src/types.ts index a8b5799..30bf385 100644 --- a/src/types.ts +++ b/src/types.ts @@ -50,8 +50,8 @@ export type WebProjectionRequest = captureID?: string; captureOwner?: string; captureDelayMs?: number; - /** iOS: defer idle acquisition while scrolling. Native-view contact stops momentum, - * then measures and captures before UIKit handles the contact. Defaults to false. + /** Native-view contact stops momentum before measuring and acquiring pixels. + * iOS also defers idle acquisition while scrolling. Defaults to false. */ stopScrollBeforeCapture?: boolean; } @@ -69,7 +69,7 @@ export type WebProjectionRequest = | { action: 'change'; captureOwner?: string; - /** Scroll changes retire old pixels while keeping live, unacquired geometry available for contact. */ + /** Visual changes retire old pixels while keeping live, unacquired geometry available for contact. */ retainPending?: boolean; }; /** Empty prepare results mean capture is unavailable; a partial capture is never valid. */ @@ -90,6 +90,8 @@ export interface WebProjectionInteraction { active: boolean; /** Advertised only by a native window with a pre-dispatch hook. */ capturePolicy?: 'preinteraction'; + /** Native can arm a request during a gesture, while deferring idle acquisition until release. */ + prepareDuringInteraction?: boolean; /** Immutable pixels selected before this contact changes the screen. */ captureID?: string; captureOwner?: string; diff --git a/tests/native/session.spec.ts b/tests/native/session.spec.ts index 8a23ef9..97fc59a 100644 --- a/tests/native/session.spec.ts +++ b/tests/native/session.spec.ts @@ -434,6 +434,77 @@ for (const fallback of ['web', 'native'] as const) { }); } +for (const result of ['success', 'failure', 'none', 'timeout'] as const) { + test(`a pinned contact capture survives web fallback until ${result}`, async ({ page }) => { + await boot(page, false); + if (result === 'timeout') await page.clock.install(); + await page.evaluate(() => { + const w = window as any; + w.holdPrepare = true; + document.body.innerHTML = '
'; + }); + await expect.poll(() => page.evaluate(() => (window as any).pendingPrepare.length)).toBe(1); + await page.evaluate(() => { + const w = window as any; + const request = w.calls.find((call: any) => call.action === 'prepare'); + w.interaction({ active: false, capturePolicy: 'preinteraction' }); + w.interaction({ active: true, captureID: request.captureID, captureOwner: request.captureOwner, captureState: 'processing' }); + w.interaction({ active: false }); + w.calls = []; + // The SDK activates while immutable contact pixels are still being converted. + w.events.moving({ slot: 1 }); + }); + await expect(page.locator('#slot')).toHaveAttribute('data-native-ui-destination', 'web'); + await page.evaluate(() => { + (window as any).events.updated({ slot: 1 }); + return new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(() => resolve()))); + }); + expect(await page.evaluate(() => (window as any).calls.some((call: any) => call.action === 'visibility' && !call.visible))).toBe(false); + if (result === 'none') { + await page.evaluate(() => (window as any).events.hidden({ slot: 1 })); + await expect(page.locator('#slot')).toHaveAttribute('data-native-ui-session-surface', 'none'); + await expect + .poll(() => page.evaluate(() => (window as any).calls.some((call: any) => call.action === 'visibility' && !call.visible))) + .toBe(true); + } + if (result === 'timeout') await page.clock.runFor(5100); + else { + await page.evaluate((result) => { + (window as any).pendingPrepare.shift()( + result === 'failure' + ? {} + : { token: 'contact-pixels', parts: [{ ids: ['view'], url: 'file:///contact.png', x: 8, y: 8, width: 120, height: 40 }] }, + ); + }, result); + } + if (result === 'success') { + await expect(page.locator('#slot')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + expect(await page.evaluate(() => (window as any).calls.filter((call: any) => call.action === 'cover'))).toContainEqual( + expect.objectContaining({ token: 'contact-pixels' }), + ); + expect(await page.evaluate(() => (window as any).calls.some((call: any) => call.action === 'visibility' && !call.visible))).toBe( + false, + ); + } else { + await expect + .poll(() => page.evaluate(() => (window as any).calls.some((call: any) => call.action === 'visibility' && !call.visible))) + .toBe(true); + await expect(page.locator('#slot')).not.toHaveAttribute('data-native-ui-snapshot-visible'); + expect(await page.evaluate(() => (window as any).calls.some((call: any) => call.action === 'cover'))).toBe(false); + if (result === 'timeout') { + await page.evaluate(() => (window as any).pendingPrepare.shift()({ token: 'contact-pixels' })); + } + if (result === 'none' || result === 'timeout') { + await expect + .poll(() => + page.evaluate(() => (window as any).calls.some((call: any) => call.action === 'release' && call.token === 'contact-pixels')), + ) + .toBe(true); + } else await expect(page.locator('#slot button')).toBeVisible(); + } + }); +} + test('native failed fallback recovers rejected handoff without capturing on motion', async ({ page }) => { await boot(page, true, '#slot', 'native'); await expect(page.locator('#slot')).toHaveAttribute('data-native-ui-snapshot', /.+/); diff --git a/tests/projection/web.spec.ts b/tests/projection/web.spec.ts index f0e995c..6d96d2e 100644 --- a/tests/projection/web.spec.ts +++ b/tests/projection/web.spec.ts @@ -1612,6 +1612,128 @@ const beginDelayedReturn = async (page: Page) => { await expect.poll(() => page.evaluate(() => !!(window as any).releaseReply)).toBe(true); }; +test('a contact-capable host stays armed while a Web gesture and virtual changes continue', async ({ page }) => { + await boot(page); + await page.evaluate(async () => { + const w = window as any; + const source = document.querySelector('#source')!; + const rect = source.getBoundingClientRect(); + w.calls = []; + w.manager = await w.engine.createWebProjection( + w.engine.createWebProjectionScope(source, 'test'), + { + listen: async (listener: any) => { + w.interact = listener; + return { remove() {} }; + }, + send: async (request: any) => { + w.calls.push(request); + if (request.action === 'prepare') { + w.pending?.resolve({}); + return new Promise((resolve) => { + w.pending = { request, resolve }; + }); + } + return { shown: true }; + }, + }, + { wake() {}, blocked: () => !!w.blocked, captureDelayMs: 500 }, + ); + w.interact({ active: false, capturePolicy: 'preinteraction', prepareDuringInteraction: true }); + w.manager.consider(1, [{ element: source, control: { id: 'action', ...rect.toJSON() } }]); + }); + await expect.poll(() => page.evaluate(() => !!(window as any).pending)).toBe(true); + const initial = await page.evaluate(() => (window as any).pending.request.captureID); + await page.evaluate(() => { + const w = window as any; + w.interact({ active: true }); + document.querySelector('#scroll')!.lastElementChild!.textContent = 'Recycled during Web drag'; + w.manager.changed(); + }); + await expect.poll(() => page.evaluate(() => (window as any).pending.request.captureID)).not.toBe(initial); + await page.evaluate(() => { + const w = window as any; + const pending = w.pending; + w.interact({ active: false }); + w.interact({ + active: true, + captureID: pending.request.captureID, + captureOwner: pending.request.captureOwner, + captureState: 'processing', + }); + const rect = document.querySelector('#source')!.getBoundingClientRect(); + pending.resolve({ token: 'contact', parts: [{ ids: ['action'], url: 'file:///image.png', ...rect.toJSON() }] }); + w.blocked = true; + w.manager.begin(); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await expect(page.locator('img[data-native-ui-projection-image]')).toHaveCSS('opacity', '1'); + await destroyProjection(page); +}); + +test('virtual list recycling during momentum preserves contact capture for an overlay', async ({ page }) => { + await boot(page); + await page.evaluate(async () => { + const w = window as any; + const source = document.querySelector('#source')!; + // A fixed action bar is independent of the list recycled underneath it. + document.body.append(source); + const rect = source.getBoundingClientRect(); + w.pending = undefined; + w.manager = await w.engine.createWebProjection( + w.engine.createWebProjectionScope(source, 'test'), + { + listen: async (listener: any) => { + w.interact = listener; + return { remove() {} }; + }, + send: async (request: any) => { + if (request.action === 'prepare') + return new Promise((resolve) => { + w.pending = { request, resolve }; + }); + if (request.action === 'change' && !request.retainPending) { + w.pending?.resolve({}); + w.pending = undefined; + } + return { shown: true }; + }, + }, + { wake() {}, blocked: () => false, captureDelayMs: 500 }, + ); + w.interact({ active: false, capturePolicy: 'preinteraction' }); + w.manager.consider(1, [{ element: source, control: { id: 'action', x: rect.x, y: rect.y, width: rect.width, height: rect.height } }]); + }); + await expect.poll(() => page.evaluate(() => !!(window as any).pending)).toBe(true); + await page.evaluate(async () => { + const w = window as any; + const list = document.querySelector('#scroll')!; + list.dispatchEvent(new Event('scroll', { bubbles: true })); + // CDK virtual scrolling updates both the wrapper transform and recycled rows. + list.firstElementChild!.setAttribute('style', 'height:100px;transform:translateY(200px)'); + list.lastElementChild!.textContent = 'New visible wine rows'; + await Promise.resolve(); + const pending = w.pending; + if (pending) { + w.interact({ + active: true, + captureID: pending.request.captureID, + captureOwner: pending.request.captureOwner, + captureState: 'waiting', + }); + const rect = document.querySelector('#source')!.getBoundingClientRect(); + pending.resolve({ + token: 'contact', + parts: [{ ids: ['action'], url: 'file:///image.png', x: rect.x, y: rect.y, width: rect.width, height: rect.height }], + }); + } else w.interact({ active: true }); + w.manager.begin(); + }); + await expect(page.locator('#source')).toHaveAttribute('data-native-ui-snapshot-visible', ''); + await expect(page.locator('img[data-native-ui-projection-image]')).toHaveCSS('opacity', '1'); + await destroyProjection(page); +}); + test('scroll changes retain pending native acquisition while source invalidation still cancels it', async ({ page }) => { await prepare(page); await page.evaluate(() => {