diff --git a/android/app/src/main/java/com/oney/WebRTCModule/RTCFsrVideoView.java b/android/app/src/main/java/com/oney/WebRTCModule/RTCFsrVideoView.java index ef10e2c8..508ce67a 100644 --- a/android/app/src/main/java/com/oney/WebRTCModule/RTCFsrVideoView.java +++ b/android/app/src/main/java/com/oney/WebRTCModule/RTCFsrVideoView.java @@ -1,6 +1,7 @@ package com.oney.WebRTCModule; import android.annotation.SuppressLint; +import android.app.ActivityManager; import android.content.Context; import android.graphics.Color; import android.graphics.Point; @@ -52,6 +53,12 @@ public class RTCFsrVideoView extends ViewGroup { private int frameWidth; private boolean mirror; private boolean rendererAttached; + private boolean rendererCleanupInProgress; + private boolean hasAppliedLayout; + private int appliedLayoutLeft; + private int appliedLayoutTop; + private int appliedLayoutRight; + private int appliedLayoutBottom; private ScalingType scalingType; private int videoFormatMode = VIDEO_FORMAT_MODE_AUTO; private float videoFormatAspectRatio; @@ -61,10 +68,17 @@ public class RTCFsrVideoView extends ViewGroup { private VideoTrack videoTrack; private boolean fsrEnabled = true; + private boolean autoDisableFsrOnLowMemory; + private boolean fsrFallbackActive; private float fsrSharpness = 2f; private boolean fsrInitialized; + private float appliedFsrSharpness = Float.NaN; + private int appliedSurfaceWidth = -1; + private int appliedSurfaceHeight = -1; private Object fsrDrawerProxy; private Object fallbackDrawer; + private Method fallbackDrawOesMethod; + private Method fallbackReleaseMethod; private final FsrVideoProcessor fsrVideoProcessor; @@ -228,34 +242,58 @@ protected void onLayout(boolean changed, int l, int t, int r, int b) { } } + if (hasAppliedLayout + && appliedLayoutLeft == l + && appliedLayoutTop == t + && appliedLayoutRight == r + && appliedLayoutBottom == b) { + return; + } + appliedLayoutLeft = l; + appliedLayoutTop = t; + appliedLayoutRight = r; + appliedLayoutBottom = b; + hasAppliedLayout = true; surfaceViewRenderer.layout(l, t, r, b); applyRendererLayoutAspectRatio(r - l, b - t); } private void removeRendererFromVideoTrack() { - if (rendererAttached) { - if (videoTrack != null) { - ThreadUtils.runOnExecutor(() -> { - try { - videoTrack.removeSink(surfaceViewRenderer); - } catch (Throwable ignored) { - // Ignore track lifecycle race. + if (!rendererAttached || rendererCleanupInProgress) { + return; + } + + rendererAttached = false; + rendererCleanupInProgress = true; + VideoTrack track = videoTrack; + ThreadUtils.runOnExecutor(() -> { + try { + if (track != null) { + track.removeSink(surfaceViewRenderer); + } + } catch (Throwable ignored) { + // Ignore track lifecycle race. + } finally { + surfaceViewRenderer.release(); + releaseDrawerResources(); + if (surfaceViewRendererInstances > 0) { + surfaceViewRendererInstances--; + } + post(() -> { + rendererCleanupInProgress = false; + synchronized (layoutSyncRoot) { + frameHeight = 0; + frameRotation = 0; + frameWidth = 0; + } + hasAppliedLayout = false; + requestSurfaceViewRendererLayout(); + if (videoTrack != null && ViewCompat.isAttachedToWindow(this)) { + tryAddRendererToVideoTrack(); } }); } - - surfaceViewRenderer.release(); - releaseDrawerResources(); - surfaceViewRendererInstances--; - rendererAttached = false; - - synchronized (layoutSyncRoot) { - frameHeight = 0; - frameRotation = 0; - frameWidth = 0; - } - requestSurfaceViewRendererLayout(); - } + }); } @SuppressLint("WrongCall") @@ -489,16 +527,40 @@ public void setZOrder(int zOrder) { public void setFsrEnabled(boolean enabled) { fsrEnabled = enabled; - fsrVideoProcessor.setFsrEnabled(enabled); + if (enabled) { + fsrFallbackActive = false; + } + applyEffectiveFsrEnabled(); + } + + public void setAutoDisableFsrOnLowMemory(boolean enabled) { + autoDisableFsrOnLowMemory = enabled; + applyEffectiveFsrEnabled(); + } + + private void applyEffectiveFsrEnabled() { + boolean lowMemoryDevice = false; + if (autoDisableFsrOnLowMemory) { + ActivityManager activityManager = + (ActivityManager) getContext().getSystemService(Context.ACTIVITY_SERVICE); + lowMemoryDevice = activityManager != null && activityManager.isLowRamDevice(); + } + boolean effectiveEnabled = fsrEnabled && !lowMemoryDevice; + if (effectiveEnabled) { + fsrFallbackActive = false; + } + fsrVideoProcessor.setFsrEnabled(effectiveEnabled); } public void setFsrSharpness(float sharpness) { fsrSharpness = sharpness; + appliedFsrSharpness = Float.NaN; fsrVideoProcessor.setSharpness(sharpness / 10f); } private void tryAddRendererToVideoTrack() { - if (!rendererAttached && videoTrack != null && ViewCompat.isAttachedToWindow(this)) { + if (!rendererAttached && !rendererCleanupInProgress + && videoTrack != null && ViewCompat.isAttachedToWindow(this)) { EglBase.Context sharedContext = EglUtils.getRootEglBaseContext(); if (sharedContext == null) { @@ -627,7 +689,7 @@ private void handleDrawOes(Object[] args) { return; } - if (!fsrEnabled) { + if (!fsrEnabled || fsrFallbackActive) { invokeFallbackDrawer("drawOes", args); return; } @@ -652,11 +714,18 @@ private void handleDrawOes(Object[] args) { } ensureFsrInitialized(); - fsrVideoProcessor.setSharpness(fsrSharpness / 10f); - if (viewportWidth > 0 && viewportHeight > 0) { - fsrVideoProcessor.setSurfaceSize(viewportWidth, viewportHeight); - } else if (frameWidth > 0 && frameHeight > 0) { - fsrVideoProcessor.setSurfaceSize(frameWidth, frameHeight); + float requestedSharpness = fsrSharpness / 10f; + if (Float.compare(appliedFsrSharpness, requestedSharpness) != 0) { + fsrVideoProcessor.setSharpness(requestedSharpness); + appliedFsrSharpness = requestedSharpness; + } + int surfaceWidth = viewportWidth > 0 ? viewportWidth : frameWidth; + int surfaceHeight = viewportHeight > 0 ? viewportHeight : frameHeight; + if (surfaceWidth > 0 && surfaceHeight > 0 + && (surfaceWidth != appliedSurfaceWidth || surfaceHeight != appliedSurfaceHeight)) { + fsrVideoProcessor.setSurfaceSize(surfaceWidth, surfaceHeight); + appliedSurfaceWidth = surfaceWidth; + appliedSurfaceHeight = surfaceHeight; } boolean rendered = fsrVideoProcessor.draw( @@ -667,10 +736,15 @@ private void handleDrawOes(Object[] args) { texMatrix ); if (!rendered) { + // Release FSR resources and keep using the stable fallback after a failed draw. + releaseDrawerResources(); + fsrFallbackActive = true; invokeFallbackDrawer("drawOes", args); } } catch (Throwable t) { Log.e(TAG, "FSR draw failed, fallback to default OES drawer.", t); + releaseDrawerResources(); + fsrFallbackActive = true; invokeFallbackDrawer("drawOes", args); } } @@ -711,7 +785,7 @@ private synchronized void ensureFsrInitialized() { String extensions = GLES20.glGetString(GLES20.GL_EXTENSIONS); fsrVideoProcessor.initialize(major, minor, extensions == null ? "" : extensions); - fsrVideoProcessor.setFsrEnabled(fsrEnabled); + applyEffectiveFsrEnabled(); fsrVideoProcessor.setSharpness(fsrSharpness / 10f); fsrInitialized = true; } @@ -743,13 +817,21 @@ private void invokeFallbackDrawer(String methodName, Object[] args) { if (drawer == null) { return; } - try { - Method target = findByNameAndArgCount(drawer.getClass(), methodName, args == null ? 0 : args.length); + Method target; + if ("drawOes".equals(methodName)) { + if (fallbackDrawOesMethod == null) { + fallbackDrawOesMethod = findByNameAndArgCount(drawer.getClass(), methodName, args == null ? 0 : args.length); + } + target = fallbackDrawOesMethod; + } else { + target = findByNameAndArgCount(drawer.getClass(), methodName, args == null ? 0 : args.length); + } if (target == null) { return; } target.invoke(drawer, args == null ? new Object[0] : args); + } catch (Throwable t) { Log.e(TAG, "Fallback drawer invocation failed: " + methodName, t); } @@ -772,18 +854,26 @@ private synchronized void releaseDrawerResources() { Log.w(TAG, "Failed to release FSR processor", t); } fsrInitialized = false; + fsrFallbackActive = false; + appliedFsrSharpness = Float.NaN; + appliedSurfaceWidth = -1; + appliedSurfaceHeight = -1; } - if (fallbackDrawer != null) { try { - Method release = findByNameAndArgCount(fallbackDrawer.getClass(), "release", 0); - if (release != null) { - release.invoke(fallbackDrawer); + if (fallbackReleaseMethod == null) { + fallbackReleaseMethod = findByNameAndArgCount(fallbackDrawer.getClass(), "release", 0); } + if (fallbackReleaseMethod != null) { + fallbackReleaseMethod.invoke(fallbackDrawer); + } + } catch (Throwable t) { Log.w(TAG, "Failed to release fallback drawer", t); } fallbackDrawer = null; + fallbackDrawOesMethod = null; + fallbackReleaseMethod = null; } } } diff --git a/android/app/src/main/java/com/oney/WebRTCModule/RTCFsrVideoViewManager.java b/android/app/src/main/java/com/oney/WebRTCModule/RTCFsrVideoViewManager.java index 4ad18b31..acd3f41e 100644 --- a/android/app/src/main/java/com/oney/WebRTCModule/RTCFsrVideoViewManager.java +++ b/android/app/src/main/java/com/oney/WebRTCModule/RTCFsrVideoViewManager.java @@ -47,6 +47,11 @@ public void setFsrEnabled(RTCFsrVideoView view, boolean enabled) { view.setFsrEnabled(enabled); } + @ReactProp(name = "autoDisableFsrOnLowMemory", defaultBoolean = false) + public void setAutoDisableFsrOnLowMemory(RTCFsrVideoView view, boolean enabled) { + view.setAutoDisableFsrOnLowMemory(enabled); + } + @ReactProp(name = "fsrSharpness", defaultFloat = 2f) public void setFsrSharpness(RTCFsrVideoView view, float sharpness) { view.setFsrSharpness(sharpness); diff --git a/android/app/src/main/java/com/xstreaming/fsr/FsrVideoProcessor.java b/android/app/src/main/java/com/xstreaming/fsr/FsrVideoProcessor.java index 9f5cdb05..42ff3136 100644 --- a/android/app/src/main/java/com/xstreaming/fsr/FsrVideoProcessor.java +++ b/android/app/src/main/java/com/xstreaming/fsr/FsrVideoProcessor.java @@ -58,14 +58,19 @@ public class FsrVideoProcessor implements VideoProcessor { private boolean mobileHasSharpness; private boolean mobileHasHdrToneMap; private boolean twoPassFailureLogged; + private long lastGlErrorLogTimestamp; private boolean fsrEnabled = true; private boolean hdrInputEnabled; private boolean usingPqWindow; + private boolean softwareHdrToneMap; private int outputWidth = -1; private int outputHeight = -1; - private float[] outputSize = new float[2]; + private final float[] outputSize = new float[2]; + private final float[] inputTextureSize = new float[2]; + private int inputTextureWidth = -1; + private int inputTextureHeight = -1; public FsrVideoProcessor(Context context) { this.context = context.getApplicationContext(); @@ -94,9 +99,6 @@ public void initialize(int glMajorVersion, int glMinorVersion, String extensions Log.w(TAG, "GLES3 context without GL_OES_EGL_image_external_essl3, force FSR 2.0 shaders"); } - Log.i(TAG, "FSR preferred shader dir: " + preferredDir); - Log.i(TAG, "OpenGL extensions: " + extensions); - boolean skipTwoPassForDriverStability = !ENABLE_TWO_PASS_PIPELINE; if (skipTwoPassForDriverStability) { Log.w(TAG, "Skip two-pass FSR globally, use mobile pipeline for stability"); @@ -149,10 +151,10 @@ public void setSurfaceSize(int width, int height) { if (outputWidth == width && outputHeight == height) { return; } - Log.i(TAG, "setSurfaceSize(" + width + "," + height + ")"); outputWidth = width; outputHeight = height; - outputSize = new float[]{width, height}; + outputSize[0] = width; + outputSize[1] = height; if (pipelineMode == PIPELINE_TWO_PASS) { deleteFramebuffer(); @@ -190,12 +192,7 @@ public void release() { public void setHdrToneMappingEnabled(boolean enabled) { hdrInputEnabled = enabled; - if (enabled && !FORCE_SOFTWARE_HDR_TONE_MAP) { - Log.i( - TAG, - "HDR stream detected; software HDR tone-map disabled." - ); - } + updateSoftwareHdrToneMapState(); } public void setSharpness(float value) { @@ -203,13 +200,6 @@ public void setSharpness(float value) { mobileSharpness = clamped; // Map [0..2] (stronger as larger) to RCAS stop domain [2..0]. rcasSharpness = 2f - clamped; - Log.i( - TAG, - "Sharpness request=" + value + ", clamped=" + clamped - + ", mobileApplied=" + mobileSharpness - + ", rcasApplied=" + rcasSharpness - ); - logEffectiveSharpness("setSharpness"); } public void resetSharpness() { @@ -347,17 +337,18 @@ private boolean drawTwoPass(int frameTexture, } } - float[] inputTextureSize = null; - if (needInputSize) { - inputTextureSize = (frameWidth > 0 && frameHeight > 0) - ? new float[]{frameWidth, frameHeight} - : new float[]{0f, 0f}; + if (needInputSize + && (frameWidth != inputTextureWidth || frameHeight != inputTextureHeight)) { + inputTextureSize[0] = frameWidth > 0 ? frameWidth : 0f; + inputTextureSize[1] = frameHeight > 0 ? frameHeight : 0f; + inputTextureWidth = frameWidth; + inputTextureHeight = frameHeight; } GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, framebuffers[0]); try { easu.setSamplerTexIdUniform("inputTexture", frameTexture, 0); - if (inputTextureSize != null) { + if (needInputSize) { easu.setFloatsUniform("inputTextureSize", inputTextureSize); } easu.setFloatsUniform("outputTextureSize", outputSize); @@ -425,16 +416,17 @@ private boolean drawMobileSinglePass(int frameTexture, return drawPassthrough(frameTexture, transformMatrix); } - float[] inputTextureSize = null; - if (needInputSize) { - inputTextureSize = (frameWidth > 0 && frameHeight > 0) - ? new float[]{frameWidth, frameHeight} - : new float[]{0f, 0f}; + if (needInputSize + && (frameWidth != inputTextureWidth || frameHeight != inputTextureHeight)) { + inputTextureSize[0] = frameWidth > 0 ? frameWidth : 0f; + inputTextureSize[1] = frameHeight > 0 ? frameHeight : 0f; + inputTextureWidth = frameWidth; + inputTextureHeight = frameHeight; } try { program.setSamplerTexIdUniform("inputTexture", frameTexture, 0); - if (inputTextureSize != null) { + if (needInputSize) { program.setFloatsUniform("inputTextureSize", inputTextureSize); } program.setFloatsUniform("outputTextureSize", outputSize); @@ -612,7 +604,11 @@ private boolean checkGlError(String message) { GlUtil.checkGlError(); return true; } catch (GlException e) { - Log.e(TAG, message, e); + long now = System.currentTimeMillis(); + if (now - lastGlErrorLogTimestamp >= 5000L) { + lastGlErrorLogTimestamp = now; + Log.e(TAG, message, e); + } return false; } } @@ -638,18 +634,16 @@ private void logEffectiveSharpness(String reason) { } private boolean shouldApplySoftwareHdrToneMap() { - if (!hdrInputEnabled) { - return false; - } - if (usingPqWindow) { - // PQ output window should preserve HDR signal and avoid SDR tone map in this shader path. - return false; - } - return FORCE_SOFTWARE_HDR_TONE_MAP; + return softwareHdrToneMap; + } + + private void updateSoftwareHdrToneMapState() { + softwareHdrToneMap = hdrInputEnabled && !usingPqWindow && FORCE_SOFTWARE_HDR_TONE_MAP; } private void detectHdrWindowState() { usingPqWindow = false; + updateSoftwareHdrToneMapState(); android.opengl.EGLDisplay display = EGL14.eglGetCurrentDisplay(); android.opengl.EGLSurface drawSurface = EGL14.eglGetCurrentSurface(EGL14.EGL_DRAW); if (display == null || display == EGL14.EGL_NO_DISPLAY @@ -670,6 +664,7 @@ private void detectHdrWindowState() { if (EGL14.eglQuerySurface(display, drawSurface, EGL_GL_COLORSPACE_KHR, colorspace, 0)) { usingPqWindow = colorspace[0] == EGL_GL_COLORSPACE_BT2020_PQ_EXT; } + updateSoftwareHdrToneMapState(); Log.i( TAG, diff --git a/src/components/RTCFsrView.tsx b/src/components/RTCFsrView.tsx index 8e1db6fc..db20216f 100644 --- a/src/components/RTCFsrView.tsx +++ b/src/components/RTCFsrView.tsx @@ -14,6 +14,7 @@ type Props = ViewProps & { zOrder?: number; videoFormat?: string; fsrEnabled?: boolean; + autoDisableFsrOnLowMemory?: boolean; fsrSharpness?: number; }; diff --git a/src/pages/NativeStream.tsx b/src/pages/NativeStream.tsx index 13ddcf38..a9f94745 100644 --- a/src/pages/NativeStream.tsx +++ b/src/pages/NativeStream.tsx @@ -55,6 +55,7 @@ const FAILED = 'failed'; const DUALSENSE = 'DualSenseController'; const LIVE_GAMEPAD_PROFILE = 'LiveLayout'; const PICTURE_IN_PICTURE_MODE_CHANGED = 'pictureInPictureModeChanged'; +const isAndroidTv = Platform.OS === 'android' && Platform.isTV === true; const { FullScreenManager, @@ -226,6 +227,7 @@ export function NativeStreamScreenBase({ React.useState(false); const [openMicro, setOpenMicro] = React.useState(false); const [isInPictureInPicture, setIsInPictureInPicture] = React.useState(false); + const [appState, setAppState] = React.useState(AppState.currentState); const xHomeApiRef = React.useRef(undefined); const xCloudApiRef = React.useRef(undefined); const isRumbling = React.useRef(false); @@ -241,6 +243,7 @@ export function NativeStreamScreenBase({ const audioGainRef = React.useRef(1); const keepaliveInterval = React.useRef(null); const performanceInterval = React.useRef(null); + const performanceRequestInFlight = React.useRef(false); const connectStateRef = React.useRef(''); const gpDownEventListener = React.useRef(undefined); @@ -305,22 +308,16 @@ export function NativeStreamScreenBase({ ); React.useEffect(() => { - let layoutTimer: any = null; const lockTimer = setTimeout(() => { if (portraitMode) { Orientation.lockToPortrait(); } else { Orientation.lockToLandscape(); } - - layoutTimer = setTimeout(() => {}, 100); }, 500); return () => { clearTimeout(lockTimer); - if (layoutTimer) { - clearTimeout(layoutTimer); - } Orientation.unlockAllOrientations(); }; }, [route.params?.sessionId, route.params?.streamType, portraitMode]); @@ -754,6 +751,7 @@ export function NativeStreamScreenBase({ appStateSubscription.current = AppState.addEventListener( 'change', async state => { + setAppState(state); if ( !portraitMode && state === 'background' && @@ -1090,7 +1088,7 @@ export function NativeStreamScreenBase({ } // Sensor - if (_settings.sensor) { + if (_settings.sensor && !isAndroidTv) { const sensorManager = _settings.sensor === 2 ? GamepadSensorModule : SensorModule; @@ -1334,7 +1332,7 @@ export function NativeStreamScreenBase({ ); // Alway show virtual gamepad - if (portraitMode || _settings.show_virtual_gamead) { + if (!isAndroidTv && (portraitMode || _settings.show_virtual_gamead)) { setShowVirtualGamepad(true); } @@ -1394,7 +1392,7 @@ export function NativeStreamScreenBase({ ); } }); - }, 16); + }, 50); } } else if (state === CLOSED) { if (isRequestExit.current) { @@ -1702,6 +1700,7 @@ export function NativeStreamScreenBase({ JSON.stringify(iceDetails), ); webrtcClient.setIceCandidates(iceDetails); + webrtcClient.clearIceCandidates?.(); setLoadingText(`${t('Exchange ICE successfully...')}`); }) .catch(e => { @@ -1790,6 +1789,9 @@ export function NativeStreamScreenBase({ FullScreenManager.immersiveModeOff(); stopVibrate(); webrtcClient && webrtcClient.close(); + remoteStream.current?.getTracks?.().forEach(track => track.stop?.()); + remoteStream.current = null; + performanceRequestInFlight.current = false; usbGpEventListener.current && usbGpEventListener.current.remove(); gpDownEventListener.current && gpDownEventListener.current.remove(); gpUpEventListener.current && gpUpEventListener.current.remove(); @@ -1868,6 +1870,7 @@ export function NativeStreamScreenBase({ React.useEffect(() => { if ( connectState !== CONNECTED || + appState !== 'active' || !showPerformance || !webrtcClient || typeof webrtcClient.getStreamState !== 'function' @@ -1880,16 +1883,38 @@ export function NativeStreamScreenBase({ } const updatePerformance = () => { + if (performanceRequestInFlight.current) { + return; + } + performanceRequestInFlight.current = true; webrtcClient .getStreamState() .then(res => { - setPerformance(res); + setPerformance(previous => { + if ( + previous.resolution === res.resolution && + previous.rtt === res.rtt && + previous.jit === res.jit && + previous.fps === res.fps && + previous.pl === res.pl && + previous.fl === res.fl && + previous.br === res.br && + previous.decode === res.decode + ) { + return previous; + } + return res; + }); }) - .catch(() => {}); + .catch(() => {}) + .finally(() => { + performanceRequestInFlight.current = false; + }); }; updatePerformance(); - performanceInterval.current = setInterval(updatePerformance, 1000); + // Telemetry is auxiliary; 2 Hz reduces JS-thread work during gameplay. + performanceInterval.current = setInterval(updatePerformance, 2000); return () => { if (performanceInterval.current) { @@ -1897,7 +1922,7 @@ export function NativeStreamScreenBase({ performanceInterval.current = null; } }; - }, [connectState, showPerformance, webrtcClient]); + }, [appState, connectState, showPerformance, webrtcClient]); const handlePowerOff = React.useCallback(async () => { const webApi = new WebApi(webToken); @@ -2234,7 +2259,7 @@ export function NativeStreamScreenBase({ setShowVirtualGamepad(false); webrtcClient && webrtcClient.close(); setShowModal(false); - if (settings.sensor) { + if (settings.sensor && !isAndroidTv) { SensorModule.stopSensor(); GamepadSensorModule.stopSensor(); } @@ -2470,7 +2495,7 @@ export function NativeStreamScreenBase({ }, [openOptionsModal, showModal]); const renderVirtualGamepad = () => { - if (portraitMode) { + if (portraitMode || isAndroidTv) { return null; } if (isInPictureInPicture || !showVirtualGamepad) { @@ -2533,18 +2558,19 @@ export function NativeStreamScreenBase({ const useFsrRenderer = !!settings.fsr; const fsrSharpness = settings.fsr_display_options?.sharpness ?? 2; + const nativeTouchEnabled = !!settings.native_touch && !isAndroidTv; const handleNativePointerInput = React.useCallback( (event: PointerWireData) => { - if (!webrtcClient || !settings.native_touch) { + if (!webrtcClient || !nativeTouchEnabled) { return; } webrtcClient.getChannelProcessor('input')?.queuePointerInput([event]); }, - [settings.native_touch, webrtcClient], + [nativeTouchEnabled, webrtcClient], ); - const video_format = settings.native_touch ? '' : settings.video_format; + const video_format = nativeTouchEnabled ? '' : settings.video_format; const loadingPosterUrl = typeof route.params?.postUrl === 'string' ? route.params.postUrl : ''; const showLoadingPoster = loading && !!loadingPosterUrl; @@ -2571,13 +2597,16 @@ export function NativeStreamScreenBase({ streamURL={remote} videoFormat={video_format || ''} fsrEnabled={true} + autoDisableFsrOnLowMemory={true} fsrSharpness={fsrSharpness} /> - + {nativeTouchEnabled && !isInPictureInPicture ? ( + + ) : null} ); } @@ -2591,17 +2620,19 @@ export function NativeStreamScreenBase({ streamURL={remote} videoFormat={video_format || ''} /> - + {nativeTouchEnabled && !isInPictureInPicture ? ( + + ) : null} ); }; const renderPortraitVirtualGamepad = () => { - if (!portraitMode || connectState !== CONNECTED) { + if (!portraitMode || isAndroidTv || connectState !== CONNECTED) { return null; } @@ -2728,12 +2759,14 @@ export function NativeStreamScreenBase({ {renderVirtualGamepad()} - setShowGamepadEditor(false)} - /> + {!isAndroidTv ? ( + setShowGamepadEditor(false)} + /> + ) : null} {renderMenu()} diff --git a/src/pages/Stream.tsx b/src/pages/Stream.tsx index 05b6d1f6..65f02c76 100644 --- a/src/pages/Stream.tsx +++ b/src/pages/Stream.tsx @@ -57,6 +57,7 @@ const CONNECTED = 'connected'; const DUALSENSE = 'DualSenseController'; const LIVE_GAMEPAD_PROFILE = 'LiveLayout'; const PICTURE_IN_PICTURE_MODE_CHANGED = 'pictureInPictureModeChanged'; +const isAndroidTv = Platform.OS === 'android' && Platform.isTV === true; const { FullScreenManager, @@ -151,21 +152,14 @@ function StreamScreen({navigation, route}) { const webviewRef = React.useRef(null); React.useEffect(() => { - let layoutTimer: any = null; const lockTimer = setTimeout(() => { Orientation.lockToLandscape(); - - layoutTimer = setTimeout(() => { - const {height: dHeight} = Dimensions.get('window'); - setModalMaxHeight(dHeight - 50); - }, 100); + const {height: dHeight} = Dimensions.get('window'); + setModalMaxHeight(dHeight - 50); }, 500); return () => { clearTimeout(lockTimer); - if (layoutTimer) { - clearTimeout(layoutTimer); - } Orientation.unlockAllOrientations(); }; }, [route.params?.sessionId, route.params?.streamType]); @@ -574,7 +568,7 @@ function StreamScreen({navigation, route}) { }, 1000 / _settings.polling_rate); } - if (_settings.sensor) { + if (_settings.sensor && !isAndroidTv) { const sensorManager = _settings.sensor === 2 ? GamepadSensorModule : SensorModule; @@ -1424,6 +1418,9 @@ function StreamScreen({navigation, route}) { const isNativeLikeKernel = settings.gamepad_kernal === 'Native' || settings.gamepad_kernal === 'SDL'; // Close + if (isAndroidTv) { + return; + } if (showVirtualGamepad) { clearMacroTimers(); setShowVirtualGamepad(false); @@ -1446,10 +1443,11 @@ function StreamScreen({navigation, route}) { setShowVirtualGamepad(false); postData2Webview('disconnect', {}); setShowModal(false); - if (settings.sensor) { + if (settings.sensor && !isAndroidTv) { SensorModule.stopSensor(); GamepadSensorModule.stopSensor(); } + }; const background = { @@ -1505,7 +1503,7 @@ function StreamScreen({navigation, route}) { }; const renderVirtualGamepad = () => { - if (isInPictureInPicture || !showVirtualGamepad) { + if (isAndroidTv || isInPictureInPicture || !showVirtualGamepad) { return null; } const useCustomVirtualGamepad = settings.custom_virtual_gamepad !== ''; @@ -1545,14 +1543,16 @@ function StreamScreen({navigation, route}) { )} - {renderVirtualGamepad()} + {!isAndroidTv && renderVirtualGamepad()} - setShowGamepadEditor(false)} - /> + {!isAndroidTv ? ( + setShowGamepadEditor(false)} + /> + ) : null} { handleWebviewMessage(event); diff --git a/src/webrtc/Channel/Input.ts b/src/webrtc/Channel/Input.ts index 3bdb951f..cdf4d5d3 100644 --- a/src/webrtc/Channel/Input.ts +++ b/src/webrtc/Channel/Input.ts @@ -60,6 +60,8 @@ export interface KeyboardFrame { key: string; } +const MAX_FRAME_METADATA_QUEUE = 60; + export default class InputChannel extends BaseChannel { _inputSequenceNum = 0; @@ -132,10 +134,12 @@ export default class InputChannel extends BaseChannel { } onMessage(event: any) { - console.log( - 'Channel/Input.ts - [' + this._channelName + '] onMessage:', - event, - ); + if (__DEV__) { + console.log( + 'Channel/Input.ts - [' + this._channelName + '] onMessage:', + event, + ); + } const dataView = new DataView(event.data); @@ -187,12 +191,18 @@ export default class InputChannel extends BaseChannel { onClose(event: any) { clearInterval(this._inputInterval); - + this._inputInterval = null; + this._frameMetadataQueue.length = 0; + this._gamepadFrames.length = 0; + this._pointerFrames.length = 0; super.onClose(event); - console.log( - 'Channel/Input.ts - [' + this._channelName + '] onClose:', - event, - ); + + if (__DEV__) { + console.log( + 'Channel/Input.ts - [' + this._channelName + '] onClose:', + event, + ); + } } getGamepadQueue(size = 30) { @@ -259,14 +269,22 @@ export default class InputChannel extends BaseChannel { destroy() { clearInterval(this._inputInterval); + this._inputInterval = null; + this._frameMetadataQueue.length = 0; + this._gamepadFrames.length = 0; + this._pointerFrames.length = 0; super.destroy(); } - addProcessedFrame(frame: any) { frame.frameRenderedTimeMs = performance.now(); + if (this._frameMetadataQueue.length >= MAX_FRAME_METADATA_QUEUE) { + // Drop stale feedback instead of allowing an unbounded backlog. + this._frameMetadataQueue.shift(); + } this._frameMetadataQueue.push(frame); } + getMetadataQueue(size = 30) { return this._frameMetadataQueue.splice(0, size - 1); } diff --git a/src/webrtc/Packet/index.ts b/src/webrtc/Packet/index.ts index e335bee4..ae2ca966 100644 --- a/src/webrtc/Packet/index.ts +++ b/src/webrtc/Packet/index.ts @@ -400,13 +400,11 @@ export default class InputPacket { } _convertToInt16(e: any) { - const int = new Int16Array(1); - return (int[0] = e), int[0]; + return (e << 16) >> 16; } _convertToUInt16(e: any) { - const int = new Uint16Array(1); - return (int[0] = e), int[0]; + return e & 0xffff; } _clampUint8(value: number) { diff --git a/src/webrtc/index.ts b/src/webrtc/index.ts index a7c75566..7cc36cfc 100644 --- a/src/webrtc/index.ts +++ b/src/webrtc/index.ts @@ -83,6 +83,7 @@ class webRTCClient { _webrtcDataChannels: any = {}; _webrtcChannelProcessors: any = {}; + _baseIceServerCount = this._webrtcConfiguration.iceServers.length; _isResetting = false; _gamepad_deadzone = 0.2; @@ -116,6 +117,11 @@ class webRTCClient { _audioLevel = 0; _audioEnergySnapshot: AudioEnergySnapshot | null = null; _hasAudioLevelSample = false; + // Share getStats() between the audio and video timers. + _statsCache: any = null; + _statsCacheAt = 0; + _statsRequest: Promise | null = null; + _statsCacheTtlMs = 250; _hasVideoTrack = false; _hasAudioTrack = false; _hasSentInitialVideoKeyframeRequest = false; @@ -127,9 +133,16 @@ class webRTCClient { } init() { + this._isResetting = false; const settings = getSettings(); this._resetAudioLevelTracking(); this._resetVideoTrackState(); + this._iceCandidates.length = 0; + this._webrtcStates.iceCandidates = []; + this._webrtcConfiguration.iceServers = this._webrtcConfiguration.iceServers.slice( + 0, + this._baseIceServerCount, + ); // Use custom STUN/TURN server if ( @@ -210,7 +223,7 @@ class webRTCClient { 'connectionstatechange:', this._webrtcClient?.connectionState, ); - this._connectedHandler(this._webrtcClient?.connectionState); + this._connectedHandler?.(this._webrtcClient?.connectionState); }); // this._webrtcClient.addEventListener('iceconnectionstatechange', _ => { @@ -284,6 +297,11 @@ class webRTCClient { return this._iceCandidates; } + clearIceCandidates() { + this._iceCandidates.length = 0; + this._webrtcStates.iceCandidates = []; + } + setIceCandidates(iceDetails: any) { for (const candidate in iceDetails) { if (iceDetails[candidate].candidate === 'a=end-of-candidates') { @@ -311,7 +329,19 @@ class webRTCClient { this._isResetting = true; this._webrtcClient?.close(); this._resetAudioLevelTracking(); + this._clearStatsCache(); + globalThis._lastStat = null; + this._webrtcClient = undefined; this._resetVideoTrackState(); + this._iceCandidates.length = 0; + this._webrtcStates.iceCandidates = []; + + for (const name in this._webrtcDataChannels) { + try { + this._webrtcDataChannels[name]?.close?.(); + } catch (_) {} + } + this._webrtcDataChannels = {}; for (const name in this._webrtcChannelProcessors) { this._webrtcChannelProcessors[name].destroy(); @@ -319,7 +349,15 @@ class webRTCClient { this._webrtcChannelProcessors = {}; - this._inputDriver.stop(); + this._inputDriver?.stop(); + this._inputDriver = undefined; + this._trackHandler = undefined; + this._trackAddHandler = undefined; + this._connectedHandler = undefined; + this._sdpHandler = undefined; + this._rumbleHandler = undefined; + this._systemUiHandler = undefined; + this._messageHandler = undefined; } } @@ -358,7 +396,7 @@ class webRTCClient { _sdpHandler: any; sdpNegotiationChat() { this.createOffer().then(offer => { - this._sdpHandler(this, offer); + this._sdpHandler?.(this, offer); }); } @@ -530,13 +568,44 @@ class webRTCClient { this._gpState = gpState; } + _getStatsCached() { + const now = Date.now(); + if (this._statsCache && now - this._statsCacheAt < this._statsCacheTtlMs) { + return Promise.resolve(this._statsCache); + } + if (this._statsRequest) { + return this._statsRequest; + } + if (!this._webrtcClient) { + return Promise.resolve(null); + } + this._statsRequest = this._webrtcClient + .getStats() + .then(stats => { + this._statsCache = stats; + this._statsCacheAt = Date.now(); + return stats; + }) + .finally(() => { + this._statsRequest = null; + }); + return this._statsRequest; + } + _clearStatsCache() { + this._statsCache = null; + this._statsCacheAt = 0; + this._statsRequest = null; + } getAudioVolume() { return new Promise(resolve => { let volume = 0; if (this._webrtcClient) { - this._webrtcClient - .getStats() + this._getStatsCached() .then(stats => { + if (!stats) { + resolve(volume); + return; + } stats.forEach((stat: any) => { if ( stat.type === 'inbound-rtp' && @@ -575,9 +644,12 @@ class webRTCClient { decode: '', }; if (this._webrtcClient) { - this._webrtcClient - .getStats() + this._getStatsCached() .then(stats => { + if (!stats) { + resove(performances); + return; + } stats.forEach((stat: any) => { if ( stat.type === 'inbound-rtp' &&