diff --git a/.changeset/stereo-answer-munging.md b/.changeset/stereo-answer-munging.md new file mode 100644 index 000000000..6b7cba1a0 --- /dev/null +++ b/.changeset/stereo-answer-munging.md @@ -0,0 +1,7 @@ +--- +'livekit-android': patch +--- + +Negotiate stereo Opus on the subscriber answer: add `stereo=1` to the fmtp line for media sections where the server offer advertised `sprop-stereo=1`. Without this, a stereo track published by another participant is decoded as mono on Android. + +Also adds a `stereo` option to `LocalAudioTrackOptions` (surfaced as the `TF_STEREO` track feature) and `AudioTrackPublishOptions` (sent as `stereo` in the AddTrackRequest), mirroring client-sdk-js `forceStereo`. diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/RTCEngine.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/RTCEngine.kt index e0e64e7db..056e73262 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/RTCEngine.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/RTCEngine.kt @@ -54,6 +54,7 @@ import io.livekit.android.webrtc.DataChannelManager import io.livekit.android.webrtc.DataPacketBuffer import io.livekit.android.webrtc.DataPacketItem import io.livekit.android.webrtc.RTCStatsGetter +import io.livekit.android.webrtc.ensureStereoOpus import io.livekit.android.webrtc.copy import io.livekit.android.webrtc.isConnected import io.livekit.android.webrtc.isDisconnected @@ -1158,11 +1159,24 @@ internal constructor( return@launch } + val stereoAnswer = answer.ensureStereoOpus(sessionDescription) + run { - when (val outcome = subscriber?.withPeerConnection { setLocalDescription(answer) }.nullSafe()) { + when (val outcome = subscriber?.withPeerConnection { setLocalDescription(stereoAnswer) }.nullSafe()) { is Either.Left -> Unit is Either.Right -> { - LKLog.e { "error setting local description for answer: ${outcome.value}" } + LKLog.e { "error setting local description for munged answer: ${outcome.value}" } + // Fall back to the un-munged answer rather than leaving the + // subscriber without a local description (mirrors + // PeerConnectionTransport.setMungedSdp). + when (val fallback = subscriber?.withPeerConnection { setLocalDescription(answer) }.nullSafe()) { + is Either.Left -> Unit + is Either.Right -> { + LKLog.e { "error setting local description for answer: ${fallback.value}" } + return@launch + } + } + client.sendAnswer(answer, offerId) return@launch } } @@ -1171,7 +1185,7 @@ internal constructor( if (isClosed) { return@launch } - client.sendAnswer(answer, offerId) + client.sendAnswer(stereoAnswer, offerId) } } diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index 75ac69c11..4f8d7daf4 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -499,6 +499,7 @@ internal constructor( disableRed = !options.red addAllAudioFeatures(options.getFeaturesList()) source = options.source?.toProto() ?: LivekitModels.TrackSource.MICROPHONE + stereo = options.stereo || track.options.stereo }, encodings = encodings, publishListener = publishListener, @@ -1697,6 +1698,10 @@ data class AudioTrackPublishOptions( override val audioBitrate: Int? = null, override val dtx: Boolean = true, override val red: Boolean = true, + /** + * Publish the track as stereo, sent as `stereo` in the AddTrackRequest. + */ + val stereo: Boolean = false, override val source: Track.Source? = null, override val stream: String? = null, override val preconnect: Boolean = false, diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrack.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrack.kt index 8c83c165a..d458da47d 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrack.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrack.kt @@ -204,7 +204,12 @@ constructor( if (options.autoGainControl) { features.add(AudioTrackFeature.TF_AUTO_GAIN_CONTROL) } - // TODO: Handle getting other info from JavaAudioDeviceModule + if (options.stereo) { + features.add(AudioTrackFeature.TF_STEREO) + } + // Note: stereo is sourced from options, not introspected from + // JavaAudioDeviceModule; callers that enable stereo input via a module + // customizer should set LocalAudioTrackOptions.stereo to match. return features } diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrackOptions.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrackOptions.kt index 25be2db8b..7530f50c6 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrackOptions.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalAudioTrackOptions.kt @@ -24,6 +24,7 @@ data class LocalAudioTrackOptions( val autoGainControl: Boolean = true, val highPassFilter: Boolean = true, val typingNoiseDetection: Boolean = true, + val stereo: Boolean = false, ) internal fun LocalAudioTrackOptions.toAudioProcessingOptions(): AudioProcessingOptions { diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/StereoSdpMunging.kt b/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/StereoSdpMunging.kt new file mode 100644 index 000000000..7ea6a2f32 --- /dev/null +++ b/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/StereoSdpMunging.kt @@ -0,0 +1,152 @@ +/* + * Copyright 2023-2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.webrtc + +import android.javax.sdp.MediaDescription +import android.javax.sdp.SdpException +import android.javax.sdp.SdpFactory +import android.javax.sdp.SdpParseException +import androidx.annotation.VisibleForTesting +import io.livekit.android.util.LKLog +import livekit.org.webrtc.SessionDescription + +private const val OPUS_CODEC = "opus" +private const val STEREO_FMTP_PARAM = "stereo=1" +private const val SPROP_STEREO_FMTP_PARAM = "sprop-stereo=1" +private const val MID_ATTRIBUTE = "mid" + +/** + * Adds `stereo=1` to the Opus fmtp line of the answer for every audio media + * section where [offer] advertised `sprop-stereo=1`. + * + * The native Opus decoder downmixes incoming stereo packets to mono unless the + * local answer negotiates `stereo=1`, so without this munging a stereo track + * published by another participant is received as mono. + * + * Mirrors the behavior of client-sdk-js, which extracts `remoteStereoMids` from + * the server offer and rewrites the matching fmtp lines when creating the answer. + * + * @suppress + */ +@VisibleForTesting +internal fun SessionDescription.ensureStereoOpus(offer: SessionDescription): SessionDescription { + val sdpFactory = SdpFactory.getInstance() + val parsedAnswer = parseSessionDescription(sdpFactory, description) ?: return this + val parsedOffer = parseSessionDescription(sdpFactory, offer.description) ?: return this + + val stereoMids = findStereoMids(parsedOffer) + if (stereoMids.isEmpty()) { + return this + } + + for (mediaDesc in mediaDescriptionsOf(parsedAnswer)) { + val mid = midOf(mediaDesc) ?: continue + if (mid !in stereoMids) continue + + val payloadType = findOpusPayloadType(mediaDesc) ?: continue + ensureStereoFmtpParam(mediaDesc, payloadType) + } + + return try { + SessionDescription(type, parsedAnswer.toString()) + } catch (_: SdpException) { + this + } +} + +private fun parseSessionDescription( + sdpFactory: SdpFactory, + description: String, +): android.javax.sdp.SessionDescription? = try { + sdpFactory.createSessionDescription(description) +} catch (_: SdpParseException) { + LKLog.w { "stereo munging: could not parse sdp" } + null +} + +private fun mediaDescriptionsOf(parsed: android.javax.sdp.SessionDescription): List { + val raw = try { + parsed.getMediaDescriptions(true) + } catch (_: SdpException) { + return emptyList() + } + return raw.filterIsInstance() +} + +private fun findStereoMids(offer: android.javax.sdp.SessionDescription): List { + val mids = mutableListOf() + for (mediaDesc in mediaDescriptionsOf(offer)) { + if (isPublisherStereo(mediaDesc)) { + midOf(mediaDesc)?.let { mids.add(it) } + } + } + return mids +} + +private fun midOf(mediaDesc: MediaDescription): String? = try { + mediaDesc.getAttribute(MID_ATTRIBUTE) +} catch (_: SdpParseException) { + null +} + +private fun isPublisherStereo(mediaDesc: MediaDescription): Boolean { + val payloadType = findOpusPayloadType(mediaDesc) ?: return false + for ((_, fmtp) in mediaDesc.getFmtps()) { + if (fmtp.payload == payloadType && fmtp.config.split(";").any { it.trim() == SPROP_STEREO_FMTP_PARAM }) { + return true + } + } + return false +} + +private fun findOpusPayloadType(mediaDesc: MediaDescription): Long? { + for ((_, rtp) in mediaDesc.getRtps()) { + if (rtp.codec.equals(OPUS_CODEC, ignoreCase = true)) { + return rtp.payload + } + } + return null +} + +/* The native Opus decoder requires both sides of the negotiation to carry +stereo=1. The server only puts sprop-stereo=1 into its offer; the answer must +add the stereo=1 parameter itself or received packets are decoded as mono. +*/ +private fun ensureStereoFmtpParam(mediaDesc: MediaDescription, payloadType: Long) { + var fmtpFound = false + for ((attribute, fmtp) in mediaDesc.getFmtps()) { + if (fmtp.payload != payloadType) { + continue + } + fmtpFound = true + if (!fmtp.config.split(";").any { it.trim() == STEREO_FMTP_PARAM }) { + try { + attribute.setValue("${fmtp.payload} ${fmtp.config};$STEREO_FMTP_PARAM") + } catch (_: SdpException) { + LKLog.w { "stereo munging: failed to update opus fmtp line" } + } + } + break + } + + // Not found, add manually + if (!fmtpFound) { + mediaDesc.addAttribute( + SdpFmtp(payloadType, STEREO_FMTP_PARAM).toAttributeField(), + ) + } +} diff --git a/livekit-android-test/src/test/java/io/livekit/android/webrtc/StereoSdpMungingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/webrtc/StereoSdpMungingTest.kt new file mode 100644 index 000000000..d16604316 --- /dev/null +++ b/livekit-android-test/src/test/java/io/livekit/android/webrtc/StereoSdpMungingTest.kt @@ -0,0 +1,146 @@ +/* + * Copyright 2023-2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.webrtc + +import android.javax.sdp.MediaDescription +import android.javax.sdp.SdpFactory +import livekit.org.webrtc.SessionDescription +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class StereoSdpMungingTest { + + private val sdpFactory = SdpFactory.getInstance() + + @Test + fun ensureStereoOpusAddsStereoWhenOfferHasSpropStereo() { + val answer = answerOf(STEREO_OFFER_DESCRIPTION) + val offer = offerOf(STEREO_OFFER_DESCRIPTION) + + val munged = answer.ensureStereoOpus(offer) + + val audioMedia = audioMid(munged, "1") + val fmtp = audioMedia.getFmtps().first { (_, fmtp) -> fmtp.payload == 111L } + assertTrue(fmtp.second.config.split(";").any { it.trim() == "stereo=1" }) + } + + @Test + fun ensureStereoOpusDoesNothingWhenOfferLacksSpropStereo() { + val answer = answerOf(NO_STEREO_OFFER_DESCRIPTION) + val offer = offerOf(NO_STEREO_OFFER_DESCRIPTION) + + val munged = answer.ensureStereoOpus(offer) + + assertEquals(NO_STEREO_OFFER_DESCRIPTION, munged.description) + } + + @Test + fun ensureStereoOpusDoesNotDuplicateStereoParam() { + val answer = answerOf(ANSWER_WITH_STEREO_DESCRIPTION) + val offer = offerOf(STEREO_OFFER_DESCRIPTION) + + val munged = answer.ensureStereoOpus(offer) + + val audioMedia = audioMid(munged, "1") + val stereoFmtps = audioMedia.getFmtps() + .filter { (_, fmtp) -> + fmtp.payload == 111L && fmtp.config.split(";").any { it.trim() == "stereo=1" } + } + assertEquals(1, stereoFmtps.size) + } + + private fun answerOf(sdp: String): SessionDescription = + SessionDescription(SessionDescription.Type.ANSWER, sdp) + + private fun offerOf(sdp: String): SessionDescription = + SessionDescription(SessionDescription.Type.OFFER, sdp) + + private fun audioMid(munged: SessionDescription, mid: String): MediaDescription = + sdpFactory.createSessionDescription(munged.description) + .getMediaDescriptions(true) + .filterIsInstance() + .first { it.getAttribute("mid") == mid } + + companion object { + // Mirrors a LiveKit subscriber offer: audio mid "1" carries opus with + // sprop-stereo=1 (publisher published a stereo track). + private const val STEREO_OFFER_DESCRIPTION = "v=0\r\n" + + "o=- 8980856298632007851 1787470315 IN IP4 0.0.0.0\r\n" + + "s=-\r\n" + + "t=0 0\r\n" + + "a=msid-semantic:WMS *\r\n" + + "a=group:BUNDLE 0 1\r\n" + + "m=application 9 UDP/DTLS/SCTP webrtc-datachannel\r\n" + + "c=IN IP4 0.0.0.0\r\n" + + "a=mid:0\r\n" + + "a=sendrecv\r\n" + + "a=sctp-port:5000\r\n" + + "m=audio 9 UDP/TLS/RTP/SAVPF 111 63\r\n" + + "c=IN IP4 0.0.0.0\r\n" + + "a=mid:1\r\n" + + "a=rtcp-mux\r\n" + + "a=rtpmap:111 opus/48000/2\r\n" + + "a=fmtp:111 minptime=10;useinbandfec=1;sprop-stereo=1\r\n" + + "a=rtpmap:63 red/48000/2\r\n" + + "a=fmtp:63 111/111\r\n" + + "a=recvonly\r\n" + + // Publisher published a mono track: no sprop-stereo anywhere. + private const val NO_STEREO_OFFER_DESCRIPTION = "v=0\r\n" + + "o=- 8980856298632007851 1787470315 IN IP4 0.0.0.0\r\n" + + "s=-\r\n" + + "t=0 0\r\n" + + "a=msid-semantic:WMS *\r\n" + + "a=group:BUNDLE 0 1\r\n" + + "m=application 9 UDP/DTLS/SCTP webrtc-datachannel\r\n" + + "c=IN IP4 0.0.0.0\r\n" + + "a=mid:0\r\n" + + "a=sendrecv\r\n" + + "a=sctp-port:5000\r\n" + + "m=audio 9 UDP/TLS/RTP/SAVPF 111 63\r\n" + + "c=IN IP4 0.0.0.0\r\n" + + "a=mid:1\r\n" + + "a=rtcp-mux\r\n" + + "a=rtpmap:111 opus/48000/2\r\n" + + "a=fmtp:111 minptime=10;useinbandfec=1\r\n" + + "a=rtpmap:63 red/48000/2\r\n" + + "a=fmtp:63 111/111\r\n" + + "a=recvonly\r\n" + + // The answer already negotiated stereo=1: munging must not duplicate it. + private const val ANSWER_WITH_STEREO_DESCRIPTION = "v=0\r\n" + + "o=- 3119613797835240840 4 IN IP4 127.0.0.1\r\n" + + "s=-\r\n" + + "t=0 0\r\n" + + "a=group:BUNDLE 0 1\r\n" + + "a=msid-semantic: WMS\r\n" + + "m=application 9 UDP/DTLS/SCTP webrtc-datachannel\r\n" + + "c=IN IP4 0.0.0.0\r\n" + + "a=mid:0\r\n" + + "a=sctp-port:5000\r\n" + + "m=audio 9 UDP/TLS/RTP/SAVPF 111 63\r\n" + + "c=IN IP4 0.0.0.0\r\n" + + "a=mid:1\r\n" + + "a=rtcp-mux\r\n" + + "a=rtpmap:111 opus/48000/2\r\n" + + "a=fmtp:111 minptime=10;useinbandfec=1;stereo=1\r\n" + + "a=rtpmap:63 red/48000/2\r\n" + + "a=fmtp:63 111/111\r\n" + + "a=recvonly\r\n" + } +}