From 23e2ec1c298f2055316143fe2d6a0b0839947acd Mon Sep 17 00:00:00 2001 From: unohee Date: Thu, 20 Aug 2026 00:47:11 +0900 Subject: [PATCH 1/2] fix(au): let the main input stream format select a layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The input scope was only ever validated against the layout the *output* channel count implied, so a layout was reachable only if the host configured the output bus first. Hosts do not agree on that order: auval's channel tests set the input format first, so a plugin declaring both {2,2} and {1,1} was told FormatNotSupported for a mono input it fully supports, and validation then died with -10868 in Render Preparation — making the mono layout unusable in practice. The main input bus now selects a matching layout the same way the output bus already did, moving the output channel count with it. Auxiliary sidechain elements keep the strict check: their counts never imply a different main layout, and single-layout plugins are unaffected because the lookup simply finds nothing and still rejects. Covered by three wrapper tests over a stereo+mono effect; the layout selection test fails with exactly -10868 when the change is reverted. --- src/wrapper/au/wrapper.rs | 236 +++++++++++++++++++++++++++++++++++++- 1 file changed, 235 insertions(+), 1 deletion(-) diff --git a/src/wrapper/au/wrapper.rs b/src/wrapper/au/wrapper.rs index 27ddb034..343bf866 100644 --- a/src/wrapper/au/wrapper.rs +++ b/src/wrapper/au/wrapper.rs @@ -1614,7 +1614,38 @@ impl Wrapper

{ None => return au::kAudioUnitErr_InvalidElement, }; if expected != req_ch { - return au::kAudioUnitErr_FormatNotSupported; + // The main input bus selects a layout as well; it is + // not merely validated against whatever the output + // bus already picked. + // + // AU negotiates one scope at a time and hosts + // disagree on the order. `auval`'s channel tests set + // the *input* format first, so a plugin declaring + // both {2,2} and {1,1} was told FormatNotSupported + // for a mono input it fully supports, purely because + // the output scope was still stereo at that moment — + // making the mono layout unreachable and failing + // validation with -10868 in Render Preparation. + // + // Only the main input may do this. Auxiliary + // sidechain elements stay strictly validated: their + // counts never imply a different main layout. + let is_main_input = + current_layout::

(this.n_channels()).is_some_and(|layout| { + layout.main_input_channels.is_some() && element == 0 + }); + let req = NonZeroU32::new(req_ch).expect("channel count checked above"); + let selected = if is_main_input { + layout_for_main_input::

(req) + } else { + None + }; + match selected.and_then(|layout| layout.main_output_channels) { + Some(out_ch) => { + this.n_channels.store(out_ch.get(), Ordering::Release) + } + None => return au::kAudioUnitErr_FormatNotSupported, + } } } au::kAudioUnitScope_Output => return au::kAudioUnitErr_InvalidElement, @@ -2558,6 +2589,18 @@ fn layout_for_output(channels: NonZeroU32) -> Option<&'static Audio .find(|layout| layout.main_output_channels == Some(channels)) } +/// Find the declared layout for an AU main *input* channel count. +/// +/// The counterpart of [`layout_for_output`]. Because AU configures one scope at +/// a time and hosts differ on which they set first, a layout that is only +/// reachable through the output scope is effectively unreachable for any host +/// that starts from the input side. +fn layout_for_main_input(channels: NonZeroU32) -> Option<&'static AudioIOLayout> { + P::AUDIO_IO_LAYOUTS + .iter() + .find(|layout| layout.main_input_channels == Some(channels)) +} + fn current_layout(output_channels: u32) -> Option<&'static AudioIOLayout> { NonZeroU32::new(output_channels).and_then(layout_for_output::

) } @@ -3378,4 +3421,195 @@ mod tests { au::kAudioUnitParameterUnit_Generic ); } + + #[derive(Default)] + struct EffectParams; + + unsafe impl Params for EffectParams { + fn param_map(&self) -> Vec<(String, ParamPtr, String)> { + Vec::new() + } + } + + #[derive(Default)] + struct TestEffect { + params: Arc, + } + + impl Plugin for TestEffect { + const NAME: &'static str = "AU Effect Test"; + const VENDOR: &'static str = "NIH-plug"; + const URL: &'static str = "https://github.com/robbert-vdh/nih-plug"; + const EMAIL: &'static str = "test@example.com"; + const VERSION: &'static str = "0.0.0"; + // Stereo first, mono second — the shape every stock example plugin uses. + const AUDIO_IO_LAYOUTS: &'static [AudioIOLayout] = &[ + AudioIOLayout { + main_input_channels: NonZeroU32::new(2), + main_output_channels: NonZeroU32::new(2), + ..AudioIOLayout::const_default() + }, + AudioIOLayout { + main_input_channels: NonZeroU32::new(1), + main_output_channels: NonZeroU32::new(1), + ..AudioIOLayout::const_default() + }, + ]; + + type SysExMessage = (); + type BackgroundTask = (); + + fn params(&self) -> Arc { + self.params.clone() + } + + fn process( + &mut self, + _buffer: &mut Buffer, + _aux: &mut AuxiliaryBuffers, + _context: &mut impl ProcessContext, + ) -> ProcessStatus { + ProcessStatus::Normal + } + } + + impl AuPlugin for TestEffect { + const AU_TYPE: [u8; 4] = *b"aufx"; + const AU_SUBTYPE: [u8; 4] = *b"TstE"; + const AU_MANUFACTURER: [u8; 4] = *b"Test"; + } + + fn float_asbd(channels: u32) -> au::AudioStreamBasicDescription { + au::AudioStreamBasicDescription { + mSampleRate: 44100.0, + mFormatID: au::kAudioFormatLinearPCM, + mFormatFlags: au::kAudioFormatFlagIsFloat + | au::kAudioFormatFlagIsPacked + | au::kAudioFormatFlagIsNonInterleaved, + mBytesPerPacket: 4, + mFramesPerPacket: 1, + mBytesPerFrame: 4, + mChannelsPerFrame: channels, + mBitsPerChannel: 32, + mReserved: 0, + } + } + + fn set_stream_format( + wrapper: *mut c_void, + scope: au::AudioUnitScope, + channels: u32, + ) -> au::OSStatus { + let asbd = float_asbd(channels); + unsafe { + Wrapper::::set_property( + wrapper, + au::kAudioUnitProperty_StreamFormat, + scope, + 0, + &asbd as *const au::AudioStreamBasicDescription as *const c_void, + std::mem::size_of::() as u32, + ) + } + } + + fn stream_format_channels(wrapper: *mut c_void, scope: au::AudioUnitScope) -> u32 { + let mut asbd = float_asbd(0); + let mut size = std::mem::size_of::() as u32; + assert_eq!( + unsafe { + Wrapper::::get_property( + wrapper, + au::kAudioUnitProperty_StreamFormat, + scope, + 0, + &mut asbd as *mut au::AudioStreamBasicDescription as *mut c_void, + &mut size, + ) + }, + au::noErr + ); + asbd.mChannelsPerFrame + } + + /// Setting the main *input* format must be able to select a layout, not + /// just be checked against the one the output scope already implies. + /// + /// The wrapper starts at two output channels, so before this the only way + /// to reach a `{1,1}` layout was to configure the output scope first. Hosts + /// do not agree on that order — `auval`'s channel tests set the input format + /// first, so a mono layout was reported as `FormatNotSupported` and + /// validation died later with `-10868` in Render Preparation. + #[test] + fn main_input_stream_format_selects_a_matching_layout() { + let wrapper = Wrapper::::new() as *mut c_void; + + // Default is the stereo layout at index 0. + assert_eq!( + stream_format_channels(wrapper, au::kAudioUnitScope_Output), + 2 + ); + + // Input-first mono negotiation: accepted, and it moves the whole layout. + assert_eq!( + set_stream_format(wrapper, au::kAudioUnitScope_Input, 1), + au::noErr, + "mono input must be accepted while the output is still stereo" + ); + assert_eq!( + stream_format_channels(wrapper, au::kAudioUnitScope_Output), + 1, + "selecting the mono input layout must move the output bus with it" + ); + assert_eq!( + stream_format_channels(wrapper, au::kAudioUnitScope_Input), + 1 + ); + + // And back, so the negotiation is not one-way. + assert_eq!( + set_stream_format(wrapper, au::kAudioUnitScope_Input, 2), + au::noErr + ); + assert_eq!( + stream_format_channels(wrapper, au::kAudioUnitScope_Output), + 2 + ); + } + + /// A channel count no declared layout offers must still be refused — + /// the relaxation above is a layout lookup, not a blanket "accept anything". + #[test] + fn main_input_stream_format_still_rejects_undeclared_channel_counts() { + let wrapper = Wrapper::::new() as *mut c_void; + + assert_eq!( + set_stream_format(wrapper, au::kAudioUnitScope_Input, 4), + au::kAudioUnitErr_FormatNotSupported + ); + assert_eq!( + stream_format_channels(wrapper, au::kAudioUnitScope_Output), + 2, + "a rejected format must leave the selected layout untouched" + ); + } + + /// The output scope keeps selecting layouts exactly as before. + #[test] + fn output_stream_format_selection_is_unchanged() { + let wrapper = Wrapper::::new() as *mut c_void; + + assert_eq!( + set_stream_format(wrapper, au::kAudioUnitScope_Output, 1), + au::noErr + ); + assert_eq!( + stream_format_channels(wrapper, au::kAudioUnitScope_Input), + 1 + ); + assert_eq!( + set_stream_format(wrapper, au::kAudioUnitScope_Output, 7), + au::kAudioUnitErr_FormatNotSupported + ); + } } From 5faea2819d0a4500f77e734fdc340b1dca4854dc Mon Sep 17 00:00:00 2001 From: unohee Date: Thu, 20 Aug 2026 01:15:01 +0900 Subject: [PATCH 2/2] fix(au): refuse an input-selected layout the wrapper cannot reproduce MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit let the main input stream format select a layout, but persisted only its output channel count. Everything downstream re-derives the layout from that count through `layout_for_output`, which takes the first match — so for a plugin whose layouts share an output count but differ on the input side, e.g. [{in:2,out:2}, {in:1,out:2}], setting the input to 1 channel returned noErr while the derivation still landed on {2,2}. The wrapper reported success for a format it would not honor; before the relaxation this case was rejected. Accept a selection only when `layout_for_output` resolves back to the same layout. Symmetric layouts — the shape every stock example plugin and the AU-supported configuration use — are unaffected. Also correcting an overclaim in the previous commit message: single-layout plugins are unaffected only when their output count is 2. A mono-only plugin still cannot negotiate input-first, because `n_channels` starts at a hardcoded 2 and `current_layout(2)` finds nothing, so the input branch returns InvalidElement before the new lookup is reachable. That is pre-existing, not a regression — but the fix makes declared layouts reachable from a resolvable starting state, not in general. The new test returns noErr instead of -10868 when the round-trip check is removed. --- src/wrapper/au/wrapper.rs | 114 +++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-) diff --git a/src/wrapper/au/wrapper.rs b/src/wrapper/au/wrapper.rs index 343bf866..a48e00e9 100644 --- a/src/wrapper/au/wrapper.rs +++ b/src/wrapper/au/wrapper.rs @@ -1640,7 +1640,26 @@ impl Wrapper

{ } else { None }; - match selected.and_then(|layout| layout.main_output_channels) { + // Only accept a selection the rest of the wrapper + // can reproduce. Everything downstream re-derives + // the layout from the output channel count through + // `layout_for_output`, which takes the *first* + // match. For a plugin whose layouts share an output + // count but differ on the input side — say + // `[{in:2,out:2}, {in:1,out:2}]` — storing the + // input-matched layout would leave the derivation + // landing on the other one, so the wrapper would + // report success for a format it then silently + // refuses to honor. Rejecting is what the old code + // did here, and it stays correct. + let accepted = selected.and_then(|layout| { + let out_ch = layout.main_output_channels?; + match layout_for_output::

(out_ch) { + Some(resolved) if std::ptr::eq(resolved, layout) => Some(out_ch), + _ => None, + } + }); + match accepted { Some(out_ch) => { this.n_channels.store(out_ch.get(), Ordering::Release) } @@ -3594,6 +3613,99 @@ mod tests { ); } + #[derive(Default)] + struct AsymmetricEffect { + params: Arc, + } + + impl Plugin for AsymmetricEffect { + const NAME: &'static str = "AU Asymmetric Test"; + const VENDOR: &'static str = "NIH-plug"; + const URL: &'static str = "https://github.com/robbert-vdh/nih-plug"; + const EMAIL: &'static str = "test@example.com"; + const VERSION: &'static str = "0.0.0"; + // Two layouts sharing an output count. `layout_for_output(2)` can only + // ever resolve to the first one, so the second is not representable + // through the wrapper's output-keyed derivation. + const AUDIO_IO_LAYOUTS: &'static [AudioIOLayout] = &[ + AudioIOLayout { + main_input_channels: NonZeroU32::new(2), + main_output_channels: NonZeroU32::new(2), + ..AudioIOLayout::const_default() + }, + AudioIOLayout { + main_input_channels: NonZeroU32::new(1), + main_output_channels: NonZeroU32::new(2), + ..AudioIOLayout::const_default() + }, + ]; + + type SysExMessage = (); + type BackgroundTask = (); + + fn params(&self) -> Arc { + self.params.clone() + } + + fn process( + &mut self, + _buffer: &mut Buffer, + _aux: &mut AuxiliaryBuffers, + _context: &mut impl ProcessContext, + ) -> ProcessStatus { + ProcessStatus::Normal + } + } + + impl AuPlugin for AsymmetricEffect { + const AU_TYPE: [u8; 4] = *b"aufx"; + const AU_SUBTYPE: [u8; 4] = *b"TstA"; + const AU_MANUFACTURER: [u8; 4] = *b"Test"; + } + + /// Selecting by input count must not report success for a layout the + /// wrapper cannot then reproduce. + /// + /// Everything downstream re-derives the layout from the output channel + /// count via `layout_for_output`, which takes the first match. When two + /// layouts share an output count, the input-matched one is unreachable — + /// accepting it would tell the host "yes, 1 channel in" and then run + /// `{2,2}` anyway. Rejecting is what the pre-existing code did, and the + /// input-side relaxation must not weaken it. + #[test] + fn main_input_selection_rejects_layouts_it_cannot_reproduce() { + let wrapper = Wrapper::::new() as *mut c_void; + let asbd = au::AudioStreamBasicDescription { + mSampleRate: 44100.0, + mFormatID: au::kAudioFormatLinearPCM, + mFormatFlags: au::kAudioFormatFlagIsFloat + | au::kAudioFormatFlagIsPacked + | au::kAudioFormatFlagIsNonInterleaved, + mBytesPerPacket: 4, + mFramesPerPacket: 1, + mBytesPerFrame: 4, + mChannelsPerFrame: 1, + mBitsPerChannel: 32, + mReserved: 0, + }; + + let status = unsafe { + Wrapper::::set_property( + wrapper, + au::kAudioUnitProperty_StreamFormat, + au::kAudioUnitScope_Input, + 0, + &asbd as *const au::AudioStreamBasicDescription as *const c_void, + std::mem::size_of::() as u32, + ) + }; + assert_eq!( + status, + au::kAudioUnitErr_FormatNotSupported, + "a layout that output-keyed derivation cannot reach must be refused" + ); + } + /// The output scope keeps selecting layouts exactly as before. #[test] fn output_stream_format_selection_is_unchanged() {