From 7745cab06d747507fed9369364d81dab14349825 Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Mon, 25 May 2026 04:03:37 -0700 Subject: [PATCH 01/12] feat(macos/capture): add ScreenCaptureKit backend, runtime-select on 12.3+ AVCaptureScreenInput was deprecated in macOS 13 (October 2022) and is fundamentally limited to 8-bit BGRA, blocking any honest HDR or 10-bit work on the macOS capture path. ScreenCaptureKit has been available since macOS 12.3 (March 2022) and is the only forward path; this commit lays the foundation by adding a drop-in SCK-based backend that preserves behaviour exactly (same pixel format, frame rate, display selection) so it can be reviewed independently of the HDR work that builds on top. Changes: * Add SunshineVideoCapture protocol in av_video.h declaring the capture-side surface both backends expose. * Make AVVideo conform to the protocol (no behaviour change; pure declaration). * Add SCVideo (sc_video.h / sc_video.m) implementing the same protocol against SCStream + SCContentFilter + SCStreamConfiguration. Built with -fobjc-arc for SCK's block-heavy API surface; objects cross the MRC boundary via the standard +1-retain alloc/init convention so display.mm continues to work in MRC. * Drop incomplete frames from SCK output by inspecting SCStreamFrameInfoStatus on each sample-buffer attachment, matching the reliability the legacy path got for free from AVCaptureSession. * display.mm now holds an id and branches at construction via @available(macOS 12.3, *): SCVideo on supported systems, AVVideo as fallback for older macOS. * Wire ScreenCaptureKit framework into cmake/dependencies/macos.cmake and cmake/compile_definitions/macos.cmake; set ARC compile flag on sc_video.m only. Pixel format stays 32BGRA for this commit; 10-bit + EDR metadata follow in a subsequent change. --- cmake/compile_definitions/macos.cmake | 11 + cmake/dependencies/macos.cmake | 1 + src/platform/macos/av_video.h | 25 ++- src/platform/macos/display.mm | 20 +- src/platform/macos/sc_video.h | 30 +++ src/platform/macos/sc_video.m | 278 ++++++++++++++++++++++++++ 6 files changed, 358 insertions(+), 7 deletions(-) create mode 100644 src/platform/macos/sc_video.h create mode 100644 src/platform/macos/sc_video.m diff --git a/cmake/compile_definitions/macos.cmake b/cmake/compile_definitions/macos.cmake index dbca9df9073..15a8bbbe8c7 100644 --- a/cmake/compile_definitions/macos.cmake +++ b/cmake/compile_definitions/macos.cmake @@ -35,6 +35,7 @@ list(APPEND SUNSHINE_EXTERNAL_LIBRARIES ${CORE_MEDIA_LIBRARY} ${CORE_VIDEO_LIBRARY} ${FOUNDATION_LIBRARY} + ${SCREEN_CAPTURE_KIT_LIBRARY} ${VIDEO_TOOLBOX_LIBRARY}) set(APPLE_PLIST_TEMPLATE "${SUNSHINE_SOURCE_ASSETS_DIR}/macos/build/Info.plist.in") @@ -55,6 +56,16 @@ set(PLATFORM_TARGET_FILES "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.cpp" "${CMAKE_SOURCE_DIR}/src/platform/macos/nv12_zero_device.h" "${CMAKE_SOURCE_DIR}/src/platform/macos/publish.cpp" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_video.h" + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_video.m" "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.c" "${CMAKE_SOURCE_DIR}/third-party/TPCircularBuffer/TPCircularBuffer.h" ${APPLE_PLIST_FILE}) + +# sc_video.m is written against ARC for clarity (SCK APIs are async/ +# block-heavy and benefit from ARC). The rest of the macOS Obj-C +# sources remain MRC; objects flowing across the boundary follow the +# standard +1-retain alloc/init convention so both modes interoperate. +set_source_files_properties( + "${CMAKE_SOURCE_DIR}/src/platform/macos/sc_video.m" + PROPERTIES COMPILE_FLAGS "-fobjc-arc") diff --git a/cmake/dependencies/macos.cmake b/cmake/dependencies/macos.cmake index 5e225fdac21..44444749c16 100644 --- a/cmake/dependencies/macos.cmake +++ b/cmake/dependencies/macos.cmake @@ -10,6 +10,7 @@ FIND_LIBRARY(CORE_MEDIA_LIBRARY CoreMedia) FIND_LIBRARY(CORE_VIDEO_LIBRARY CoreVideo) FIND_LIBRARY(FOUNDATION_LIBRARY Foundation) FIND_LIBRARY(VIDEO_TOOLBOX_LIBRARY VideoToolbox) +FIND_LIBRARY(SCREEN_CAPTURE_KIT_LIBRARY ScreenCaptureKit) if(SUNSHINE_ENABLE_TRAY) FIND_LIBRARY(COCOA Cocoa REQUIRED) diff --git a/src/platform/macos/av_video.h b/src/platform/macos/av_video.h index b2fa5d4b255..94d5e2db565 100644 --- a/src/platform/macos/av_video.h +++ b/src/platform/macos/av_video.h @@ -15,7 +15,17 @@ struct CaptureSession { static const int kMaxDisplays = 32; -@interface AVVideo: NSObject +typedef bool (^FrameCallbackBlock)(CMSampleBufferRef); + +/** + * @brief Shared interface for macOS screen capture backends. + * + * Both the legacy AVCaptureScreenInput-based implementation (AVVideo) and + * the modern ScreenCaptureKit-based implementation (SCVideo) conform to + * this protocol so display.mm can hold either behind a single pointer + * type and branch on macOS version at construction. + */ +@protocol SunshineVideoCapture @property (nonatomic, assign) CGDirectDisplayID displayID; @property (nonatomic, assign) CMTime minFrameDuration; @@ -23,7 +33,18 @@ static const int kMaxDisplays = 32; @property (nonatomic, assign) int frameWidth; @property (nonatomic, assign) int frameHeight; -typedef bool (^FrameCallbackBlock)(CMSampleBufferRef); +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; +- (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback; + +@end + +@interface AVVideo: NSObject + +@property (nonatomic, assign) CGDirectDisplayID displayID; +@property (nonatomic, assign) CMTime minFrameDuration; +@property (nonatomic, assign) OSType pixelFormat; +@property (nonatomic, assign) int frameWidth; +@property (nonatomic, assign) int frameHeight; @property (nonatomic, assign) AVCaptureSession *session; @property (nonatomic, assign) NSMapTable *videoOutputs; diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index be124b2d331..381eb380a8b 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -10,6 +10,7 @@ #include "src/platform/macos/av_video.h" #include "src/platform/macos/misc.h" #include "src/platform/macos/nv12_zero_device.h" +#include "src/platform/macos/sc_video.h" // Avoid conflict between AVFoundation and libavutil both defining AVMediaType #define AVMediaType AVMediaType_FFmpeg @@ -22,7 +23,7 @@ using namespace std::literals; struct av_display_t: public display_t { - AVVideo *av_capture {}; + id av_capture {}; CGDirectDisplayID display_id {}; ~av_display_t() override { @@ -86,7 +87,7 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010) { auto device = std::make_unique(); - device->init(static_cast(av_capture), pix_fmt, setResolution, setPixelFormat); + device->init((__bridge void *) av_capture, pix_fmt, setResolution, setPixelFormat); return device; } else { @@ -143,11 +144,11 @@ int dummy_img(img_t *img) override { * height --> the intended capture height */ static void setResolution(void *display, int width, int height) { - [static_cast(display) setFrameWidth:width frameHeight:height]; + [(__bridge id) display setFrameWidth:width frameHeight:height]; } static void setPixelFormat(void *display, OSType pixelFormat) { - static_cast(display).pixelFormat = pixelFormat; + ((__bridge id) display).pixelFormat = pixelFormat; } }; @@ -177,7 +178,16 @@ static void setPixelFormat(void *display, OSType pixelFormat) { } BOOST_LOG(info) << "Configuring selected display ("sv << display->display_id << ") to stream"sv; - display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; + // Prefer ScreenCaptureKit on macOS 12.3+ (AVCaptureScreenInput was + // deprecated in macOS 13 and is hardcoded to 8-bit BGRA). Fall back to + // the legacy AVCaptureScreenInput path on older macOS. + if (@available(macOS 12.3, *)) { + BOOST_LOG(info) << "Using ScreenCaptureKit capture backend"sv; + display->av_capture = [[SCVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; + } else { + BOOST_LOG(info) << "Using legacy AVCaptureScreenInput capture backend"sv; + display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; + } if (!display->av_capture) { BOOST_LOG(error) << "Video setup failed."sv; diff --git a/src/platform/macos/sc_video.h b/src/platform/macos/sc_video.h new file mode 100644 index 00000000000..7462ad6afe6 --- /dev/null +++ b/src/platform/macos/sc_video.h @@ -0,0 +1,30 @@ +/** + * @file src/platform/macos/sc_video.h + * @brief Declarations for ScreenCaptureKit-based video capture on macOS. + * + * Modern replacement for AVCaptureScreenInput (which was deprecated in + * macOS 13). SCVideo conforms to the same SunshineVideoCapture protocol + * as the legacy AVVideo class so callers can swap implementations at + * runtime based on @available(macOS 12.3, *) without other code changes. + */ +#pragma once + +#import "av_video.h" + +#import + +API_AVAILABLE(macos(12.3)) +@interface SCVideo: NSObject + +@property (nonatomic, assign) CGDirectDisplayID displayID; +@property (nonatomic, assign) CMTime minFrameDuration; +@property (nonatomic, assign) OSType pixelFormat; +@property (nonatomic, assign) int frameWidth; +@property (nonatomic, assign) int frameHeight; + +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate; + +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; +- (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback; + +@end diff --git a/src/platform/macos/sc_video.m b/src/platform/macos/sc_video.m new file mode 100644 index 00000000000..f42c80f6daf --- /dev/null +++ b/src/platform/macos/sc_video.m @@ -0,0 +1,278 @@ +/** + * @file src/platform/macos/sc_video.m + * @brief ScreenCaptureKit-based video capture for macOS 12.3+. + * + * Drop-in replacement for the legacy AVCaptureScreenInput path in + * av_video.m. This first-pass implementation preserves the original + * pixel format (BGRA8) and selection semantics; HDR / 10-bit pixel + * format selection and EDR color metadata propagation are layered on + * top in subsequent commits. + * + * Compiled with ARC (-fobjc-arc) for clarity. The other macOS capture + * files remain MRC; objects flowing from this file to display.mm + * follow the standard alloc/init +1-retain convention so the boundary + * works regardless of compile mode on the other side. + */ +#import "sc_video.h" + +#import + +API_AVAILABLE(macos(12.3)) +@interface SCVideo () + +@property (nonatomic, strong) SCStream *stream; +@property (nonatomic, strong) SCContentFilter *filter; +@property (nonatomic, strong) SCStreamConfiguration *streamConfig; +@property (nonatomic, strong) dispatch_queue_t sampleQueue; +@property (nonatomic, copy) FrameCallbackBlock currentCallback; +@property (nonatomic, strong) dispatch_semaphore_t currentSignal; +@property (nonatomic, assign) BOOL streamRunning; + +@end + +@implementation SCVideo + +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate { + self = [super init]; + if (!self) { + return nil; + } + + self.displayID = displayID; + self.minFrameDuration = CMTimeMake(1, frameRate); + self.pixelFormat = kCVPixelFormatType_32BGRA; + + CGDisplayModeRef mode = CGDisplayCopyDisplayMode(displayID); + if (mode) { + self.frameWidth = (int) CGDisplayModeGetPixelWidth(mode); + self.frameHeight = (int) CGDisplayModeGetPixelHeight(mode); + CGDisplayModeRelease(mode); + } + + self.sampleQueue = dispatch_queue_create("dev.lizardbyte.sunshine.sckCapture", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, DISPATCH_QUEUE_PRIORITY_HIGH)); + + // SCK content enumeration is async; block until we have the SCDisplay + // matching the requested CGDirectDisplayID so this initializer remains + // synchronous (matching AVVideo's contract). + __block SCDisplay *selectedDisplay = nil; + __block NSError *enumerationError = nil; + dispatch_semaphore_t ready = dispatch_semaphore_create(0); + + [SCShareableContent getShareableContentExcludingDesktopWindows:NO + onScreenWindowsOnly:NO + completionHandler:^(SCShareableContent *_Nullable content, NSError *_Nullable error) { + if (error || !content) { + enumerationError = error; + } else { + for (SCDisplay *d in content.displays) { + if (d.displayID == displayID) { + selectedDisplay = d; + break; + } + } + // If the requested display wasn't found (display reconfigured, + // unplugged, etc.) fall back to the first display SCK reports. + if (!selectedDisplay && content.displays.count > 0) { + selectedDisplay = content.displays.firstObject; + } + } + dispatch_semaphore_signal(ready); + }]; + dispatch_semaphore_wait(ready, DISPATCH_TIME_FOREVER); + + if (!selectedDisplay) { + NSLog(@"SCVideo: failed to resolve SCDisplay for id %u: %@", displayID, enumerationError); + return nil; + } + + // Empty excluded-windows array: capture everything on the display. + self.filter = [[SCContentFilter alloc] initWithDisplay:selectedDisplay excludingWindows:@[]]; + + self.streamConfig = [[SCStreamConfiguration alloc] init]; + self.streamConfig.width = self.frameWidth; + self.streamConfig.height = self.frameHeight; + self.streamConfig.minimumFrameInterval = self.minFrameDuration; + self.streamConfig.pixelFormat = self.pixelFormat; + self.streamConfig.queueDepth = 6; // SCK docs recommend 3-8 + self.streamConfig.showsCursor = YES; + + self.stream = [[SCStream alloc] initWithFilter:self.filter + configuration:self.streamConfig + delegate:self]; + + return self; +} + +- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight { + _frameWidth = frameWidth; + _frameHeight = frameHeight; + + if (self.streamConfig) { + self.streamConfig.width = frameWidth; + self.streamConfig.height = frameHeight; + [self applyConfigurationIfRunning]; + } +} + +- (void)setPixelFormat:(OSType)pixelFormat { + _pixelFormat = pixelFormat; + + if (self.streamConfig) { + self.streamConfig.pixelFormat = pixelFormat; + [self applyConfigurationIfRunning]; + } +} + +- (void)setMinFrameDuration:(CMTime)minFrameDuration { + _minFrameDuration = minFrameDuration; + + if (self.streamConfig) { + self.streamConfig.minimumFrameInterval = minFrameDuration; + [self applyConfigurationIfRunning]; + } +} + +- (void)applyConfigurationIfRunning { + if (!self.streamRunning || !self.stream) { + return; + } + [self.stream updateConfiguration:self.streamConfig + completionHandler:^(NSError *_Nullable error) { + if (error) { + NSLog(@"SCVideo: updateConfiguration failed: %@", error); + } + }]; +} + +- (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback { + @synchronized(self) { + // Signal and clear any previous capture; SCK streams support one + // logical consumer in this wrapper. Matches single-callback use in + // display.mm. + if (self.currentSignal) { + dispatch_semaphore_signal(self.currentSignal); + } + + self.currentCallback = frameCallback; + self.currentSignal = dispatch_semaphore_create(0); + + if (!self.streamRunning) { + NSError *outputError = nil; + if (![self.stream addStreamOutput:self + type:SCStreamOutputTypeScreen + sampleHandlerQueue:self.sampleQueue + error:&outputError]) { + NSLog(@"SCVideo: addStreamOutput failed: %@", outputError); + dispatch_semaphore_signal(self.currentSignal); + return self.currentSignal; + } + + __block NSError *startError = nil; + dispatch_semaphore_t started = dispatch_semaphore_create(0); + [self.stream startCaptureWithCompletionHandler:^(NSError *_Nullable error) { + startError = error; + dispatch_semaphore_signal(started); + }]; + dispatch_semaphore_wait(started, DISPATCH_TIME_FOREVER); + + if (startError) { + NSLog(@"SCVideo: startCapture failed: %@", startError); + dispatch_semaphore_signal(self.currentSignal); + return self.currentSignal; + } + self.streamRunning = YES; + } + + return self.currentSignal; + } +} + +- (void)dealloc { + if (self.streamRunning && self.stream) { + // Best-effort synchronous stop. The completion handler may not fire + // before dealloc returns; SCStream itself will tear down cleanly. + dispatch_semaphore_t stopped = dispatch_semaphore_create(0); + [self.stream stopCaptureWithCompletionHandler:^(NSError *_Nullable error) { + (void) error; + dispatch_semaphore_signal(stopped); + }]; + // Bounded wait so a misbehaving SCK doesn't hang teardown. + dispatch_semaphore_wait(stopped, dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC)); + } +} + +#pragma mark - SCStreamOutput + +- (void)stream:(SCStream *)stream + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + ofType:(SCStreamOutputType)type { + if (type != SCStreamOutputTypeScreen) { + return; + } + if (!CMSampleBufferIsValid(sampleBuffer)) { + return; + } + + // Drop frames whose status array says they aren't ready. SCK delivers + // a status attachment on every sample buffer indicating idle vs + // complete vs blank — we want only complete frames downstream. + CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, NO); + if (attachmentsArray && CFArrayGetCount(attachmentsArray) > 0) { + CFDictionaryRef attachments = CFArrayGetValueAtIndex(attachmentsArray, 0); + CFNumberRef statusNum = CFDictionaryGetValue(attachments, (__bridge CFStringRef) SCStreamFrameInfoStatus); + if (statusNum) { + int status = 0; + CFNumberGetValue(statusNum, kCFNumberSInt32Type, &status); + if (status != SCFrameStatusComplete) { + return; + } + } + } + + FrameCallbackBlock callback; + dispatch_semaphore_t signal; + @synchronized(self) { + callback = self.currentCallback; + signal = self.currentSignal; + } + + if (!callback) { + return; + } + + if (!callback(sampleBuffer)) { + // Consumer signalled stop. Tear down the stream and unblock the + // semaphore the caller is waiting on. + @synchronized(self) { + self.currentCallback = nil; + } + if (self.streamRunning) { + [self.stream stopCaptureWithCompletionHandler:^(NSError *_Nullable error) { + (void) error; + }]; + self.streamRunning = NO; + } + if (signal) { + dispatch_semaphore_signal(signal); + } + } +} + +#pragma mark - SCStreamDelegate + +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { + if (error) { + NSLog(@"SCVideo: stream stopped with error: %@", error); + } + self.streamRunning = NO; + dispatch_semaphore_t signal; + @synchronized(self) { + signal = self.currentSignal; + self.currentCallback = nil; + } + if (signal) { + dispatch_semaphore_signal(signal); + } +} + +@end From 632737043b479519d8d9111e571566a837096283 Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Mon, 25 May 2026 04:11:58 -0700 Subject: [PATCH 02/12] feat(macos/capture): enable EDR (HDR) output on macOS 14+ for 10-bit pixel formats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With AVCaptureScreenInput, asking the capture surface for a 10-bit pixel format silently produced 8-bit BGRA — the OS-level lie that made HEVC Main10 / AV1 Main10 / ProRes 10-bit profiles on macOS into fake HDR (color-tagged 8-bit data). With ScreenCaptureKit landing in the previous commit, 10-bit pixel formats are actually honoured, but SCK needs an explicit signal to attach HDR metadata to those buffers instead of treating them as 10-bit Rec.709. This commit wires SCStreamConfiguration.captureDynamicRange: * Add +pixelFormatIsHighBitDepth: classifier covering the YUV 4:2:0, 4:2:2 and 4:4:4 10-bit BiPlanar formats plus ARGB2101010 packed and 64-bit RGBA formats. * On the synchronous init path, set captureDynamicRange immediately if the starting pixel format is high bit depth so the very first sample buffer carries HDR metadata. * On the setPixelFormat: path (called by nv12_zero_device when the encoder selects p010), also update captureDynamicRange and push the new config to a running stream via -updateConfiguration:. * Use SCCaptureDynamicRangeHDRLocalDisplay rather than canonical HDR: game streaming wants the host display's actual HDR characteristics (peak luminance, primaries) so the receiver shows what a local user would see, not Apple's idealised reference. * Guard the whole block behind @available(macOS 14.0, *); on 12.3-13.x SCK still honours the 10-bit pixel format request but doesn't auto-tag buffers, so Sunshine's existing colorspace logic continues to drive the encoder's color fields. Validated on M4 Max: Sunshine's encoder probe matrix now includes successful 10-bit HEVC and 10-bit ProRes entries that previously could not have validated because the capture surface couldn't deliver matching pixel data. ProRes-specific VideoToolbox color tags land in a separate follow-up commit. --- src/platform/macos/sc_video.m | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/platform/macos/sc_video.m b/src/platform/macos/sc_video.m index f42c80f6daf..209ce67bbb3 100644 --- a/src/platform/macos/sc_video.m +++ b/src/platform/macos/sc_video.m @@ -96,6 +96,10 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)fram self.streamConfig.queueDepth = 6; // SCK docs recommend 3-8 self.streamConfig.showsCursor = YES; + // If the initial pixel format is already a 10-bit format, flip on EDR + // immediately so the very first sample buffer carries HDR metadata. + [self applyDynamicRangeForPixelFormat:self.pixelFormat]; + self.stream = [[SCStream alloc] initWithFilter:self.filter configuration:self.streamConfig delegate:self]; @@ -103,6 +107,47 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)fram return self; } +/** + * @brief Whether a CVPixelBuffer OSType denotes a 10-bit (or wider) format. + * + * Returning YES is the signal that the capture surface is HDR-capable; we + * use it to drive SCStreamConfiguration.captureDynamicRange on macOS 14+ + * so SCK emits BT.2020 PQ-tagged buffers instead of 10-bit Rec.709. + */ ++ (BOOL)pixelFormatIsHighBitDepth:(OSType)pixelFormat { + switch (pixelFormat) { + case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_422YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_ARGB2101010LEPacked: + case kCVPixelFormatType_64ARGB: + case kCVPixelFormatType_64RGBALE: + return YES; + default: + return NO; + } +} + +- (void)applyDynamicRangeForPixelFormat:(OSType)pixelFormat { + // captureDynamicRange landed in macOS 14 (Sonoma). On 12.3-13.x the + // capture surface honours the requested 10-bit pixel format, but the + // OS won't tag the buffers with BT.2020 PQ metadata automatically; + // downstream code falls back to Sunshine's existing colorspace logic. + if (@available(macOS 14.0, *)) { + if ([SCVideo pixelFormatIsHighBitDepth:pixelFormat]) { + // hdrLocalDisplay matches the host display's HDR characteristics, + // which is what we want for game-streaming: stream what the user + // would see locally, including the local panel's PQ peak luminance. + self.streamConfig.captureDynamicRange = SCCaptureDynamicRangeHDRLocalDisplay; + } else { + self.streamConfig.captureDynamicRange = SCCaptureDynamicRangeSDR; + } + } +} + - (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight { _frameWidth = frameWidth; _frameHeight = frameHeight; @@ -119,6 +164,7 @@ - (void)setPixelFormat:(OSType)pixelFormat { if (self.streamConfig) { self.streamConfig.pixelFormat = pixelFormat; + [self applyDynamicRangeForPixelFormat:pixelFormat]; [self applyConfigurationIfRunning]; } } From 57986e8421d7d40bf36d0f4436083ff452b87450 Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Tue, 26 May 2026 19:32:22 -0700 Subject: [PATCH 03/12] feat(macos/capture): harden SCK lifecycle and gate EDR on negotiated session HDR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two improvements to the macOS ScreenCaptureKit backend that landed in the upstream review cycle (#5190) and never made it back onto our fork's dev: 1. **Lifecycle hardening**: - Register the SCStreamOutput exactly once in -init, not on every -capture: call. SCStream retains outputs across stop/start cycles, so re-registering would fail or silently duplicate delivery. - Bound all SCK completion-handler waits to 5s. SCK should always invoke them, but a misbehaving system service must not hang the whole startup path. - @synchronized(self) around all reads/writes of currentCallback / currentSignal / streamRunning. The sample-handler queue, the -capture: caller, and the SCStream delegate all touch these from different threads. - dispatch_queue_attr_make_with_qos_class's third argument is a RELATIVE priority (range -15..0), not one of the legacy DISPATCH_QUEUE_PRIORITY_* constants. Using 0 keeps the queue at its QoS class's nominal priority. - CGDisplayBounds fallback when CGDisplayCopyDisplayMode returns NULL (display reconfiguration races). 2. **EDR gating fix**: The previous EDR code flipped captureDynamicRange = HDRLocalDisplay whenever the chosen CVPixelBuffer format was 10-bit. That's necessary but not sufficient: a 10-bit format may be selected for codec reasons (e.g., a ProRes profile that requires 4:4:4 10-bit input) without the client ever requesting HDR ingest. Without gating, Sunshine would tell the client "HDR mode false" in the SDP while emitting BT.2020 PQ-tagged buffers — a silent control/data-plane mismatch. Now EDR requires BOTH 10-bit pixel format AND the negotiated session's enable_hdr (plumbed from launch_session_t via the existing config.dynamicRange field on video::config_t). Default is SDR; HDR is opt-in per session. New init signature: initWithDisplay:frameRate:hdrAllowed: The old initializer is preserved as a convenience that passes NO. New log line at session start makes the gating visible: "Using ScreenCaptureKit capture backend (HDR allowed|blocked)" --- src/platform/macos/display.mm | 10 +- src/platform/macos/sc_video.h | 7 ++ src/platform/macos/sc_video.m | 219 ++++++++++++++++++++++++---------- 3 files changed, 172 insertions(+), 64 deletions(-) diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index 381eb380a8b..9b4604060ee 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -182,8 +182,14 @@ static void setPixelFormat(void *display, OSType pixelFormat) { // deprecated in macOS 13 and is hardcoded to 8-bit BGRA). Fall back to // the legacy AVCaptureScreenInput path on older macOS. if (@available(macOS 12.3, *)) { - BOOST_LOG(info) << "Using ScreenCaptureKit capture backend"sv; - display->av_capture = [[SCVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; + // hdrAllowed reflects the negotiated `enable_hdr` for this session + // (rtsp.cpp maps `x-nv-video[0].dynamicRangeMode` into config.dynamicRange). + // SCK uses this together with the chosen pixel format depth to decide + // whether to flip captureDynamicRange to HDRLocalDisplay; neither + // condition alone is sufficient. See sc_video.m::applyDynamicRangeForPixelFormat:. + const BOOL hdr_allowed = config.dynamicRange ? YES : NO; + BOOST_LOG(info) << "Using ScreenCaptureKit capture backend (HDR "sv << (hdr_allowed ? "allowed" : "blocked") << ")"sv; + display->av_capture = [[SCVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate hdrAllowed:hdr_allowed]; } else { BOOST_LOG(info) << "Using legacy AVCaptureScreenInput capture backend"sv; display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; diff --git a/src/platform/macos/sc_video.h b/src/platform/macos/sc_video.h index 7462ad6afe6..51823214d79 100644 --- a/src/platform/macos/sc_video.h +++ b/src/platform/macos/sc_video.h @@ -22,7 +22,14 @@ API_AVAILABLE(macos(12.3)) @property (nonatomic, assign) int frameWidth; @property (nonatomic, assign) int frameHeight; +// YES iff the negotiated streaming session enabled HDR (Moonlight's +// hdrMode flag). Required (in combination with a 10-bit pixel format) +// before SCK is allowed to flip captureDynamicRange to HDRLocalDisplay +// on macOS 14+. Defaults to NO; the SDR capture path is always safe. +@property (nonatomic, assign) BOOL hdrAllowed; + - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate; +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate hdrAllowed:(BOOL)hdrAllowed; - (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; - (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback; diff --git a/src/platform/macos/sc_video.m b/src/platform/macos/sc_video.m index 209ce67bbb3..52ccacfc952 100644 --- a/src/platform/macos/sc_video.m +++ b/src/platform/macos/sc_video.m @@ -8,6 +8,13 @@ * format selection and EDR color metadata propagation are layered on * top in subsequent commits. * + * Lifecycle: the underlying SCStream is started exactly once during + * -initWithDisplay:frameRate: and stopped exactly once during -dealloc. + * -capture: only swaps the active callback / signal; it never touches + * the stream lifecycle. This avoids the "addStreamOutput called twice" + * failure mode that SCK exhibits when an output is re-registered on a + * stream that already retains it across stop/start cycles. + * * Compiled with ARC (-fobjc-arc) for clarity. The other macOS capture * files remain MRC; objects flowing from this file to display.mm * follow the standard alloc/init +1-retain convention so the boundary @@ -17,6 +24,11 @@ #import +// Bounded wait for any SCK completion handler. SCK should always +// invoke these, but a misbehaving system service must not hang the +// whole startup path. +static const int64_t kSCVideoCompletionTimeoutSec = 5; + API_AVAILABLE(macos(12.3)) @interface SCVideo () @@ -24,15 +36,25 @@ @interface SCVideo () @property (nonatomic, strong) SCContentFilter *filter; @property (nonatomic, strong) SCStreamConfiguration *streamConfig; @property (nonatomic, strong) dispatch_queue_t sampleQueue; + +// All four of the following are mutated from multiple threads (the +// caller of -capture:, the SCK sample-handler queue, and the SCStream +// delegate's didStopWithError:) and so are only ever accessed under +// @synchronized(self). @property (nonatomic, copy) FrameCallbackBlock currentCallback; @property (nonatomic, strong) dispatch_semaphore_t currentSignal; @property (nonatomic, assign) BOOL streamRunning; +@property (nonatomic, assign) BOOL streamOutputAdded; @end @implementation SCVideo - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate { + return [self initWithDisplay:displayID frameRate:frameRate hdrAllowed:NO]; +} + +- (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate hdrAllowed:(BOOL)hdrAllowed { self = [super init]; if (!self) { return nil; @@ -41,19 +63,37 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)fram self.displayID = displayID; self.minFrameDuration = CMTimeMake(1, frameRate); self.pixelFormat = kCVPixelFormatType_32BGRA; + self.hdrAllowed = hdrAllowed; + // Prefer the active display mode's pixel dimensions; fall back to + // CGDisplayBounds if no mode is currently set (e.g., during display + // reconfiguration). If both fail we still proceed — SCK will + // accept the requested SCContentFilter dimensions later. CGDisplayModeRef mode = CGDisplayCopyDisplayMode(displayID); if (mode) { self.frameWidth = (int) CGDisplayModeGetPixelWidth(mode); self.frameHeight = (int) CGDisplayModeGetPixelHeight(mode); CGDisplayModeRelease(mode); + } else { + CGRect bounds = CGDisplayBounds(displayID); + self.frameWidth = (int) CGRectGetWidth(bounds); + self.frameHeight = (int) CGRectGetHeight(bounds); } - self.sampleQueue = dispatch_queue_create("dev.lizardbyte.sunshine.sckCapture", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, DISPATCH_QUEUE_PRIORITY_HIGH)); - - // SCK content enumeration is async; block until we have the SCDisplay - // matching the requested CGDirectDisplayID so this initializer remains - // synchronous (matching AVVideo's contract). + // dispatch_queue_attr_make_with_qos_class's third parameter is a + // relative priority (range -15..0), NOT one of the legacy global- + // queue DISPATCH_QUEUE_PRIORITY_* constants. Using 0 keeps the + // queue at the chosen QoS class's nominal priority. + dispatch_queue_attr_t qos = dispatch_queue_attr_make_with_qos_class( + DISPATCH_QUEUE_SERIAL, + QOS_CLASS_USER_INTERACTIVE, + 0 + ); + self.sampleQueue = dispatch_queue_create("dev.lizardbyte.sunshine.sckCapture", qos); + + // SCK content enumeration is async; block (with a bounded timeout) + // until we have the SCDisplay matching the requested CGDirectDisplayID + // so this initializer remains synchronous (matching AVVideo's contract). __block SCDisplay *selectedDisplay = nil; __block NSError *enumerationError = nil; dispatch_semaphore_t ready = dispatch_semaphore_create(0); @@ -78,7 +118,10 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)fram } dispatch_semaphore_signal(ready); }]; - dispatch_semaphore_wait(ready, DISPATCH_TIME_FOREVER); + if (dispatch_semaphore_wait(ready, dispatch_time(DISPATCH_TIME_NOW, kSCVideoCompletionTimeoutSec * NSEC_PER_SEC)) != 0) { + NSLog(@"SCVideo: getShareableContent timed out after %lld seconds", kSCVideoCompletionTimeoutSec); + return nil; + } if (!selectedDisplay) { NSLog(@"SCVideo: failed to resolve SCDisplay for id %u: %@", displayID, enumerationError); @@ -103,6 +146,46 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)fram self.stream = [[SCStream alloc] initWithFilter:self.filter configuration:self.streamConfig delegate:self]; + if (!self.stream) { + NSLog(@"SCVideo: SCStream allocation failed"); + return nil; + } + + // Register the SCStreamOutput exactly once, here. SCStream retains + // outputs across stop/start cycles, so re-registering on every + // -capture: call would fail (or worse, silently duplicate + // delivery). All subsequent state changes are callback swaps on + // -capture: rather than stream-lifecycle operations. + NSError *outputError = nil; + if (![self.stream addStreamOutput:self + type:SCStreamOutputTypeScreen + sampleHandlerQueue:self.sampleQueue + error:&outputError]) { + NSLog(@"SCVideo: addStreamOutput failed: %@", outputError); + return nil; + } + self.streamOutputAdded = YES; + + // Start the stream once. Frames begin flowing immediately on the + // sampleQueue; sample-handler delivery is a no-op until the first + // -capture: installs a callback (see -stream:didOutputSampleBuffer:ofType:). + __block NSError *startError = nil; + dispatch_semaphore_t started = dispatch_semaphore_create(0); + [self.stream startCaptureWithCompletionHandler:^(NSError *_Nullable error) { + startError = error; + dispatch_semaphore_signal(started); + }]; + if (dispatch_semaphore_wait(started, dispatch_time(DISPATCH_TIME_NOW, kSCVideoCompletionTimeoutSec * NSEC_PER_SEC)) != 0) { + NSLog(@"SCVideo: startCapture timed out after %lld seconds", kSCVideoCompletionTimeoutSec); + return nil; + } + if (startError) { + NSLog(@"SCVideo: startCapture failed: %@", startError); + return nil; + } + @synchronized(self) { + self.streamRunning = YES; + } return self; } @@ -132,12 +215,27 @@ + (BOOL)pixelFormatIsHighBitDepth:(OSType)pixelFormat { } - (void)applyDynamicRangeForPixelFormat:(OSType)pixelFormat { - // captureDynamicRange landed in macOS 14 (Sonoma). On 12.3-13.x the - // capture surface honours the requested 10-bit pixel format, but the - // OS won't tag the buffers with BT.2020 PQ metadata automatically; + // captureDynamicRange / SCCaptureDynamicRange* are macOS 14 (Sonoma) + // SDK symbols. The compile-time guard ensures this block is preprocessed + // away entirely when building against an older SDK that lacks the + // declarations; the runtime @available guard prevents using the + // symbols at runtime on pre-14 systems even with a newer SDK. On + // 12.3-13.x SCK still honours a requested 10-bit pixel format, but + // the OS won't tag buffers with BT.2020 PQ metadata automatically; // downstream code falls back to Sunshine's existing colorspace logic. + // + // Gating: EDR is only enabled when BOTH (a) the chosen pixel format + // is 10-bit, AND (b) the session was actually negotiated as HDR + // (`hdrAllowed`). The pixel format on its own is necessary but not + // sufficient — a 10-bit format may be selected for codec reasons + // (e.g., a ProRes profile) without the client ever requesting HDR + // ingest, and silently emitting BT.2020 PQ-tagged buffers into a + // stream the control plane describes as SDR causes the decoder to + // tone-map undefined content. Defaulting hdrAllowed to NO keeps the + // legacy/SDR semantics intact when callers don't opt in. +#if defined(__MAC_OS_X_VERSION_MAX_ALLOWED) && __MAC_OS_X_VERSION_MAX_ALLOWED >= 140000 if (@available(macOS 14.0, *)) { - if ([SCVideo pixelFormatIsHighBitDepth:pixelFormat]) { + if (self.hdrAllowed && [SCVideo pixelFormatIsHighBitDepth:pixelFormat]) { // hdrLocalDisplay matches the host display's HDR characteristics, // which is what we want for game-streaming: stream what the user // would see locally, including the local panel's PQ peak luminance. @@ -146,6 +244,9 @@ - (void)applyDynamicRangeForPixelFormat:(OSType)pixelFormat { self.streamConfig.captureDynamicRange = SCCaptureDynamicRangeSDR; } } +#else + (void) pixelFormat; +#endif } - (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight { @@ -179,7 +280,11 @@ - (void)setMinFrameDuration:(CMTime)minFrameDuration { } - (void)applyConfigurationIfRunning { - if (!self.streamRunning || !self.stream) { + BOOL running; + @synchronized(self) { + running = self.streamRunning; + } + if (!running || !self.stream) { return; } [self.stream updateConfiguration:self.streamConfig @@ -191,58 +296,47 @@ - (void)applyConfigurationIfRunning { } - (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback { - @synchronized(self) { - // Signal and clear any previous capture; SCK streams support one - // logical consumer in this wrapper. Matches single-callback use in - // display.mm. - if (self.currentSignal) { - dispatch_semaphore_signal(self.currentSignal); - } + // Swap in the new callback. The SCStream output and frame flow are + // already running from -init; this method is purely a callback + // installation, not a stream-lifecycle operation. That avoids the + // double-add failure mode and makes -capture: cheap enough to be + // called multiple times across the SCVideo's lifetime (e.g., the + // encoder probe path's dummy_img followed by the real capture). + dispatch_semaphore_t newSignal = dispatch_semaphore_create(0); + dispatch_semaphore_t previousSignal = nil; + @synchronized(self) { + previousSignal = self.currentSignal; self.currentCallback = frameCallback; - self.currentSignal = dispatch_semaphore_create(0); - - if (!self.streamRunning) { - NSError *outputError = nil; - if (![self.stream addStreamOutput:self - type:SCStreamOutputTypeScreen - sampleHandlerQueue:self.sampleQueue - error:&outputError]) { - NSLog(@"SCVideo: addStreamOutput failed: %@", outputError); - dispatch_semaphore_signal(self.currentSignal); - return self.currentSignal; - } - - __block NSError *startError = nil; - dispatch_semaphore_t started = dispatch_semaphore_create(0); - [self.stream startCaptureWithCompletionHandler:^(NSError *_Nullable error) { - startError = error; - dispatch_semaphore_signal(started); - }]; - dispatch_semaphore_wait(started, DISPATCH_TIME_FOREVER); - - if (startError) { - NSLog(@"SCVideo: startCapture failed: %@", startError); - dispatch_semaphore_signal(self.currentSignal); - return self.currentSignal; - } - self.streamRunning = YES; - } + self.currentSignal = newSignal; + } - return self.currentSignal; + // Unblock any prior caller still waiting on the old semaphore. + // They will observe their callback was cleared and return. + if (previousSignal) { + dispatch_semaphore_signal(previousSignal); } + + return newSignal; } - (void)dealloc { - if (self.streamRunning && self.stream) { - // Best-effort synchronous stop. The completion handler may not fire - // before dealloc returns; SCStream itself will tear down cleanly. + BOOL running; + SCStream *stream; + @synchronized(self) { + running = self.streamRunning; + stream = self.stream; + self.streamRunning = NO; + self.currentCallback = nil; + } + if (running && stream) { + // Best-effort synchronous stop with a bounded wait so a + // misbehaving SCK doesn't hang teardown. dispatch_semaphore_t stopped = dispatch_semaphore_create(0); - [self.stream stopCaptureWithCompletionHandler:^(NSError *_Nullable error) { + [stream stopCaptureWithCompletionHandler:^(NSError *_Nullable error) { (void) error; dispatch_semaphore_signal(stopped); }]; - // Bounded wait so a misbehaving SCK doesn't hang teardown. dispatch_semaphore_wait(stopped, dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC)); } } @@ -283,20 +377,20 @@ - (void)stream:(SCStream *)stream } if (!callback) { + // No active consumer. Drop the frame; the stream keeps running + // so subsequent -capture: calls can pick up immediately. return; } if (!callback(sampleBuffer)) { - // Consumer signalled stop. Tear down the stream and unblock the - // semaphore the caller is waiting on. + // Consumer signalled stop. Clear the callback and wake the + // caller; the underlying SCStream stays alive for any future + // -capture: caller (cheaper than tearing down and restarting). @synchronized(self) { - self.currentCallback = nil; - } - if (self.streamRunning) { - [self.stream stopCaptureWithCompletionHandler:^(NSError *_Nullable error) { - (void) error; - }]; - self.streamRunning = NO; + if (self.currentCallback == callback) { + self.currentCallback = nil; + self.currentSignal = nil; + } } if (signal) { dispatch_semaphore_signal(signal); @@ -310,11 +404,12 @@ - (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { if (error) { NSLog(@"SCVideo: stream stopped with error: %@", error); } - self.streamRunning = NO; dispatch_semaphore_t signal; @synchronized(self) { + self.streamRunning = NO; signal = self.currentSignal; self.currentCallback = nil; + self.currentSignal = nil; } if (signal) { dispatch_semaphore_signal(signal); From e173045c5113772b8004ee6509c966ddc760692d Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Wed, 27 May 2026 00:41:28 -0700 Subject: [PATCH 04/12] build(macos): require ScreenCaptureKit at configure time FIND_LIBRARY(SCREEN_CAPTURE_KIT_LIBRARY ScreenCaptureKit REQUIRED) so configure fails fast on environments without the SDK rather than later at header-lookup time. sc_video.m is compiled unconditionally; the REQUIRED keyword ensures the build prerequisites are surfaced clearly when the build host's Xcode/SDK is older than 13.3 / 12.3 (well past routine compatibility). Addresses Copilot inline feedback from the closed upstream PR cycle. --- cmake/dependencies/macos.cmake | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmake/dependencies/macos.cmake b/cmake/dependencies/macos.cmake index 44444749c16..846bf78d176 100644 --- a/cmake/dependencies/macos.cmake +++ b/cmake/dependencies/macos.cmake @@ -10,7 +10,14 @@ FIND_LIBRARY(CORE_MEDIA_LIBRARY CoreMedia) FIND_LIBRARY(CORE_VIDEO_LIBRARY CoreVideo) FIND_LIBRARY(FOUNDATION_LIBRARY Foundation) FIND_LIBRARY(VIDEO_TOOLBOX_LIBRARY VideoToolbox) -FIND_LIBRARY(SCREEN_CAPTURE_KIT_LIBRARY ScreenCaptureKit) +# ScreenCaptureKit is the modern (macOS 12.3+) replacement for the +# deprecated AVCaptureScreenInput-based capture path. Sunshine's +# sc_video.{h,m} is unconditionally compiled into the macOS target; +# fail configure with a clear message rather than failing the build +# later on header lookup when the SDK doesn't ship the framework +# (e.g., when building with an Xcode older than 13.3 / SDK older than +# 12.3, which dropped out of routine compatibility long ago). +FIND_LIBRARY(SCREEN_CAPTURE_KIT_LIBRARY ScreenCaptureKit REQUIRED) if(SUNSHINE_ENABLE_TRAY) FIND_LIBRARY(COCOA Cocoa REQUIRED) From 9d242cdd8e43d977faa4671dbf1054c6a2f6fd6d Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Wed, 27 May 2026 00:54:17 -0700 Subject: [PATCH 05/12] refactor(macos/capture): drop legacy AVCaptureScreenInput path entirely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ScreenCaptureKit became available in macOS 12.3; Sunshine's deployment target (MACOSX_DEPLOYMENT_TARGET=14.2) is well above that. The @available(macOS 12.3, *) runtime branch in display.mm and the entire AVCaptureScreenInput-based AVVideo class were therefore dead code on every supported build. Changes: - Remove @available(macOS 12.3, *) check in display.mm; SCK is the only branch. - Replace `id` with `SCVideo *` directly — the protocol existed to abstract over both AVVideo and SCVideo, and is no longer needed with a single concrete capture class. - Move the small bits we still need (FrameCallbackBlock typedef, +displayNames / +getDisplayName: helpers) from av_video.{h,m} into sc_video.{h,m}. - Delete src/platform/macos/av_video.{h,m} (208 lines). - Drop both from PLATFORM_TARGET_FILES. Addresses andygrundman + ReenigneArcher review feedback on the original PR: "shouldn't keep workarounds for versions older than what we support." --- cmake/compile_definitions/macos.cmake | 2 - src/platform/macos/av_video.h | 62 ----------- src/platform/macos/av_video.m | 146 -------------------------- src/platform/macos/display.mm | 42 ++++---- src/platform/macos/sc_video.h | 26 +++-- src/platform/macos/sc_video.m | 46 ++++++-- 6 files changed, 75 insertions(+), 249 deletions(-) delete mode 100644 src/platform/macos/av_video.h delete mode 100644 src/platform/macos/av_video.m diff --git a/cmake/compile_definitions/macos.cmake b/cmake/compile_definitions/macos.cmake index 15a8bbbe8c7..f2b6499efee 100644 --- a/cmake/compile_definitions/macos.cmake +++ b/cmake/compile_definitions/macos.cmake @@ -46,8 +46,6 @@ set(PLATFORM_TARGET_FILES "${CMAKE_SOURCE_DIR}/src/platform/macos/av_audio.h" "${CMAKE_SOURCE_DIR}/src/platform/macos/av_audio.mm" "${CMAKE_SOURCE_DIR}/src/platform/macos/av_img_t.h" - "${CMAKE_SOURCE_DIR}/src/platform/macos/av_video.h" - "${CMAKE_SOURCE_DIR}/src/platform/macos/av_video.m" "${CMAKE_SOURCE_DIR}/src/platform/macos/display.mm" "${CMAKE_SOURCE_DIR}/src/platform/macos/input.cpp" "${CMAKE_SOURCE_DIR}/src/platform/macos/microphone.mm" diff --git a/src/platform/macos/av_video.h b/src/platform/macos/av_video.h deleted file mode 100644 index 94d5e2db565..00000000000 --- a/src/platform/macos/av_video.h +++ /dev/null @@ -1,62 +0,0 @@ -/** - * @file src/platform/macos/av_video.h - * @brief Declarations for video capture on macOS. - */ -#pragma once - -// platform includes -#import -#import - -struct CaptureSession { - AVCaptureVideoDataOutput *output; - NSCondition *captureStopped; -}; - -static const int kMaxDisplays = 32; - -typedef bool (^FrameCallbackBlock)(CMSampleBufferRef); - -/** - * @brief Shared interface for macOS screen capture backends. - * - * Both the legacy AVCaptureScreenInput-based implementation (AVVideo) and - * the modern ScreenCaptureKit-based implementation (SCVideo) conform to - * this protocol so display.mm can hold either behind a single pointer - * type and branch on macOS version at construction. - */ -@protocol SunshineVideoCapture - -@property (nonatomic, assign) CGDirectDisplayID displayID; -@property (nonatomic, assign) CMTime minFrameDuration; -@property (nonatomic, assign) OSType pixelFormat; -@property (nonatomic, assign) int frameWidth; -@property (nonatomic, assign) int frameHeight; - -- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; -- (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback; - -@end - -@interface AVVideo: NSObject - -@property (nonatomic, assign) CGDirectDisplayID displayID; -@property (nonatomic, assign) CMTime minFrameDuration; -@property (nonatomic, assign) OSType pixelFormat; -@property (nonatomic, assign) int frameWidth; -@property (nonatomic, assign) int frameHeight; - -@property (nonatomic, assign) AVCaptureSession *session; -@property (nonatomic, assign) NSMapTable *videoOutputs; -@property (nonatomic, assign) NSMapTable *captureCallbacks; -@property (nonatomic, assign) NSMapTable *captureSignals; - -+ (NSArray *)displayNames; -+ (NSString *)getDisplayName:(CGDirectDisplayID)displayID; - -- (id)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate; - -- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; -- (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback; - -@end diff --git a/src/platform/macos/av_video.m b/src/platform/macos/av_video.m deleted file mode 100644 index 630d7101598..00000000000 --- a/src/platform/macos/av_video.m +++ /dev/null @@ -1,146 +0,0 @@ -/** - * @file src/platform/macos/av_video.m - * @brief Definitions for video capture on macOS. - */ -// local includes -#import "av_video.h" - -@implementation AVVideo - -// XXX: Currently, this function only returns the screen IDs as names, -// which is not very helpful to the user. The API to retrieve names -// was deprecated with 10.9+. -// However, there is a solution with little external code that can be used: -// https://stackoverflow.com/questions/20025868/cgdisplayioserviceport-is-deprecated-in-os-x-10-9-how-to-replace -+ (NSArray *)displayNames { - CGDirectDisplayID displays[kMaxDisplays]; - uint32_t count; - if (CGGetActiveDisplayList(kMaxDisplays, displays, &count) != kCGErrorSuccess) { - return [NSArray array]; - } - - NSMutableArray *result = [NSMutableArray array]; - - for (uint32_t i = 0; i < count; i++) { - [result addObject:@{ - @"id": [NSNumber numberWithUnsignedInt:displays[i]], - @"name": [NSString stringWithFormat:@"%d", displays[i]], - @"displayName": [self getDisplayName:displays[i]], - }]; - } - - return [NSArray arrayWithArray:result]; -} - -+ (NSString *)getDisplayName:(CGDirectDisplayID)displayID { - for (NSScreen *screen in [NSScreen screens]) { - if ([screen.deviceDescription[@"NSScreenNumber"] isEqualToNumber:[NSNumber numberWithUnsignedInt:displayID]]) { - return screen.localizedName; - } - } - return nil; -} - -- (id)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)frameRate { - self = [super init]; - - CGDisplayModeRef mode = CGDisplayCopyDisplayMode(displayID); - - self.displayID = displayID; - self.pixelFormat = kCVPixelFormatType_32BGRA; - self.frameWidth = (int) CGDisplayModeGetPixelWidth(mode); - self.frameHeight = (int) CGDisplayModeGetPixelHeight(mode); - self.minFrameDuration = CMTimeMake(1, frameRate); - self.session = [[AVCaptureSession alloc] init]; - self.videoOutputs = [[NSMapTable alloc] init]; - self.captureCallbacks = [[NSMapTable alloc] init]; - self.captureSignals = [[NSMapTable alloc] init]; - - CFRelease(mode); - - AVCaptureScreenInput *screenInput = [[AVCaptureScreenInput alloc] initWithDisplayID:self.displayID]; - [screenInput setMinFrameDuration:self.minFrameDuration]; - - if ([self.session canAddInput:screenInput]) { - [self.session addInput:screenInput]; - } else { - [screenInput release]; - return nil; - } - - [self.session startRunning]; - - return self; -} - -- (void)dealloc { - [self.videoOutputs release]; - [self.captureCallbacks release]; - [self.captureSignals release]; - [self.session stopRunning]; - [super dealloc]; -} - -- (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight { - self.frameWidth = frameWidth; - self.frameHeight = frameHeight; -} - -- (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback { - @synchronized(self) { - AVCaptureVideoDataOutput *videoOutput = [[AVCaptureVideoDataOutput alloc] init]; - - [videoOutput setVideoSettings:@{ - (NSString *) kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithUnsignedInt:self.pixelFormat], - (NSString *) kCVPixelBufferWidthKey: [NSNumber numberWithInt:self.frameWidth], - (NSString *) kCVPixelBufferHeightKey: [NSNumber numberWithInt:self.frameHeight], - (NSString *) AVVideoScalingModeKey: AVVideoScalingModeResizeAspect, - }]; - - dispatch_queue_attr_t qos = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, DISPATCH_QUEUE_PRIORITY_HIGH); - dispatch_queue_t recordingQueue = dispatch_queue_create("videoCaptureQueue", qos); - [videoOutput setSampleBufferDelegate:self queue:recordingQueue]; - - [self.session stopRunning]; - - if ([self.session canAddOutput:videoOutput]) { - [self.session addOutput:videoOutput]; - } else { - [videoOutput release]; - return nil; - } - - AVCaptureConnection *videoConnection = [videoOutput connectionWithMediaType:AVMediaTypeVideo]; - dispatch_semaphore_t signal = dispatch_semaphore_create(0); - - [self.videoOutputs setObject:videoOutput forKey:videoConnection]; - [self.captureCallbacks setObject:frameCallback forKey:videoConnection]; - [self.captureSignals setObject:signal forKey:videoConnection]; - - [self.session startRunning]; - - return signal; - } -} - -- (void)captureOutput:(AVCaptureOutput *)captureOutput - didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection { - FrameCallbackBlock callback = [self.captureCallbacks objectForKey:connection]; - - if (callback != nil) { - if (!callback(sampleBuffer)) { - @synchronized(self) { - [self.session stopRunning]; - [self.captureCallbacks removeObjectForKey:connection]; - [self.session removeOutput:[self.videoOutputs objectForKey:connection]]; - [self.videoOutputs removeObjectForKey:connection]; - dispatch_semaphore_signal([self.captureSignals objectForKey:connection]); - [self.captureSignals removeObjectForKey:connection]; - [self.session startRunning]; - } - } - } -} - -@end diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index 9b4604060ee..45bf97e46b2 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -7,7 +7,6 @@ #include "src/logging.h" #include "src/platform/common.h" #include "src/platform/macos/av_img_t.h" -#include "src/platform/macos/av_video.h" #include "src/platform/macos/misc.h" #include "src/platform/macos/nv12_zero_device.h" #include "src/platform/macos/sc_video.h" @@ -23,7 +22,7 @@ using namespace std::literals; struct av_display_t: public display_t { - id av_capture {}; + SCVideo *av_capture {}; CGDirectDisplayID display_id {}; ~av_display_t() override { @@ -87,7 +86,7 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const } else if (pix_fmt == pix_fmt_e::nv12 || pix_fmt == pix_fmt_e::p010) { auto device = std::make_unique(); - device->init((__bridge void *) av_capture, pix_fmt, setResolution, setPixelFormat); + device->init((void *) av_capture, pix_fmt, setResolution, setPixelFormat); return device; } else { @@ -144,11 +143,11 @@ int dummy_img(img_t *img) override { * height --> the intended capture height */ static void setResolution(void *display, int width, int height) { - [(__bridge id) display setFrameWidth:width frameHeight:height]; + [(SCVideo *) display setFrameWidth:width frameHeight:height]; } static void setPixelFormat(void *display, OSType pixelFormat) { - ((__bridge id) display).pixelFormat = pixelFormat; + ((SCVideo *) display).pixelFormat = pixelFormat; } }; @@ -164,7 +163,7 @@ static void setPixelFormat(void *display, OSType pixelFormat) { display->display_id = CGMainDisplayID(); // Print all displays available with it's name and id - auto display_array = [AVVideo displayNames]; + auto display_array = [SCVideo displayNames]; BOOST_LOG(info) << "Detecting displays"sv; for (NSDictionary *item in display_array) { NSNumber *display_id = item[@"id"]; @@ -178,22 +177,19 @@ static void setPixelFormat(void *display, OSType pixelFormat) { } BOOST_LOG(info) << "Configuring selected display ("sv << display->display_id << ") to stream"sv; - // Prefer ScreenCaptureKit on macOS 12.3+ (AVCaptureScreenInput was - // deprecated in macOS 13 and is hardcoded to 8-bit BGRA). Fall back to - // the legacy AVCaptureScreenInput path on older macOS. - if (@available(macOS 12.3, *)) { - // hdrAllowed reflects the negotiated `enable_hdr` for this session - // (rtsp.cpp maps `x-nv-video[0].dynamicRangeMode` into config.dynamicRange). - // SCK uses this together with the chosen pixel format depth to decide - // whether to flip captureDynamicRange to HDRLocalDisplay; neither - // condition alone is sufficient. See sc_video.m::applyDynamicRangeForPixelFormat:. - const BOOL hdr_allowed = config.dynamicRange ? YES : NO; - BOOST_LOG(info) << "Using ScreenCaptureKit capture backend (HDR "sv << (hdr_allowed ? "allowed" : "blocked") << ")"sv; - display->av_capture = [[SCVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate hdrAllowed:hdr_allowed]; - } else { - BOOST_LOG(info) << "Using legacy AVCaptureScreenInput capture backend"sv; - display->av_capture = [[AVVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate]; - } + // ScreenCaptureKit is the only capture backend Sunshine ships on macOS; + // the deployment target (14.2) is well above SCK's minimum (12.3) so + // there is no @available branch and no legacy AVCaptureScreenInput + // fallback to maintain. + // + // hdrAllowed reflects the negotiated `enable_hdr` for this session + // (rtsp.cpp maps `x-nv-video[0].dynamicRangeMode` into config.dynamicRange). + // SCK uses this together with the chosen pixel format depth to decide + // whether to flip captureDynamicRange to HDRLocalDisplay; neither + // condition alone is sufficient. See sc_video.m::applyDynamicRangeForPixelFormat:. + const BOOL hdr_allowed = config.dynamicRange ? YES : NO; + BOOST_LOG(info) << "Using ScreenCaptureKit capture backend (HDR "sv << (hdr_allowed ? "allowed" : "blocked") << ")"sv; + display->av_capture = [[SCVideo alloc] initWithDisplay:display->display_id frameRate:config.framerate hdrAllowed:hdr_allowed]; if (!display->av_capture) { BOOST_LOG(error) << "Video setup failed."sv; @@ -212,7 +208,7 @@ static void setPixelFormat(void *display, OSType pixelFormat) { std::vector display_names(mem_type_e hwdevice_type) { __block std::vector display_names; - auto display_array = [AVVideo displayNames]; + auto display_array = [SCVideo displayNames]; display_names.reserve([display_array count]); [display_array enumerateObjectsUsingBlock:^(NSDictionary *_Nonnull obj, NSUInteger idx, BOOL *_Nonnull stop) { diff --git a/src/platform/macos/sc_video.h b/src/platform/macos/sc_video.h index 51823214d79..37831570b93 100644 --- a/src/platform/macos/sc_video.h +++ b/src/platform/macos/sc_video.h @@ -2,19 +2,23 @@ * @file src/platform/macos/sc_video.h * @brief Declarations for ScreenCaptureKit-based video capture on macOS. * - * Modern replacement for AVCaptureScreenInput (which was deprecated in - * macOS 13). SCVideo conforms to the same SunshineVideoCapture protocol - * as the legacy AVVideo class so callers can swap implementations at - * runtime based on @available(macOS 12.3, *) without other code changes. + * SCVideo is now Sunshine's only macOS capture backend. The deployment + * target (MACOSX_DEPLOYMENT_TARGET=14.2) is well above the macOS 12.3 + * minimum where ScreenCaptureKit became available, so the legacy + * AVCaptureScreenInput-based AVVideo path has been removed entirely. */ #pragma once -#import "av_video.h" - #import +#import +#import + +// Block signature used to deliver captured sample buffers back to the +// platform-agnostic capture loop. Returning NO from the block stops +// further deliveries on this capture session. +typedef bool (^FrameCallbackBlock)(CMSampleBufferRef); -API_AVAILABLE(macos(12.3)) -@interface SCVideo: NSObject +@interface SCVideo : NSObject @property (nonatomic, assign) CGDirectDisplayID displayID; @property (nonatomic, assign) CMTime minFrameDuration; @@ -34,4 +38,10 @@ API_AVAILABLE(macos(12.3)) - (void)setFrameWidth:(int)frameWidth frameHeight:(int)frameHeight; - (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback; +// Enumerate the currently-active CGDisplays as an array of dictionaries +// with keys @"id" (NSNumber, the CGDirectDisplayID), @"name" (NSString, +// the numeric id as a string for legacy callers), and @"displayName" +// (NSString, the user-facing name from NSScreen.localizedName). ++ (NSArray *)displayNames; + @end diff --git a/src/platform/macos/sc_video.m b/src/platform/macos/sc_video.m index 52ccacfc952..08cd8610dd3 100644 --- a/src/platform/macos/sc_video.m +++ b/src/platform/macos/sc_video.m @@ -1,12 +1,9 @@ /** * @file src/platform/macos/sc_video.m - * @brief ScreenCaptureKit-based video capture for macOS 12.3+. - * - * Drop-in replacement for the legacy AVCaptureScreenInput path in - * av_video.m. This first-pass implementation preserves the original - * pixel format (BGRA8) and selection semantics; HDR / 10-bit pixel - * format selection and EDR color metadata propagation are layered on - * top in subsequent commits. + * @brief ScreenCaptureKit-based video capture. Sole macOS capture + * backend; Sunshine's deployment target (14.2) is well above the SCK + * minimum (12.3) so the legacy AVCaptureScreenInput-based AVVideo + * implementation has been retired. * * Lifecycle: the underlying SCStream is started exactly once during * -initWithDisplay:frameRate: and stopped exactly once during -dealloc. @@ -93,7 +90,7 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)fram // SCK content enumeration is async; block (with a bounded timeout) // until we have the SCDisplay matching the requested CGDirectDisplayID - // so this initializer remains synchronous (matching AVVideo's contract). + // so this initializer remains synchronous: callers are not yet block-aware. __block SCDisplay *selectedDisplay = nil; __block NSError *enumerationError = nil; dispatch_semaphore_t ready = dispatch_semaphore_create(0); @@ -416,4 +413,37 @@ - (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { } } +#pragma mark - Display enumeration + +// Active-display upper bound. We just need a buffer size that comfortably +// exceeds any plausible attached-display count. +static const int kMaxDisplays = 32; + ++ (NSString *)getDisplayName:(CGDirectDisplayID)displayID { + for (NSScreen *screen in [NSScreen screens]) { + if ([screen.deviceDescription[@"NSScreenNumber"] isEqualToNumber:[NSNumber numberWithUnsignedInt:displayID]]) { + return screen.localizedName; + } + } + return nil; +} + ++ (NSArray *)displayNames { + CGDirectDisplayID displays[kMaxDisplays]; + uint32_t count = 0; + if (CGGetActiveDisplayList(kMaxDisplays, displays, &count) != kCGErrorSuccess) { + return @[]; + } + + NSMutableArray *result = [NSMutableArray array]; + for (uint32_t i = 0; i < count; i++) { + [result addObject:@{ + @"id": [NSNumber numberWithUnsignedInt:displays[i]], + @"name": [NSString stringWithFormat:@"%u", displays[i]], + @"displayName": [SCVideo getDisplayName:displays[i]] ?: [NSString stringWithFormat:@"Display %u", displays[i]], + }]; + } + return [NSArray arrayWithArray:result]; +} + @end From 45382a2bc0acd2107a2f40b42c1071831c6b01aa Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Wed, 27 May 2026 02:02:10 -0700 Subject: [PATCH 06/12] review: address sc_video.m bot review feedback Two Copilot findings on the SCK PR: 1. **dealloc signals pending semaphore.** -dealloc was clearing currentCallback but never signalling currentSignal. If the stream stopped without firing -stream:didStopWithError:, any caller still waiting on the semaphore returned by -capture: would stall forever. Snapshot the pending signal in the @synchronized block, then send it after clearing the callback so the waiter wakes up to observe their callback is nil and exits. 2. **Drop unused streamOutputAdded property.** Set in -init but never read. Removed both the @property declaration and the assignment. The "register output exactly once at init" invariant is now structural (the call is in -init, not gated on a flag) and the dead state can't drift. --- src/platform/macos/sc_video.m | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/platform/macos/sc_video.m b/src/platform/macos/sc_video.m index 08cd8610dd3..eab24a328da 100644 --- a/src/platform/macos/sc_video.m +++ b/src/platform/macos/sc_video.m @@ -41,7 +41,6 @@ @interface SCVideo () @property (nonatomic, copy) FrameCallbackBlock currentCallback; @property (nonatomic, strong) dispatch_semaphore_t currentSignal; @property (nonatomic, assign) BOOL streamRunning; -@property (nonatomic, assign) BOOL streamOutputAdded; @end @@ -161,7 +160,6 @@ - (instancetype)initWithDisplay:(CGDirectDisplayID)displayID frameRate:(int)fram NSLog(@"SCVideo: addStreamOutput failed: %@", outputError); return nil; } - self.streamOutputAdded = YES; // Start the stream once. Frames begin flowing immediately on the // sampleQueue; sample-handler delivery is a no-op until the first @@ -320,12 +318,26 @@ - (dispatch_semaphore_t)capture:(FrameCallbackBlock)frameCallback { - (void)dealloc { BOOL running; SCStream *stream; + dispatch_semaphore_t pendingSignal; @synchronized(self) { running = self.streamRunning; stream = self.stream; + pendingSignal = self.currentSignal; self.streamRunning = NO; self.currentCallback = nil; + self.currentSignal = nil; + } + + // Unblock any caller still waiting on the semaphore that -capture: + // returned. Without this, if the stream stops without triggering + // -stream:didStopWithError: (or the delegate callback can't be + // delivered during teardown), the waiting thread would stall + // forever. Signalling after clearing currentCallback means the + // caller wakes up to observe their callback was cleared and exits. + if (pendingSignal) { + dispatch_semaphore_signal(pendingSignal); } + if (running && stream) { // Best-effort synchronous stop with a bounded wait so a // misbehaving SCK doesn't hang teardown. From 28b5d0d1bbf63bb39ad2d0bfc4e6af1aa97305d9 Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Sat, 23 May 2026 16:15:11 -0700 Subject: [PATCH 07/12] feat: add experimental ProRes VideoToolbox support for macOS - Introduced `prores_mode` configuration option to enable custom clients to request ProRes video streams. - Added `prores_profile` configuration option to set the FFmpeg ProRes profile. - Updated changelog to reflect the addition of ProRes encoder plumbing. - Enhanced configuration documentation with details on ProRes options. - Implemented necessary changes in the codebase to support ProRes encoding, including updates to video format handling and encoder capabilities. - Added tests to validate ProRes configuration and protocol behavior. --- docs/changelog.md | 6 + docs/configuration.md | 87 +++++++++++ src/config.cpp | 23 +++ src/config.h | 3 + src/nvenc/nvenc_base.cpp | 9 +- src/nvhttp.cpp | 15 +- src/platform/windows/display_vram.cpp | 4 +- src/rtsp.cpp | 29 +++- src/video.cpp | 142 +++++++++++++++--- src/video.h | 33 +++- src_assets/common/assets/web/config.html | 2 + .../assets/web/configs/tabs/Advanced.vue | 11 ++ .../tabs/encoders/VideotoolboxEncoder.vue | 12 ++ .../assets/web/public/assets/locale/en.json | 13 ++ tests/unit/test_video.cpp | 64 ++++++++ 15 files changed, 412 insertions(+), 41 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 9cf35f26907..4c89af85b58 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +- Added disabled-by-default experimental macOS ProRes VideoToolbox encoder + plumbing for custom clients. This is Sunshine-side protocol and encoder + support only and does not add stock Moonlight ProRes decoder compatibility. + @htmlonly +### prores_mode + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Description + Allows custom clients to request experimental macOS ProRes VideoToolbox video streams. + @warning{This does not add stock Moonlight client decoder support and should remain disabled unless + a custom client is explicitly being tested.} +
Default@code{} + 0 + @endcode
Example@code{} + prores_mode = 1 + @endcode
Choices0disabled
1accept an explicit ProRes request from a custom client
2force ProRes for local development sessions
+ ### capture @@ -3004,6 +3042,55 @@ editing the `conf` file in a text editor. Use the examples as reference.
+### prores_profile + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Description + Sets the FFmpeg `prores_videotoolbox` profile when experimental ProRes is enabled. + @note{This option only applies when using macOS.} +
Default@code{} + lt + @endcode
Example@code{} + prores_profile = hq + @endcode
ChoicesproxyProRes 422 Proxy
ltProRes 422 LT
standardProRes 422
hqProRes 422 HQ
4444ProRes 4444
xqProRes 4444 XQ
+ ## VA-API Encoder ### vaapi_strict_rc_buffer diff --git a/src/config.cpp b/src/config.cpp index 6d266c0ef22..b7af4bdbad2 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -350,6 +350,25 @@ namespace config { } // namespace vt + namespace prores { + + std::string profile_from_view(const std::string_view profile) { +#define _CONVERT_(x) \ + if (profile == #x##sv) \ + return #x + _CONVERT_(proxy); + _CONVERT_(lt); + _CONVERT_(standard); + _CONVERT_(hq); + _CONVERT_(4444); + _CONVERT_(xq); +#undef _CONVERT_ + BOOST_LOG(warning) << "config: unknown prores_profile value: " << profile; + return "lt"; + } + + } // namespace prores + namespace sw { int svtav1_preset_from_view(const std::string_view &preset) { #define _CONVERT_(x, y) \ @@ -454,6 +473,8 @@ namespace config { 0, // hevc_mode 0, // av1_mode + 0, // prores_mode + "lt"s, // prores_profile 2, // min_threads { @@ -1101,6 +1122,8 @@ namespace config { int_f(vars, "qp", video.qp); int_between_f(vars, "hevc_mode", video.hevc_mode, {0, 3}); int_between_f(vars, "av1_mode", video.av1_mode, {0, 3}); + int_between_f(vars, "prores_mode", video.prores_mode, {0, 2}); + generic_f(vars, "prores_profile", video.prores_profile, prores::profile_from_view); int_f(vars, "min_threads", video.min_threads); string_f(vars, "sw_preset", video.sw.sw_preset); if (!video.sw.sw_preset.empty()) { diff --git a/src/config.h b/src/config.h index e0e40501d0b..b999ff6ae19 100644 --- a/src/config.h +++ b/src/config.h @@ -31,6 +31,7 @@ namespace config { }; void log_config_settings(const std::unordered_map &vars, bool save); + void apply_config(std::unordered_map &&vars); struct video_t { // ffmpeg params @@ -38,6 +39,8 @@ namespace config { int hevc_mode; int av1_mode; + int prores_mode; + std::string prores_profile; int min_threads; // Minimum number of threads/slices for CPU encoding diff --git a/src/nvenc/nvenc_base.cpp b/src/nvenc/nvenc_base.cpp index ab8aa7a91ff..acd1004fdfc 100644 --- a/src/nvenc/nvenc_base.cpp +++ b/src/nvenc/nvenc_base.cpp @@ -453,10 +453,11 @@ namespace nvenc { } { - auto video_format_string = client_config.videoFormat == 0 ? "H.264 " : - client_config.videoFormat == 1 ? "HEVC " : - client_config.videoFormat == 2 ? "AV1 " : - " "; + auto video_format_string = client_config.videoFormat == video::SUNSHINE_FORMAT_H264 ? "H.264 " : + client_config.videoFormat == video::SUNSHINE_FORMAT_HEVC ? "HEVC " : + client_config.videoFormat == video::SUNSHINE_FORMAT_AV1 ? "AV1 " : + client_config.videoFormat == video::SUNSHINE_FORMAT_PRORES ? "ProRes " : + " "; std::string extra; if (init_params.enableEncodeAsync) { extra += " async"; diff --git a/src/nvhttp.cpp b/src/nvhttp.cpp index 72e9dc5cd05..bdea2430940 100644 --- a/src/nvhttp.cpp +++ b/src/nvhttp.cpp @@ -736,34 +736,39 @@ namespace nvhttp { } uint32_t codec_mode_flags = SCM_H264; - if (video::last_encoder_probe_supported_yuv444_for_codec[0]) { + if (video::last_encoder_probe_supported_yuv444_for_codec[video::SUNSHINE_FORMAT_H264]) { codec_mode_flags |= SCM_H264_HIGH8_444; } if (video::active_hevc_mode >= 2) { codec_mode_flags |= SCM_HEVC; - if (video::last_encoder_probe_supported_yuv444_for_codec[1]) { + if (video::last_encoder_probe_supported_yuv444_for_codec[video::SUNSHINE_FORMAT_HEVC]) { codec_mode_flags |= SCM_HEVC_REXT8_444; } } if (video::active_hevc_mode >= 3) { codec_mode_flags |= SCM_HEVC_MAIN10; - if (video::last_encoder_probe_supported_yuv444_for_codec[1]) { + if (video::last_encoder_probe_supported_yuv444_for_codec[video::SUNSHINE_FORMAT_HEVC]) { codec_mode_flags |= SCM_HEVC_REXT10_444; } } if (video::active_av1_mode >= 2) { codec_mode_flags |= SCM_AV1_MAIN8; - if (video::last_encoder_probe_supported_yuv444_for_codec[2]) { + if (video::last_encoder_probe_supported_yuv444_for_codec[video::SUNSHINE_FORMAT_AV1]) { codec_mode_flags |= SCM_AV1_HIGH8_444; } } if (video::active_av1_mode >= 3) { codec_mode_flags |= SCM_AV1_MAIN10; - if (video::last_encoder_probe_supported_yuv444_for_codec[2]) { + if (video::last_encoder_probe_supported_yuv444_for_codec[video::SUNSHINE_FORMAT_AV1]) { codec_mode_flags |= SCM_AV1_HIGH10_444; } } tree.put("root.ServerCodecModeSupport", codec_mode_flags); + if (video::active_prores_mode > 0) { + tree.put("root.SunshineExperimentalProRes", "1"); + tree.put("root.SunshineExperimentalProResVideoFormat", video::SUNSHINE_FORMAT_PRORES); + tree.put("root.SunshineExperimentalProResProfile", config::video.prores_profile); + } if (!config::nvhttp.external_ip.empty()) { tree.put("root.ExternalIP", config::nvhttp.external_ip); diff --git a/src/platform/windows/display_vram.cpp b/src/platform/windows/display_vram.cpp index d659c4bbed7..264e6c54bd3 100644 --- a/src/platform/windows/display_vram.cpp +++ b/src/platform/windows/display_vram.cpp @@ -1861,7 +1861,7 @@ namespace platf::dxgi { amf_uint64 version; auto result = fnAMFQueryVersion(&version); if (result == AMF_OK) { - if (config.videoFormat == 2 && version < AMF_MAKE_FULL_VERSION(1, 4, 30, 0)) { + if (config.videoFormat == video::SUNSHINE_FORMAT_AV1 && version < AMF_MAKE_FULL_VERSION(1, 4, 30, 0)) { // AMF 1.4.30 adds ultra low latency mode for AV1. Don't use AV1 on earlier versions. // This corresponds to driver version 23.5.2 (23.10.01.45) or newer. BOOST_LOG(warning) << "AV1 encoding is disabled on AMF version "sv @@ -1898,7 +1898,7 @@ namespace platf::dxgi { return false; } if (config.chromaSamplingType == 1) { - if (config.videoFormat == 0 || config.videoFormat == 2) { + if (config.videoFormat == video::SUNSHINE_FORMAT_H264 || config.videoFormat == video::SUNSHINE_FORMAT_AV1) { // QSV doesn't support 4:4:4 in H.264 or AV1 return false; } diff --git a/src/rtsp.cpp b/src/rtsp.cpp index 0953cd59f60..53af3c3fd5a 100644 --- a/src/rtsp.cpp +++ b/src/rtsp.cpp @@ -815,6 +815,12 @@ namespace rtsp_stream { ss << "a=rtpmap:98 AV1/90000"sv << std::endl; } + if (video::active_prores_mode > 0) { + ss << "a=x-sunshine-prores:1"sv << std::endl; + ss << "a=x-sunshine-prores-profile:"sv << config::video.prores_profile << std::endl; + ss << "a=rtpmap:99 prores/90000"sv << std::endl; + } + if (!session.surround_params.empty()) { // If we have our own surround parameters, advertise them twice first ss << "a=fmtp:97 surround-params="sv << session.surround_params << std::endl; @@ -1115,20 +1121,39 @@ namespace rtsp_stream { config.monitor.bitrate = (int) configuredBitrateKbps; } - if (config.monitor.videoFormat == 1 && video::active_hevc_mode == 1) { + if (video::active_prores_mode == 2 && config.monitor.videoFormat != video::SUNSHINE_FORMAT_PRORES) { + BOOST_LOG(warning) << "Forcing experimental ProRes because prores_mode is 2"sv; + config.monitor.videoFormat = video::SUNSHINE_FORMAT_PRORES; + } + + if (!video::is_known_video_format(config.monitor.videoFormat)) { + BOOST_LOG(warning) << "Client requested unknown video format "sv << config.monitor.videoFormat; + + respond(sock, session, &option, 400, "BAD REQUEST", req->sequenceNumber, {}); + return; + } + + if (config.monitor.videoFormat == video::SUNSHINE_FORMAT_HEVC && video::active_hevc_mode == 1) { BOOST_LOG(warning) << "HEVC is disabled, yet the client requested HEVC"sv; respond(sock, session, &option, 400, "BAD REQUEST", req->sequenceNumber, {}); return; } - if (config.monitor.videoFormat == 2 && video::active_av1_mode == 1) { + if (config.monitor.videoFormat == video::SUNSHINE_FORMAT_AV1 && video::active_av1_mode == 1) { BOOST_LOG(warning) << "AV1 is disabled, yet the client requested AV1"sv; respond(sock, session, &option, 400, "BAD REQUEST", req->sequenceNumber, {}); return; } + if (!video::is_video_format_enabled_by_prores_gate(config.monitor.videoFormat, video::active_prores_mode)) { + BOOST_LOG(warning) << "Experimental ProRes is disabled, yet the client requested ProRes"sv; + + respond(sock, session, &option, 400, "BAD REQUEST", req->sequenceNumber, {}); + return; + } + // Check that any required encryption is enabled auto encryption_mode = net::encryption_mode_for_address(sock.remote_endpoint().address()); if (encryption_mode == config::ENCRYPTION_MODE_MANDATORY && diff --git a/src/video.cpp b/src/video.cpp index 00af66c3a69..486a2fd2194 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -119,6 +119,26 @@ namespace video { } // namespace qsv + int prores_profile_from_config() { + const auto &profile = config::video.prores_profile; + if (profile == "proxy"sv) { + return AV_PROFILE_PRORES_PROXY; + } + if (profile == "standard"sv) { + return AV_PROFILE_PRORES_STANDARD; + } + if (profile == "hq"sv) { + return AV_PROFILE_PRORES_HQ; + } + if (profile == "4444"sv) { + return AV_PROFILE_PRORES_4444; + } + if (profile == "xq"sv) { + return AV_PROFILE_PRORES_XQ; + } + return AV_PROFILE_PRORES_LT; + } + util::Either dxgi_init_avcodec_hardware_input_buffer(platf::avcodec_encode_device_t *); util::Either vaapi_init_avcodec_hardware_input_buffer(platf::avcodec_encode_device_t *); util::Either cuda_init_avcodec_hardware_input_buffer(platf::avcodec_encode_device_t *); @@ -513,6 +533,7 @@ namespace video { {}, // Fallback options "h264_nvenc"s, }, + {}, PARALLEL_ENCODING | REF_FRAMES_INVALIDATION | YUV444_SUPPORT | ASYNC_TEARDOWN // flags }; #elif !defined(__APPLE__) @@ -610,6 +631,7 @@ namespace video { {}, // Fallback options "h264_nvenc"s, }, + {}, PARALLEL_ENCODING }; #endif @@ -720,6 +742,7 @@ namespace video { }, "h264_qsv"s, }, + {}, PARALLEL_ENCODING | CBR_WITH_VBR | RELAXED_COMPLIANCE | NO_RC_BUF_LIMIT | YUV444_SUPPORT }; @@ -826,6 +849,7 @@ namespace video { }, "h264_amf"s, }, + {}, PARALLEL_ENCODING }; @@ -883,6 +907,7 @@ namespace video { {}, // Fallback options "h264_mf"s, }, + {}, PARALLEL_ENCODING | FIXED_GOP_SIZE // MF encoder doesn't support on-demand IDR frames }; #endif @@ -954,6 +979,7 @@ namespace video { {}, // Fallback options "libx264"s, }, + {}, H264_ONLY | PARALLEL_ENCODING | ALWAYS_REPROBE | YUV444_SUPPORT }; @@ -1011,6 +1037,7 @@ namespace video { {}, // Fallback options "h264_vaapi"s, }, + {}, // RC buffer size will be set in platform code if supported LIMITED_GOP_SIZE | PARALLEL_ENCODING | NO_RC_BUF_LIMIT }; @@ -1082,6 +1109,7 @@ namespace video { {}, // Fallback options "h264_vulkan"s, }, + {}, LIMITED_GOP_SIZE | PARALLEL_ENCODING }; #endif // SUNSHINE_BUILD_VULKAN @@ -1151,6 +1179,20 @@ namespace video { }, "h264_videotoolbox"s, }, + { + // Common options + { + {"allow_sw"s, 0}, + {"realtime"s, 1}, + {"profile"s, &config::video.prores_profile}, + }, + {}, // SDR-specific options + {}, // HDR-specific options + {}, // YUV444 SDR-specific options + {}, // YUV444 HDR-specific options + {}, // Fallback options + "prores_videotoolbox"s, + }, DEFAULT }; #endif @@ -1179,8 +1221,9 @@ namespace video { static encoder_t *chosen_encoder; int active_hevc_mode; int active_av1_mode; + int active_prores_mode; bool last_encoder_probe_supported_ref_frames_invalidation = false; - std::array last_encoder_probe_supported_yuv444_for_codec = {}; + std::array last_encoder_probe_supported_yuv444_for_codec = {}; void reset_display(std::shared_ptr &disp, const platf::mem_type_e &type, const std::string &display_name, const config_t &config) { // We try this twice, in case we still get an error on reinitialization @@ -1675,13 +1718,13 @@ namespace video { } switch (config.videoFormat) { - case 0: + case SUNSHINE_FORMAT_H264: // 10-bit h264 encoding is not supported by our streaming protocol assert(!config.dynamicRange); ctx->profile = (config.chromaSamplingType == 1) ? AV_PROFILE_H264_HIGH_444_PREDICTIVE : AV_PROFILE_H264_HIGH; break; - case 1: + case SUNSHINE_FORMAT_HEVC: if (config.chromaSamplingType == 1) { // HEVC uses the same RExt profile for both 8 and 10 bit YUV 4:4:4 encoding ctx->profile = AV_PROFILE_HEVC_REXT; @@ -1690,11 +1733,15 @@ namespace video { } break; - case 2: + case SUNSHINE_FORMAT_AV1: // AV1 supports both 8 and 10 bit encoding with the same Main profile // but YUV 4:4:4 sampling requires High profile ctx->profile = (config.chromaSamplingType == 1) ? AV_PROFILE_AV1_HIGH : AV_PROFILE_AV1_MAIN; break; + + case SUNSHINE_FORMAT_PRORES: + ctx->profile = prores_profile_from_config(); + break; } // B-frames delay decoder output, so never use them @@ -1877,7 +1924,7 @@ namespace video { } if (!(encoder.flags & NO_RC_BUF_LIMIT)) { - if (!hardware && (ctx->slices > 1 || config.videoFormat == 1)) { + if (!hardware && (ctx->slices > 1 || config.videoFormat == SUNSHINE_FORMAT_HEVC)) { // Use a larger rc_buffer_size for software encoding when slices are enabled, // because libx264 can severely degrade quality if the buffer is too small. // libx265 encounters this issue more frequently, so always scale the @@ -1989,7 +2036,7 @@ namespace video { std::move(encode_device_final), // 0 ==> don't inject, 1 ==> inject for h264, 2 ==> inject for hevc - config.videoFormat <= 1 ? (1 - (int) video_format[encoder_t::VUI_PARAMETERS]) * (1 + config.videoFormat) : 0 + config.videoFormat <= SUNSHINE_FORMAT_HEVC ? (1 - (int) video_format[encoder_t::VUI_PARAMETERS]) * (1 + config.videoFormat) : 0 ); return session; @@ -2598,9 +2645,9 @@ namespace video { int flag = 0; // This check only applies for H.264 and HEVC - if (config.videoFormat <= 1) { + if (config.videoFormat <= SUNSHINE_FORMAT_HEVC) { if (auto packet_avcodec = dynamic_cast(packet.get())) { - if (cbs::validate_sps(packet_avcodec->av_packet, config.videoFormat ? AV_CODEC_ID_H265 : AV_CODEC_ID_H264)) { + if (cbs::validate_sps(packet_avcodec->av_packet, config.videoFormat == SUNSHINE_FORMAT_HEVC ? AV_CODEC_ID_H265 : AV_CODEC_ID_H264)) { flag |= VUI_PARAMS; } } else { @@ -2623,10 +2670,12 @@ namespace video { auto test_hevc = active_hevc_mode >= 2 || (active_hevc_mode == 0 && !(encoder.flags & H264_ONLY)); auto test_av1 = active_av1_mode >= 2 || (active_av1_mode == 0 && !(encoder.flags & H264_ONLY)); + auto test_prores = active_prores_mode > 0; encoder.h264.capabilities.set(); encoder.hevc.capabilities.set(); encoder.av1.capabilities.set(); + encoder.prores.capabilities.set(); // First, test encoder viability config_t config_max_ref_frames {1920, 1080, 60, 6000, 1000, 1, 1, 1, 0, 0, 0}; @@ -2666,8 +2715,8 @@ namespace video { encoder.h264[encoder_t::PASSED] = true; if (test_hevc) { - config_max_ref_frames.videoFormat = 1; - config_autoselect.videoFormat = 1; + config_max_ref_frames.videoFormat = SUNSHINE_FORMAT_HEVC; + config_autoselect.videoFormat = SUNSHINE_FORMAT_HEVC; if (disp->is_codec_supported(encoder.hevc.name, config_autoselect)) { auto max_ref_frames_hevc = validate_config(disp, encoder, config_max_ref_frames); @@ -2694,8 +2743,8 @@ namespace video { } if (test_av1) { - config_max_ref_frames.videoFormat = 2; - config_autoselect.videoFormat = 2; + config_max_ref_frames.videoFormat = SUNSHINE_FORMAT_AV1; + config_autoselect.videoFormat = SUNSHINE_FORMAT_AV1; if (disp->is_codec_supported(encoder.av1.name, config_autoselect)) { auto max_ref_frames_av1 = validate_config(disp, encoder, config_max_ref_frames); @@ -2721,6 +2770,26 @@ namespace video { encoder.av1.capabilities.reset(); } + if (test_prores) { + config_max_ref_frames.videoFormat = SUNSHINE_FORMAT_PRORES; + config_autoselect.videoFormat = SUNSHINE_FORMAT_PRORES; + + if (disp->is_codec_supported(encoder.prores.name, config_autoselect)) { + auto max_ref_frames_prores = validate_config(disp, encoder, config_max_ref_frames); + auto autoselect_prores = max_ref_frames_prores >= 0 ? + max_ref_frames_prores : + validate_config(disp, encoder, config_autoselect); + + encoder.prores[encoder_t::REF_FRAMES_RESTRICT] = max_ref_frames_prores >= 0; + encoder.prores[encoder_t::PASSED] = max_ref_frames_prores >= 0 || autoselect_prores >= 0; + } else { + BOOST_LOG(info) << "Encoder ["sv << encoder.prores.name << "] is not supported on this GPU"sv; + encoder.prores.capabilities.reset(); + } + } else { + encoder.prores.capabilities.reset(); + } + // Test HDR and YUV444 support { // H.264 is special because encoders may support YUV 4:4:4 without supporting 10-bit color depth @@ -2772,8 +2841,9 @@ namespace video { // HDR is not supported with H.264. Don't bother even trying it. encoder.h264[encoder_t::DYNAMIC_RANGE] = false; - test_hdr_and_yuv444(encoder.hevc, 1); - test_hdr_and_yuv444(encoder.av1, 2); + test_hdr_and_yuv444(encoder.hevc, SUNSHINE_FORMAT_HEVC); + test_hdr_and_yuv444(encoder.av1, SUNSHINE_FORMAT_AV1); + test_hdr_and_yuv444(encoder.prores, SUNSHINE_FORMAT_PRORES); } encoder.h264[encoder_t::VUI_PARAMETERS] = encoder.h264[encoder_t::VUI_PARAMETERS] && !config::sunshine.flags[config::flag::FORCE_VIDEO_HEADER_REPLACE]; @@ -2808,6 +2878,8 @@ namespace video { chosen_encoder = nullptr; active_hevc_mode = config::video.hevc_mode; active_av1_mode = config::video.av1_mode; + active_prores_mode = config::video.prores_mode; + const bool require_prores = active_prores_mode >= 2; last_encoder_probe_supported_ref_frames_invalidation = false; auto adjust_encoder_constraints = [&](encoder_t *encoder) { @@ -2827,6 +2899,11 @@ namespace video { BOOST_LOG(warning) << "Encoder ["sv << encoder->name << "] does not support AV1 on this system"sv; active_av1_mode = 0; } + + if (active_prores_mode > 0 && !encoder->prores[encoder_t::PASSED]) { + BOOST_LOG(warning) << "Encoder ["sv << encoder->name << "] does not support experimental ProRes on this system"sv; + active_prores_mode = 0; + } }; if (!config::video.encoder.empty()) { @@ -2859,7 +2936,7 @@ namespace video { BOOST_LOG(info) << "// Testing for available encoders, this may generate errors. You can safely ignore those errors. //"sv; // If we haven't found an encoder yet, but we want one with specific codec support, search for that now. - if (chosen_encoder == nullptr && (active_hevc_mode >= 2 || active_av1_mode >= 2)) { + if (chosen_encoder == nullptr && (active_hevc_mode >= 2 || active_av1_mode >= 2 || require_prores)) { KITTY_WHILE_LOOP(auto pos = std::begin(encoder_list), pos != std::end(encoder_list), { auto encoder = *pos; @@ -2870,7 +2947,9 @@ namespace video { } // Skip it if it doesn't support the specified codec at all - if ((active_hevc_mode >= 2 && !encoder->hevc[encoder_t::PASSED]) || (active_av1_mode >= 2 && !encoder->av1[encoder_t::PASSED])) { + if ((active_hevc_mode >= 2 && !encoder->hevc[encoder_t::PASSED]) || + (active_av1_mode >= 2 && !encoder->av1[encoder_t::PASSED]) || + (require_prores && !encoder->prores[encoder_t::PASSED])) { pos++; continue; } @@ -2886,7 +2965,7 @@ namespace video { }); if (chosen_encoder == nullptr) { - BOOST_LOG(error) << "Couldn't find any working encoder that meets HEVC/AV1 requirements"sv; + BOOST_LOG(error) << "Couldn't find any working encoder that meets HEVC/AV1/forced-ProRes requirements"sv; } } @@ -2930,12 +3009,14 @@ namespace video { auto &encoder = *chosen_encoder; last_encoder_probe_supported_ref_frames_invalidation = (encoder.flags & REF_FRAMES_INVALIDATION); - last_encoder_probe_supported_yuv444_for_codec[0] = encoder.h264[encoder_t::PASSED] && - encoder.h264[encoder_t::YUV444]; - last_encoder_probe_supported_yuv444_for_codec[1] = encoder.hevc[encoder_t::PASSED] && - encoder.hevc[encoder_t::YUV444]; - last_encoder_probe_supported_yuv444_for_codec[2] = encoder.av1[encoder_t::PASSED] && - encoder.av1[encoder_t::YUV444]; + last_encoder_probe_supported_yuv444_for_codec[SUNSHINE_FORMAT_H264] = encoder.h264[encoder_t::PASSED] && + encoder.h264[encoder_t::YUV444]; + last_encoder_probe_supported_yuv444_for_codec[SUNSHINE_FORMAT_HEVC] = encoder.hevc[encoder_t::PASSED] && + encoder.hevc[encoder_t::YUV444]; + last_encoder_probe_supported_yuv444_for_codec[SUNSHINE_FORMAT_AV1] = encoder.av1[encoder_t::PASSED] && + encoder.av1[encoder_t::YUV444]; + last_encoder_probe_supported_yuv444_for_codec[SUNSHINE_FORMAT_PRORES] = encoder.prores[encoder_t::PASSED] && + encoder.prores[encoder_t::YUV444]; BOOST_LOG(debug) << "------ h264 ------"sv; for (int x = 0; x < encoder_t::MAX_FLAGS; ++x) { @@ -2967,6 +3048,17 @@ namespace video { BOOST_LOG(info) << "Found AV1 encoder: "sv << encoder.av1.name << " ["sv << encoder.name << ']'; } + if (encoder.prores[encoder_t::PASSED]) { + BOOST_LOG(debug) << "------ prores -----"sv; + for (int x = 0; x < encoder_t::MAX_FLAGS; ++x) { + auto flag = (encoder_t::flag_e) x; + BOOST_LOG(debug) << encoder_t::from_flag(flag) << (encoder.prores[flag] ? ": supported"sv : ": unsupported"sv); + } + BOOST_LOG(debug) << "-------------------"sv; + + BOOST_LOG(info) << "Found ProRes encoder: "sv << encoder.prores.name << " ["sv << encoder.name << ']'; + } + if (active_hevc_mode == 0) { active_hevc_mode = encoder.hevc[encoder_t::PASSED] ? (encoder.hevc[encoder_t::DYNAMIC_RANGE] ? 3 : 2) : 1; } @@ -2975,6 +3067,10 @@ namespace video { active_av1_mode = encoder.av1[encoder_t::PASSED] ? (encoder.av1[encoder_t::DYNAMIC_RANGE] ? 3 : 2) : 1; } + if (active_prores_mode > 0 && !encoder.prores[encoder_t::PASSED]) { + active_prores_mode = 0; + } + return 0; } diff --git a/src/video.h b/src/video.h index 8fa25850036..78883012543 100644 --- a/src/video.h +++ b/src/video.h @@ -19,6 +19,25 @@ struct AVPacket; namespace video { + inline constexpr int SUNSHINE_FORMAT_H264 = 0; + inline constexpr int SUNSHINE_FORMAT_HEVC = 1; + inline constexpr int SUNSHINE_FORMAT_AV1 = 2; + inline constexpr int SUNSHINE_FORMAT_PRORES = 3; + inline constexpr std::size_t SUNSHINE_FORMAT_COUNT = 4; + + static_assert(SUNSHINE_FORMAT_H264 == 0); + static_assert(SUNSHINE_FORMAT_HEVC == 1); + static_assert(SUNSHINE_FORMAT_AV1 == 2); + static_assert(SUNSHINE_FORMAT_PRORES == 3); + + inline bool is_known_video_format(int video_format) { + return video_format >= SUNSHINE_FORMAT_H264 && video_format <= SUNSHINE_FORMAT_PRORES; + } + + inline bool is_video_format_enabled_by_prores_gate(int video_format, int prores_mode) { + return video_format != SUNSHINE_FORMAT_PRORES || prores_mode > 0; + } + /* Encoding configuration requested by remote client */ struct config_t { int width; // Video width in pixels @@ -34,7 +53,7 @@ namespace video { SDR encoding colorspace (encoderCscMode >> 1) : 0 - BT.601, 1 - BT.709, 2 - BT.2020 */ int encoderCscMode; - int videoFormat; // 0 - H.264, 1 - HEVC, 2 - AV1 + int videoFormat; // 0 - H.264, 1 - HEVC, 2 - AV1, 3 - ProRes experimental /* Encoding color depth (bit depth): 0 - 8-bit, 1 - 10-bit HDR encoding activates when color depth is higher than 8-bit and the display which is being captured is operating in HDR mode */ @@ -189,18 +208,21 @@ namespace video { codec_t av1; codec_t hevc; codec_t h264; + codec_t prores; const codec_t &codec_from_config(const config_t &config) const { switch (config.videoFormat) { default: BOOST_LOG(error) << "Unknown video format " << config.videoFormat << ", falling back to H.264"; // fallthrough - case 0: + case SUNSHINE_FORMAT_H264: return h264; - case 1: + case SUNSHINE_FORMAT_HEVC: return hevc; - case 2: + case SUNSHINE_FORMAT_AV1: return av1; + case SUNSHINE_FORMAT_PRORES: + return prores; } } @@ -343,8 +365,9 @@ namespace video { extern int active_hevc_mode; extern int active_av1_mode; + extern int active_prores_mode; extern bool last_encoder_probe_supported_ref_frames_invalidation; - extern std::array last_encoder_probe_supported_yuv444_for_codec; // 0 - H.264, 1 - HEVC, 2 - AV1 + extern std::array last_encoder_probe_supported_yuv444_for_codec; void capture( safe::mail_t mail, diff --git a/src_assets/common/assets/web/config.html b/src_assets/common/assets/web/config.html index 27d84205dc4..466191a6e88 100644 --- a/src_assets/common/assets/web/config.html +++ b/src_assets/common/assets/web/config.html @@ -277,6 +277,7 @@

{{ $t('config.configuration') }}

"min_threads": 2, "hevc_mode": 0, "av1_mode": 0, + "prores_mode": 0, "capture": "", "encoder": "", }, @@ -325,6 +326,7 @@

{{ $t('config.configuration') }}

"vt_coder": "auto", "vt_software": "auto", "vt_realtime": "enabled", + "prores_profile": "lt", }, }, { diff --git a/src_assets/common/assets/web/configs/tabs/Advanced.vue b/src_assets/common/assets/web/configs/tabs/Advanced.vue index 108d2b3362d..3e13cd73ef7 100644 --- a/src_assets/common/assets/web/configs/tabs/Advanced.vue +++ b/src_assets/common/assets/web/configs/tabs/Advanced.vue @@ -58,6 +58,17 @@ const config = ref(props.config)
{{ $t('config.av1_mode_desc') }}
+ +
+ + +
{{ $t('config.prores_mode_desc') }}
+
+
diff --git a/src_assets/common/assets/web/configs/tabs/encoders/VideotoolboxEncoder.vue b/src_assets/common/assets/web/configs/tabs/encoders/VideotoolboxEncoder.vue index 084e5f1def0..09d6dbf2411 100644 --- a/src_assets/common/assets/web/configs/tabs/encoders/VideotoolboxEncoder.vue +++ b/src_assets/common/assets/web/configs/tabs/encoders/VideotoolboxEncoder.vue @@ -37,6 +37,18 @@ const config = ref(props.config) v-model="config.vt_realtime" default="true" > +
+ + +
{{ $t('config.prores_profile_desc') }}
+
diff --git a/src_assets/common/assets/web/public/assets/locale/en.json b/src_assets/common/assets/web/public/assets/locale/en.json index f4eee2cdccf..5059dcc5b48 100644 --- a/src_assets/common/assets/web/public/assets/locale/en.json +++ b/src_assets/common/assets/web/public/assets/locale/en.json @@ -153,6 +153,11 @@ "av1_mode_2": "Sunshine will advertise support for AV1 Main 8-bit profile", "av1_mode_3": "Sunshine will advertise support for AV1 Main 8-bit and 10-bit (HDR) profiles", "av1_mode_desc": "Allows the client to request AV1 Main 8-bit or 10-bit video streams. AV1 is more CPU-intensive to encode, so enabling this may reduce performance when using software encoding.", + "prores_mode": "Experimental ProRes Support", + "prores_mode_0": "Disabled (default)", + "prores_mode_1": "Accept an explicit ProRes request from a custom client", + "prores_mode_2": "Force ProRes for local development sessions", + "prores_mode_desc": "Allows custom clients to request experimental macOS ProRes VideoToolbox streams. This does not add stock Moonlight client decoder support.", "back_button_timeout": "Home/Guide Button Emulation Timeout", "back_button_timeout_desc": "If the Back/Select button is held down for the specified number of milliseconds, a Home/Guide button press is emulated. If set to a value < 0 (default), holding the Back/Select button will not emulate the Home/Guide button.", "bind_address": "Bind address", @@ -404,6 +409,14 @@ "virtual_sink_desc": "Manually specify a virtual audio device to use. If unset, the device is chosen automatically. We strongly recommend leaving this field blank to use automatic device selection!", "virtual_sink_placeholder": "Steam Streaming Speakers", "vt_coder": "VideoToolbox Coder", + "prores_profile": "ProRes Profile", + "prores_profile_4444": "4444", + "prores_profile_desc": "Sets the FFmpeg prores_videotoolbox profile when experimental ProRes is enabled.", + "prores_profile_hq": "HQ", + "prores_profile_lt": "LT (default)", + "prores_profile_proxy": "Proxy", + "prores_profile_standard": "Standard", + "prores_profile_xq": "XQ", "vt_realtime": "VideoToolbox Realtime Encoding", "vt_software": "VideoToolbox Software Encoding", "vt_software_allowed": "Allowed", diff --git a/tests/unit/test_video.cpp b/tests/unit/test_video.cpp index ed578f7d8db..16e851fb2c2 100644 --- a/tests/unit/test_video.cpp +++ b/tests/unit/test_video.cpp @@ -4,8 +4,15 @@ */ #include "../tests_common.h" +#include #include +static_assert(video::SUNSHINE_FORMAT_H264 == 0); +static_assert(video::SUNSHINE_FORMAT_HEVC == 1); +static_assert(video::SUNSHINE_FORMAT_AV1 == 2); +static_assert(video::SUNSHINE_FORMAT_PRORES == 3); +static_assert(video::SUNSHINE_FORMAT_COUNT == 4); + struct EncoderTest: PlatformTestSuite, testing::WithParamInterface { void SetUp() override { auto &encoder = *GetParam(); @@ -45,6 +52,63 @@ INSTANTIATE_TEST_SUITE_P( } ); +TEST(ProResConfigTest, DefaultsDisabled) { + EXPECT_EQ(config::video.prores_mode, 0); + EXPECT_EQ(config::video.prores_profile, "lt"); +} + +TEST(ProResConfigTest, ParsesExplicitModeAndProfile) { + auto mode = config::video.prores_mode; + auto profile = config::video.prores_profile; + + config::apply_config({ + {"prores_mode", "1"}, + {"prores_profile", "hq"}, + }); + + EXPECT_EQ(config::video.prores_mode, 1); + EXPECT_EQ(config::video.prores_profile, "hq"); + + config::video.prores_mode = mode; + config::video.prores_profile = profile; +} + +TEST(ProResConfigTest, NormalizesInvalidValues) { + auto mode = config::video.prores_mode; + auto profile = config::video.prores_profile; + + config::video.prores_mode = 0; + config::video.prores_profile = "lt"; + config::apply_config({ + {"prores_mode", "9"}, + {"prores_profile", "bad_profile"}, + }); + + EXPECT_EQ(config::video.prores_mode, 0); + EXPECT_EQ(config::video.prores_profile, "lt"); + + config::video.prores_mode = mode; + config::video.prores_profile = profile; +} + +TEST(ProResProtocolGateTest, KnownFormatsIncludeExperimentalProRes) { + EXPECT_TRUE(video::is_known_video_format(video::SUNSHINE_FORMAT_H264)); + EXPECT_TRUE(video::is_known_video_format(video::SUNSHINE_FORMAT_HEVC)); + EXPECT_TRUE(video::is_known_video_format(video::SUNSHINE_FORMAT_AV1)); + EXPECT_TRUE(video::is_known_video_format(video::SUNSHINE_FORMAT_PRORES)); + EXPECT_FALSE(video::is_known_video_format(video::SUNSHINE_FORMAT_PRORES + 1)); +} + +TEST(ProResProtocolGateTest, ProResRequestsAreRejectedWhenDisabled) { + EXPECT_TRUE(video::is_video_format_enabled_by_prores_gate(video::SUNSHINE_FORMAT_H264, 0)); + EXPECT_FALSE(video::is_video_format_enabled_by_prores_gate(video::SUNSHINE_FORMAT_PRORES, 0)); +} + +TEST(ProResProtocolGateTest, ProResRequestsAreAcceptedWhenEnabled) { + EXPECT_TRUE(video::is_video_format_enabled_by_prores_gate(video::SUNSHINE_FORMAT_PRORES, 1)); + EXPECT_TRUE(video::is_video_format_enabled_by_prores_gate(video::SUNSHINE_FORMAT_PRORES, 2)); +} + TEST_P(EncoderTest, ValidateEncoder) { // todo:: test something besides fixture setup } From 4fa2de9c86879ca784888a59a68d2adecccc231a Mon Sep 17 00:00:00 2001 From: Jason Lu Date: Mon, 25 May 2026 04:16:51 -0700 Subject: [PATCH 08/12] ui(prores): move prores_mode toggle next to prores_profile on VideoToolbox tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The experimental ProRes mode selector landed in the generic Advanced tab while prores_profile (its inseparable companion — the mode toggle only makes sense if you also pick a profile) lived on the VideoToolbox Encoder tab. Splitting the two knobs across tabs hurts discoverability: users configuring VideoToolbox don't realise ProRes exists, and users on the Advanced tab can't see the profile that controls bit depth and chroma subsampling. Move prores_mode into VideoToolboxEncoder.vue directly above prores_profile so both controls sit together in their codec-relevant section. The macOS-only platform guard previously around prores_mode in Advanced.vue is implicit on the VideoToolbox tab (which is itself macOS-only), so it can be dropped. No backend or config change; pure UI relocation. --- .../common/assets/web/configs/tabs/Advanced.vue | 11 ----------- .../web/configs/tabs/encoders/VideotoolboxEncoder.vue | 11 +++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src_assets/common/assets/web/configs/tabs/Advanced.vue b/src_assets/common/assets/web/configs/tabs/Advanced.vue index 3e13cd73ef7..108d2b3362d 100644 --- a/src_assets/common/assets/web/configs/tabs/Advanced.vue +++ b/src_assets/common/assets/web/configs/tabs/Advanced.vue @@ -58,17 +58,6 @@ const config = ref(props.config)
{{ $t('config.av1_mode_desc') }}
- -
- - -
{{ $t('config.prores_mode_desc') }}
-
-
diff --git a/src_assets/common/assets/web/configs/tabs/encoders/VideotoolboxEncoder.vue b/src_assets/common/assets/web/configs/tabs/encoders/VideotoolboxEncoder.vue index 09d6dbf2411..3a856614949 100644 --- a/src_assets/common/assets/web/configs/tabs/encoders/VideotoolboxEncoder.vue +++ b/src_assets/common/assets/web/configs/tabs/encoders/VideotoolboxEncoder.vue @@ -37,6 +37,17 @@ const config = ref(props.config) v-model="config.vt_realtime" default="true" > + + +
+ + +
{{ $t('config.prores_mode_desc') }}
+