Skip to content

fix(linux/vulkan): harden encoder fallback #5174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
codeafridi wants to merge 1 commit into
base: master
Choose a base branch
from
+153 −31
Open

fix(linux/vulkan): harden encoder fallback #5174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions src/platform/linux/kmsgrab.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -611,6 +611,11 @@ namespace platf {
int init(const std::string &display_name, const ::video::config_t &config) {
delay = std::chrono::nanoseconds {1s} / config.framerate;

if (kms::card_descriptors.empty()) {
BOOST_LOG(error) << "No KMS monitor descriptors are available; aborting monitor lookup for ["sv << display_name << ']';
return -1;
}

int monitor_index = util::from_view(display_name);
int monitor = 0;

Expand Down Expand Up @@ -777,7 +782,7 @@ namespace platf {
}
}

BOOST_LOG(error) << "Couldn't find monitor ["sv << monitor_index << ']';
BOOST_LOG(error) << "Couldn't find monitor ["sv << monitor_index << "] in "sv << monitor << " enumerated KMS monitor(s)"sv;
return -1;

// Neatly break from nested for loop
Expand Down Expand Up @@ -1616,6 +1621,12 @@ namespace platf {
std::vector<std::string> kms_display_names(mem_type_e hwdevice_type) {
int count = 0;

kms::env_width = 0;
kms::env_height = 0;
kms::env_logical_width = 0;
kms::env_logical_height = 0;
kms::card_descriptors.clear();

if (!fs::exists("/dev/dri")) {
BOOST_LOG(warning) << "Couldn't find /dev/dri, kmsgrab won't be enabled"sv;
return {};
Expand Down Expand Up @@ -1728,13 +1739,14 @@ namespace platf {
correlate_to_wayland(cds);
}

// Deduce the full virtual desktop size
kms::env_width = 0;
kms::env_height = 0;
BOOST_LOG(debug) << "Enumerated "sv << display_names.size() << " KMS monitor(s)"sv;

kms::env_logical_width = 0;
kms::env_logical_height = 0;
if (display_names.empty()) {
BOOST_LOG(error) << "No KMS monitors were found during enumeration"sv;
return {};
}

// Deduce the full virtual desktop size
for (auto &card_descriptor : cds) {
for (auto &[_, monitor_descriptor] : card_descriptor.crtc_to_monitor) {
BOOST_LOG(debug) << "Monitor description"sv;
Expand Down
68 changes: 67 additions & 1 deletion src/platform/linux/vulkan_encode.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,64 @@ namespace vk {
return -1;
}

static bool physical_device_supports_h264_encode(VkPhysicalDevice physical_device) {
uint32_t count = 0;
if (vkEnumerateDeviceExtensionProperties(physical_device, nullptr, &count, nullptr) != VK_SUCCESS) {
return false;
}

std::vector<VkExtensionProperties> extensions(count);
if (vkEnumerateDeviceExtensionProperties(physical_device, nullptr, &count, extensions.data()) != VK_SUCCESS) {
return false;
}

bool video_queue = false;
bool encode_queue = false;
bool encode_h264 = false;

for (const auto &extension : extensions) {
const std::string_view name {extension.extensionName};
video_queue = video_queue || name == "VK_KHR_video_queue"sv;
encode_queue = encode_queue || name == "VK_KHR_video_encode_queue"sv;
encode_h264 = encode_h264 || name == "VK_KHR_video_encode_h264"sv;
}

return video_queue && encode_queue && encode_h264;
}

static bool has_h264_encode_physical_device() {
VkApplicationInfo app = {VK_STRUCTURE_TYPE_APPLICATION_INFO};
app.apiVersion = VK_API_VERSION_1_1;

VkInstanceCreateInfo ci = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO};
ci.pApplicationInfo = &app;

VkInstance instance = VK_NULL_HANDLE;
if (vkCreateInstance(&ci, nullptr, &instance) != VK_SUCCESS) {
return false;
}

uint32_t count = 0;
if (vkEnumeratePhysicalDevices(instance, &count, nullptr) != VK_SUCCESS || count == 0) {
vkDestroyInstance(instance, nullptr);
return false;
}

std::vector<VkPhysicalDevice> physical_devices(count);
if (vkEnumeratePhysicalDevices(instance, &count, physical_devices.data()) != VK_SUCCESS) {
vkDestroyInstance(instance, nullptr);
return false;
}

const bool supported = std::any_of(
std::begin(physical_devices),
std::end(physical_devices),
physical_device_supports_h264_encode
);
vkDestroyInstance(instance, nullptr);
return supported;
}

struct PushConstants {
std::array<float, 4> color_vec_y;
std::array<float, 4> color_vec_u;
Expand Down Expand Up @@ -1025,11 +1083,19 @@ namespace vk {
}

bool validate() {
if (!avcodec_find_encoder_by_name("h264_vulkan") && !avcodec_find_encoder_by_name("hevc_vulkan")) {
if (!avcodec_find_encoder_by_name("h264_vulkan")) {
BOOST_LOG(info) << "FFmpeg h264_vulkan encoder is not available"sv;
return false;
}

if (!has_h264_encode_physical_device()) {
BOOST_LOG(info) << "Vulkan H.264 video encode is not supported by this device"sv;
return false;
}

AVBufferRef *dev = nullptr;
if (create_vulkan_hwdevice(&dev) < 0) {
BOOST_LOG(info) << "Failed to create Vulkan hardware device for encoder validation"sv;
return false;
}
av_buffer_unref(&dev);
Expand Down
92 changes: 68 additions & 24 deletions src/video.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ extern "C" {
#include "sync.h"
#include "video.h"

#ifdef SUNSHINE_BUILD_VULKAN
#if defined(__linux__) || defined(linux) || defined(__linux) || defined(__FreeBSD__)
#include "platform/linux/vulkan_encode.h"
#endif
#endif

#ifdef _WIN32
extern "C" {
#include <libavutil/hwcontext_d3d11va.h>
Expand Down Expand Up @@ -1181,6 +1187,8 @@ namespace video {
int active_av1_mode;
bool last_encoder_probe_supported_ref_frames_invalidation = false;
std::array<bool, 3> last_encoder_probe_supported_yuv444_for_codec = {};
constexpr auto no_display_available_msg = "Unable to start capture because no display is available"sv;
constexpr auto no_encoder_selected_msg = "No encoder selected; aborting capture instead of using invalid encoder state"sv;

void reset_display(std::shared_ptr<platf::display_t> &disp, const platf::mem_type_e &type, const std::string &display_name, const config_t &config) {
// We try this twice, in case we still get an error on reinitialization
Expand Down Expand Up @@ -1214,16 +1222,18 @@ namespace video {
}

// Refresh the display names
auto old_display_names = std::move(display_names);
auto had_display_names = !display_names.empty();
display_names = platf::display_names(dev_type);

// If we now have no displays, let's put the old display array back and fail
if (display_names.empty() && !old_display_names.empty()) {
BOOST_LOG(error) << "No displays were found after reenumeration!"sv;
display_names = std::move(old_display_names);
return;
} else if (display_names.empty()) {
display_names.emplace_back(output_name);
// If we now have no displays, fail instead of reusing stale display names.
if (display_names.empty()) {
if (!had_display_names && !output_name.empty()) {
display_names.emplace_back(output_name);
} else {
BOOST_LOG(error) << (had_display_names ? "No displays were found after reenumeration!"sv : "No displays were found during enumeration!"sv);
current_display_index = -1;
return;
}
}

// We now have a new display name list, so reset the index back to 0
Expand Down Expand Up @@ -1284,6 +1294,10 @@ namespace video {
std::vector<std::string> display_names;
int display_p = -1;
refresh_displays(encoder.platform_formats->dev_type, display_names, display_p);
if (display_p < 0 || display_names.empty()) {
BOOST_LOG(error) << no_display_available_msg;
return;
}
auto disp = platf::display(encoder.platform_formats->dev_type, display_names[display_p], capture_ctxs.front().config);
if (!disp) {
return;
Expand Down Expand Up @@ -1897,27 +1911,34 @@ namespace video {
// Allow the encoding device a final opportunity to set/unset or override any options
encode_device->init_codec_options(ctx.get(), &options);

if (auto status = avcodec_open2(ctx.get(), codec, &options)) {
char err_str[AV_ERROR_MAX_STRING_SIZE] {0};
auto status = avcodec_open2(ctx.get(), codec, &options);
if (!status) {
// Successfully opened the codec
break;
}

if (!video_format.fallback_options.empty() && retries == 0) {
BOOST_LOG(info)
<< "Retrying with fallback configuration options for ["sv << video_format.name << "] after error: "sv
<< av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
char err_str[AV_ERROR_MAX_STRING_SIZE] {0};
if (!video_format.fallback_options.empty() && retries == 0) {
BOOST_LOG(info)
<< "Retrying with fallback configuration options for ["sv << video_format.name << "] after error: "sv
<< av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);

continue;
} else {
BOOST_LOG(error)
<< "Could not open codec ["sv
<< video_format.name << "]: "sv
<< av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);
continue;
}

return nullptr;
}
BOOST_LOG(error)
<< "Could not open codec ["sv
<< video_format.name << "]: "sv
<< av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, status);

if (encoder.name == "vulkan"sv) {
BOOST_LOG(error)
<< "Vulkan encoder setup failed for ["sv << video_format.name
<< "]; this GPU or driver does not expose the required Vulkan video encode capability. "
<< "Sunshine will discard this encoder and continue probing fallback encoders."sv;
}

// Successfully opened the codec
break;
return nullptr;
}

avcodec_frame_t frame {av_frame_alloc()};
Expand Down Expand Up @@ -2286,6 +2307,10 @@ namespace video {
while (encode_session_ctx_queue.running()) {
// Refresh display names since a display removal might have caused the reinitialization
refresh_displays(encoder.platform_formats->dev_type, display_names, display_p);
if (display_p < 0 || display_names.empty()) {
BOOST_LOG(error) << no_display_available_msg;
return encode_e::error;
}

// Process any pending display switch with the new list of displays
if (switch_display_event->peek()) {
Expand Down Expand Up @@ -2491,6 +2516,11 @@ namespace video {
display = ref->display_wp->lock();
}

if (!chosen_encoder) {
BOOST_LOG(error) << no_encoder_selected_msg;
return;
}

auto &encoder = *chosen_encoder;

auto encode_device = make_encode_device(*display, encoder, config);
Expand Down Expand Up @@ -2533,6 +2563,11 @@ namespace video {
) {
auto idr_events = mail->event<bool>(mail::idr);

if (!chosen_encoder) {
BOOST_LOG(error) << no_encoder_selected_msg;
return;
}

idr_events->raise(true);
if (chosen_encoder->flags & PARALLEL_ENCODING) {
capture_async(std::move(mail), config, channel_data);
Expand Down Expand Up @@ -2628,6 +2663,14 @@ namespace video {
encoder.hevc.capabilities.set();
encoder.av1.capabilities.set();

#ifdef SUNSHINE_BUILD_VULKAN
if (encoder.name == "vulkan"sv && !vk::validate()) {
fg.disable();
BOOST_LOG(info) << "Encoder ["sv << encoder.name << "] is not supported on this GPU"sv;
return false;
}
#endif

// First, test encoder viability
config_t config_max_ref_frames {1920, 1080, 60, 6000, 1000, 1, 1, 1, 0, 0, 0};
config_t config_autoselect {1920, 1080, 60, 6000, 1000, 1, 0, 1, 0, 0, 0};
Expand Down Expand Up @@ -3013,6 +3056,7 @@ namespace video {

if (encode_device && encode_device->data) {
if (((vulkan_init_avcodec_hardware_input_buffer_fn) encode_device->data)(encode_device, &hw_device_buf)) {
BOOST_LOG(error) << "Failed to create a Vulkan device from the capture backend; aborting Vulkan encoder setup"sv;
return -1;
}
return hw_device_buf;
Expand Down
Loading