From ac41d6c56aee7d7f104582c629204c21fd6001fb Mon Sep 17 00:00:00 2001 From: luanweslley77 Date: Sat, 29 Aug 2026 00:36:14 -0300 Subject: [PATCH 1/4] feat(video,platform): declare explicit capture pixel formats Add pix_fmt_e pixel_format to platf::img_t and make all capture backends declare the negotiated format. The software encoder now prefers the declared format and falls back to the pixel_pitch heuristic only for unknown. VAAPI/CUDA RAM paths fail loudly for formats they cannot represent. Includes the 2 zero-cost 10-bit identity mappings (abgr/argb2101010 -> X2BGR10LE/X2RGB10LE) and the 2 shift-formats (bgra/rgba1010102) mapped unconditionally via native libswscale support, which the bundled build-deps FFmpeg provides. Sync cuda.cu duplicate img_t and fix kmsgrab cursor to bgra, handle P010 plane and fail loudly for unsupported declared formats. Advertise NV12 on MemPtr for 8/8 capture. --- src/platform/common.h | 29 +++++++ src/platform/linux/cuda.cpp | 10 +++ src/platform/linux/cuda.cu | 25 ++++++ src/platform/linux/graphics.cpp | 12 ++- src/platform/linux/graphics.h | 5 +- src/platform/linux/kmsgrab.cpp | 3 + src/platform/linux/pipewire.cpp | 37 +++++++- src/platform/linux/vaapi.cpp | 4 +- src/platform/linux/wlgrab.cpp | 1 + src/platform/linux/x11grab.cpp | 3 + src/platform/macos/display.mm | 21 +++++ src/platform/windows/display_ram.cpp | 1 + src/platform/windows/display_vram.cpp | 1 + src/video.cpp | 117 +++++++++++++++++++++----- src/video.h | 2 + 15 files changed, 247 insertions(+), 24 deletions(-) diff --git a/src/platform/common.h b/src/platform/common.h index f4006f85b54..8077de05e08 100644 --- a/src/platform/common.h +++ b/src/platform/common.h @@ -345,6 +345,13 @@ namespace platf { yuv444p16, ///< Planar 10-bit (shifted to 16-bit) YUV 4:4:4 yuv444p, ///< Planar 8-bit YUV 4:4:4 y410, ///< Y410 + bgr0, ///< Packed 8-bit B,G,R,0 + bgra, ///< Packed 8-bit B,G,R,A + xbgr2101010, ///< Packed 10-bit X,B,G,R + bgra1010102, ///< Packed 10-bit B,G,R,A + rgba1010102, ///< Packed 10-bit R,G,B,A + abgr2101010, ///< Packed 10-bit A,B,G,R + argb2101010, ///< Packed 10-bit A,R,G,B unknown ///< Unknown }; @@ -370,6 +377,13 @@ namespace platf { _CONVERT(yuv444p16); _CONVERT(yuv444p); _CONVERT(y410); + _CONVERT(bgr0); + _CONVERT(bgra); + _CONVERT(xbgr2101010); + _CONVERT(bgra1010102); + _CONVERT(rgba1010102); + _CONVERT(abgr2101010); + _CONVERT(argb2101010); _CONVERT(unknown); } #undef _CONVERT @@ -377,6 +391,20 @@ namespace platf { return "unknown"sv; } + /** + * @brief Check whether a capture pixel format can be uploaded as packed 8-bit BGR. + * @note Unknown formats are treated as BGR0, the historical assumption for + * capture buffers that do not declare a format. + * + * @param pix_fmt Capture pixel format to check. + * @return True when the format is representable as packed 8-bit BGR. + */ + inline bool is_bgr_capture_format(pix_fmt_e pix_fmt) { + using enum pix_fmt_e; + + return pix_fmt == unknown || pix_fmt == bgr0 || pix_fmt == bgra; + } + // Dimensions for touchscreen input /** * @brief Touchscreen coordinate bounds used to scale absolute input. @@ -536,6 +564,7 @@ namespace platf { std::int32_t height {}; ///< Image height in pixels. std::int32_t pixel_pitch {}; ///< Bytes per pixel in the image buffer. std::int32_t row_pitch {}; ///< Bytes between consecutive image rows. + pix_fmt_e pixel_format {pix_fmt_e::unknown}; ///< Declared pixel format of the captured image. std::optional frame_timestamp; ///< Capture timestamp associated with the frame. diff --git a/src/platform/linux/cuda.cpp b/src/platform/linux/cuda.cpp index 6623c9e95a0..1e859c14cda 100644 --- a/src/platform/linux/cuda.cpp +++ b/src/platform/linux/cuda.cpp @@ -243,6 +243,7 @@ namespace cuda { img.height = height; img.pixel_pitch = 4; img.row_pitch = img.width * img.pixel_pitch; + img.pixel_format = platf::pix_fmt_e::bgr0; std::vector image_data; image_data.resize(img.row_pitch * img.height); @@ -296,6 +297,14 @@ namespace cuda { * @return Conversion status. */ int convert(platf::img_t &img) override { + // This copy path only understands packed 8-bit BGR captures. Fail loudly + // for declared formats it cannot represent (e.g. NV12 or 10-bit formats) + // instead of silently copying them as 8-bit RGBA. + if (!platf::is_bgr_capture_format(img.pixel_format)) { + BOOST_LOG(error) << "RAM capture conversion does not support pixel format: "sv << platf::from_pix_fmt(img.pixel_format); + return -1; + } + if (is_yuv444) { return sws.load_ram(img, tex.array) || sws.convert_yuv444(frame->data[0], frame->data[1], frame->data[2], frame->linesize[0], tex_obj(tex), stream.get()); } @@ -1227,6 +1236,7 @@ namespace cuda { img->height = height; img->pixel_pitch = 4; img->row_pitch = img->width * img->pixel_pitch; + img->pixel_format = platf::pix_fmt_e::bgr0; auto tex_opt = tex_t::make(height, width * img->pixel_pitch); if (!tex_opt) { diff --git a/src/platform/linux/cuda.cu b/src/platform/linux/cuda.cu index fbc5401fe3a..cf42c18dc9b 100644 --- a/src/platform/linux/cuda.cu +++ b/src/platform/linux/cuda.cu @@ -49,6 +49,28 @@ using namespace std::literals; * Not pretty and extremely error-prone, fix at earliest convenience. */ namespace platf { + /** + * @brief Keep in sync with src/platform/common.h:529 platf::img_t. + */ + enum class pix_fmt_e { + yuv420p, + yuv420p10, + nv12, + p010, + ayuv, + yuv444p16, + yuv444p, + y410, + bgr0, + bgra, + xbgr2101010, + bgra1010102, + rgba1010102, + abgr2101010, + argb2101010, + unknown + }; + struct img_t: std::enable_shared_from_this { public: std::uint8_t *data {}; @@ -56,6 +78,7 @@ namespace platf { std::int32_t height {}; std::int32_t pixel_pitch {}; std::int32_t row_pitch {}; + pix_fmt_e pixel_format {pix_fmt_e::unknown}; ///< Keep in sync with src/platform/common.h:529 std::optional frame_timestamp; @@ -65,6 +88,8 @@ namespace platf { // End special declarations +static_assert(sizeof(platf::pix_fmt_e) == 4, "pix_fmt_e must be 4 bytes"); + namespace cuda { struct alignas(16) cuda_color_t { diff --git a/src/platform/linux/graphics.cpp b/src/platform/linux/graphics.cpp index 1ef27c7166e..9f132c39c41 100644 --- a/src/platform/linux/graphics.cpp +++ b/src/platform/linux/graphics.cpp @@ -1258,11 +1258,21 @@ namespace egl { return make_nv12(in_width, in_height, out_width, out_height, std::move(tex)); } - void sws_t::load_ram(platf::img_t &img) { + int sws_t::load_ram(platf::img_t &img) { + // This upload path only understands packed 8-bit BGR captures. Fail loudly + // for declared formats it cannot represent (e.g. NV12 or 10-bit formats) + // instead of silently uploading them as BGR0. + if (!platf::is_bgr_capture_format(img.pixel_format)) { + BOOST_LOG(error) << "RAM capture conversion does not support pixel format: "sv << platf::from_pix_fmt(img.pixel_format); + return -1; + } + loaded_texture = tex[0]; gl::ctx.BindTexture(GL_TEXTURE_2D, loaded_texture); gl::ctx.TexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, img.width, img.height, GL_BGRA, GL_UNSIGNED_BYTE, img.data); + + return 0; } void sws_t::load_vram(img_descriptor_t &img, int offset_x, int offset_y, int texture, bool is_yuv444) { diff --git a/src/platform/linux/graphics.h b/src/platform/linux/graphics.h index 2a2a06429c1..7a885e11a04 100644 --- a/src/platform/linux/graphics.h +++ b/src/platform/linux/graphics.h @@ -721,10 +721,13 @@ namespace egl { /** * @brief Load ram data from the backing API or store. + * @note Only packed 8-bit BGR captures are supported; other declared + * formats fail with a negative status instead of being misread. * * @param img Image or frame object to read from or populate. + * @return 0 on success; negative on unsupported formats. */ - void load_ram(platf::img_t &img); + int load_ram(platf::img_t &img); /** * @brief Load vram data from the backing API or store. * diff --git a/src/platform/linux/kmsgrab.cpp b/src/platform/linux/kmsgrab.cpp index 9575e2c6097..945b90722b8 100644 --- a/src/platform/linux/kmsgrab.cpp +++ b/src/platform/linux/kmsgrab.cpp @@ -1699,6 +1699,7 @@ namespace platf { img->height = height; img->pixel_pitch = 4; img->row_pitch = img->pixel_pitch * width; + img->pixel_format = platf::pix_fmt_e::bgr0; img->data = new std::uint8_t[height * img->row_pitch]; return img; @@ -1775,6 +1776,7 @@ namespace platf { img->serial = std::numeric_limitsserial)>::max(); img->data = nullptr; img->pixel_pitch = 4; + img->pixel_format = platf::pix_fmt_e::bgr0; img->sequence = 0; std::fill_n(img->sd.fds, 4, -1); @@ -1877,6 +1879,7 @@ namespace platf { img->height = captured_cursor.dst_h; img->pixel_pitch = 4; img->row_pitch = img->pixel_pitch * img->width; + img->pixel_format = platf::pix_fmt_e::bgra; img->data = img->buffer.data(); } else { img->data = nullptr; diff --git a/src/platform/linux/pipewire.cpp b/src/platform/linux/pipewire.cpp index 8a6b45e6dce..2913d6605cf 100644 --- a/src/platform/linux/pipewire.cpp +++ b/src/platform/linux/pipewire.cpp @@ -50,6 +50,37 @@ namespace { constexpr int MAX_PARAMS = 200; constexpr int MAX_DMABUF_FORMATS = 200; constexpr int MAX_DMABUF_MODIFIERS = 200; + + /** + * @brief Map a PipeWire SPA video format to Sunshine's pixel format enum. + * + * @param spa_format PipeWire SPA video format identifier. + * @return Sunshine pixel format; unknown for unmapped formats. + */ + platf::pix_fmt_e map_spa_pix_fmt(int32_t spa_format) { + using enum platf::pix_fmt_e; + + switch (spa_format) { + case SPA_VIDEO_FORMAT_NV12: + return nv12; + case SPA_VIDEO_FORMAT_BGRx: + return bgr0; + case SPA_VIDEO_FORMAT_BGRA: + return bgra; + case SPA_VIDEO_FORMAT_xBGR_210LE: + return xbgr2101010; + case SPA_VIDEO_FORMAT_ARGB_210LE: + return bgra1010102; + case SPA_VIDEO_FORMAT_ABGR_210LE: + return rgba1010102; + case SPA_VIDEO_FORMAT_RGBA_102LE: + return abgr2101010; + case SPA_VIDEO_FORMAT_BGRA_102LE: + return argb2101010; + default: + return unknown; + } + } } // namespace using namespace std::literals; @@ -63,7 +94,8 @@ namespace pipewire { int32_t pw_format; ///< Matching PipeWire SPA video format. }; - static constexpr std::array format_map = {{ + static constexpr std::array format_map = {{ + {DRM_FORMAT_NV12, SPA_VIDEO_FORMAT_NV12}, {DRM_FORMAT_XBGR2101010, SPA_VIDEO_FORMAT_xBGR_210LE}, {DRM_FORMAT_BGRA1010102, SPA_VIDEO_FORMAT_ARGB_210LE}, {DRM_FORMAT_RGBA1010102, SPA_VIDEO_FORMAT_ABGR_210LE}, @@ -447,6 +479,7 @@ namespace pipewire { // NV12 is the only 1-byte-per-pixel format delivered on the memory // path; every other negotiated format is packed 4 bytes per pixel. img->pixel_pitch = (stream_data.format.info.raw.format == SPA_VIDEO_FORMAT_NV12) ? 1 : 4; + img->pixel_format = map_spa_pix_fmt(stream_data.format.info.raw.format); } } @@ -955,6 +988,7 @@ namespace pipewire { img->height = height; img->pixel_pitch = 4; img->row_pitch = img->pixel_pitch * width; + img->pixel_format = platf::pix_fmt_e::bgr0; img->sequence = 0; img->serial = std::numeric_limitsserial)>::max(); img->data = nullptr; @@ -1095,6 +1129,7 @@ namespace pipewire { static_cast(img)->data_owned = true; img->row_pitch = w * 4; img->pixel_pitch = 4; + img->pixel_format = platf::pix_fmt_e::bgr0; } } return 0; diff --git a/src/platform/linux/vaapi.cpp b/src/platform/linux/vaapi.cpp index a9190a7bb55..66af53193ef 100644 --- a/src/platform/linux/vaapi.cpp +++ b/src/platform/linux/vaapi.cpp @@ -531,7 +531,9 @@ namespace va { * @return Conversion status. */ int convert(platf::img_t &img) override { - sws.load_ram(img); + if (sws.load_ram(img) < 0) { + return -1; + } sws.convert_nv12(nv12->buf); return 0; diff --git a/src/platform/linux/wlgrab.cpp b/src/platform/linux/wlgrab.cpp index 65020d6262f..8d4b2c8c29a 100644 --- a/src/platform/linux/wlgrab.cpp +++ b/src/platform/linux/wlgrab.cpp @@ -348,6 +348,7 @@ namespace wl { img->height = height; img->pixel_pitch = 4; img->row_pitch = img->pixel_pitch * width; + img->pixel_format = platf::pix_fmt_e::bgr0; img->data = new std::uint8_t[height * img->row_pitch]; return img; diff --git a/src/platform/linux/x11grab.cpp b/src/platform/linux/x11grab.cpp index 2fe6a1d12ed..c1e37e31df7 100644 --- a/src/platform/linux/x11grab.cpp +++ b/src/platform/linux/x11grab.cpp @@ -635,6 +635,7 @@ namespace platf { img->data = (uint8_t *) x_img->data; img->row_pitch = x_img->bytes_per_line; img->pixel_pitch = x_img->bits_per_pixel / 8; + img->pixel_format = (img->pixel_pitch == 4) ? platf::pix_fmt_e::bgr0 : platf::pix_fmt_e::unknown; img->img.reset(x_img); if (cursor) { @@ -830,6 +831,7 @@ namespace platf { img->height = height; img->pixel_pitch = 4; img->row_pitch = img->pixel_pitch * width; + img->pixel_format = platf::pix_fmt_e::bgr0; img->data = new std::uint8_t[height * img->row_pitch]; return img; @@ -1052,6 +1054,7 @@ namespace platf { img.y = xcursor->y - xcursor->yhot; img.pixel_pitch = 4; img.row_pitch = img.pixel_pitch * img.width; + img.pixel_format = platf::pix_fmt_e::bgr0; img.serial = xcursor->cursor_serial; } diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index d5fe3d024b2..3a1e1bb8668 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -52,6 +52,25 @@ OSType videotoolbox_pixel_format(const video::config_t &config) { const auto colorspace {video::colorspace_from_client_config(config, false)}; return colorspace.bit_depth == 10 ? kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; } + + /** + * @brief Map a Core Video pixel format to Sunshine's pixel format enum. + * + * @param cv_format Core Video pixel format type. + * @return Sunshine pixel format; unknown for unmapped formats. + */ + platf::pix_fmt_e pix_fmt_from_cv_pixel_format(OSType cv_format) { + switch (cv_format) { + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + return platf::pix_fmt_e::nv12; + case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: + return platf::pix_fmt_e::p010; + case kCVPixelFormatType_32BGRA: + return platf::pix_fmt_e::bgra; + default: + return platf::pix_fmt_e::unknown; + } + } } // namespace /** @@ -93,6 +112,7 @@ capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const img_out->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf); img_out->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf); img_out->pixel_pitch = img_out->row_pitch / img_out->width; + img_out->pixel_format = pix_fmt_from_cv_pixel_format(av_capture.pixelFormat); old_data_retainer = nullptr; @@ -177,6 +197,7 @@ int dummy_img(img_t *img) override { img->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf); img->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf); img->pixel_pitch = img->row_pitch / img->width; + img->pixel_format = pix_fmt_from_cv_pixel_format(av_capture.pixelFormat); old_data_retainer = nullptr; diff --git a/src/platform/windows/display_ram.cpp b/src/platform/windows/display_ram.cpp index 9a4553bff93..5f8f0f121b5 100644 --- a/src/platform/windows/display_ram.cpp +++ b/src/platform/windows/display_ram.cpp @@ -374,6 +374,7 @@ namespace platf::dxgi { } img->pixel_pitch = get_pixel_pitch(); + img->pixel_format = (img->pixel_pitch == 4) ? platf::pix_fmt_e::bgr0 : platf::pix_fmt_e::unknown; if (dummy && !img->row_pitch) { // Assume our dummy image will have no padding diff --git a/src/platform/windows/display_vram.cpp b/src/platform/windows/display_vram.cpp index 5eefc3ea8f4..eccd2a66dad 100644 --- a/src/platform/windows/display_vram.cpp +++ b/src/platform/windows/display_vram.cpp @@ -1947,6 +1947,7 @@ namespace platf::dxgi { // Initialize format-dependent fields img->pixel_pitch = get_pixel_pitch(); + img->pixel_format = (img->pixel_pitch == 4) ? platf::pix_fmt_e::bgr0 : platf::pix_fmt_e::unknown; img->row_pitch = img->pixel_pitch * img->width; img->dummy = dummy; img->format = (capture_format == DXGI_FORMAT_UNKNOWN) ? DXGI_FORMAT_B8G8R8A8_UNORM : capture_format; diff --git a/src/video.cpp b/src/video.cpp index 6e99ada4ce4..bfa9d1d3bcc 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -75,6 +75,94 @@ namespace video { BOOST_LOG(error) << "No display devices are active at the moment! Cannot probe the encoders."; return false; } + + /** + * @brief Map a declared capture pixel format to its FFmpeg equivalent. + * @note Formats without an FFmpeg equivalent in the bundled build-deps + * FFmpeg map to AV_PIX_FMT_NONE, letting callers fail loudly. + * + * @param pix_fmt Declared capture pixel format. + * @return FFmpeg pixel format; AV_PIX_FMT_NONE when the value is not a capture format. + */ + AVPixelFormat map_capture_pix_fmt(platf::pix_fmt_e pix_fmt) { + using enum platf::pix_fmt_e; + + switch (pix_fmt) { + case nv12: + return AV_PIX_FMT_NV12; + case p010: + return AV_PIX_FMT_P010; + case bgr0: + return AV_PIX_FMT_BGR0; + case bgra: + return AV_PIX_FMT_BGRA; + case xbgr2101010: + return AV_PIX_FMT_X2BGR10LE; + case abgr2101010: + return AV_PIX_FMT_X2BGR10LE; + case argb2101010: + return AV_PIX_FMT_X2RGB10LE; + case bgra1010102: + return AV_PIX_FMT_BGRA1010102LE; + case rgba1010102: + return AV_PIX_FMT_RGBA1010102LE; + default: + return AV_PIX_FMT_NONE; + } + } + + /** + * @brief Resolve the FFmpeg pixel format for a capture image. + * @note Uses the format declared by the capture backend when it knows it, + * falling back to deriving the format from the bytes per pixel otherwise. + * PipeWire-based captures (KWin screencast / XDG portal) deliver NV12 with + * 1 byte per pixel, while KMS/DMABUF captures deliver BGR0 (4 bytes per + * pixel). The capture backend reports bytes per pixel in img.pixel_pitch; + * fall back to the row pitch heuristic when it is unavailable. + * + * @param img Capture image to resolve the input format for. + * @return FFmpeg pixel format; AV_PIX_FMT_NONE when the declared format is unsupported. + */ + AVPixelFormat resolve_input_fmt(const platf::img_t &img) { + using enum platf::pix_fmt_e; + + if (img.pixel_format == unknown) { + // No declared format; derive the format from the bytes per pixel. + const auto pixel_pitch = img.pixel_pitch > 0 ? img.pixel_pitch : (img.row_pitch / std::max(img.width, 1)); + return (pixel_pitch == 1) ? AV_PIX_FMT_NV12 : AV_PIX_FMT_BGR0; + } + + auto input_fmt = map_capture_pix_fmt(img.pixel_format); + if (input_fmt == AV_PIX_FMT_NONE) { + BOOST_LOG(error) << "Unsupported capture pixel format: "sv << platf::from_pix_fmt(img.pixel_format); + } + return input_fmt; + } + + /** + * @brief Copy the scaled output frame into the padded output frame. + * @note Used when aspect ratio padding is required; copies line by line to + * preserve the leading padding for each row of every plane. + * + * @param padded_frame Destination frame carrying the aspect ratio padding. + * @param output_frame Source frame produced by the scaler. + * @param offset_w Horizontal offset in pixels of the scaled region. + * @param offset_h Vertical offset in pixels of the scaled region. + */ + void copy_padded_frame(const AVFrame &padded_frame, const AVFrame &output_frame, int offset_w, int offset_h) { + auto fmt_desc = av_pix_fmt_desc_get(static_cast(output_frame.format)); + auto planes = av_pix_fmt_count_planes(static_cast(output_frame.format)); + for (int plane = 0; plane < planes; plane++) { + auto shift_h = plane == 0 ? 0 : fmt_desc->log2_chroma_h; + auto shift_w = plane == 0 ? 0 : fmt_desc->log2_chroma_w; + auto offset = ((offset_w >> shift_w) * fmt_desc->comp[plane].step) + (offset_h >> shift_h) * padded_frame.linesize[plane]; + + // Copy line-by-line to preserve leading padding for each row + for (int line = 0; line < output_frame.height >> shift_h; line++) { + memcpy(padded_frame.data[plane] + offset + (line * padded_frame.linesize[plane]), output_frame.data[plane] + (line * output_frame.linesize[plane]), static_cast(output_frame.width >> shift_w) * fmt_desc->comp[plane].step); + } + } + } } // namespace /** @@ -225,13 +313,13 @@ namespace video { // If we need to add aspect ratio padding, we need to scale into an intermediate output buffer bool requires_padding = (sw_frame->width != sws_output_frame->width || sw_frame->height != sws_output_frame->height); - // Detect the actual capture pixel format. PipeWire-based captures (KWin - // screencast / XDG portal) deliver NV12 with 1 byte per pixel, while - // KMS/DMABUF captures deliver BGR0 (4 bytes per pixel). The capture - // backend reports bytes per pixel in img.pixel_pitch; fall back to the row - // pitch heuristic when it is unavailable. - const auto pixel_pitch = img.pixel_pitch > 0 ? img.pixel_pitch : (img.row_pitch / std::max(img.width, 1)); - const auto input_fmt = (pixel_pitch == 1) ? AV_PIX_FMT_NV12 : AV_PIX_FMT_BGR0; + // Resolve the capture pixel format: use the format declared by the + // capture backend when available, deriving it from the bytes per pixel + // otherwise. + AVPixelFormat input_fmt = resolve_input_fmt(img); + if (input_fmt == AV_PIX_FMT_NONE) { + return -1; + } // The sws context is created with the default BGR0 source format; // recreate it once if the capture is actually NV12. @@ -247,7 +335,7 @@ namespace video { // Setup the input frame using the caller's img_t sws_input_frame->data[0] = img.data; sws_input_frame->linesize[0] = img.row_pitch; - if (input_fmt == AV_PIX_FMT_NV12) { + if (input_fmt == AV_PIX_FMT_NV12 || input_fmt == AV_PIX_FMT_P010) { sws_input_frame->data[1] = img.data + static_cast(img.row_pitch) * img.height; sws_input_frame->linesize[1] = img.row_pitch; } else { @@ -269,18 +357,7 @@ namespace video { // If we require aspect ratio padding, copy the output frame into the final padded frame if (requires_padding) { - auto fmt_desc = av_pix_fmt_desc_get(static_cast(sws_output_frame->format)); - auto planes = av_pix_fmt_count_planes(static_cast(sws_output_frame->format)); - for (int plane = 0; plane < planes; plane++) { - auto shift_h = plane == 0 ? 0 : fmt_desc->log2_chroma_h; - auto shift_w = plane == 0 ? 0 : fmt_desc->log2_chroma_w; - auto offset = ((offsetW >> shift_w) * fmt_desc->comp[plane].step) + (offsetH >> shift_h) * sw_frame->linesize[plane]; - - // Copy line-by-line to preserve leading padding for each row - for (int line = 0; line < sws_output_frame->height >> shift_h; line++) { - memcpy(sw_frame->data[plane] + offset + (line * sw_frame->linesize[plane]), sws_output_frame->data[plane] + (line * sws_output_frame->linesize[plane]), static_cast(sws_output_frame->width >> shift_w) * fmt_desc->comp[plane].step); - } - } + copy_padded_frame(*sw_frame, *sws_output_frame, offsetW, offsetH); } // If frame is not a software frame, it means we still need to transfer from main memory diff --git a/src/video.h b/src/video.h index 63f3cf55cbf..fa04a2aef68 100644 --- a/src/video.h +++ b/src/video.h @@ -126,6 +126,8 @@ namespace video { public: /** * @brief Convert a captured image into the encoder input representation. + * @note Uses the capture pixel format declared in `img.pixel_format` when + * available, falling back to a heuristic based on `img.pixel_pitch` otherwise. * * @param img Image or frame object to read from or populate. * @return Conversion status. From 27a9af663399613ec1c59874194e7fc58dec4454 Mon Sep 17 00:00:00 2001 From: luanweslley77 Date: Sat, 29 Aug 2026 00:36:14 -0300 Subject: [PATCH 2/4] test(video,platform): cover explicit capture formats Exercise declared BGR0/NV12/BGRA/xbgr2101010, the 2 identity 10-bit formats and the 2 shift-formats with padded strides, plus FromPixFmt and IsBgrCaptureFormat full matrices. A declared format with no capture mapping must fail loudly. --- tests/unit/platform/test_common.cpp | 43 ++++++++++ tests/unit/test_video.cpp | 127 +++++++++++++++++++++++++++- 2 files changed, 168 insertions(+), 2 deletions(-) diff --git a/tests/unit/platform/test_common.cpp b/tests/unit/platform/test_common.cpp index b497d35815f..2ad07963578 100644 --- a/tests/unit/platform/test_common.cpp +++ b/tests/unit/platform/test_common.cpp @@ -11,3 +11,46 @@ TEST(HostnameTests, TestAsioEquality) { // These should be equivalent on all platforms for ASCII hostnames ASSERT_EQ(platf::get_host_name(), boost::asio::ip::host_name()); } + +/** + * @brief Capture formats representable as packed 8-bit BGR are accepted by the RAM + * conversion paths, while other declared formats are rejected. + */ +TEST(PixelFormatTests, IsBgrCaptureFormat) { + EXPECT_TRUE(platf::is_bgr_capture_format(platf::pix_fmt_e::unknown)); + EXPECT_TRUE(platf::is_bgr_capture_format(platf::pix_fmt_e::bgr0)); + EXPECT_TRUE(platf::is_bgr_capture_format(platf::pix_fmt_e::bgra)); + + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::nv12)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::p010)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::yuv420p)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::yuv420p10)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::ayuv)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::yuv444p)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::yuv444p16)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::y410)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::xbgr2101010)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::bgra1010102)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::rgba1010102)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::abgr2101010)); + EXPECT_FALSE(platf::is_bgr_capture_format(platf::pix_fmt_e::argb2101010)); +} + +TEST(PixelFormatTests, FromPixFmt) { + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::yuv420p), "yuv420p"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::yuv420p10), "yuv420p10"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::nv12), "nv12"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::p010), "p010"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::ayuv), "ayuv"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::yuv444p16), "yuv444p16"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::yuv444p), "yuv444p"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::y410), "y410"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::bgr0), "bgr0"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::bgra), "bgra"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::xbgr2101010), "xbgr2101010"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::bgra1010102), "bgra1010102"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::rgba1010102), "rgba1010102"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::abgr2101010), "abgr2101010"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::argb2101010), "argb2101010"); + EXPECT_EQ(platf::from_pix_fmt(platf::pix_fmt_e::unknown), "unknown"); +} diff --git a/tests/unit/test_video.cpp b/tests/unit/test_video.cpp index 3d36b2e49c5..308f130967b 100644 --- a/tests/unit/test_video.cpp +++ b/tests/unit/test_video.cpp @@ -314,8 +314,8 @@ INSTANTIATE_TEST_SUITE_P( ); /** - * @brief Software encoder converts BGR0 and NV12 frames, including padded strides and - * backends that don't report the pixel pitch. + * @brief Software encoder converts declared capture formats, including padded strides, + * 10-bit formats, and backends that don't report the pixel pitch. */ TEST(SoftwareEncoderConversion, Bgr0AndNv12) { constexpr int w = 320; @@ -365,6 +365,129 @@ TEST(SoftwareEncoderConversion, Bgr0AndNv12) { padded_nv12_img.pixel_pitch = 1; EXPECT_EQ(device.convert(padded_nv12_img), 0); + // Capture backends that declare the negotiated format explicitly take + // precedence over the pixel_pitch heuristic. + platf::img_t explicit_bgr0_img {}; + explicit_bgr0_img.data = bgr0_buffer.data(); + explicit_bgr0_img.width = w; + explicit_bgr0_img.height = h; + explicit_bgr0_img.row_pitch = w * 4; + explicit_bgr0_img.pixel_pitch = 4; + explicit_bgr0_img.pixel_format = platf::pix_fmt_e::bgr0; + EXPECT_EQ(device.convert(explicit_bgr0_img), 0); + + // NV12 declared explicitly -- even with a misleading pixel_pitch the declared + // format wins and the interleaved UV plane is read correctly. + platf::img_t explicit_nv12_img {}; + explicit_nv12_img.data = nv12_buffer.data(); + explicit_nv12_img.width = w; + explicit_nv12_img.height = h; + explicit_nv12_img.row_pitch = w; + explicit_nv12_img.pixel_pitch = 4; + explicit_nv12_img.pixel_format = platf::pix_fmt_e::nv12; + EXPECT_EQ(device.convert(explicit_nv12_img), 0); + + // BGRA capture (alpha channel present) -- 4 bytes per pixel, declared explicitly. + std::vector bgra_buffer(static_cast(w) * h * 4); + platf::img_t bgra_img {}; + bgra_img.data = bgra_buffer.data(); + bgra_img.width = w; + bgra_img.height = h; + bgra_img.row_pitch = w * 4; + bgra_img.pixel_pitch = 4; + bgra_img.pixel_format = platf::pix_fmt_e::bgra; + EXPECT_EQ(device.convert(bgra_img), 0); + + // Packed 10-bit RGB (e.g. XBGR2101010 from an HDR PipeWire capture) -- 4 + // bytes per pixel, declared explicitly so it is not misread as BGR0. + std::vector xbgr10_buffer(static_cast(w) * h * 4); + platf::img_t xbgr10_img {}; + xbgr10_img.data = xbgr10_buffer.data(); + xbgr10_img.width = w; + xbgr10_img.height = h; + xbgr10_img.row_pitch = w * 4; + xbgr10_img.pixel_pitch = 4; + xbgr10_img.pixel_format = platf::pix_fmt_e::xbgr2101010; + EXPECT_EQ(device.convert(xbgr10_img), 0); + + // Padded-stride 10-bit -- the declared format keeps the conversion correct + // even when the row pitch is larger than width * bytes-per-pixel. + constexpr int padded10_stride = w * 4 + 32; + std::vector padded_xbgr10_buffer(static_cast(padded10_stride) * h); + platf::img_t padded_xbgr10_img {}; + padded_xbgr10_img.data = padded_xbgr10_buffer.data(); + padded_xbgr10_img.width = w; + padded_xbgr10_img.height = h; + padded_xbgr10_img.row_pitch = padded10_stride; + padded_xbgr10_img.pixel_pitch = 4; + padded_xbgr10_img.pixel_format = platf::pix_fmt_e::xbgr2101010; + EXPECT_EQ(device.convert(padded_xbgr10_img), 0); + + // 10-bit with alpha in high bits — bit-identical to X2RGB10LE/X2BGR10LE, zero-cost. + std::vector abgr10_buffer(static_cast(w) * h * 4); + platf::img_t abgr10_img {}; + abgr10_img.data = abgr10_buffer.data(); + abgr10_img.width = w; + abgr10_img.height = h; + abgr10_img.row_pitch = w * 4; + abgr10_img.pixel_pitch = 4; + abgr10_img.pixel_format = platf::pix_fmt_e::abgr2101010; + EXPECT_EQ(device.convert(abgr10_img), 0); + + std::vector argb10_buffer(static_cast(w) * h * 4); + platf::img_t argb10_img {}; + argb10_img.data = argb10_buffer.data(); + argb10_img.width = w; + argb10_img.height = h; + argb10_img.row_pitch = w * 4; + argb10_img.pixel_pitch = 4; + argb10_img.pixel_format = platf::pix_fmt_e::argb2101010; + EXPECT_EQ(device.convert(argb10_img), 0); + + // 10-bit with alpha in low bits (BGRA1010102/RGBA1010102), provided by the + // patched FFmpeg from build-deps. + std::vector bgra1010102_buffer(static_cast(w) * h * 4); + platf::img_t bgra1010102_img {}; + bgra1010102_img.data = bgra1010102_buffer.data(); + bgra1010102_img.width = w; + bgra1010102_img.height = h; + bgra1010102_img.row_pitch = w * 4; + bgra1010102_img.pixel_pitch = 4; + bgra1010102_img.pixel_format = platf::pix_fmt_e::bgra1010102; + EXPECT_EQ(device.convert(bgra1010102_img), 0); + + std::vector rgba1010102_buffer(static_cast(w) * h * 4); + platf::img_t rgba1010102_img {}; + rgba1010102_img.data = rgba1010102_buffer.data(); + rgba1010102_img.width = w; + rgba1010102_img.height = h; + rgba1010102_img.row_pitch = w * 4; + rgba1010102_img.pixel_pitch = 4; + rgba1010102_img.pixel_format = platf::pix_fmt_e::rgba1010102; + EXPECT_EQ(device.convert(rgba1010102_img), 0); + + // A declared format with no capture mapping must fail loudly instead of + // falling back to the pixel pitch heuristic. + platf::img_t unsupported_img {}; + unsupported_img.data = bgr0_buffer.data(); + unsupported_img.width = w; + unsupported_img.height = h; + unsupported_img.row_pitch = w * 4; + unsupported_img.pixel_pitch = 4; + unsupported_img.pixel_format = platf::pix_fmt_e::yuv420p; + EXPECT_NE(device.convert(unsupported_img), 0); + + // P010 (10-bit YUV semi-planar) — used on macOS. + std::vector p010_buffer(static_cast(w) * h * 2 + static_cast(w) * h); + platf::img_t p010_img {}; + p010_img.data = p010_buffer.data(); + p010_img.width = w; + p010_img.height = h; + p010_img.row_pitch = w * 2; + p010_img.pixel_pitch = 2; + p010_img.pixel_format = platf::pix_fmt_e::p010; + EXPECT_EQ(device.convert(p010_img), 0); + // Capture backends that don't report pixel_pitch fall back to deriving it // from the row pitch (1 byte per pixel = NV12, 4 = BGR0). platf::img_t fallback_bgr0_img {}; From 5df6a9d3ded5818bd1b3e12633dcb94551dd2439 Mon Sep 17 00:00:00 2001 From: luanweslley77 Date: Sat, 29 Aug 2026 00:36:14 -0300 Subject: [PATCH 3/4] chore(deps): bump build-deps to aabd22d Bump third-party/build-deps a9a9277 -> aabd22d (fix/ffmpeg-patch-fallback rebased onto upstream/master 4d864e1). Includes in build-deps: - feat(ffmpeg): add BGRA1010102/RGBA1010102 pixel formats for 10-bit HDR PipeWire captures (SPA ARGB_210LE/ABGR_210LE) on MemPtr - build(ffmpeg): wire libavutil patches into the shared patch list (new BUILD_FFMPEG_LIBAVUTIL_PATCHES option) --- third-party/build-deps | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third-party/build-deps b/third-party/build-deps index a9a9277cdaf..aabd22d5daf 160000 --- a/third-party/build-deps +++ b/third-party/build-deps @@ -1 +1 @@ -Subproject commit a9a9277cdafe8a0ff9f197915fe43b383ed4f36b +Subproject commit aabd22d5dafc4cd2d77523bfd5a98df1a8c17300 From 29e068ce9f98dea70d2ed2ce6226232598bdf4cb Mon Sep 17 00:00:00 2001 From: luanweslley77 Date: Tue, 1 Sep 2026 14:40:34 -0300 Subject: [PATCH 4/4] fix(pipewire): correct 1:1 DRM<->SPA for HDR 10-bit BGRA1010102->BGRA_102LE (B:G:R:A), RGBA->RGBA_102LE (R:G:B:A), ABGR2101010->ABGR_210LE (A:B:G:R), ARGB->ARGB_210LE (A:R:G:B). Fixes R/B swap and 210<->102 swap. Verified against xdg-desktop-portal-wlr and KWin 1:1 mapping. --- src/platform/linux/pipewire.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/platform/linux/pipewire.cpp b/src/platform/linux/pipewire.cpp index 2913d6605cf..0eb9012ce96 100644 --- a/src/platform/linux/pipewire.cpp +++ b/src/platform/linux/pipewire.cpp @@ -70,13 +70,13 @@ namespace { case SPA_VIDEO_FORMAT_xBGR_210LE: return xbgr2101010; case SPA_VIDEO_FORMAT_ARGB_210LE: - return bgra1010102; + return argb2101010; case SPA_VIDEO_FORMAT_ABGR_210LE: - return rgba1010102; - case SPA_VIDEO_FORMAT_RGBA_102LE: return abgr2101010; + case SPA_VIDEO_FORMAT_RGBA_102LE: + return rgba1010102; case SPA_VIDEO_FORMAT_BGRA_102LE: - return argb2101010; + return bgra1010102; default: return unknown; } @@ -97,10 +97,10 @@ namespace pipewire { static constexpr std::array format_map = {{ {DRM_FORMAT_NV12, SPA_VIDEO_FORMAT_NV12}, {DRM_FORMAT_XBGR2101010, SPA_VIDEO_FORMAT_xBGR_210LE}, - {DRM_FORMAT_BGRA1010102, SPA_VIDEO_FORMAT_ARGB_210LE}, - {DRM_FORMAT_RGBA1010102, SPA_VIDEO_FORMAT_ABGR_210LE}, - {DRM_FORMAT_ABGR2101010, SPA_VIDEO_FORMAT_RGBA_102LE}, - {DRM_FORMAT_ARGB2101010, SPA_VIDEO_FORMAT_BGRA_102LE}, + {DRM_FORMAT_BGRA1010102, SPA_VIDEO_FORMAT_BGRA_102LE}, + {DRM_FORMAT_RGBA1010102, SPA_VIDEO_FORMAT_RGBA_102LE}, + {DRM_FORMAT_ABGR2101010, SPA_VIDEO_FORMAT_ABGR_210LE}, + {DRM_FORMAT_ARGB2101010, SPA_VIDEO_FORMAT_ARGB_210LE}, {DRM_FORMAT_ARGB8888, SPA_VIDEO_FORMAT_BGRA}, {DRM_FORMAT_XRGB8888, SPA_VIDEO_FORMAT_BGRx}, }};