diff --git a/packages/camera/camera_windows/CHANGELOG.md b/packages/camera/camera_windows/CHANGELOG.md index 56dd79f33fce..781f04fc46a7 100644 --- a/packages/camera/camera_windows/CHANGELOG.md +++ b/packages/camera/camera_windows/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.2.6+6 + +* Fixes `CameraException(camera_error, Failed to initialize video preview)` on + multi-stream USB cameras. + ## 0.2.6+5 * Updates pigeon dev_dependency to ^27.3.2 for analyzer 14 compatibility. diff --git a/packages/camera/camera_windows/pubspec.yaml b/packages/camera/camera_windows/pubspec.yaml index a17da257bd42..aeea3eb8c9e9 100644 --- a/packages/camera/camera_windows/pubspec.yaml +++ b/packages/camera/camera_windows/pubspec.yaml @@ -2,7 +2,7 @@ name: camera_windows description: A Flutter plugin for getting information about and controlling the camera on Windows. repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_windows issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22 -version: 0.2.6+5 +version: 0.2.6+6 environment: sdk: ^3.10.0 diff --git a/packages/camera/camera_windows/windows/capture_controller.cpp b/packages/camera/camera_windows/windows/capture_controller.cpp index 1afa5c0a8c45..af25ba709384 100644 --- a/packages/camera/camera_windows/windows/capture_controller.cpp +++ b/packages/camera/camera_windows/windows/capture_controller.cpp @@ -284,6 +284,12 @@ void CaptureControllerImpl::ResetCaptureController() { video_source_ = nullptr; base_preview_media_type_ = nullptr; base_capture_media_type_ = nullptr; + preview_source_stream_index_ = static_cast( + MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW); + record_source_stream_index_ = static_cast( + MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_RECORD); + photo_source_stream_index_ = + static_cast(MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_PHOTO); if (dxgi_device_manager_) { dxgi_device_manager_->ResetDevice(dx11_device_.Get(), @@ -371,7 +377,8 @@ void CaptureControllerImpl::TakePicture(const std::string& file_path) { // Check MF_CAPTURE_ENGINE_PHOTO_TAKEN event handling // for response process. hr = photo_handler_->TakePhoto(file_path, capture_engine_.Get(), - base_capture_media_type_.Get()); + base_capture_media_type_.Get(), + photo_source_stream_index_); if (FAILED(hr)) { // Destroy photo handler on error cases to make sure state is resetted. photo_handler_ = nullptr; @@ -470,22 +477,65 @@ HRESULT CaptureControllerImpl::FindBaseMediaTypes() { return FindBaseMediaTypesForSource(source.Get()); } +void CaptureControllerImpl::ResolveSourceStreamIndices( + IMFCaptureSource* source) { + // Devices that don't support the preferred-stream selectors fail this probe, + // typically with MF_E_INVALIDSTREAMNUMBER. + ComPtr probe; + HRESULT hr = source->GetAvailableDeviceMediaType( + preview_source_stream_index_, 0, probe.GetAddressOf()); + if (SUCCEEDED(hr)) { + return; + } + + DWORD stream_count = 0; + if (FAILED(source->GetDeviceStreamCount(&stream_count))) { + return; + } + + // Prefer a video-preview stream, then video-capture. + DWORD video_stream_index = MAXDWORD; + for (DWORD i = 0; i < stream_count; i++) { + MF_CAPTURE_ENGINE_STREAM_CATEGORY category; + if (FAILED(source->GetDeviceStreamCategory(i, &category))) { + continue; + } + if (category == MF_CAPTURE_ENGINE_STREAM_CATEGORY_VIDEO_PREVIEW) { + video_stream_index = i; + break; + } + if (category == MF_CAPTURE_ENGINE_STREAM_CATEGORY_VIDEO_CAPTURE && + video_stream_index == MAXDWORD) { + video_stream_index = i; + } + } + if (video_stream_index == MAXDWORD) { + // No matching stream found. + return; + } + + // One stream for every role keeps the media types consistent with the sinks. + preview_source_stream_index_ = video_stream_index; + record_source_stream_index_ = video_stream_index; + photo_source_stream_index_ = video_stream_index; +} + HRESULT CaptureControllerImpl::FindBaseMediaTypesForSource( IMFCaptureSource* source) { + ResolveSourceStreamIndices(source); + // Find base media type for previewing. - if (!FindBestMediaType( - (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW, - source, base_preview_media_type_.GetAddressOf(), - GetMaxPreviewHeight(), &preview_frame_width_, - &preview_frame_height_)) { + if (!FindBestMediaType(preview_source_stream_index_, source, + base_preview_media_type_.GetAddressOf(), + GetMaxPreviewHeight(), &preview_frame_width_, + &preview_frame_height_)) { return E_FAIL; } // Find base media type for record and photo capture. - if (!FindBestMediaType( - (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_RECORD, - source, base_capture_media_type_.GetAddressOf(), 0xffffffff, nullptr, - nullptr)) { + if (!FindBestMediaType(record_source_stream_index_, source, + base_capture_media_type_.GetAddressOf(), 0xffffffff, + nullptr, nullptr)) { return E_FAIL; } @@ -524,7 +574,8 @@ void CaptureControllerImpl::StartRecord(const std::string& file_path) { // Check MF_CAPTURE_ENGINE_RECORD_STARTED event handling for response // process. hr = record_handler_->StartRecord(file_path, capture_engine_.Get(), - base_capture_media_type_.Get()); + base_capture_media_type_.Get(), + record_source_stream_index_); if (FAILED(hr)) { // Destroy record handler on error cases to make sure state is resetted. record_handler_ = nullptr; @@ -587,9 +638,8 @@ void CaptureControllerImpl::StartPreview() { } } - hr = source->SetCurrentDeviceMediaType( - (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW, - base_preview_media_type_.Get()); + hr = source->SetCurrentDeviceMediaType(preview_source_stream_index_, + base_preview_media_type_.Get()); if (FAILED(hr)) { return OnPreviewStarted(GetCameraResult(hr), "Failed to set video preview output format"); @@ -612,7 +662,8 @@ void CaptureControllerImpl::StartPreview() { // process. hr = preview_handler_->StartPreview(capture_engine_.Get(), base_preview_media_type_.Get(), - capture_engine_callback_handler_.Get()); + capture_engine_callback_handler_.Get(), + preview_source_stream_index_); if (FAILED(hr)) { // Destroy preview handler on error cases to make sure state is resetted. diff --git a/packages/camera/camera_windows/windows/capture_controller.h b/packages/camera/camera_windows/windows/capture_controller.h index 7b804bb1ebc6..0958e673709a 100644 --- a/packages/camera/camera_windows/windows/capture_controller.h +++ b/packages/camera/camera_windows/windows/capture_controller.h @@ -185,6 +185,11 @@ class CaptureControllerImpl : public CaptureController, // for a given source. HRESULT FindBaseMediaTypesForSource(IMFCaptureSource* source); + // Resolves the source stream indices for preview, record and photo, falling + // back to a physical stream for devices that reject the + // MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_* selectors. + void ResolveSourceStreamIndices(IMFCaptureSource* source); + // Stops preview. Called internally on camera reset and dispose. HRESULT StopPreview(); @@ -231,6 +236,14 @@ class CaptureControllerImpl : public CaptureController, ComPtr dx11_device_; ComPtr base_capture_media_type_; ComPtr base_preview_media_type_; + + // Resolved by ResolveSourceStreamIndices(). + DWORD preview_source_stream_index_ = static_cast( + MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW); + DWORD record_source_stream_index_ = static_cast( + MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_RECORD); + DWORD photo_source_stream_index_ = + static_cast(MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_PHOTO); ComPtr video_source_; ComPtr audio_source_; diff --git a/packages/camera/camera_windows/windows/photo_handler.cpp b/packages/camera/camera_windows/windows/photo_handler.cpp index 7544b5d25ab1..574194691384 100644 --- a/packages/camera/camera_windows/windows/photo_handler.cpp +++ b/packages/camera/camera_windows/windows/photo_handler.cpp @@ -50,7 +50,8 @@ HRESULT BuildMediaTypeForPhotoCapture(IMFMediaType* src_media_type, } HRESULT PhotoHandler::InitPhotoSink(IMFCaptureEngine* capture_engine, - IMFMediaType* base_media_type) { + IMFMediaType* base_media_type, + DWORD source_stream_index) { assert(capture_engine); assert(base_media_type); @@ -99,9 +100,8 @@ HRESULT PhotoHandler::InitPhotoSink(IMFCaptureEngine* capture_engine, } DWORD photo_sink_stream_index; - hr = photo_sink_->AddStream( - (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_PHOTO, - photo_media_type.Get(), nullptr, &photo_sink_stream_index); + hr = photo_sink_->AddStream(source_stream_index, photo_media_type.Get(), + nullptr, &photo_sink_stream_index); if (FAILED(hr)) { photo_sink_ = nullptr; return hr; @@ -118,14 +118,16 @@ HRESULT PhotoHandler::InitPhotoSink(IMFCaptureEngine* capture_engine, HRESULT PhotoHandler::TakePhoto(const std::string& file_path, IMFCaptureEngine* capture_engine, - IMFMediaType* base_media_type) { + IMFMediaType* base_media_type, + DWORD source_stream_index) { assert(!file_path.empty()); assert(capture_engine); assert(base_media_type); file_path_ = file_path; - HRESULT hr = InitPhotoSink(capture_engine, base_media_type); + HRESULT hr = + InitPhotoSink(capture_engine, base_media_type, source_stream_index); if (FAILED(hr)) { return hr; } diff --git a/packages/camera/camera_windows/windows/photo_handler.h b/packages/camera/camera_windows/windows/photo_handler.h index 60dd5d8ddf83..690a4e40b357 100644 --- a/packages/camera/camera_windows/windows/photo_handler.h +++ b/packages/camera/camera_windows/windows/photo_handler.h @@ -47,9 +47,11 @@ class PhotoHandler { // base_media_type: A pointer to base media type used as a base // for the actual photo capture media type. // file_path: A string that hold file path for photo capture. + // source_stream_index: The source stream to connect to the photo sink. HRESULT TakePhoto(const std::string& file_path, IMFCaptureEngine* capture_engine, - IMFMediaType* base_media_type); + IMFMediaType* base_media_type, + DWORD source_stream_index); // Set the photo handler recording state to: kIdle. void OnPhotoTaken(); @@ -68,7 +70,8 @@ class PhotoHandler { private: // Initializes record sink for video file capture. HRESULT InitPhotoSink(IMFCaptureEngine* capture_engine, - IMFMediaType* base_media_type); + IMFMediaType* base_media_type, + DWORD source_stream_index); std::string file_path_; PhotoState photo_state_ = PhotoState::kNotStarted; diff --git a/packages/camera/camera_windows/windows/preview_handler.cpp b/packages/camera/camera_windows/windows/preview_handler.cpp index 771f1a44e1ca..300e731bbcba 100644 --- a/packages/camera/camera_windows/windows/preview_handler.cpp +++ b/packages/camera/camera_windows/windows/preview_handler.cpp @@ -51,7 +51,7 @@ HRESULT BuildMediaTypeForVideoPreview(IMFMediaType* src_media_type, HRESULT PreviewHandler::InitPreviewSink( IMFCaptureEngine* capture_engine, IMFMediaType* base_media_type, - CaptureEngineListener* sample_callback) { + CaptureEngineListener* sample_callback, DWORD source_stream_index) { assert(capture_engine); assert(base_media_type); assert(sample_callback); @@ -94,9 +94,8 @@ HRESULT PreviewHandler::InitPreviewSink( } DWORD preview_sink_stream_index; - hr = preview_sink_->AddStream( - (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW, - preview_media_type.Get(), nullptr, &preview_sink_stream_index); + hr = preview_sink_->AddStream(source_stream_index, preview_media_type.Get(), + nullptr, &preview_sink_stream_index); if (FAILED(hr)) { return hr; @@ -115,12 +114,13 @@ HRESULT PreviewHandler::InitPreviewSink( HRESULT PreviewHandler::StartPreview(IMFCaptureEngine* capture_engine, IMFMediaType* base_media_type, - CaptureEngineListener* sample_callback) { + CaptureEngineListener* sample_callback, + DWORD source_stream_index) { assert(capture_engine); assert(base_media_type); - HRESULT hr = - InitPreviewSink(capture_engine, base_media_type, sample_callback); + HRESULT hr = InitPreviewSink(capture_engine, base_media_type, sample_callback, + source_stream_index); if (FAILED(hr)) { return hr; diff --git a/packages/camera/camera_windows/windows/preview_handler.h b/packages/camera/camera_windows/windows/preview_handler.h index b3e5557f3a9c..a965a66b3d51 100644 --- a/packages/camera/camera_windows/windows/preview_handler.h +++ b/packages/camera/camera_windows/windows/preview_handler.h @@ -52,9 +52,11 @@ class PreviewHandler { // for the actual video capture media type. // sample_callback: A pointer to capture engine listener. // This is set as sample callback for preview sink. + // source_stream_index: The source stream to connect to the preview sink. HRESULT StartPreview(IMFCaptureEngine* capture_engine, IMFMediaType* base_media_type, - CaptureEngineListener* sample_callback); + CaptureEngineListener* sample_callback, + DWORD source_stream_index); // Stops existing recording. // @@ -90,7 +92,8 @@ class PreviewHandler { // Initializes record sink for video file capture. HRESULT InitPreviewSink(IMFCaptureEngine* capture_engine, IMFMediaType* base_media_type, - CaptureEngineListener* sample_callback); + CaptureEngineListener* sample_callback, + DWORD source_stream_index); PreviewState preview_state_ = PreviewState::kNotStarted; ComPtr preview_sink_; diff --git a/packages/camera/camera_windows/windows/record_handler.cpp b/packages/camera/camera_windows/windows/record_handler.cpp index 3de5d698ca2f..d3cf925272d0 100644 --- a/packages/camera/camera_windows/windows/record_handler.cpp +++ b/packages/camera/camera_windows/windows/record_handler.cpp @@ -131,7 +131,8 @@ inline HRESULT SetAudioBitrate(IMFMediaType* pType, UINT32 bitrate) { } HRESULT RecordHandler::InitRecordSink(IMFCaptureEngine* capture_engine, - IMFMediaType* base_media_type) { + IMFMediaType* base_media_type, + DWORD source_stream_index) { assert(!file_path_.empty()); assert(capture_engine); assert(base_media_type); @@ -189,9 +190,8 @@ HRESULT RecordHandler::InitRecordSink(IMFCaptureEngine* capture_engine, } DWORD video_record_sink_stream_index; - hr = record_sink_->AddStream( - (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_RECORD, - video_record_media_type.Get(), nullptr, &video_record_sink_stream_index); + hr = record_sink_->AddStream(source_stream_index, video_record_media_type.Get(), + nullptr, &video_record_sink_stream_index); if (FAILED(hr)) { return hr; } @@ -228,7 +228,8 @@ HRESULT RecordHandler::InitRecordSink(IMFCaptureEngine* capture_engine, HRESULT RecordHandler::StartRecord(const std::string& file_path, IMFCaptureEngine* capture_engine, - IMFMediaType* base_media_type) { + IMFMediaType* base_media_type, + DWORD source_stream_index) { assert(!file_path.empty()); assert(capture_engine); assert(base_media_type); @@ -237,7 +238,8 @@ HRESULT RecordHandler::StartRecord(const std::string& file_path, recording_start_timestamp_us_ = -1; recording_duration_us_ = 0; - HRESULT hr = InitRecordSink(capture_engine, base_media_type); + HRESULT hr = + InitRecordSink(capture_engine, base_media_type, source_stream_index); if (FAILED(hr)) { return hr; } diff --git a/packages/camera/camera_windows/windows/record_handler.h b/packages/camera/camera_windows/windows/record_handler.h index 79b308558333..5371bfad7b22 100644 --- a/packages/camera/camera_windows/windows/record_handler.h +++ b/packages/camera/camera_windows/windows/record_handler.h @@ -48,9 +48,10 @@ class RecordHandler { // the actual recording. // base_media_type: A pointer to base media type used as a base // for the actual video capture media type. + // source_stream_index: The source stream to connect to the record sink. HRESULT StartRecord(const std::string& file_path, IMFCaptureEngine* capture_engine, - IMFMediaType* base_media_type); + IMFMediaType* base_media_type, DWORD source_stream_index); // Stops existing recording. // @@ -83,7 +84,8 @@ class RecordHandler { private: // Initializes record sink for video file capture. HRESULT InitRecordSink(IMFCaptureEngine* capture_engine, - IMFMediaType* base_media_type); + IMFMediaType* base_media_type, + DWORD source_stream_index); const PlatformMediaSettings media_settings_; int64_t recording_start_timestamp_us_ = -1; diff --git a/packages/camera/camera_windows/windows/test/capture_controller_test.cpp b/packages/camera/camera_windows/windows/test/capture_controller_test.cpp index 442f405910bb..97a2ef8e6499 100644 --- a/packages/camera/camera_windows/windows/test/capture_controller_test.cpp +++ b/packages/camera/camera_windows/windows/test/capture_controller_test.cpp @@ -567,6 +567,122 @@ TEST(CaptureController, StartPreviewStartsProcessingSamples) { texture_registrar = nullptr; } +TEST(CaptureController, StartPreviewFallsBackToPhysicalStreamWhenSelectorRejected) { + ComPtr engine = new MockCaptureEngine(); + std::unique_ptr camera = + std::make_unique(MOCK_DEVICE_ID); + std::unique_ptr capture_controller = + std::make_unique(camera.get()); + std::unique_ptr texture_registrar = + std::make_unique(); + + int64_t mock_texture_id = 1234; + + MockInitCaptureController(capture_controller.get(), texture_registrar.get(), + engine.Get(), camera.get(), mock_texture_id); + + ComPtr capture_source = new MockCaptureSource(); + EXPECT_CALL(*engine.Get(), GetSource) + .Times(1) + .WillOnce([src_source = capture_source.Get()]( + IMFCaptureSource** target_source) { + *target_source = src_source; + src_source->AddRef(); + return S_OK; + }); + + const DWORD kPhysicalStreamIndex = 1; + uint32_t mock_preview_width = 2; + uint32_t mock_preview_height = 1; + + // The device rejects the preferred-stream selectors... + EXPECT_CALL( + *capture_source.Get(), + GetAvailableDeviceMediaType( + Eq((DWORD) + MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW), + _, _)) + .WillRepeatedly(Return(MF_E_INVALIDSTREAMNUMBER)); + EXPECT_CALL( + *capture_source.Get(), + GetAvailableDeviceMediaType( + Eq((DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_RECORD), + _, _)) + .WillRepeatedly(Return(MF_E_INVALIDSTREAMNUMBER)); + + // ...so it falls back to the stream reported as video-capture. + EXPECT_CALL(*capture_source.Get(), GetDeviceStreamCount) + .WillRepeatedly([](DWORD* count) { + *count = 2; + return S_OK; + }); + EXPECT_CALL(*capture_source.Get(), GetDeviceStreamCategory(Eq(0u), _)) + .WillRepeatedly([](DWORD, MF_CAPTURE_ENGINE_STREAM_CATEGORY* category) { + *category = MF_CAPTURE_ENGINE_STREAM_CATEGORY_PHOTO_DEPENDENT; + return S_OK; + }); + EXPECT_CALL(*capture_source.Get(), + GetDeviceStreamCategory(Eq(kPhysicalStreamIndex), _)) + .WillRepeatedly([](DWORD, MF_CAPTURE_ENGINE_STREAM_CATEGORY* category) { + *category = MF_CAPTURE_ENGINE_STREAM_CATEGORY_VIDEO_CAPTURE; + return S_OK; + }); + EXPECT_CALL(*capture_source.Get(), + GetAvailableDeviceMediaType(Eq(kPhysicalStreamIndex), _, _)) + .WillRepeatedly([mock_preview_width, mock_preview_height]( + DWORD, DWORD media_type_index, + IMFMediaType** media_type) { + if (media_type_index != 0) return MF_E_NO_MORE_TYPES; + *media_type = + new FakeMediaType(MFMediaType_Video, MFVideoFormat_RGB32, + mock_preview_width, mock_preview_height); + (*media_type)->AddRef(); + return S_OK; + }); + + // The resolved physical stream must be used from here on, not the rejected + // selector. + EXPECT_CALL(*capture_source.Get(), + SetCurrentDeviceMediaType(Eq(kPhysicalStreamIndex), _)) + .Times(1) + .WillOnce(Return(S_OK)); + + ComPtr preview_sink = new MockCapturePreviewSink(); + EXPECT_CALL(*engine.Get(), GetSink(MF_CAPTURE_ENGINE_SINK_TYPE_PREVIEW, _)) + .Times(1) + .WillOnce([src_sink = preview_sink.Get()]( + MF_CAPTURE_ENGINE_SINK_TYPE, + IMFCaptureSink** target_sink) { + *target_sink = src_sink; + src_sink->AddRef(); + return S_OK; + }); + EXPECT_CALL(*preview_sink.Get(), RemoveAllStreams) + .Times(1) + .WillOnce(Return(S_OK)); + EXPECT_CALL(*preview_sink.Get(), AddStream(Eq(kPhysicalStreamIndex), _, _, _)) + .Times(1) + .WillOnce([](DWORD, IMFMediaType*, IMFAttributes*, + DWORD* sink_stream_index) { + *sink_stream_index = 0; + return S_OK; + }); + EXPECT_CALL(*preview_sink.Get(), SetSampleCallback) + .Times(1) + .WillOnce(Return(S_OK)); + + EXPECT_CALL(*engine.Get(), StartPreview()).Times(1).WillOnce(Return(S_OK)); + EXPECT_CALL(*engine.Get(), StopPreview()).Times(1).WillOnce(Return(S_OK)); + EXPECT_CALL(*camera, OnStartPreviewFailed).Times(0); + + capture_controller->StartPreview(); + + capture_controller = nullptr; + engine = nullptr; + camera = nullptr; + texture_registrar = nullptr; +} + TEST(CaptureController, ReportsStartPreviewError) { ComPtr engine = new MockCaptureEngine(); std::unique_ptr camera =