[camera_windows] Fall back to a physical stream for cameras that reject preferred-stream selectors

packages/camera/camera_windows/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.2.6+6
+
+* Fixes `CameraException(camera_error, Failed to initialize video preview)` on
+  multi-stream USB cameras.
+
 ## 0.2.6+5
 
 * Updates pigeon dev_dependency to ^27.3.2 for analyzer 14 compatibility.

packages/camera/camera_windows/pubspec.yaml

@@ -2,7 +2,7 @@ name: camera_windows
 description: A Flutter plugin for getting information about and controlling the camera on Windows.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_windows
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.2.6+5
+version: 0.2.6+6
 
 environment:
   sdk: ^3.10.0

packages/camera/camera_windows/windows/capture_controller.cpp

@@ -284,6 +284,12 @@ void CaptureControllerImpl::ResetCaptureController() {
   video_source_ = nullptr;
   base_preview_media_type_ = nullptr;
   base_capture_media_type_ = nullptr;
+  preview_source_stream_index_ = static_cast<DWORD>(
+      MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW);
+  record_source_stream_index_ = static_cast<DWORD>(
+      MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_RECORD);
+  photo_source_stream_index_ =
+      static_cast<DWORD>(MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_PHOTO);
 
   if (dxgi_device_manager_) {
     dxgi_device_manager_->ResetDevice(dx11_device_.Get(),
@@ -371,7 +377,8 @@ void CaptureControllerImpl::TakePicture(const std::string& file_path) {
   // Check MF_CAPTURE_ENGINE_PHOTO_TAKEN event handling
   // for response process.
   hr = photo_handler_->TakePhoto(file_path, capture_engine_.Get(),
-                                 base_capture_media_type_.Get());
+                                 base_capture_media_type_.Get(),
+                                 photo_source_stream_index_);
   if (FAILED(hr)) {
     // Destroy photo handler on error cases to make sure state is resetted.
     photo_handler_ = nullptr;
@@ -470,22 +477,65 @@ HRESULT CaptureControllerImpl::FindBaseMediaTypes() {
   return FindBaseMediaTypesForSource(source.Get());
 }
 
+void CaptureControllerImpl::ResolveSourceStreamIndices(
+    IMFCaptureSource* source) {
+  // Devices that don't support the preferred-stream selectors fail this probe,
+  // typically with MF_E_INVALIDSTREAMNUMBER.
+  ComPtr<IMFMediaType> probe;
+  HRESULT hr = source->GetAvailableDeviceMediaType(
+      preview_source_stream_index_, 0, probe.GetAddressOf());
+  if (SUCCEEDED(hr)) {
+    return;
+  }
+
+  DWORD stream_count = 0;
+  if (FAILED(source->GetDeviceStreamCount(&stream_count))) {
+    return;
+  }
+
+  // Prefer a video-preview stream, then video-capture.
+  DWORD video_stream_index = MAXDWORD;
+  for (DWORD i = 0; i < stream_count; i++) {
+    MF_CAPTURE_ENGINE_STREAM_CATEGORY category;
+    if (FAILED(source->GetDeviceStreamCategory(i, &category))) {
+      continue;
+    }
+    if (category == MF_CAPTURE_ENGINE_STREAM_CATEGORY_VIDEO_PREVIEW) {
+      video_stream_index = i;
+      break;
+    }
+    if (category == MF_CAPTURE_ENGINE_STREAM_CATEGORY_VIDEO_CAPTURE &&
+        video_stream_index == MAXDWORD) {
+      video_stream_index = i;
+    }
+  }
+  if (video_stream_index == MAXDWORD) {
+    // No matching stream found.
+    return;
+  }
+
+  // One stream for every role keeps the media types consistent with the sinks.
+  preview_source_stream_index_ = video_stream_index;
+  record_source_stream_index_ = video_stream_index;
+  photo_source_stream_index_ = video_stream_index;
+}
+
 HRESULT CaptureControllerImpl::FindBaseMediaTypesForSource(
     IMFCaptureSource* source) {
+  ResolveSourceStreamIndices(source);
+
   // Find base media type for previewing.
-  if (!FindBestMediaType(
-          (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW,
-          source, base_preview_media_type_.GetAddressOf(),
-          GetMaxPreviewHeight(), &preview_frame_width_,
-          &preview_frame_height_)) {
+  if (!FindBestMediaType(preview_source_stream_index_, source,
+                         base_preview_media_type_.GetAddressOf(),
+                         GetMaxPreviewHeight(), &preview_frame_width_,
+                         &preview_frame_height_)) {
     return E_FAIL;
   }
 
   // Find base media type for record and photo capture.
-  if (!FindBestMediaType(
-          (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_RECORD,
-          source, base_capture_media_type_.GetAddressOf(), 0xffffffff, nullptr,
-          nullptr)) {
+  if (!FindBestMediaType(record_source_stream_index_, source,
+                         base_capture_media_type_.GetAddressOf(), 0xffffffff,
+                         nullptr, nullptr)) {
     return E_FAIL;
   }
 
@@ -524,7 +574,8 @@ void CaptureControllerImpl::StartRecord(const std::string& file_path) {
   // Check MF_CAPTURE_ENGINE_RECORD_STARTED event handling for response
   // process.
   hr = record_handler_->StartRecord(file_path, capture_engine_.Get(),
-                                    base_capture_media_type_.Get());
+                                    base_capture_media_type_.Get(),
+                                    record_source_stream_index_);
   if (FAILED(hr)) {
     // Destroy record handler on error cases to make sure state is resetted.
     record_handler_ = nullptr;
@@ -587,9 +638,8 @@ void CaptureControllerImpl::StartPreview() {
     }
   }
 
-  hr = source->SetCurrentDeviceMediaType(
-      (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW,
-      base_preview_media_type_.Get());
+  hr = source->SetCurrentDeviceMediaType(preview_source_stream_index_,
+                                         base_preview_media_type_.Get());
   if (FAILED(hr)) {
     return OnPreviewStarted(GetCameraResult(hr),
                             "Failed to set video preview output format");
@@ -612,7 +662,8 @@ void CaptureControllerImpl::StartPreview() {
   // process.
   hr = preview_handler_->StartPreview(capture_engine_.Get(),
                                       base_preview_media_type_.Get(),
-                                      capture_engine_callback_handler_.Get());
+                                      capture_engine_callback_handler_.Get(),
+                                      preview_source_stream_index_);
 
   if (FAILED(hr)) {
     // Destroy preview handler on error cases to make sure state is resetted.

packages/camera/camera_windows/windows/capture_controller.h

@@ -185,6 +185,11 @@ class CaptureControllerImpl : public CaptureController,
   // for a given source.
   HRESULT FindBaseMediaTypesForSource(IMFCaptureSource* source);
 
+  // Resolves the source stream indices for preview, record and photo, falling
+  // back to a physical stream for devices that reject the
+  // MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_* selectors.
+  void ResolveSourceStreamIndices(IMFCaptureSource* source);
+
   // Stops preview. Called internally on camera reset and dispose.
   HRESULT StopPreview();
 
@@ -231,6 +236,14 @@ class CaptureControllerImpl : public CaptureController,
   ComPtr<ID3D11Device> dx11_device_;
   ComPtr<IMFMediaType> base_capture_media_type_;
   ComPtr<IMFMediaType> base_preview_media_type_;
+
+  // Resolved by ResolveSourceStreamIndices().
+  DWORD preview_source_stream_index_ = static_cast<DWORD>(
+      MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW);
+  DWORD record_source_stream_index_ = static_cast<DWORD>(
+      MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_RECORD);
+  DWORD photo_source_stream_index_ =
+      static_cast<DWORD>(MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_PHOTO);
   ComPtr<IMFMediaSource> video_source_;
   ComPtr<IMFMediaSource> audio_source_;
 

packages/camera/camera_windows/windows/photo_handler.cpp

@@ -50,7 +50,8 @@ HRESULT BuildMediaTypeForPhotoCapture(IMFMediaType* src_media_type,
 }
 
 HRESULT PhotoHandler::InitPhotoSink(IMFCaptureEngine* capture_engine,
-                                    IMFMediaType* base_media_type) {
+                                    IMFMediaType* base_media_type,
+                                    DWORD source_stream_index) {
   assert(capture_engine);
   assert(base_media_type);
 
@@ -99,9 +100,8 @@ HRESULT PhotoHandler::InitPhotoSink(IMFCaptureEngine* capture_engine,
   }
 
   DWORD photo_sink_stream_index;
-  hr = photo_sink_->AddStream(
-      (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_PHOTO,
-      photo_media_type.Get(), nullptr, &photo_sink_stream_index);
+  hr = photo_sink_->AddStream(source_stream_index, photo_media_type.Get(),
+                              nullptr, &photo_sink_stream_index);
   if (FAILED(hr)) {
     photo_sink_ = nullptr;
     return hr;
@@ -118,14 +118,16 @@ HRESULT PhotoHandler::InitPhotoSink(IMFCaptureEngine* capture_engine,
 
 HRESULT PhotoHandler::TakePhoto(const std::string& file_path,
                                 IMFCaptureEngine* capture_engine,
-                                IMFMediaType* base_media_type) {
+                                IMFMediaType* base_media_type,
+                                DWORD source_stream_index) {
   assert(!file_path.empty());
   assert(capture_engine);
   assert(base_media_type);
 
   file_path_ = file_path;
 
-  HRESULT hr = InitPhotoSink(capture_engine, base_media_type);
+  HRESULT hr =
+      InitPhotoSink(capture_engine, base_media_type, source_stream_index);
   if (FAILED(hr)) {
     return hr;
   }

packages/camera/camera_windows/windows/photo_handler.h

@@ -47,9 +47,11 @@ class PhotoHandler {
   // base_media_type: A pointer to base media type used as a base
   //                  for the actual photo capture media type.
   // file_path:       A string that hold file path for photo capture.
+  // source_stream_index: The source stream to connect to the photo sink.
   HRESULT TakePhoto(const std::string& file_path,
                     IMFCaptureEngine* capture_engine,
-                    IMFMediaType* base_media_type);
+                    IMFMediaType* base_media_type,
+                    DWORD source_stream_index);
 
   // Set the photo handler recording state to: kIdle.
   void OnPhotoTaken();
@@ -68,7 +70,8 @@ class PhotoHandler {
  private:
   // Initializes record sink for video file capture.
   HRESULT InitPhotoSink(IMFCaptureEngine* capture_engine,
-                        IMFMediaType* base_media_type);
+                        IMFMediaType* base_media_type,
+                        DWORD source_stream_index);
 
   std::string file_path_;
   PhotoState photo_state_ = PhotoState::kNotStarted;

packages/camera/camera_windows/windows/preview_handler.cpp

@@ -51,7 +51,7 @@ HRESULT BuildMediaTypeForVideoPreview(IMFMediaType* src_media_type,
 
 HRESULT PreviewHandler::InitPreviewSink(
     IMFCaptureEngine* capture_engine, IMFMediaType* base_media_type,
-    CaptureEngineListener* sample_callback) {
+    CaptureEngineListener* sample_callback, DWORD source_stream_index) {
   assert(capture_engine);
   assert(base_media_type);
   assert(sample_callback);
@@ -94,9 +94,8 @@ HRESULT PreviewHandler::InitPreviewSink(
   }
 
   DWORD preview_sink_stream_index;
-  hr = preview_sink_->AddStream(
-      (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_PREVIEW,
-      preview_media_type.Get(), nullptr, &preview_sink_stream_index);
+  hr = preview_sink_->AddStream(source_stream_index, preview_media_type.Get(),
+                                nullptr, &preview_sink_stream_index);
 
   if (FAILED(hr)) {
     return hr;
@@ -115,12 +114,13 @@ HRESULT PreviewHandler::InitPreviewSink(
 
 HRESULT PreviewHandler::StartPreview(IMFCaptureEngine* capture_engine,
                                      IMFMediaType* base_media_type,
-                                     CaptureEngineListener* sample_callback) {
+                                     CaptureEngineListener* sample_callback,
+                                     DWORD source_stream_index) {
   assert(capture_engine);
   assert(base_media_type);
 
-  HRESULT hr =
-      InitPreviewSink(capture_engine, base_media_type, sample_callback);
+  HRESULT hr = InitPreviewSink(capture_engine, base_media_type, sample_callback,
+                               source_stream_index);
 
   if (FAILED(hr)) {
     return hr;

packages/camera/camera_windows/windows/preview_handler.h

@@ -52,9 +52,11 @@ class PreviewHandler {
   //                  for the actual video capture media type.
   // sample_callback: A pointer to capture engine listener.
   //                  This is set as sample callback for preview sink.
+  // source_stream_index: The source stream to connect to the preview sink.
   HRESULT StartPreview(IMFCaptureEngine* capture_engine,
                        IMFMediaType* base_media_type,
-                       CaptureEngineListener* sample_callback);
+                       CaptureEngineListener* sample_callback,
+                       DWORD source_stream_index);
 
   // Stops existing recording.
   //
@@ -90,7 +92,8 @@ class PreviewHandler {
   // Initializes record sink for video file capture.
   HRESULT InitPreviewSink(IMFCaptureEngine* capture_engine,
                           IMFMediaType* base_media_type,
-                          CaptureEngineListener* sample_callback);
+                          CaptureEngineListener* sample_callback,
+                          DWORD source_stream_index);
 
   PreviewState preview_state_ = PreviewState::kNotStarted;
   ComPtr<IMFCapturePreviewSink> preview_sink_;

packages/camera/camera_windows/windows/record_handler.cpp

@@ -131,7 +131,8 @@ inline HRESULT SetAudioBitrate(IMFMediaType* pType, UINT32 bitrate) {
 }
 
 HRESULT RecordHandler::InitRecordSink(IMFCaptureEngine* capture_engine,
-                                      IMFMediaType* base_media_type) {
+                                      IMFMediaType* base_media_type,
+                                      DWORD source_stream_index) {
   assert(!file_path_.empty());
   assert(capture_engine);
   assert(base_media_type);
@@ -189,9 +190,8 @@ HRESULT RecordHandler::InitRecordSink(IMFCaptureEngine* capture_engine,
   }
 
   DWORD video_record_sink_stream_index;
-  hr = record_sink_->AddStream(
-      (DWORD)MF_CAPTURE_ENGINE_PREFERRED_SOURCE_STREAM_FOR_VIDEO_RECORD,
-      video_record_media_type.Get(), nullptr, &video_record_sink_stream_index);
+  hr = record_sink_->AddStream(source_stream_index, video_record_media_type.Get(),
+                               nullptr, &video_record_sink_stream_index);
   if (FAILED(hr)) {
     return hr;
   }
@@ -228,7 +228,8 @@ HRESULT RecordHandler::InitRecordSink(IMFCaptureEngine* capture_engine,
 
 HRESULT RecordHandler::StartRecord(const std::string& file_path,
                                    IMFCaptureEngine* capture_engine,
-                                   IMFMediaType* base_media_type) {
+                                   IMFMediaType* base_media_type,
+                                   DWORD source_stream_index) {
   assert(!file_path.empty());
   assert(capture_engine);
   assert(base_media_type);
@@ -237,7 +238,8 @@ HRESULT RecordHandler::StartRecord(const std::string& file_path,
   recording_start_timestamp_us_ = -1;
   recording_duration_us_ = 0;
 
-  HRESULT hr = InitRecordSink(capture_engine, base_media_type);
+  HRESULT hr =
+      InitRecordSink(capture_engine, base_media_type, source_stream_index);
   if (FAILED(hr)) {
     return hr;
   }

packages/camera/camera_windows/windows/record_handler.h

@@ -48,9 +48,10 @@ class RecordHandler {
   //                  the actual recording.
   // base_media_type: A pointer to base media type used as a base
   //                  for the actual video capture media type.
+  // source_stream_index: The source stream to connect to the record sink.
   HRESULT StartRecord(const std::string& file_path,
                       IMFCaptureEngine* capture_engine,
-                      IMFMediaType* base_media_type);
+                      IMFMediaType* base_media_type, DWORD source_stream_index);
 
   // Stops existing recording.
   //
@@ -83,7 +84,8 @@ class RecordHandler {
  private:
   // Initializes record sink for video file capture.
   HRESULT InitRecordSink(IMFCaptureEngine* capture_engine,
-                         IMFMediaType* base_media_type);
+                         IMFMediaType* base_media_type,
+                         DWORD source_stream_index);
 
   const PlatformMediaSettings media_settings_;
   int64_t recording_start_timestamp_us_ = -1;