From dea547be73f5ad6d9bef56e179ac5caf210a6628 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Wed, 20 Oct 2021 17:00:41 +0800 Subject: [PATCH 01/30] Enable IC background blur feature --- build_overrides/owt.gni | 5 + scripts/build-win.py | 23 ++--- talk/owt/BUILD.gn | 11 ++- talk/owt/sdk/base/cameravideocapturer.cc | 25 ++++- talk/owt/sdk/base/cameravideocapturer.h | 8 ++ talk/owt/sdk/base/icmanager.cc | 39 ++++++++ talk/owt/sdk/base/icmanager.h | 36 +++++++ .../sdk/base/localcamerastreamparameters.cc | 3 + .../base/peerconnectiondependencyfactory.cc | 1 + talk/owt/sdk/base/stream.cc | 11 ++- talk/owt/sdk/base/videoframepostprocessing.h | 23 +++++ talk/owt/sdk/ic/BUILD.gn | 93 +++++++++++++++++++ talk/owt/sdk/ic/backgroundblur.cc | 61 ++++++++++++ talk/owt/sdk/ic/backgroundblur.h | 28 ++++++ talk/owt/sdk/ic/postprocessing.cc | 14 +++ talk/owt/sdk/ic/postprocessing.h | 15 +++ talk/owt/sdk/ic/selfiesegmentation.cc | 62 +++++++++++++ talk/owt/sdk/ic/selfiesegmentation.h | 35 +++++++ talk/owt/sdk/ic/test_background_blur.cc | 60 ++++++++++++ .../owt/base/localcamerastreamparameters.h | 7 ++ thirdpartylicense.txt | 18 ++++ 21 files changed, 558 insertions(+), 20 deletions(-) create mode 100644 build_overrides/owt.gni create mode 100644 talk/owt/sdk/base/icmanager.cc create mode 100644 talk/owt/sdk/base/icmanager.h create mode 100644 talk/owt/sdk/base/videoframepostprocessing.h create mode 100644 talk/owt/sdk/ic/BUILD.gn create mode 100644 talk/owt/sdk/ic/backgroundblur.cc create mode 100644 talk/owt/sdk/ic/backgroundblur.h create mode 100644 talk/owt/sdk/ic/postprocessing.cc create mode 100644 talk/owt/sdk/ic/postprocessing.h create mode 100644 talk/owt/sdk/ic/selfiesegmentation.cc create mode 100644 talk/owt/sdk/ic/selfiesegmentation.h create mode 100644 talk/owt/sdk/ic/test_background_blur.cc diff --git a/build_overrides/owt.gni b/build_overrides/owt.gni new file mode 100644 index 000000000..ae8286b0d --- /dev/null +++ b/build_overrides/owt.gni @@ -0,0 +1,5 @@ +declare_args() { + owt_msdk_lib_root = "" + owt_msdk_header_root = "" + owt_build_ic = false +} diff --git a/scripts/build-win.py b/scripts/build-win.py index 4b6d77eb1..ad68f727e 100644 --- a/scripts/build-win.py +++ b/scripts/build-win.py @@ -35,7 +35,7 @@ ] -def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests): +def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests): gn_args = list(GN_ARGS) gn_args.append('target_cpu="%s"' % arch) using_llvm = False @@ -76,6 +76,9 @@ def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests): else: return False gn_args.append('owt_use_quic=true') + if openvino_root: + gn_args.append(f'owt_build_ic=true') + gn_args.append(f'owt_openvino_root="{openvino_root}"') if tests: gn_args.append('rtc_include_tests=true') gn_args.append('owt_include_tests=true') @@ -183,6 +186,7 @@ def main(): parser.add_argument('--ssl_root', help='Path for OpenSSL.') parser.add_argument('--msdk_root', help='Path for MSDK.') parser.add_argument('--quic_root', help='Path to QUIC library') + parser.add_argument('--openvino_root', default="", help='Path for OpenVINO.') parser.add_argument('--scheme', default='debug', choices=('debug', 'release'), help='Schemes for building. Supported value: debug, release') parser.add_argument('--gn_gen', default=False, action='store_true', @@ -195,19 +199,16 @@ def main(): help='To generate the API document.') parser.add_argument('--output_path', help='Path to copy sdk.') opts = parser.parse_args() - if opts.ssl_root and not os.path.exists(os.path.expanduser(opts.ssl_root)): - print('Invalid ssl_root.') - return 1 - if opts.msdk_root and not os.path.exists(os.path.expanduser(opts.msdk_root)): - print('Invalid msdk_root') - return 1 - if opts.quic_root and not os.path.exists(os.path.expanduser(opts.quic_root)): - print('Invalid quic_root') - return 1 + for name in ['ssl', 'msdk', 'quic', 'openvino']: + attr = name + '_root' + path = getattr(opts, attr) + if path is not None and not os.path.exists(os.path.expanduser(path)): + print('Invalid {}.'.format(attr)) + return 1 print(opts) if opts.gn_gen: if not gngen(opts.arch, opts.ssl_root, opts.msdk_root, opts.quic_root, - opts.scheme, opts.tests): + opts.openvino_root, opts.scheme, opts.tests): return 1 if opts.sdk: if not ninjabuild(opts.arch, opts.scheme): diff --git a/talk/owt/BUILD.gn b/talk/owt/BUILD.gn index 792b2dd88..ee29cf81c 100644 --- a/talk/owt/BUILD.gn +++ b/talk/owt/BUILD.gn @@ -3,12 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 import("//build_overrides/webrtc.gni") +import("//build_overrides/owt.gni") import("//testing/test.gni") declare_args() { include_internal_audio_device = true - owt_msdk_lib_root = "" - owt_msdk_header_root = "" } # Introduced for using libvpx config files. We only enable libvpx rate @@ -83,6 +82,11 @@ if (!is_ios) { "//third_party/webrtc:webrtc", "//third_party/webrtc/api:libjingle_peerconnection_api", ] + if (owt_build_ic) { + data_deps = [ + "sdk/ic:owt_ic", + ] + } complete_static_lib = true } } @@ -99,6 +103,8 @@ static_library("owt_sdk_base") { "sdk/base/functionalobserver.cc", "sdk/base/functionalobserver.h", "sdk/base/globalconfiguration.cc", + "sdk/base/icmanager.cc", + "sdk/base/icmanager.h", "sdk/base/localcamerastreamparameters.cc", "sdk/base/logging.cc", "sdk/base/logsinks.cc", @@ -118,6 +124,7 @@ static_library("owt_sdk_base") { "sdk/base/sysinfo.h", "sdk/base/vcmcapturer.cc", "sdk/base/vcmcapturer.h", + "sdk/base/videoframepostprocessing.h", "sdk/base/webrtcaudiorendererimpl.cc", "sdk/base/webrtcaudiorendererimpl.h", "sdk/include/cpp/owt/base/audioplayerinterface.h", diff --git a/talk/owt/sdk/base/cameravideocapturer.cc b/talk/owt/sdk/base/cameravideocapturer.cc index 62f270666..22a0d8d7e 100644 --- a/talk/owt/sdk/base/cameravideocapturer.cc +++ b/talk/owt/sdk/base/cameravideocapturer.cc @@ -36,18 +36,28 @@ void CameraVideoCapturer::OnFrame(const webrtc::VideoFrame& frame) { return; } + rtc::scoped_refptr buffer = frame.video_frame_buffer(); + if (out_height != frame.height() || out_width != frame.width()) { // Video adapter has requested a down-scale. Allocate a new buffer and // return scaled version. rtc::scoped_refptr scaled_buffer = webrtc::I420Buffer::Create(out_width, out_height); - scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); + scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); + buffer = scaled_buffer; + } + + for (auto &pp : video_frame_post_processings_) { + buffer = pp->Process(buffer); + } + + if (buffer != frame.video_frame_buffer()) { broadcaster_.OnFrame(webrtc::VideoFrame::Builder() - .set_video_frame_buffer(scaled_buffer) + .set_video_frame_buffer(buffer) .set_rotation(webrtc::kVideoRotation_0) .set_timestamp_us(frame.timestamp_us()) .set_id(frame.id()) - .build()); + .build()); } else { // No adaptations needed, just return the frame as is. broadcaster_.OnFrame(frame); @@ -71,9 +81,14 @@ void CameraVideoCapturer::RemoveSink( UpdateVideoAdapter(); } +void CameraVideoCapturer::AddVideoFramePostProcessing( + std::shared_ptr post_processing) { + video_frame_post_processings_.emplace_back(post_processing); +} + void CameraVideoCapturer::UpdateVideoAdapter() { video_adapter_.OnSinkWants(broadcaster_.wants()); } -} // namespace test -} // namespace webrtc +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/base/cameravideocapturer.h b/talk/owt/sdk/base/cameravideocapturer.h index 0941190fe..c8aa32771 100644 --- a/talk/owt/sdk/base/cameravideocapturer.h +++ b/talk/owt/sdk/base/cameravideocapturer.h @@ -13,12 +13,15 @@ #include #include +#include #include "api/video/video_frame.h" #include "api/video/video_source_interface.h" #include "media/base/video_adapter.h" #include "media/base/video_broadcaster.h" +#include "videoframepostprocessing.h" + // This file is borrowed from webrtc project namespace owt { namespace base { @@ -32,6 +35,9 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface const rtc::VideoSinkWants& wants) override; void RemoveSink(rtc::VideoSinkInterface* sink) override; + void AddVideoFramePostProcessing( + const std::shared_ptr post_processing); + protected: void OnFrame(const webrtc::VideoFrame& frame); rtc::VideoSinkWants GetSinkWants(); @@ -41,6 +47,8 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface rtc::VideoBroadcaster broadcaster_; cricket::VideoAdapter video_adapter_; + std::vector> + video_frame_post_processings_; }; } // namespace base } // namespace owt diff --git a/talk/owt/sdk/base/icmanager.cc b/talk/owt/sdk/base/icmanager.cc new file mode 100644 index 000000000..63027eabc --- /dev/null +++ b/talk/owt/sdk/base/icmanager.cc @@ -0,0 +1,39 @@ +#include "icmanager.h" + +#include "webrtc/rtc_base/logging.h" + +namespace owt { +namespace base { + +ICManager* ICManager::GetInstance() { + static ICManager instance; + return &instance; +} + +std::shared_ptr ICManager::CreatePostProcessing( + const char* name) { + return create_post_processing_ + ? std::shared_ptr( + create_post_processing_(name), + [](VideoFramePostProcessing* ptr) { ptr->Release(); }) + : nullptr; +} + +ICManager::ICManager() { + if (owt_ic_dll_ = LoadLibrary(L"owt_ic.dll")) { + RTC_LOG(INFO) << "owt_ic.dll is loaded."; + create_post_processing_ = + (CREATE_POST_PROCESSING)GetProcAddress(owt_ic_dll_, "CreatePostProcessing"); + } else { + RTC_LOG(WARNING) << "owt_ic.dll is not loaded."; + } +} + +ICManager::~ICManager() { + if (owt_ic_dll_) { + FreeLibrary(owt_ic_dll_); + } +} + +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/base/icmanager.h b/talk/owt/sdk/base/icmanager.h new file mode 100644 index 000000000..87acf7de0 --- /dev/null +++ b/talk/owt/sdk/base/icmanager.h @@ -0,0 +1,36 @@ +#ifndef OWT_BASE_ICMANAGER_H_ +#define OWT_BASE_ICMANAGER_H_ + +#include +#include + +#include "api/scoped_refptr.h" +#include "base/macros.h" +#include "videoframepostprocessing.h" + +namespace owt { +namespace base { + +class ICManager { + public: + static ICManager* GetInstance(); + std::shared_ptr CreatePostProcessing( + const char* name); + + private: + ICManager(); + ~ICManager(); + + typedef owt::base::VideoFramePostProcessing* (*CREATE_POST_PROCESSING)( + const char* name); + + HMODULE owt_ic_dll_ = nullptr; + CREATE_POST_PROCESSING create_post_processing_ = nullptr; + + DISALLOW_COPY_AND_ASSIGN(ICManager); +}; + +} // namespace base +} // namespace owt + +#endif // OWT_BASE_ICMANAGER_H_ \ No newline at end of file diff --git a/talk/owt/sdk/base/localcamerastreamparameters.cc b/talk/owt/sdk/base/localcamerastreamparameters.cc index ed40131c6..4e3be76fc 100644 --- a/talk/owt/sdk/base/localcamerastreamparameters.cc +++ b/talk/owt/sdk/base/localcamerastreamparameters.cc @@ -19,6 +19,9 @@ LocalCameraStreamParameters::LocalCameraStreamParameters(bool audio_enabled, void LocalCameraStreamParameters::Fps(int fps) { fps_ = fps; } +void LocalCameraStreamParameters::BackgroundBlur(bool enable) { + background_blur_ = enable; +} void LocalCameraStreamParameters::CameraId(const std::string& camera_id) { camera_id_ = camera_id; } diff --git a/talk/owt/sdk/base/peerconnectiondependencyfactory.cc b/talk/owt/sdk/base/peerconnectiondependencyfactory.cc index cf9e3210a..b9cd097c7 100644 --- a/talk/owt/sdk/base/peerconnectiondependencyfactory.cc +++ b/talk/owt/sdk/base/peerconnectiondependencyfactory.cc @@ -47,6 +47,7 @@ #pragma comment(lib, "wmcodecdspuuid.lib") #pragma comment(lib, "amstrmid.lib") #pragma comment(lib, "strmiids.lib") +#pragma comment(lib, "dcomp.lib") #ifdef OWT_USE_MSDK #pragma comment(lib, "mf.lib") #pragma comment(lib, "mfplat.lib") diff --git a/talk/owt/sdk/base/stream.cc b/talk/owt/sdk/base/stream.cc index 4f9a7872e..c03ef075e 100644 --- a/talk/owt/sdk/base/stream.cc +++ b/talk/owt/sdk/base/stream.cc @@ -11,8 +11,10 @@ #include "webrtc/sdk/media_constraints.h" #include "talk/owt/sdk/base/customizedframescapturer.h" +#include "talk/owt/sdk/base/icmanager.h" #include "talk/owt/sdk/base/webrtcaudiorendererimpl.h" #include "talk/owt/sdk/base/webrtcvideorendererimpl.h" +#include "talk/owt/sdk/ic/backgroundblur.h" #include "talk/owt/sdk/include/cpp/owt/base/framegeneratorinterface.h" #if defined(WEBRTC_WIN) @@ -45,7 +47,8 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { const size_t width, const size_t height, const size_t fps, - int capture_device_idx) { + int capture_device_idx, + bool background_blur_enabled = false) { std::unique_ptr capturer; std::unique_ptr info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -56,6 +59,9 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { for (int i = 0; i < num_devices; ++i) { capturer = absl::WrapUnique(owt::base::VcmCapturer::Create( width, height, fps, capture_device_idx)); + if (background_blur_enabled) { + capturer->AddVideoFramePostProcessing(ICManager::GetInstance()->CreatePostProcessing("background_blur")); + } if (capturer) { return new rtc::RefCountedObject( std::move(capturer)); @@ -513,7 +519,8 @@ LocalStream::LocalStream(const LocalCameraStreamParameters& parameters, CapturerTrackSource::Create( parameters.ResolutionWidth(), parameters.ResolutionHeight(), parameters.Fps(), - DeviceUtils::GetVideoCaptureDeviceIndex(parameters.CameraId())); + DeviceUtils::GetVideoCaptureDeviceIndex(parameters.CameraId()), + parameters.BackgroundBlur()); #else capturer_ = ObjcVideoCapturerFactory::Create(parameters); if (!capturer_) { diff --git a/talk/owt/sdk/base/videoframepostprocessing.h b/talk/owt/sdk/base/videoframepostprocessing.h new file mode 100644 index 000000000..e803c3a57 --- /dev/null +++ b/talk/owt/sdk/base/videoframepostprocessing.h @@ -0,0 +1,23 @@ +#ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_ +#define OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_ + +#include "api/scoped_refptr.h" +#include "api/video/video_frame_buffer.h" + +namespace owt { +namespace base { + +class VideoFramePostProcessing { + public: + virtual ~VideoFramePostProcessing() = default; + + virtual rtc::scoped_refptr Process( + const rtc::scoped_refptr& buffer) = 0; + + virtual void Release() noexcept { delete this; } +}; + +} // namespace base +} // namespace owt + +#endif // OWT_IC_VIDEOFRAMEPOSTPROCESSING_H_ diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn new file mode 100644 index 000000000..08e1c3be8 --- /dev/null +++ b/talk/owt/sdk/ic/BUILD.gn @@ -0,0 +1,93 @@ +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import("//build_overrides/webrtc.gni") +import("//build_overrides/owt.gni") +import("//testing/test.gni") + +declare_args() { + owt_openvino_root = "" +} + +config("openvino") { + assert(owt_openvino_root != "", "owt_openvino_root not set") + defines = [ "USE_OPENVINO" ] + include_dirs = [ + owt_openvino_root + "/inference_engine/include", + ] + if (is_debug) { + lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Debug" ] + libs = [ + "inference_engined.lib", + "inference_engine_transformationsd.lib", + ] + } else { + lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Release" ] + libs = [ + "inference_engine.lib", + "inference_engine_transformations.lib", + ] + } + configs = [ + "//build/config/compiler:rtti", + ] +} + +config("openvino_opencv") { + assert(owt_openvino_root != "", "owt_openvino_root not set") + include_dirs = [ owt_openvino_root + "/opencv/include" ] + lib_dirs = [ owt_openvino_root + "/opencv/lib" ] + if (is_debug) { + libs = [ + "opencv_core453d.lib", + "opencv_imgproc453d.lib", + "opencv_imgcodecs453d.lib", + "opencv_highgui453d.lib", + "opencv_videoio453d.lib", + ] + } else { + libs = [ + "opencv_core453.lib", + "opencv_imgproc453.lib", + "opencv_imgcodecs453.lib", + "opencv_highgui453.lib", + "opencv_videoio453.lib", + ] + } +} + +shared_library("owt_ic") { + sources = [ + "backgroundblur.cc", + "backgroundblur.h", + "postprocessing.cc", + "postprocessing.h", + "selfiesegmentation.cc", + "selfiesegmentation.h", + ] + defines = [ "OWT_IC_IMPL" ] + configs += [ + ":openvino", + ":openvino_opencv", + ] + deps = [ + "//third_party/webrtc/api/video:video_frame", + "//third_party/libyuv:libyuv", + ] +} + +executable("test_background_blur") { + sources = [ "test_background_blur.cc" ] + include_dirs = [ "//talk/owt/sdk/include/cpp" ] + configs += [ + ":openvino", + ":openvino_opencv", + ] + deps = [ + "//talk/owt:owt_sdk_base", + ] + data_deps = [ + ":owt_ic", + ] +} diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc new file mode 100644 index 000000000..24e539ba6 --- /dev/null +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -0,0 +1,61 @@ +#include "backgroundblur.h" + +#include "api/video/i420_buffer.h" +#include "libyuv/convert.h" +#include "opencv2/imgproc.hpp" + +#include "selfiesegmentation.h" + +namespace owt { +namespace ic { + +BackgroundBlur::BackgroundBlur(int blurRadius) + : model(new SelfieSegmentation( + "C:/Users/wangzhib/Desktop/segmentation/contrib/" + "owt-selfie-segmentation-144x256.xml")), + blur_radius_(blurRadius) { + if (this->blur_radius_ < 0) { + this->blur_radius_ = 1; + } else if (this->blur_radius_ % 2 == 0) { + ++this->blur_radius_; + } +} + +BackgroundBlur::~BackgroundBlur() {} + +rtc::scoped_refptr BackgroundBlur::Process( + const rtc::scoped_refptr& buffer) { + auto i420 = buffer->GetI420(); + std::vector rgb(3ll * i420->width() * i420->height()); + libyuv::I420ToRAW(i420->DataY(), i420->StrideY(), i420->DataU(), + i420->StrideU(), i420->DataV(), i420->StrideV(), rgb.data(), + i420->width() * 3, i420->width(), i420->height()); + cv::Mat frame(buffer->height(), buffer->width(), CV_8UC3, rgb.data()); + + cv::Mat input; + frame.convertTo(input, CV_32FC3, 1. / 255); + cv::Mat mask = model->predict(input); // mask is of 8UC1 + + cv::Mat resizedMask; + cv::resize(mask, resizedMask, {buffer->width(), buffer->height()}); + cv::Mat mask3; + std::vector masks{255 - resizedMask, 255 - resizedMask, 255 - resizedMask}; + cv::merge(masks.data(), 3, mask3); + + cv::Mat background; + cv::multiply(frame, mask3, background, 1. / 255); + frame -= background; + cv::GaussianBlur(background, background, {blur_radius_, blur_radius_}, 0); + cv::add(frame, background, frame); + + auto newBuffer = webrtc::I420Buffer::Create(buffer->width(), buffer->height()); + libyuv::RAWToI420(frame.data, frame.cols * 3, newBuffer->MutableDataY(), + newBuffer->StrideY(), newBuffer->MutableDataU(), + newBuffer->StrideU(), newBuffer->MutableDataV(), + newBuffer->StrideV(), buffer->width(), buffer->height()); + + return newBuffer; +} + +} // namespace ic +} // namespace owt diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h new file mode 100644 index 000000000..4b6a3cc5f --- /dev/null +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -0,0 +1,28 @@ +#ifndef OWT_IC_BACKGROUNDBLUR_H_ +#define OWT_IC_BACKGROUNDBLUR_H_ + +#include + +#include "talk/owt/sdk/base/videoframepostprocessing.h" + +namespace owt { +namespace ic { +class SelfieSegmentation; + +class BackgroundBlur : public owt::base::VideoFramePostProcessing { + public: + BackgroundBlur(int blurRadius = 55); + ~BackgroundBlur() override; + + rtc::scoped_refptr Process( + const rtc::scoped_refptr& buffer) override; + + private: + std::unique_ptr model; + int blur_radius_; +}; + +} // namespace ic +} // namespace owt + +#endif // OWT_IC_BACKGROUNDBLUR_H_ diff --git a/talk/owt/sdk/ic/postprocessing.cc b/talk/owt/sdk/ic/postprocessing.cc new file mode 100644 index 000000000..48d2428c7 --- /dev/null +++ b/talk/owt/sdk/ic/postprocessing.cc @@ -0,0 +1,14 @@ +#include "postprocessing.h" + +#include + +#include "backgroundblur.h" + +owt::base::VideoFramePostProcessing* CreatePostProcessing( + const char* name) { + if (strcmp(name, "background_blur") == 0) { + return new owt::ic::BackgroundBlur; + } else { + return nullptr; + } +} diff --git a/talk/owt/sdk/ic/postprocessing.h b/talk/owt/sdk/ic/postprocessing.h new file mode 100644 index 000000000..a768d72fd --- /dev/null +++ b/talk/owt/sdk/ic/postprocessing.h @@ -0,0 +1,15 @@ +#ifndef OWT_IC_POSTPROCESSING_H_ +#define OWT_IC_POSTPROCESSING_H_ + +#include "../base/videoframepostprocessing.h" + +#ifdef OWT_IC_IMPL +#define OWT_IC_EXPORT extern "C" __declspec(dllexport) +#else +#define OWT_IC_EXPORT extern "C" __declspec(dllimport) +#endif + +OWT_IC_EXPORT owt::base::VideoFramePostProcessing* CreatePostProcessing( + const char* name); + +#endif // OWT_IC_POSTPROCESSING_H_ diff --git a/talk/owt/sdk/ic/selfiesegmentation.cc b/talk/owt/sdk/ic/selfiesegmentation.cc new file mode 100644 index 000000000..a0a4fab56 --- /dev/null +++ b/talk/owt/sdk/ic/selfiesegmentation.cc @@ -0,0 +1,62 @@ +#include "selfiesegmentation.h" + +#include +#include + +namespace IE = InferenceEngine; + +namespace owt { +namespace ic { + +SelfieSegmentation::SelfieSegmentation(const std::string& xmlPath, + const std::string& device) + : core(), + network(core.ReadNetwork(xmlPath)), + executableNetwork(), + request() { + for (auto& p : network.getInputsInfo()) { + inputName = p.first; + break; + } + for (auto& p : network.getOutputsInfo()) { + p.second->setPrecision(IE::Precision::U8); + outputName = p.first; + outputSize = p.second->getTensorDesc().getDims(); + break; + } + + IE::PreProcessInfo& preprocess = + network.getInputsInfo()[inputName]->getPreProcess(); + preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); + + executableNetwork = core.LoadNetwork(network, device); + request = executableNetwork.CreateInferRequest(); +} + +cv::Mat SelfieSegmentation::predict(const cv::Mat& frame) { + predictAsync(frame); + return waitForFinished(); +} + +void SelfieSegmentation::predictAsync(const cv::Mat& frame) { + CV_DbgAssert(frame.type() == CV_32FC3); + size_t height = frame.rows; + size_t width = frame.cols; + auto blob = IE::make_shared_blob( + IE::TensorDesc(IE::Precision::FP32, {1, 3, height, width}, + IE::Layout::NHWC), + (float*)frame.data); + request.SetBlob(inputName, blob); + request.StartAsync(); +} + +cv::Mat SelfieSegmentation::waitForFinished() { + request.Wait(IE::InferRequest::WaitMode::RESULT_READY); + const unsigned char* data = request.GetBlob(outputName)->buffer(); + int height = outputSize[1]; + int width = outputSize[2]; + return cv::Mat(height, width, CV_8U, (void*)data); +} + +} // namespace ic +} // namespace owt diff --git a/talk/owt/sdk/ic/selfiesegmentation.h b/talk/owt/sdk/ic/selfiesegmentation.h new file mode 100644 index 000000000..6f04c6890 --- /dev/null +++ b/talk/owt/sdk/ic/selfiesegmentation.h @@ -0,0 +1,35 @@ +#ifndef OWT_IC_SELFIESEGMENTATION_H_ +#define OWT_IC_SELFIESEGMENTATION_H_ + +#include +#include + +#include "inference_engine.hpp" +#include "opencv2/core/mat.hpp" + +namespace owt { +namespace ic { + +class SelfieSegmentation { + public: + explicit SelfieSegmentation(const std::string& xmlPath, + const std::string& device = "CPU"); + + cv::Mat predict(const cv::Mat& frame); + void predictAsync(const cv::Mat& frame); + cv::Mat waitForFinished(); + + private: + InferenceEngine::Core core; + InferenceEngine::CNNNetwork network; + InferenceEngine::ExecutableNetwork executableNetwork; + InferenceEngine::InferRequest request; + std::string inputName; + std::string outputName; + InferenceEngine::SizeVector outputSize; +}; + +} // namespace ic +} // namespace owt + +#endif // OWT_IC_SELFIE_SEGMENTATION_H_ diff --git a/talk/owt/sdk/ic/test_background_blur.cc b/talk/owt/sdk/ic/test_background_blur.cc new file mode 100644 index 000000000..5ea72ffda --- /dev/null +++ b/talk/owt/sdk/ic/test_background_blur.cc @@ -0,0 +1,60 @@ +#include "owt/base/deviceutils.h" +#include "owt/base/globalconfiguration.h" +#include "owt/base/localcamerastreamparameters.h" +#include "owt/base/stream.h" + +using namespace owt::base; + +class MainWindow : public VideoRenderWindow { + const wchar_t* class_name = L"MainWindow"; + const wchar_t* title = L"Camera"; + + public: + MainWindow() { + WNDCLASSEX cls = {}; + cls.cbSize = sizeof(WNDCLASSEX); + cls.lpfnWndProc = WindowProcedure; + cls.hInstance = GetModuleHandle(nullptr); + cls.lpszClassName = class_name; + RegisterClassEx(&cls); + HWND handle = CreateWindow( + class_name, title, WS_OVERLAPPEDWINDOW, CW_USEDEFAULT, CW_USEDEFAULT, + CW_USEDEFAULT, CW_USEDEFAULT, nullptr, nullptr, cls.hInstance, nullptr); + SetWindowHandle(handle); + } + + static LRESULT WindowProcedure(HWND window, + unsigned int msg, + WPARAM wp, + LPARAM lp) { + switch (msg) { + case WM_DESTROY: + PostQuitMessage(0); + return 0; + default: + return DefWindowProc(window, msg, wp, lp); + } + } + + int exec() { + ShowWindow(GetWindowHandle(), SW_SHOW); + MSG msg; + while (GetMessage(&msg, 0, 0, 0)) { + DispatchMessage(&msg); + } + return 0; + } +}; + +int main(int, char*[]) { + MainWindow w; + LocalCameraStreamParameters param(false, true); + param.CameraId(DeviceUtils::VideoCapturerIds()[0]); + param.Resolution(1280, 720); + param.Fps(60); + param.BackgroundBlur(true); + int error = 0; + auto stream = LocalStream::Create(param, error); + stream->AttachVideoRenderer(w); + return w.exec(); +} diff --git a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h index 1cb5847de..251382f09 100644 --- a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h +++ b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h @@ -51,6 +51,11 @@ class LocalCameraStreamParameters final { @param fps The frame rate of the video. */ void Fps(int fps); + /** + @brief Set whether to enable the background blur + @param enable Indicates if the background is enabled + */ + void BackgroundBlur(bool enable); /** @cond */ std::string CameraId() const { return camera_id_; } std::string StreamName() const { return stream_name_; } @@ -59,6 +64,7 @@ class LocalCameraStreamParameters final { int Fps() const { return fps_; } bool VideoEnabled() const { return video_enabled_; } bool AudioEnabled() const { return audio_enabled_; } + bool BackgroundBlur() const { return background_blur_; } /** @endcond */ private: std::string camera_id_; @@ -68,6 +74,7 @@ class LocalCameraStreamParameters final { int fps_; bool video_enabled_; bool audio_enabled_; + bool background_blur_; }; diff --git a/thirdpartylicense.txt b/thirdpartylicense.txt index 060243e19..489b570f3 100644 --- a/thirdpartylicense.txt +++ b/thirdpartylicense.txt @@ -63,3 +63,21 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + IC Model License + ----------------------- + +Copyright 2021 [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. From 6b305e98eca8ccf7b0c740e1960092858424bc7c Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Thu, 21 Oct 2021 11:31:38 +0800 Subject: [PATCH 02/30] Fixups --- build_overrides/owt.gni | 4 ++++ talk/owt/sdk/base/cameravideocapturer.cc | 13 +++++++------ talk/owt/sdk/base/icmanager.cc | 8 ++++++-- talk/owt/sdk/base/icmanager.h | 6 +++++- talk/owt/sdk/base/videoframepostprocessing.h | 4 ++++ talk/owt/sdk/ic/backgroundblur.cc | 10 ++++++++-- talk/owt/sdk/ic/backgroundblur.h | 4 ++++ talk/owt/sdk/ic/postprocessing.cc | 7 +++++-- talk/owt/sdk/ic/postprocessing.h | 4 ++++ talk/owt/sdk/ic/selfiesegmentation.cc | 4 ++++ talk/owt/sdk/ic/selfiesegmentation.h | 6 +++++- talk/owt/sdk/ic/test_background_blur.cc | 4 ++++ 12 files changed, 60 insertions(+), 14 deletions(-) diff --git a/build_overrides/owt.gni b/build_overrides/owt.gni index ae8286b0d..c73fb8d36 100644 --- a/build_overrides/owt.gni +++ b/build_overrides/owt.gni @@ -1,3 +1,7 @@ +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + declare_args() { owt_msdk_lib_root = "" owt_msdk_header_root = "" diff --git a/talk/owt/sdk/base/cameravideocapturer.cc b/talk/owt/sdk/base/cameravideocapturer.cc index 22a0d8d7e..99de73171 100644 --- a/talk/owt/sdk/base/cameravideocapturer.cc +++ b/talk/owt/sdk/base/cameravideocapturer.cc @@ -36,19 +36,20 @@ void CameraVideoCapturer::OnFrame(const webrtc::VideoFrame& frame) { return; } - rtc::scoped_refptr buffer = frame.video_frame_buffer(); - + rtc::scoped_refptr buffer = + frame.video_frame_buffer(); + if (out_height != frame.height() || out_width != frame.width()) { // Video adapter has requested a down-scale. Allocate a new buffer and // return scaled version. rtc::scoped_refptr scaled_buffer = webrtc::I420Buffer::Create(out_width, out_height); - scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); + scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); buffer = scaled_buffer; } - for (auto &pp : video_frame_post_processings_) { - buffer = pp->Process(buffer); + for (auto& post_processing : video_frame_post_processings_) { + buffer = post_processing->Process(buffer); } if (buffer != frame.video_frame_buffer()) { @@ -57,7 +58,7 @@ void CameraVideoCapturer::OnFrame(const webrtc::VideoFrame& frame) { .set_rotation(webrtc::kVideoRotation_0) .set_timestamp_us(frame.timestamp_us()) .set_id(frame.id()) - .build()); + .build()); } else { // No adaptations needed, just return the frame as is. broadcaster_.OnFrame(frame); diff --git a/talk/owt/sdk/base/icmanager.cc b/talk/owt/sdk/base/icmanager.cc index 63027eabc..02b79e007 100644 --- a/talk/owt/sdk/base/icmanager.cc +++ b/talk/owt/sdk/base/icmanager.cc @@ -1,3 +1,7 @@ +// Copyright (C) <2018> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #include "icmanager.h" #include "webrtc/rtc_base/logging.h" @@ -22,8 +26,8 @@ std::shared_ptr ICManager::CreatePostProcessing( ICManager::ICManager() { if (owt_ic_dll_ = LoadLibrary(L"owt_ic.dll")) { RTC_LOG(INFO) << "owt_ic.dll is loaded."; - create_post_processing_ = - (CREATE_POST_PROCESSING)GetProcAddress(owt_ic_dll_, "CreatePostProcessing"); + create_post_processing_ = (CREATE_POST_PROCESSING)GetProcAddress( + owt_ic_dll_, "CreatePostProcessing"); } else { RTC_LOG(WARNING) << "owt_ic.dll is not loaded."; } diff --git a/talk/owt/sdk/base/icmanager.h b/talk/owt/sdk/base/icmanager.h index 87acf7de0..71e6e4e6e 100644 --- a/talk/owt/sdk/base/icmanager.h +++ b/talk/owt/sdk/base/icmanager.h @@ -1,3 +1,7 @@ +// Copyright (C) <2018> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #ifndef OWT_BASE_ICMANAGER_H_ #define OWT_BASE_ICMANAGER_H_ @@ -33,4 +37,4 @@ class ICManager { } // namespace base } // namespace owt -#endif // OWT_BASE_ICMANAGER_H_ \ No newline at end of file +#endif // OWT_BASE_ICMANAGER_H_ diff --git a/talk/owt/sdk/base/videoframepostprocessing.h b/talk/owt/sdk/base/videoframepostprocessing.h index e803c3a57..546a7e13e 100644 --- a/talk/owt/sdk/base/videoframepostprocessing.h +++ b/talk/owt/sdk/base/videoframepostprocessing.h @@ -1,3 +1,7 @@ +// Copyright (C) <2018> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_ #define OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_ diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index 24e539ba6..c104b50f2 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -1,3 +1,7 @@ +// Copyright (C) <2018> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #include "backgroundblur.h" #include "api/video/i420_buffer.h" @@ -39,7 +43,8 @@ rtc::scoped_refptr BackgroundBlur::Process( cv::Mat resizedMask; cv::resize(mask, resizedMask, {buffer->width(), buffer->height()}); cv::Mat mask3; - std::vector masks{255 - resizedMask, 255 - resizedMask, 255 - resizedMask}; + std::vector masks{255 - resizedMask, 255 - resizedMask, + 255 - resizedMask}; cv::merge(masks.data(), 3, mask3); cv::Mat background; @@ -48,7 +53,8 @@ rtc::scoped_refptr BackgroundBlur::Process( cv::GaussianBlur(background, background, {blur_radius_, blur_radius_}, 0); cv::add(frame, background, frame); - auto newBuffer = webrtc::I420Buffer::Create(buffer->width(), buffer->height()); + auto newBuffer = + webrtc::I420Buffer::Create(buffer->width(), buffer->height()); libyuv::RAWToI420(frame.data, frame.cols * 3, newBuffer->MutableDataY(), newBuffer->StrideY(), newBuffer->MutableDataU(), newBuffer->StrideU(), newBuffer->MutableDataV(), diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h index 4b6a3cc5f..38a61dd02 100644 --- a/talk/owt/sdk/ic/backgroundblur.h +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -1,3 +1,7 @@ +// Copyright (C) <2018> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #ifndef OWT_IC_BACKGROUNDBLUR_H_ #define OWT_IC_BACKGROUNDBLUR_H_ diff --git a/talk/owt/sdk/ic/postprocessing.cc b/talk/owt/sdk/ic/postprocessing.cc index 48d2428c7..b02121a9a 100644 --- a/talk/owt/sdk/ic/postprocessing.cc +++ b/talk/owt/sdk/ic/postprocessing.cc @@ -1,11 +1,14 @@ +// Copyright (C) <2018> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #include "postprocessing.h" #include #include "backgroundblur.h" -owt::base::VideoFramePostProcessing* CreatePostProcessing( - const char* name) { +owt::base::VideoFramePostProcessing* CreatePostProcessing(const char* name) { if (strcmp(name, "background_blur") == 0) { return new owt::ic::BackgroundBlur; } else { diff --git a/talk/owt/sdk/ic/postprocessing.h b/talk/owt/sdk/ic/postprocessing.h index a768d72fd..309f17897 100644 --- a/talk/owt/sdk/ic/postprocessing.h +++ b/talk/owt/sdk/ic/postprocessing.h @@ -1,3 +1,7 @@ +// Copyright (C) <2018> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #ifndef OWT_IC_POSTPROCESSING_H_ #define OWT_IC_POSTPROCESSING_H_ diff --git a/talk/owt/sdk/ic/selfiesegmentation.cc b/talk/owt/sdk/ic/selfiesegmentation.cc index a0a4fab56..6d610a7e7 100644 --- a/talk/owt/sdk/ic/selfiesegmentation.cc +++ b/talk/owt/sdk/ic/selfiesegmentation.cc @@ -1,3 +1,7 @@ +// Copyright (C) <2018> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #include "selfiesegmentation.h" #include diff --git a/talk/owt/sdk/ic/selfiesegmentation.h b/talk/owt/sdk/ic/selfiesegmentation.h index 6f04c6890..c043fd631 100644 --- a/talk/owt/sdk/ic/selfiesegmentation.h +++ b/talk/owt/sdk/ic/selfiesegmentation.h @@ -1,3 +1,7 @@ +// Copyright (C) <2018> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #ifndef OWT_IC_SELFIESEGMENTATION_H_ #define OWT_IC_SELFIESEGMENTATION_H_ @@ -13,7 +17,7 @@ namespace ic { class SelfieSegmentation { public: explicit SelfieSegmentation(const std::string& xmlPath, - const std::string& device = "CPU"); + const std::string& device = "CPU"); cv::Mat predict(const cv::Mat& frame); void predictAsync(const cv::Mat& frame); diff --git a/talk/owt/sdk/ic/test_background_blur.cc b/talk/owt/sdk/ic/test_background_blur.cc index 5ea72ffda..936d240b7 100644 --- a/talk/owt/sdk/ic/test_background_blur.cc +++ b/talk/owt/sdk/ic/test_background_blur.cc @@ -1,3 +1,7 @@ +// Copyright (C) <2018> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #include "owt/base/deviceutils.h" #include "owt/base/globalconfiguration.h" #include "owt/base/localcamerastreamparameters.h" From 07e161c0a7a8397a585864e5967cfcbfbaba36fc Mon Sep 17 00:00:00 2001 From: "Wang, Zhibo" Date: Thu, 21 Oct 2021 11:35:05 +0800 Subject: [PATCH 03/30] Update thirdpartylicense.txt --- thirdpartylicense.txt | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/thirdpartylicense.txt b/thirdpartylicense.txt index 489b570f3..060243e19 100644 --- a/thirdpartylicense.txt +++ b/thirdpartylicense.txt @@ -63,21 +63,3 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - - IC Model License - ----------------------- - -Copyright 2021 [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. From 091f187c70179407914321fff07763c1d4899e01 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Thu, 21 Oct 2021 15:51:19 +0800 Subject: [PATCH 04/30] Fixups --- scripts/build-win.py | 16 +++++- talk/owt/BUILD.gn | 9 ++-- talk/owt/sdk/base/icmanager.cc | 10 ++-- talk/owt/sdk/base/icmanager.h | 13 +++-- talk/owt/sdk/base/stream.cc | 9 +++- talk/owt/sdk/ic/BUILD.gn | 68 +++++++++++++++---------- talk/owt/sdk/ic/backgroundblur.cc | 2 +- talk/owt/sdk/ic/backgroundblur.h | 2 +- talk/owt/sdk/ic/postprocessing.cc | 4 +- talk/owt/sdk/ic/postprocessing.h | 8 ++- talk/owt/sdk/ic/selfiesegmentation.cc | 2 +- talk/owt/sdk/ic/selfiesegmentation.h | 2 +- talk/owt/sdk/ic/test_background_blur.cc | 6 +-- 13 files changed, 97 insertions(+), 54 deletions(-) diff --git a/scripts/build-win.py b/scripts/build-win.py index ad68f727e..4fefe7028 100644 --- a/scripts/build-win.py +++ b/scripts/build-win.py @@ -11,6 +11,7 @@ ''' import os +import re import sys import subprocess import argparse @@ -49,9 +50,13 @@ def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests): gn_args.append('use_custom_libcxx=false') if scheme == 'release': gn_args.append('is_debug=false') + gn_args.append('lib_suffix=".lib"') + gn_args.append('dll_suffix=".dll"') else: gn_args.append('is_debug=true') gn_args.append('enable_iterator_debugging=true') + gn_args.append('lib_suffix="d.lib"') + gn_args.append('dll_suffix="d.dll"') if ssl_root: gn_args.append('owt_use_openssl=true') gn_args.append('owt_openssl_header_root="%s"' % (ssl_root + r'\include')) @@ -77,8 +82,15 @@ def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests): return False gn_args.append('owt_use_quic=true') if openvino_root: - gn_args.append(f'owt_build_ic=true') - gn_args.append(f'owt_openvino_root="{openvino_root}"') + gn_args.append('owt_build_ic=true') + gn_args.append('owt_openvino_root="{}"'.format(openvino_root)) + opencv_version_bin = os.path.join(openvino_root, 'opencv', 'bin', 'opencv_version.exe') + try: + output = subprocess.check_output([opencv_version_bin]) + opencv_version = ''.join(re.findall('\d', output.decode())) + gn_args.append('owt_openvino_opencv_version={}'.format(opencv_version)) + except FileNotFoundError as e: + print('File not found: {}'.format(opencv_version_bin)) if tests: gn_args.append('rtc_include_tests=true') gn_args.append('owt_include_tests=true') diff --git a/talk/owt/BUILD.gn b/talk/owt/BUILD.gn index ee29cf81c..ba5e0361a 100644 --- a/talk/owt/BUILD.gn +++ b/talk/owt/BUILD.gn @@ -82,11 +82,6 @@ if (!is_ios) { "//third_party/webrtc:webrtc", "//third_party/webrtc/api:libjingle_peerconnection_api", ] - if (owt_build_ic) { - data_deps = [ - "sdk/ic:owt_ic", - ] - } complete_static_lib = true } } @@ -323,6 +318,10 @@ static_library("owt_sdk_base") { "-Wno-reorder", ] } + if (owt_build_ic) { + defines += [ "OWT_BUILD_IC" ] + data_deps = [ "sdk/ic:owt_ic" ] + } } static_library("owt_sdk_p2p") { deps = [ diff --git a/talk/owt/sdk/base/icmanager.cc b/talk/owt/sdk/base/icmanager.cc index 02b79e007..471dc7a77 100644 --- a/talk/owt/sdk/base/icmanager.cc +++ b/talk/owt/sdk/base/icmanager.cc @@ -1,4 +1,4 @@ -// Copyright (C) <2018> Intel Corporation +// Copyright (C) <2021> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 @@ -14,7 +14,7 @@ ICManager* ICManager::GetInstance() { return &instance; } -std::shared_ptr ICManager::CreatePostProcessing( +std::shared_ptr ICManager::CreatePostProcessor( const char* name) { return create_post_processing_ ? std::shared_ptr( @@ -24,19 +24,23 @@ std::shared_ptr ICManager::CreatePostProcessing( } ICManager::ICManager() { +#ifdef WEBRTC_WIN if (owt_ic_dll_ = LoadLibrary(L"owt_ic.dll")) { RTC_LOG(INFO) << "owt_ic.dll is loaded."; create_post_processing_ = (CREATE_POST_PROCESSING)GetProcAddress( - owt_ic_dll_, "CreatePostProcessing"); + owt_ic_dll_, "CreatePostProcessor"); } else { RTC_LOG(WARNING) << "owt_ic.dll is not loaded."; } +#endif } ICManager::~ICManager() { +#ifdef WEBRTC_WIN if (owt_ic_dll_) { FreeLibrary(owt_ic_dll_); } +#endif } } // namespace base diff --git a/talk/owt/sdk/base/icmanager.h b/talk/owt/sdk/base/icmanager.h index 71e6e4e6e..a495b25a0 100644 --- a/talk/owt/sdk/base/icmanager.h +++ b/talk/owt/sdk/base/icmanager.h @@ -1,15 +1,17 @@ -// Copyright (C) <2018> Intel Corporation +// Copyright (C) <2021> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 #ifndef OWT_BASE_ICMANAGER_H_ #define OWT_BASE_ICMANAGER_H_ -#include #include +#ifdef WEBRTC_WIN +#include +#endif -#include "api/scoped_refptr.h" #include "base/macros.h" +#include "webrtc/api/scoped_refptr.h" #include "videoframepostprocessing.h" namespace owt { @@ -18,7 +20,7 @@ namespace base { class ICManager { public: static ICManager* GetInstance(); - std::shared_ptr CreatePostProcessing( + std::shared_ptr CreatePostProcessor( const char* name); private: @@ -27,8 +29,9 @@ class ICManager { typedef owt::base::VideoFramePostProcessing* (*CREATE_POST_PROCESSING)( const char* name); - +#ifdef WEBRTC_WIN HMODULE owt_ic_dll_ = nullptr; +#endif CREATE_POST_PROCESSING create_post_processing_ = nullptr; DISALLOW_COPY_AND_ASSIGN(ICManager); diff --git a/talk/owt/sdk/base/stream.cc b/talk/owt/sdk/base/stream.cc index c03ef075e..bfe1292dc 100644 --- a/talk/owt/sdk/base/stream.cc +++ b/talk/owt/sdk/base/stream.cc @@ -14,7 +14,6 @@ #include "talk/owt/sdk/base/icmanager.h" #include "talk/owt/sdk/base/webrtcaudiorendererimpl.h" #include "talk/owt/sdk/base/webrtcvideorendererimpl.h" -#include "talk/owt/sdk/ic/backgroundblur.h" #include "talk/owt/sdk/include/cpp/owt/base/framegeneratorinterface.h" #if defined(WEBRTC_WIN) @@ -60,7 +59,13 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { capturer = absl::WrapUnique(owt::base::VcmCapturer::Create( width, height, fps, capture_device_idx)); if (background_blur_enabled) { - capturer->AddVideoFramePostProcessing(ICManager::GetInstance()->CreatePostProcessing("background_blur")); + auto post_processing = + ICManager::GetInstance()->CreatePostProcessor("background_blur"); + if (post_processing) { + capturer->AddVideoFramePostProcessing(post_processing); + } else { + RTC_LOG(WARNING) << "Unable to create background blur instance."; + } } if (capturer) { return new rtc::RefCountedObject( diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index 08e1c3be8..70a22ffb5 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -8,26 +8,21 @@ import("//testing/test.gni") declare_args() { owt_openvino_root = "" + owt_openvino_opencv_version = "" + lib_suffix = "" + dll_suffix = "" } config("openvino") { assert(owt_openvino_root != "", "owt_openvino_root not set") - defines = [ "USE_OPENVINO" ] + defines = [ "OWT_USE_OPENVINO" ] include_dirs = [ owt_openvino_root + "/inference_engine/include", ] if (is_debug) { lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Debug" ] - libs = [ - "inference_engined.lib", - "inference_engine_transformationsd.lib", - ] } else { lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Release" ] - libs = [ - "inference_engine.lib", - "inference_engine_transformations.lib", - ] } configs = [ "//build/config/compiler:rtti", @@ -36,27 +31,15 @@ config("openvino") { config("openvino_opencv") { assert(owt_openvino_root != "", "owt_openvino_root not set") + assert(owt_openvino_opencv_version != "", "owt_openvino_opencv_version not set") + defines = [ "OWT_USE_OPENCV" ] include_dirs = [ owt_openvino_root + "/opencv/include" ] lib_dirs = [ owt_openvino_root + "/opencv/lib" ] - if (is_debug) { - libs = [ - "opencv_core453d.lib", - "opencv_imgproc453d.lib", - "opencv_imgcodecs453d.lib", - "opencv_highgui453d.lib", - "opencv_videoio453d.lib", - ] - } else { - libs = [ - "opencv_core453.lib", - "opencv_imgproc453.lib", - "opencv_imgcodecs453.lib", - "opencv_highgui453.lib", - "opencv_videoio453.lib", - ] - } } +openvino_modules = [ "ir_reader", "lp_transformations", "preproc", "transformations" ] +opencv_modules = ["core", "imgproc", "imgcodecs", "highgui", "videoio"] + shared_library("owt_ic") { sources = [ "backgroundblur.cc", @@ -71,10 +54,43 @@ shared_library("owt_ic") { ":openvino", ":openvino_opencv", ] + libs = [ + "inference_engine" + lib_suffix, + "inference_engine_transformations" + lib_suffix, + ] + foreach(module, opencv_modules) { + libs += [ "opencv_" + module + owt_openvino_opencv_version + lib_suffix ] + } deps = [ "//third_party/webrtc/api/video:video_frame", "//third_party/libyuv:libyuv", ] + data_deps = [ + ":openvino_library", + ":openvino_opencv_library", + ] +} + +copy("openvino_library") { + if (is_debug) { + bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Debug" + } else { + bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Release" + } + sources = [ bin_dir + "/inference_engine" + dll_suffix ] + foreach(module, openvino_modules) { + sources += [ bin_dir + "/inference_engine_" + module + dll_suffix ] + } + outputs = [ "$target_out_dir/{{source_file_part}}" ] +} + +copy("openvino_opencv_library") { + bin_dir = owt_openvino_root + "/opencv/bin" + sources = [] + foreach(module, opencv_modules) { + sources += [ bin_dir + "/opencv_" + module + owt_openvino_opencv_version + dll_suffix ] + } + outputs = [ "$target_out_dir/{{source_file_part}}" ] } executable("test_background_blur") { diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index c104b50f2..0890fe501 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -1,4 +1,4 @@ -// Copyright (C) <2018> Intel Corporation +// Copyright (C) <2021> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h index 38a61dd02..223183817 100644 --- a/talk/owt/sdk/ic/backgroundblur.h +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -1,4 +1,4 @@ -// Copyright (C) <2018> Intel Corporation +// Copyright (C) <2021> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/talk/owt/sdk/ic/postprocessing.cc b/talk/owt/sdk/ic/postprocessing.cc index b02121a9a..5ca546e43 100644 --- a/talk/owt/sdk/ic/postprocessing.cc +++ b/talk/owt/sdk/ic/postprocessing.cc @@ -1,4 +1,4 @@ -// Copyright (C) <2018> Intel Corporation +// Copyright (C) <2021> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 @@ -8,7 +8,7 @@ #include "backgroundblur.h" -owt::base::VideoFramePostProcessing* CreatePostProcessing(const char* name) { +owt::base::VideoFramePostProcessing* CreatePostProcessor(const char* name) { if (strcmp(name, "background_blur") == 0) { return new owt::ic::BackgroundBlur; } else { diff --git a/talk/owt/sdk/ic/postprocessing.h b/talk/owt/sdk/ic/postprocessing.h index 309f17897..4f4011b2c 100644 --- a/talk/owt/sdk/ic/postprocessing.h +++ b/talk/owt/sdk/ic/postprocessing.h @@ -1,4 +1,4 @@ -// Copyright (C) <2018> Intel Corporation +// Copyright (C) <2021> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 @@ -7,13 +7,17 @@ #include "../base/videoframepostprocessing.h" +#ifdef WEBRTC_WIN #ifdef OWT_IC_IMPL #define OWT_IC_EXPORT extern "C" __declspec(dllexport) #else #define OWT_IC_EXPORT extern "C" __declspec(dllimport) #endif +#else +#define OWT_IC_EXPORT +#endif -OWT_IC_EXPORT owt::base::VideoFramePostProcessing* CreatePostProcessing( +OWT_IC_EXPORT owt::base::VideoFramePostProcessing* CreatePostProcessor( const char* name); #endif // OWT_IC_POSTPROCESSING_H_ diff --git a/talk/owt/sdk/ic/selfiesegmentation.cc b/talk/owt/sdk/ic/selfiesegmentation.cc index 6d610a7e7..93c5cef6c 100644 --- a/talk/owt/sdk/ic/selfiesegmentation.cc +++ b/talk/owt/sdk/ic/selfiesegmentation.cc @@ -1,4 +1,4 @@ -// Copyright (C) <2018> Intel Corporation +// Copyright (C) <2021> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/talk/owt/sdk/ic/selfiesegmentation.h b/talk/owt/sdk/ic/selfiesegmentation.h index c043fd631..6b9bb6f5c 100644 --- a/talk/owt/sdk/ic/selfiesegmentation.h +++ b/talk/owt/sdk/ic/selfiesegmentation.h @@ -1,4 +1,4 @@ -// Copyright (C) <2018> Intel Corporation +// Copyright (C) <2021> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 diff --git a/talk/owt/sdk/ic/test_background_blur.cc b/talk/owt/sdk/ic/test_background_blur.cc index 936d240b7..64bb09a3e 100644 --- a/talk/owt/sdk/ic/test_background_blur.cc +++ b/talk/owt/sdk/ic/test_background_blur.cc @@ -40,7 +40,7 @@ class MainWindow : public VideoRenderWindow { } } - int exec() { + int Exec() { ShowWindow(GetWindowHandle(), SW_SHOW); MSG msg; while (GetMessage(&msg, 0, 0, 0)) { @@ -51,7 +51,6 @@ class MainWindow : public VideoRenderWindow { }; int main(int, char*[]) { - MainWindow w; LocalCameraStreamParameters param(false, true); param.CameraId(DeviceUtils::VideoCapturerIds()[0]); param.Resolution(1280, 720); @@ -59,6 +58,7 @@ int main(int, char*[]) { param.BackgroundBlur(true); int error = 0; auto stream = LocalStream::Create(param, error); + MainWindow w; stream->AttachVideoRenderer(w); - return w.exec(); + return w.Exec(); } From ec24bbf2ce8fe91a8966e1760abe3a4febe265d1 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Mon, 25 Oct 2021 09:57:48 +0800 Subject: [PATCH 05/30] Fixups --- talk/owt/BUILD.gn | 4 ++- talk/owt/sdk/base/cameravideocapturer.cc | 2 +- talk/owt/sdk/base/cameravideocapturer.h | 4 +-- talk/owt/sdk/base/icmanager.cc | 29 ++++++++--------- talk/owt/sdk/base/icmanager.h | 16 ++++------ talk/owt/sdk/base/linux/sharedobjectloader.cc | 30 +++++++++++++++++ talk/owt/sdk/base/sharedobjectloader.h | 27 ++++++++++++++++ talk/owt/sdk/base/videoframepostprocessing.h | 4 +-- talk/owt/sdk/base/win/sharedobjectloader.cc | 32 +++++++++++++++++++ talk/owt/sdk/ic/backgroundblur.cc | 27 +++++++--------- talk/owt/sdk/ic/backgroundblur.h | 8 ++--- talk/owt/sdk/ic/postprocessing.cc | 11 ++++--- talk/owt/sdk/ic/postprocessing.h | 2 +- talk/owt/sdk/ic/test_background_blur.cc | 31 ++++++++++++------ 14 files changed, 160 insertions(+), 67 deletions(-) create mode 100644 talk/owt/sdk/base/linux/sharedobjectloader.cc create mode 100644 talk/owt/sdk/base/sharedobjectloader.h create mode 100644 talk/owt/sdk/base/win/sharedobjectloader.cc diff --git a/talk/owt/BUILD.gn b/talk/owt/BUILD.gn index ba5e0361a..a1562b7d6 100644 --- a/talk/owt/BUILD.gn +++ b/talk/owt/BUILD.gn @@ -143,6 +143,7 @@ static_library("owt_sdk_base") { "sdk/base/customizedvideosource.h", "sdk/base/encodedvideoencoderfactory.cc", "sdk/base/encodedvideoencoderfactory.h", + "sdk/base/sharedobjectloader.h", "sdk/base/webrtcvideorendererimpl.cc", "sdk/base/webrtcvideorendererimpl.h", "sdk/base/windowcapturer.cc", @@ -263,6 +264,7 @@ static_library("owt_sdk_base") { "sdk/base/win/device_info_mf.cc", "sdk/base/win/video_capture_mf.h", "sdk/base/win/video_capture_mf.cc", + "sdk/base/win/sharedobjectloader.cc", ] public_deps += [ "//third_party/webrtc/modules/audio_device:audio_device_module_from_input_and_output" ] @@ -284,10 +286,10 @@ static_library("owt_sdk_base") { "sdk/base/linux/msdkvideodecoderfactory.h", "sdk/base/linux/msdkvideodecoder.cc", "sdk/base/linux/msdkvideodecoder.h", - ] } sources += [ + "sdk/base/linux/sharedobjectloader.cc", "sdk/base/linux/xwindownativeframe.h", "sdk/base/linux/videorenderlinux.cc", "sdk/base/linux/videorenderlinux.h", diff --git a/talk/owt/sdk/base/cameravideocapturer.cc b/talk/owt/sdk/base/cameravideocapturer.cc index 99de73171..c20787adb 100644 --- a/talk/owt/sdk/base/cameravideocapturer.cc +++ b/talk/owt/sdk/base/cameravideocapturer.cc @@ -83,7 +83,7 @@ void CameraVideoCapturer::RemoveSink( } void CameraVideoCapturer::AddVideoFramePostProcessing( - std::shared_ptr post_processing) { + std::shared_ptr post_processing) { video_frame_post_processings_.emplace_back(post_processing); } diff --git a/talk/owt/sdk/base/cameravideocapturer.h b/talk/owt/sdk/base/cameravideocapturer.h index c8aa32771..e6319499f 100644 --- a/talk/owt/sdk/base/cameravideocapturer.h +++ b/talk/owt/sdk/base/cameravideocapturer.h @@ -36,7 +36,7 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface void RemoveSink(rtc::VideoSinkInterface* sink) override; void AddVideoFramePostProcessing( - const std::shared_ptr post_processing); + const std::shared_ptr post_processing); protected: void OnFrame(const webrtc::VideoFrame& frame); @@ -47,7 +47,7 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface rtc::VideoBroadcaster broadcaster_; cricket::VideoAdapter video_adapter_; - std::vector> + std::vector> video_frame_post_processings_; }; } // namespace base diff --git a/talk/owt/sdk/base/icmanager.cc b/talk/owt/sdk/base/icmanager.cc index 471dc7a77..83cf9a3c6 100644 --- a/talk/owt/sdk/base/icmanager.cc +++ b/talk/owt/sdk/base/icmanager.cc @@ -14,33 +14,30 @@ ICManager* ICManager::GetInstance() { return &instance; } -std::shared_ptr ICManager::CreatePostProcessor( +std::shared_ptr ICManager::CreatePostProcessor( const char* name) { return create_post_processing_ - ? std::shared_ptr( + ? std::shared_ptr( create_post_processing_(name), - [](VideoFramePostProcessing* ptr) { ptr->Release(); }) + [](VideoFramePostProcessor* ptr) { ptr->Release(); }) : nullptr; } -ICManager::ICManager() { +ICManager::ICManager() + : so_( #ifdef WEBRTC_WIN - if (owt_ic_dll_ = LoadLibrary(L"owt_ic.dll")) { + "owt_ic.dll" +#elif WEBRTC_LINUX + "owt_ic.so" +#endif + ) { + if (so_.IsLoaded()) { RTC_LOG(INFO) << "owt_ic.dll is loaded."; - create_post_processing_ = (CREATE_POST_PROCESSING)GetProcAddress( - owt_ic_dll_, "CreatePostProcessor"); + create_post_processing_ = reinterpret_cast( + so_.GetSymbol("CreatePostProcessor")); } else { RTC_LOG(WARNING) << "owt_ic.dll is not loaded."; } -#endif -} - -ICManager::~ICManager() { -#ifdef WEBRTC_WIN - if (owt_ic_dll_) { - FreeLibrary(owt_ic_dll_); - } -#endif } } // namespace base diff --git a/talk/owt/sdk/base/icmanager.h b/talk/owt/sdk/base/icmanager.h index a495b25a0..61658c0c3 100644 --- a/talk/owt/sdk/base/icmanager.h +++ b/talk/owt/sdk/base/icmanager.h @@ -6,13 +6,11 @@ #define OWT_BASE_ICMANAGER_H_ #include -#ifdef WEBRTC_WIN -#include -#endif #include "base/macros.h" +#include "talk/owt/sdk/base/sharedobjectloader.h" +#include "talk/owt/sdk/base/videoframepostprocessing.h" #include "webrtc/api/scoped_refptr.h" -#include "videoframepostprocessing.h" namespace owt { namespace base { @@ -20,18 +18,16 @@ namespace base { class ICManager { public: static ICManager* GetInstance(); - std::shared_ptr CreatePostProcessor( + std::shared_ptr CreatePostProcessor( const char* name); private: ICManager(); - ~ICManager(); + ~ICManager() = default; - typedef owt::base::VideoFramePostProcessing* (*CREATE_POST_PROCESSING)( + typedef owt::base::VideoFramePostProcessor* (*CREATE_POST_PROCESSING)( const char* name); -#ifdef WEBRTC_WIN - HMODULE owt_ic_dll_ = nullptr; -#endif + SharedObjectLoader so_; CREATE_POST_PROCESSING create_post_processing_ = nullptr; DISALLOW_COPY_AND_ASSIGN(ICManager); diff --git a/talk/owt/sdk/base/linux/sharedobjectloader.cc b/talk/owt/sdk/base/linux/sharedobjectloader.cc new file mode 100644 index 000000000..d12168f6a --- /dev/null +++ b/talk/owt/sdk/base/linux/sharedobjectloader.cc @@ -0,0 +1,30 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "talk/owt/sdk/base/sharedobjectloader.h" + +#include + +namespace owt { +namespace base { + +SharedObjectLoader::SharedObjectLoader(const char* name) + : shared_object(dlopen(name, RTLD_NOW)) {} + +SharedObjectLoader::~SharedObjectLoader() { + if (shared_object) { + dlclose(shared_object); + } +} + +bool SharedObjectLoader::IsLoaded() const { + return shared_object_; +} + +void* SharedObjectLoader::get_symbol(const char* name) { + return shared_object_ ? dlsym(shared_object, name) : nullptr; +} + +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/base/sharedobjectloader.h b/talk/owt/sdk/base/sharedobjectloader.h new file mode 100644 index 000000000..5f83c1321 --- /dev/null +++ b/talk/owt/sdk/base/sharedobjectloader.h @@ -0,0 +1,27 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_SHAREDOBJECTLOADER_H_ +#define OWT_BASE_SHAREDOBJECTLOADER_H_ + +namespace owt { +namespace base { + +class SharedObjectLoader { + public: + explicit SharedObjectLoader(const char* name); + ~SharedObjectLoader(); + + bool IsLoaded() const; + + void* GetSymbol(const char* name); + + private: + void* shared_object_; +}; + +} // namespace base +} // namespace owt + +#endif // OWT_BASE_SHAREDOBJECTLOADER_H_ diff --git a/talk/owt/sdk/base/videoframepostprocessing.h b/talk/owt/sdk/base/videoframepostprocessing.h index 546a7e13e..6b5f906da 100644 --- a/talk/owt/sdk/base/videoframepostprocessing.h +++ b/talk/owt/sdk/base/videoframepostprocessing.h @@ -11,9 +11,9 @@ namespace owt { namespace base { -class VideoFramePostProcessing { +class VideoFramePostProcessor { public: - virtual ~VideoFramePostProcessing() = default; + virtual ~VideoFramePostProcessor() = default; virtual rtc::scoped_refptr Process( const rtc::scoped_refptr& buffer) = 0; diff --git a/talk/owt/sdk/base/win/sharedobjectloader.cc b/talk/owt/sdk/base/win/sharedobjectloader.cc new file mode 100644 index 000000000..fafa505e4 --- /dev/null +++ b/talk/owt/sdk/base/win/sharedobjectloader.cc @@ -0,0 +1,32 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "talk/owt/sdk/base/sharedobjectloader.h" + +#include + +namespace owt { +namespace base { + +SharedObjectLoader::SharedObjectLoader(const char* name) + : shared_object_(LoadLibraryA(name)) {} + +SharedObjectLoader::~SharedObjectLoader() { + if (shared_object_) { + FreeLibrary(reinterpret_cast(shared_object_)); + } +} + +bool SharedObjectLoader::IsLoaded() const { + return shared_object_; +} + +void* SharedObjectLoader::GetSymbol(const char* name) { + return shared_object_ + ? GetProcAddress(reinterpret_cast(shared_object_), name) + : nullptr; +} + +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index 0890fe501..c29cf4b70 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -4,6 +4,8 @@ #include "backgroundblur.h" +#include + #include "api/video/i420_buffer.h" #include "libyuv/convert.h" #include "opencv2/imgproc.hpp" @@ -13,19 +15,10 @@ namespace owt { namespace ic { -BackgroundBlur::BackgroundBlur(int blurRadius) +BackgroundBlur::BackgroundBlur() : model(new SelfieSegmentation( "C:/Users/wangzhib/Desktop/segmentation/contrib/" - "owt-selfie-segmentation-144x256.xml")), - blur_radius_(blurRadius) { - if (this->blur_radius_ < 0) { - this->blur_radius_ = 1; - } else if (this->blur_radius_ % 2 == 0) { - ++this->blur_radius_; - } -} - -BackgroundBlur::~BackgroundBlur() {} + "owt-selfie-segmentation-144x256.xml")) {} rtc::scoped_refptr BackgroundBlur::Process( const rtc::scoped_refptr& buffer) { @@ -37,18 +30,22 @@ rtc::scoped_refptr BackgroundBlur::Process( cv::Mat frame(buffer->height(), buffer->width(), CV_8UC3, rgb.data()); cv::Mat input; - frame.convertTo(input, CV_32FC3, 1. / 255); + frame.convertTo(input, CV_32FC3, + 1. / std::numeric_limits::max()); cv::Mat mask = model->predict(input); // mask is of 8UC1 cv::Mat resizedMask; cv::resize(mask, resizedMask, {buffer->width(), buffer->height()}); cv::Mat mask3; - std::vector masks{255 - resizedMask, 255 - resizedMask, - 255 - resizedMask}; + std::vector masks{ + std::numeric_limits::max() - resizedMask, + std::numeric_limits::max() - resizedMask, + std::numeric_limits::max() - resizedMask}; cv::merge(masks.data(), 3, mask3); cv::Mat background; - cv::multiply(frame, mask3, background, 1. / 255); + cv::multiply(frame, mask3, background, + 1. / std::numeric_limits::max()); frame -= background; cv::GaussianBlur(background, background, {blur_radius_, blur_radius_}, 0); cv::add(frame, background, frame); diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h index 223183817..3b71539ae 100644 --- a/talk/owt/sdk/ic/backgroundblur.h +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -13,17 +13,17 @@ namespace owt { namespace ic { class SelfieSegmentation; -class BackgroundBlur : public owt::base::VideoFramePostProcessing { +class BackgroundBlur : public owt::base::VideoFramePostProcessor { public: - BackgroundBlur(int blurRadius = 55); - ~BackgroundBlur() override; + BackgroundBlur(); + ~BackgroundBlur() override = default; rtc::scoped_refptr Process( const rtc::scoped_refptr& buffer) override; private: std::unique_ptr model; - int blur_radius_; + int blur_radius_ = 55; }; } // namespace ic diff --git a/talk/owt/sdk/ic/postprocessing.cc b/talk/owt/sdk/ic/postprocessing.cc index 5ca546e43..b535b628d 100644 --- a/talk/owt/sdk/ic/postprocessing.cc +++ b/talk/owt/sdk/ic/postprocessing.cc @@ -8,10 +8,11 @@ #include "backgroundblur.h" -owt::base::VideoFramePostProcessing* CreatePostProcessor(const char* name) { - if (strcmp(name, "background_blur") == 0) { - return new owt::ic::BackgroundBlur; - } else { - return nullptr; +owt::base::VideoFramePostProcessor* CreatePostProcessor(const char* name) { + if (name) { + if (strcmp(name, "background_blur") == 0) { + return new owt::ic::BackgroundBlur; + } } + return nullptr; } diff --git a/talk/owt/sdk/ic/postprocessing.h b/talk/owt/sdk/ic/postprocessing.h index 4f4011b2c..c82ca2bde 100644 --- a/talk/owt/sdk/ic/postprocessing.h +++ b/talk/owt/sdk/ic/postprocessing.h @@ -17,7 +17,7 @@ #define OWT_IC_EXPORT #endif -OWT_IC_EXPORT owt::base::VideoFramePostProcessing* CreatePostProcessor( +OWT_IC_EXPORT owt::base::VideoFramePostProcessor* CreatePostProcessor( const char* name); #endif // OWT_IC_POSTPROCESSING_H_ diff --git a/talk/owt/sdk/ic/test_background_blur.cc b/talk/owt/sdk/ic/test_background_blur.cc index 64bb09a3e..62cd1ef74 100644 --- a/talk/owt/sdk/ic/test_background_blur.cc +++ b/talk/owt/sdk/ic/test_background_blur.cc @@ -2,6 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 +#include + #include "owt/base/deviceutils.h" #include "owt/base/globalconfiguration.h" #include "owt/base/localcamerastreamparameters.h" @@ -14,16 +16,16 @@ class MainWindow : public VideoRenderWindow { const wchar_t* title = L"Camera"; public: - MainWindow() { + MainWindow(int width, int height) { WNDCLASSEX cls = {}; cls.cbSize = sizeof(WNDCLASSEX); cls.lpfnWndProc = WindowProcedure; cls.hInstance = GetModuleHandle(nullptr); cls.lpszClassName = class_name; RegisterClassEx(&cls); - HWND handle = CreateWindow( - class_name, title, WS_OVERLAPPEDWINDOW, CW_USEDEFAULT, CW_USEDEFAULT, - CW_USEDEFAULT, CW_USEDEFAULT, nullptr, nullptr, cls.hInstance, nullptr); + HWND handle = CreateWindow(class_name, title, WS_OVERLAPPEDWINDOW, + CW_USEDEFAULT, CW_USEDEFAULT, width, height, + nullptr, nullptr, cls.hInstance, nullptr); SetWindowHandle(handle); } @@ -52,13 +54,22 @@ class MainWindow : public VideoRenderWindow { int main(int, char*[]) { LocalCameraStreamParameters param(false, true); - param.CameraId(DeviceUtils::VideoCapturerIds()[0]); - param.Resolution(1280, 720); - param.Fps(60); + std::string default_id = DeviceUtils::VideoCapturerIds()[0]; + param.CameraId(default_id); + auto capability = + DeviceUtils::VideoCapturerSupportedCapabilities(default_id)[0]; + param.Resolution(capability.width, capability.height); + param.Fps(capability.frameRate); param.BackgroundBlur(true); int error = 0; auto stream = LocalStream::Create(param, error); - MainWindow w; - stream->AttachVideoRenderer(w); - return w.Exec(); + if (error == 0) { + MainWindow w(capability.width, capability.height); + stream->AttachVideoRenderer(w); + return w.Exec(); + } else { + std::cerr << "Create local stream failed, error code " << error + << std::endl; + return 0; + } } From 09530e6e2734bee228ae860c0003368eb53759e2 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Mon, 25 Oct 2021 10:34:40 +0800 Subject: [PATCH 06/30] Fixups --- scripts/build-win.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/build-win.py b/scripts/build-win.py index 4fefe7028..0f4cbb4f5 100644 --- a/scripts/build-win.py +++ b/scripts/build-win.py @@ -50,13 +50,15 @@ def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests): gn_args.append('use_custom_libcxx=false') if scheme == 'release': gn_args.append('is_debug=false') - gn_args.append('lib_suffix=".lib"') - gn_args.append('dll_suffix=".dll"') + if openvino_root: + gn_args.append('lib_suffix=".lib"') + gn_args.append('dll_suffix=".dll"') else: gn_args.append('is_debug=true') gn_args.append('enable_iterator_debugging=true') - gn_args.append('lib_suffix="d.lib"') - gn_args.append('dll_suffix="d.dll"') + if openvino_root: + gn_args.append('lib_suffix="d.lib"') + gn_args.append('dll_suffix="d.dll"') if ssl_root: gn_args.append('owt_use_openssl=true') gn_args.append('owt_openssl_header_root="%s"' % (ssl_root + r'\include')) @@ -91,6 +93,7 @@ def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests): gn_args.append('owt_openvino_opencv_version={}'.format(opencv_version)) except FileNotFoundError as e: print('File not found: {}'.format(opencv_version_bin)) + return False if tests: gn_args.append('rtc_include_tests=true') gn_args.append('owt_include_tests=true') @@ -198,7 +201,7 @@ def main(): parser.add_argument('--ssl_root', help='Path for OpenSSL.') parser.add_argument('--msdk_root', help='Path for MSDK.') parser.add_argument('--quic_root', help='Path to QUIC library') - parser.add_argument('--openvino_root', default="", help='Path for OpenVINO.') + parser.add_argument('--openvino_root', help='Path for OpenVINO.') parser.add_argument('--scheme', default='debug', choices=('debug', 'release'), help='Schemes for building. Supported value: debug, release') parser.add_argument('--gn_gen', default=False, action='store_true', From 96ab95117cf7fb3a43c6f22e93b43baf83214d1d Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Mon, 25 Oct 2021 16:52:14 +0800 Subject: [PATCH 07/30] Fixups --- talk/owt/BUILD.gn | 11 ++- talk/owt/sdk/base/cameravideocapturer.cc | 8 +- talk/owt/sdk/base/cameravideocapturer.h | 4 +- talk/owt/sdk/base/icmanager.cc | 25 ++++++- talk/owt/sdk/base/icmanager.h | 7 +- .../intelligentcollaborationparameters.cc | 23 ++++++ .../sdk/base/localcamerastreamparameters.cc | 21 +++--- talk/owt/sdk/base/stream.cc | 75 +++++++++++++------ talk/owt/sdk/config/owt_ic.json | 14 ++++ talk/owt/sdk/ic/BUILD.gn | 15 ---- .../base/intelligentcollaborationparameters.h | 30 ++++++++ .../owt/base/localcamerastreamparameters.h | 51 +++++-------- talk/owt/sdk/sample/BUILD.gn | 14 ++++ talk/owt/sdk/sample/win/BUILD.gn | 11 +++ .../win/sample_background_blur/BUILD.gn | 22 ++++++ .../sample_background_blur.cc} | 6 +- 16 files changed, 243 insertions(+), 94 deletions(-) create mode 100644 talk/owt/sdk/base/intelligentcollaborationparameters.cc create mode 100644 talk/owt/sdk/config/owt_ic.json create mode 100644 talk/owt/sdk/include/cpp/owt/base/intelligentcollaborationparameters.h create mode 100644 talk/owt/sdk/sample/BUILD.gn create mode 100644 talk/owt/sdk/sample/win/BUILD.gn create mode 100644 talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn rename talk/owt/sdk/{ic/test_background_blur.cc => sample/win/sample_background_blur/sample_background_blur.cc} (90%) diff --git a/talk/owt/BUILD.gn b/talk/owt/BUILD.gn index a1562b7d6..a800f18ed 100644 --- a/talk/owt/BUILD.gn +++ b/talk/owt/BUILD.gn @@ -98,8 +98,7 @@ static_library("owt_sdk_base") { "sdk/base/functionalobserver.cc", "sdk/base/functionalobserver.h", "sdk/base/globalconfiguration.cc", - "sdk/base/icmanager.cc", - "sdk/base/icmanager.h", + "sdk/base/intelligentcollaborationparameters.cc", "sdk/base/localcamerastreamparameters.cc", "sdk/base/logging.cc", "sdk/base/logsinks.cc", @@ -128,6 +127,7 @@ static_library("owt_sdk_base") { "sdk/include/cpp/owt/base/deviceutils.h", "sdk/include/cpp/owt/base/exception.h", "sdk/include/cpp/owt/base/framegeneratorinterface.h", + "sdk/include/cpp/owt/base/intelligentcollaborationparameters.h", "sdk/include/cpp/owt/base/localcamerastreamparameters.h", "sdk/include/cpp/owt/base/logging.h", "sdk/include/cpp/owt/base/stream.h", @@ -143,6 +143,8 @@ static_library("owt_sdk_base") { "sdk/base/customizedvideosource.h", "sdk/base/encodedvideoencoderfactory.cc", "sdk/base/encodedvideoencoderfactory.h", + "sdk/base/icmanager.cc", + "sdk/base/icmanager.h", "sdk/base/sharedobjectloader.h", "sdk/base/webrtcvideorendererimpl.cc", "sdk/base/webrtcvideorendererimpl.h", @@ -506,7 +508,10 @@ if (owt_include_tests) { # Only the root target should depend on this. visibility = [ "//:default" ] - deps = [ ":owt_unittests" ] + deps = [ + ":owt_unittests", + "//talk/owt/sdk/sample", + ] } test("owt_unittests") { testonly = true diff --git a/talk/owt/sdk/base/cameravideocapturer.cc b/talk/owt/sdk/base/cameravideocapturer.cc index c20787adb..cb765331c 100644 --- a/talk/owt/sdk/base/cameravideocapturer.cc +++ b/talk/owt/sdk/base/cameravideocapturer.cc @@ -48,8 +48,8 @@ void CameraVideoCapturer::OnFrame(const webrtc::VideoFrame& frame) { buffer = scaled_buffer; } - for (auto& post_processing : video_frame_post_processings_) { - buffer = post_processing->Process(buffer); + for (auto& post_processor : video_frame_post_processors_) { + buffer = post_processor->Process(buffer); } if (buffer != frame.video_frame_buffer()) { @@ -82,9 +82,9 @@ void CameraVideoCapturer::RemoveSink( UpdateVideoAdapter(); } -void CameraVideoCapturer::AddVideoFramePostProcessing( +void CameraVideoCapturer::AddVideoFramePostProcessor( std::shared_ptr post_processing) { - video_frame_post_processings_.emplace_back(post_processing); + video_frame_post_processors_.emplace_back(post_processing); } void CameraVideoCapturer::UpdateVideoAdapter() { diff --git a/talk/owt/sdk/base/cameravideocapturer.h b/talk/owt/sdk/base/cameravideocapturer.h index e6319499f..cad74ffa7 100644 --- a/talk/owt/sdk/base/cameravideocapturer.h +++ b/talk/owt/sdk/base/cameravideocapturer.h @@ -35,7 +35,7 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface const rtc::VideoSinkWants& wants) override; void RemoveSink(rtc::VideoSinkInterface* sink) override; - void AddVideoFramePostProcessing( + void AddVideoFramePostProcessor( const std::shared_ptr post_processing); protected: @@ -48,7 +48,7 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface rtc::VideoBroadcaster broadcaster_; cricket::VideoAdapter video_adapter_; std::vector> - video_frame_post_processings_; + video_frame_post_processors_; }; } // namespace base } // namespace owt diff --git a/talk/owt/sdk/base/icmanager.cc b/talk/owt/sdk/base/icmanager.cc index 83cf9a3c6..ce876e0af 100644 --- a/talk/owt/sdk/base/icmanager.cc +++ b/talk/owt/sdk/base/icmanager.cc @@ -4,6 +4,9 @@ #include "icmanager.h" +#include + +#include "third_party/jsoncpp/source/include/json/json.h" #include "webrtc/rtc_base/logging.h" namespace owt { @@ -14,6 +17,25 @@ ICManager* ICManager::GetInstance() { return &instance; } +bool ICManager::LoadConfig(const std::string& config_path) { + Json::Reader reader; + Json::Value config; + std::ifstream fin(config_path); + if (!fin) { + return false; + } + if (!reader.parse(fin, config)) { + return false; + } + Json::Value backgroundBlur = config.get("background_blur", Json::Value()); + Json::Value bins = backgroundBlur.get("bin", Json::Value()); + for (auto& bin : bins) { + std::string source = bin.get("source", "").asString(); + std::string target = bin.get("target", "").asString(); + // TODO WIP + } +} + std::shared_ptr ICManager::CreatePostProcessor( const char* name) { return create_post_processing_ @@ -33,8 +55,9 @@ ICManager::ICManager() ) { if (so_.IsLoaded()) { RTC_LOG(INFO) << "owt_ic.dll is loaded."; - create_post_processing_ = reinterpret_cast( + create_post_processing_ = reinterpret_cast( so_.GetSymbol("CreatePostProcessor")); + LoadConfig("config/owt_ic.json"); } else { RTC_LOG(WARNING) << "owt_ic.dll is not loaded."; } diff --git a/talk/owt/sdk/base/icmanager.h b/talk/owt/sdk/base/icmanager.h index 61658c0c3..7d918e741 100644 --- a/talk/owt/sdk/base/icmanager.h +++ b/talk/owt/sdk/base/icmanager.h @@ -6,6 +6,7 @@ #define OWT_BASE_ICMANAGER_H_ #include +#include #include "base/macros.h" #include "talk/owt/sdk/base/sharedobjectloader.h" @@ -18,6 +19,8 @@ namespace base { class ICManager { public: static ICManager* GetInstance(); + + bool LoadConfig(const std::string& config_path); std::shared_ptr CreatePostProcessor( const char* name); @@ -25,10 +28,10 @@ class ICManager { ICManager(); ~ICManager() = default; - typedef owt::base::VideoFramePostProcessor* (*CREATE_POST_PROCESSING)( + typedef owt::base::VideoFramePostProcessor* (*CREATE_POST_PROCESSOR)( const char* name); SharedObjectLoader so_; - CREATE_POST_PROCESSING create_post_processing_ = nullptr; + CREATE_POST_PROCESSOR create_post_processing_ = nullptr; DISALLOW_COPY_AND_ASSIGN(ICManager); }; diff --git a/talk/owt/sdk/base/intelligentcollaborationparameters.cc b/talk/owt/sdk/base/intelligentcollaborationparameters.cc new file mode 100644 index 000000000..d55ac8858 --- /dev/null +++ b/talk/owt/sdk/base/intelligentcollaborationparameters.cc @@ -0,0 +1,23 @@ +#include "talk/owt/sdk/include/cpp/owt/base/intelligentcollaborationparameters.h" + +namespace owt { +namespace base { + +void IntelligentCollaborationParameters::BackgroundBlur(bool enable) { + enable_background_blur_ = enable; +} + +void IntelligentCollaborationParameters::BlurRadius(int radius) { + blur_radius_ = radius; +} + +bool IntelligentCollaborationParameters::BackgroundBlur() const { + return enable_background_blur_; +} + +int IntelligentCollaborationParameters::BlurRadius() const { + return blur_radius_; +} + +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/base/localcamerastreamparameters.cc b/talk/owt/sdk/base/localcamerastreamparameters.cc index 4e3be76fc..7e30b1f3f 100644 --- a/talk/owt/sdk/base/localcamerastreamparameters.cc +++ b/talk/owt/sdk/base/localcamerastreamparameters.cc @@ -1,8 +1,8 @@ // Copyright (C) <2018> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 -#include #include "talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h" +#include namespace owt { namespace base { LocalCameraStreamParameters::LocalCameraStreamParameters(bool audio_enabled, @@ -19,8 +19,12 @@ LocalCameraStreamParameters::LocalCameraStreamParameters(bool audio_enabled, void LocalCameraStreamParameters::Fps(int fps) { fps_ = fps; } -void LocalCameraStreamParameters::BackgroundBlur(bool enable) { - background_blur_ = enable; +IntelligentCollaborationParameters& LocalCameraStreamParameters::ICParams() { + return ic_params_; +} +const IntelligentCollaborationParameters& +LocalCameraStreamParameters::ICParams() const { + return ic_params_; } void LocalCameraStreamParameters::CameraId(const std::string& camera_id) { camera_id_ = camera_id; @@ -29,12 +33,11 @@ void LocalCameraStreamParameters::Resolution(int width, int height) { resolution_width_ = width; resolution_height_ = height; } -void LocalCameraStreamParameters::StreamName(const std::string& stream_name){ +void LocalCameraStreamParameters::StreamName(const std::string& stream_name) { stream_name_ = stream_name; } -LocalDesktopStreamParameters::LocalDesktopStreamParameters( - bool audio_enabled, - bool video_enabled) +LocalDesktopStreamParameters::LocalDesktopStreamParameters(bool audio_enabled, + bool video_enabled) : video_enabled_(video_enabled), audio_enabled_(audio_enabled), fps_(30), @@ -43,5 +46,5 @@ LocalDesktopStreamParameters::LocalDesktopStreamParameters( void LocalDesktopStreamParameters::Fps(int fps) { fps_ = fps; } -} -} +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/base/stream.cc b/talk/owt/sdk/base/stream.cc index bfe1292dc..a7de19c38 100644 --- a/talk/owt/sdk/base/stream.cc +++ b/talk/owt/sdk/base/stream.cc @@ -11,7 +11,6 @@ #include "webrtc/sdk/media_constraints.h" #include "talk/owt/sdk/base/customizedframescapturer.h" -#include "talk/owt/sdk/base/icmanager.h" #include "talk/owt/sdk/base/webrtcaudiorendererimpl.h" #include "talk/owt/sdk/base/webrtcvideorendererimpl.h" #include "talk/owt/sdk/include/cpp/owt/base/framegeneratorinterface.h" @@ -35,6 +34,9 @@ #endif #include "talk/owt/sdk/include/cpp/owt/base/deviceutils.h" #include "talk/owt/sdk/include/cpp/owt/base/stream.h" +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) +#include "talk/owt/sdk/base/icmanager.h" +#endif using namespace rtc; namespace owt { @@ -47,7 +49,8 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { const size_t height, const size_t fps, int capture_device_idx, - bool background_blur_enabled = false) { + const IntelligentCollaborationParameters& ic_params = + IntelligentCollaborationParameters()) { std::unique_ptr capturer; std::unique_ptr info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -58,14 +61,19 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { for (int i = 0; i < num_devices; ++i) { capturer = absl::WrapUnique(owt::base::VcmCapturer::Create( width, height, fps, capture_device_idx)); - if (background_blur_enabled) { - auto post_processing = + if (ic_params.BackgroundBlur()) { +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) + auto post_processor = ICManager::GetInstance()->CreatePostProcessor("background_blur"); - if (post_processing) { - capturer->AddVideoFramePostProcessing(post_processing); + if (post_processor) { + capturer->AddVideoFramePostProcessor(post_processor); } else { RTC_LOG(WARNING) << "Unable to create background blur instance."; } +#else + RTC_LOG(WARNING) + << "Background blur is not supported on this platform."; +#endif } if (capturer) { return new rtc::RefCountedObject( @@ -101,7 +109,8 @@ Stream::Stream() source_(AudioSourceInfo::kUnknown, VideoSourceInfo::kUnknown), #endif ended_(false), - id_("") {} + id_("") { +} Stream::Stream(const std::string& id) : media_stream_(nullptr), renderer_impl_(nullptr), @@ -115,25 +124,44 @@ Stream::Stream(const std::string& id) source_(AudioSourceInfo::kUnknown, VideoSourceInfo::kUnknown), #endif ended_(false), - id_(id) {} + id_(id) { +} #elif defined(WEBRTC_LINUX) Stream::Stream() - : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), va_renderer_impl_(nullptr), ended_(false), id_("") {} + : media_stream_(nullptr), + renderer_impl_(nullptr), + audio_renderer_impl_(nullptr), + va_renderer_impl_(nullptr), + ended_(false), + id_("") {} Stream::Stream(MediaStreamInterface* media_stream, StreamSourceInfo source) - : media_stream_(nullptr), va_renderer_impl_(nullptr),source_(source) { + : media_stream_(nullptr), va_renderer_impl_(nullptr), source_(source) { MediaStream(media_stream); } Stream::Stream(const std::string& id) - : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), va_renderer_impl_(nullptr), ended_(false), id_(id) {} + : media_stream_(nullptr), + renderer_impl_(nullptr), + audio_renderer_impl_(nullptr), + va_renderer_impl_(nullptr), + ended_(false), + id_(id) {} #else Stream::Stream() - : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), ended_(false), id_("") {} + : media_stream_(nullptr), + renderer_impl_(nullptr), + audio_renderer_impl_(nullptr), + ended_(false), + id_("") {} Stream::Stream(MediaStreamInterface* media_stream, StreamSourceInfo source) : media_stream_(nullptr), source_(source) { MediaStream(media_stream); } Stream::Stream(const std::string& id) - : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), ended_(false), id_(id) {} + : media_stream_(nullptr), + renderer_impl_(nullptr), + audio_renderer_impl_(nullptr), + ended_(false), + id_(id) {} #endif MediaStreamInterface* Stream::MediaStream() const { @@ -525,7 +553,7 @@ LocalStream::LocalStream(const LocalCameraStreamParameters& parameters, parameters.ResolutionWidth(), parameters.ResolutionHeight(), parameters.Fps(), DeviceUtils::GetVideoCaptureDeviceIndex(parameters.CameraId()), - parameters.BackgroundBlur()); + parameters.ICParams()); #else capturer_ = ObjcVideoCapturerFactory::Create(parameters); if (!capturer_) { @@ -720,9 +748,7 @@ std::string RemoteStream::Origin() { } RemoteStream::RemoteStream(const std::string& id, const std::string& from) - :Stream(id), - origin_(from) { -} + : Stream(id), origin_(from) {} void RemoteStream::MediaStream(MediaStreamInterface* media_stream) { Stream::MediaStream(media_stream); @@ -733,13 +759,14 @@ MediaStreamInterface* RemoteStream::MediaStream() { #ifdef OWT_ENABLE_QUIC QuicStream::QuicStream(owt::quic::WebTransportStreamInterface* quic_stream, - const std::string& session_id) - : quic_stream_(quic_stream), session_id_(session_id), can_read_(true), - can_write_(true), fin_read_(false) { -} - -QuicStream::~QuicStream() { -} + const std::string& session_id) + : quic_stream_(quic_stream), + session_id_(session_id), + can_read_(true), + can_write_(true), + fin_read_(false) {} + +QuicStream::~QuicStream() {} size_t QuicStream::Write(uint8_t* data, size_t length) { if (quic_stream_ && data != nullptr && length > 0) { diff --git a/talk/owt/sdk/config/owt_ic.json b/talk/owt/sdk/config/owt_ic.json new file mode 100644 index 000000000..9c8b98c2d --- /dev/null +++ b/talk/owt/sdk/config/owt_ic.json @@ -0,0 +1,14 @@ +{ + "inference_engine": { + "xml_config_file": "ic/inference_engine/plugins.xml" + }, + "background_blur": { + "bin": [{ + "source": "https://raw.githubusercontent.com/open-webrtc-toolkit/owt-model-zoo/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.xml", + "target": "ic/model/owt-selfie-segmentation-144x256.xml" + }, { + "source": "https://raw.githubusercontent.com/open-webrtc-toolkit/owt-model-zoo/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.bin", + "target": "ic/model/owt-selfie-segmentation-144x256.bin" + }] + } +} diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index 70a22ffb5..805329c76 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -92,18 +92,3 @@ copy("openvino_opencv_library") { } outputs = [ "$target_out_dir/{{source_file_part}}" ] } - -executable("test_background_blur") { - sources = [ "test_background_blur.cc" ] - include_dirs = [ "//talk/owt/sdk/include/cpp" ] - configs += [ - ":openvino", - ":openvino_opencv", - ] - deps = [ - "//talk/owt:owt_sdk_base", - ] - data_deps = [ - ":owt_ic", - ] -} diff --git a/talk/owt/sdk/include/cpp/owt/base/intelligentcollaborationparameters.h b/talk/owt/sdk/include/cpp/owt/base/intelligentcollaborationparameters.h new file mode 100644 index 000000000..fb3622b50 --- /dev/null +++ b/talk/owt/sdk/include/cpp/owt/base/intelligentcollaborationparameters.h @@ -0,0 +1,30 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_INTELLIGENTCOLLABORATIONPARAMETERS_H_ +#define OWT_BASE_INTELLIGENTCOLLABORATIONPARAMETERS_H_ + +namespace owt { +namespace base { + +class IntelligentCollaborationParameters final { + public: + IntelligentCollaborationParameters() = default; + ~IntelligentCollaborationParameters() = default; + + void BackgroundBlur(bool enable); + void BlurRadius(int radius); + + bool BackgroundBlur() const; + int BlurRadius() const; + + private: + bool enable_background_blur_ = false; + int blur_radius_ = 55; +}; + +} // namespace base +} // namespace owt + +#endif // OWT_BASE_INTELLIGENTCOLLABORATIONPARAMETERS_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h index 251382f09..c4b926df5 100644 --- a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h +++ b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h @@ -6,8 +6,9 @@ #define OWT_BASE_LOCALCAMERASTREAMPARAMETERS_H_ #include #include "owt/base/commontypes.h" +#include "owt/base/intelligentcollaborationparameters.h" namespace owt { -namespace base{ +namespace base { /** @brief This class contains parameters and methods that needed for creating a local camera stream. @@ -52,10 +53,10 @@ class LocalCameraStreamParameters final { */ void Fps(int fps); /** - @brief Set whether to enable the background blur - @param enable Indicates if the background is enabled + @brief Get the reference of intelligent collaboration parameters */ - void BackgroundBlur(bool enable); + IntelligentCollaborationParameters& ICParams(); + const IntelligentCollaborationParameters& ICParams() const; /** @cond */ std::string CameraId() const { return camera_id_; } std::string StreamName() const { return stream_name_; } @@ -64,7 +65,6 @@ class LocalCameraStreamParameters final { int Fps() const { return fps_; } bool VideoEnabled() const { return video_enabled_; } bool AudioEnabled() const { return audio_enabled_; } - bool BackgroundBlur() const { return background_blur_; } /** @endcond */ private: std::string camera_id_; @@ -74,10 +74,9 @@ class LocalCameraStreamParameters final { int fps_; bool video_enabled_; bool audio_enabled_; - bool background_blur_; + IntelligentCollaborationParameters ic_params_; }; - #ifdef OWT_ENABLE_QUIC /** @brief This class contains parameters and methods needed for creating @@ -87,15 +86,11 @@ class LocalCameraStreamParameters final { */ class LocalDataStreamParameters final { public: - LocalDataStreamParameters(bool data_enabled) { - data_enabled_ = data_enabled; - } + LocalDataStreamParameters(bool data_enabled) { data_enabled_ = data_enabled; } ~LocalDataStreamParameters() {} /** @cond */ - bool DataEnabled() const { - return data_enabled_; - } + bool DataEnabled() const { return data_enabled_; } /** @endcond */ private: @@ -117,11 +112,11 @@ class LocalCustomizedStreamParameters final { @param video_anabled Indicates if video is enabled for this stream. */ LocalCustomizedStreamParameters(bool audio_enabled, bool video_enabled) { - video_enabled_ = video_enabled; - audio_enabled_ = audio_enabled; - fps_ = 0; - bitrate_kbps_ = 0; - resolution_width_ = resolution_height_ = 0; + video_enabled_ = video_enabled; + audio_enabled_ = audio_enabled; + fps_ = 0; + bitrate_kbps_ = 0; + resolution_width_ = resolution_height_ = 0; } ~LocalCustomizedStreamParameters() {} /** @@ -141,16 +136,12 @@ class LocalCustomizedStreamParameters final { will failed. @param fps The frame rate of the video. */ - void Fps(int fps) { - fps_ = fps; - } + void Fps(int fps) { fps_ = fps; } /** @brief Set the bitrate of encoded frame. @param bitrate_kbps The bitrate expected for the encoded stream. */ - void Bitrate(int bitrate_kbps) { - bitrate_kbps_ = bitrate_kbps; - } + void Bitrate(int bitrate_kbps) { bitrate_kbps_ = bitrate_kbps; } /** @cond */ int ResolutionWidth() const { return resolution_width_; } int ResolutionHeight() const { return resolution_height_; } @@ -211,9 +202,7 @@ class LocalDesktopStreamParameters final { @param soruce_type Indicates if capture from screen or an app. @param capture_policy the OR of any of the DesktopCapturePolicy options. */ - LocalDesktopStreamParameters( - bool audio_enabled, - bool video_enabled); + LocalDesktopStreamParameters(bool audio_enabled, bool video_enabled); ~LocalDesktopStreamParameters() {} /** @brief Get video is enabled or not for this stream. @@ -230,9 +219,7 @@ class LocalDesktopStreamParameters final { @param source_type Indicate if capturing the full screen or an application. */ - void SourceType(DesktopSourceType source_type) { - source_type_ = source_type; - } + void SourceType(DesktopSourceType source_type) { source_type_ = source_type; } /** @brief Set the capturer features @params capture_policy Indicate the feature used by the capturer. @@ -258,6 +245,6 @@ class LocalDesktopStreamParameters final { DesktopSourceType source_type_; DesktopCapturePolicy capture_policy_; }; -} -} +} // namespace base +} // namespace owt #endif // OWT_BASE_LOCALCAMERASTREAMPARAMETERS_H_ diff --git a/talk/owt/sdk/sample/BUILD.gn b/talk/owt/sdk/sample/BUILD.gn new file mode 100644 index 000000000..8816d474b --- /dev/null +++ b/talk/owt/sdk/sample/BUILD.gn @@ -0,0 +1,14 @@ +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import("//build_overrides/webrtc.gni") +import("//build_overrides/owt.gni") +import("//testing/test.gni") + +group("sample") { + deps = [] + if (is_win) { + deps += [ "win:sample_win" ] + } +} diff --git a/talk/owt/sdk/sample/win/BUILD.gn b/talk/owt/sdk/sample/win/BUILD.gn new file mode 100644 index 000000000..2f045b9d6 --- /dev/null +++ b/talk/owt/sdk/sample/win/BUILD.gn @@ -0,0 +1,11 @@ +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import("//build_overrides/webrtc.gni") +import("//build_overrides/owt.gni") +import("//testing/test.gni") + +group("sample_win") { + deps = [ "sample_background_blur" ] +} diff --git a/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn new file mode 100644 index 000000000..d22e60ce4 --- /dev/null +++ b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn @@ -0,0 +1,22 @@ +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import("//build_overrides/webrtc.gni") +import("//build_overrides/owt.gni") +import("//testing/test.gni") + +executable("sample_background_blur") { + sources = [ "sample_background_blur.cc" ] + include_dirs = [ "//talk/owt/sdk/include/cpp" ] + configs += [ + "//talk/owt/sdk/ic:openvino", + "//talk/owt/sdk/ic:openvino_opencv", + ] + deps = [ + "//talk/owt:owt_sdk_base", + ] + data_deps = [ + "//talk/owt/sdk/ic:owt_ic", + ] +} diff --git a/talk/owt/sdk/ic/test_background_blur.cc b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc similarity index 90% rename from talk/owt/sdk/ic/test_background_blur.cc rename to talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc index 62cd1ef74..3656ade87 100644 --- a/talk/owt/sdk/ic/test_background_blur.cc +++ b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc @@ -1,7 +1,8 @@ -// Copyright (C) <2018> Intel Corporation +// Copyright (C) <2021> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 +#include #include #include "owt/base/deviceutils.h" @@ -60,7 +61,8 @@ int main(int, char*[]) { DeviceUtils::VideoCapturerSupportedCapabilities(default_id)[0]; param.Resolution(capability.width, capability.height); param.Fps(capability.frameRate); - param.BackgroundBlur(true); + param.ICParams().BackgroundBlur(true); + param.ICParams().BlurRadius(55); int error = 0; auto stream = LocalStream::Create(param, error); if (error == 0) { From 572b0f4b9036212228ad28c0dc4a70b6947920e3 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Thu, 28 Oct 2021 14:49:09 +0800 Subject: [PATCH 08/30] Fixups --- .gitignore | 1 + talk/owt/BUILD.gn | 10 +-- talk/owt/sdk/base/cameravideocapturer.cc | 4 +- talk/owt/sdk/base/cameravideocapturer.h | 4 +- talk/owt/sdk/base/icmanager.cc | 67 ------------------- talk/owt/sdk/base/icmanager.h | 42 ------------ .../intelligentcollaborationparameters.cc | 24 +++---- talk/owt/sdk/base/linux/sharedobjectloader.cc | 16 ++--- .../sdk/base/localcamerastreamparameters.cc | 7 +- talk/owt/sdk/base/pluginmanager.cc | 27 ++++++++ talk/owt/sdk/base/sharedobjectloader.h | 14 ++-- talk/owt/sdk/base/sharedobjectpointer.h | 61 +++++++++++++++++ talk/owt/sdk/base/stream.cc | 20 +----- talk/owt/sdk/base/win/sharedobjectloader.cc | 26 +++---- talk/owt/sdk/config/owt_ic.json | 14 ---- talk/owt/sdk/ic/BUILD.gn | 4 +- talk/owt/sdk/ic/backgroundblur.cc | 47 ++++++++----- talk/owt/sdk/ic/backgroundblur.h | 7 +- talk/owt/sdk/ic/icmanager.cc | 37 ++++++++++ talk/owt/sdk/ic/icmanager.h | 37 ++++++++++ talk/owt/sdk/ic/postprocessing.cc | 18 ----- talk/owt/sdk/ic/postprocessing.h | 23 ------- talk/owt/sdk/ic/selfiesegmentation.cc | 17 ++--- .../include/cpp/owt/base/icmanagerinterface.h | 31 +++++++++ .../base/intelligentcollaborationparameters.h | 30 --------- .../owt/base/localcamerastreamparameters.h | 8 +-- .../sdk/include/cpp/owt/base/pluginmanager.h | 23 +++++++ .../cpp/owt/base/videoframepostprocessor.h} | 63 +++++++++-------- .../include/cpp/owt/ic/icmanagerinterface.h | 40 +++++++++++ .../ic/intelligentcollaborationparameters.h | 34 ++++++++++ .../sample_background_blur.cc | 45 +++++++++---- 31 files changed, 466 insertions(+), 335 deletions(-) delete mode 100644 talk/owt/sdk/base/icmanager.cc delete mode 100644 talk/owt/sdk/base/icmanager.h create mode 100644 talk/owt/sdk/base/pluginmanager.cc create mode 100644 talk/owt/sdk/base/sharedobjectpointer.h delete mode 100644 talk/owt/sdk/config/owt_ic.json create mode 100644 talk/owt/sdk/ic/icmanager.cc create mode 100644 talk/owt/sdk/ic/icmanager.h delete mode 100644 talk/owt/sdk/ic/postprocessing.cc delete mode 100644 talk/owt/sdk/ic/postprocessing.h create mode 100644 talk/owt/sdk/include/cpp/owt/base/icmanagerinterface.h delete mode 100644 talk/owt/sdk/include/cpp/owt/base/intelligentcollaborationparameters.h create mode 100644 talk/owt/sdk/include/cpp/owt/base/pluginmanager.h rename talk/owt/sdk/{base/videoframepostprocessing.h => include/cpp/owt/base/videoframepostprocessor.h} (57%) create mode 100644 talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h create mode 100644 talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h diff --git a/.gitignore b/.gitignore index bcbbdd677..32a92ce49 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /tools /dist /talk/owt/docs/ios/html +/.vscode/settings.json diff --git a/talk/owt/BUILD.gn b/talk/owt/BUILD.gn index a800f18ed..258c6fbf0 100644 --- a/talk/owt/BUILD.gn +++ b/talk/owt/BUILD.gn @@ -109,6 +109,7 @@ static_library("owt_sdk_base") { "sdk/base/peerconnectionchannel.h", "sdk/base/peerconnectiondependencyfactory.cc", "sdk/base/peerconnectiondependencyfactory.h", + "sdk/base/pluginmanager.cc", "sdk/base/sdputils.cc", "sdk/base/sdputils.h", "sdk/base/stream.cc", @@ -118,7 +119,6 @@ static_library("owt_sdk_base") { "sdk/base/sysinfo.h", "sdk/base/vcmcapturer.cc", "sdk/base/vcmcapturer.h", - "sdk/base/videoframepostprocessing.h", "sdk/base/webrtcaudiorendererimpl.cc", "sdk/base/webrtcaudiorendererimpl.h", "sdk/include/cpp/owt/base/audioplayerinterface.h", @@ -127,11 +127,13 @@ static_library("owt_sdk_base") { "sdk/include/cpp/owt/base/deviceutils.h", "sdk/include/cpp/owt/base/exception.h", "sdk/include/cpp/owt/base/framegeneratorinterface.h", - "sdk/include/cpp/owt/base/intelligentcollaborationparameters.h", "sdk/include/cpp/owt/base/localcamerastreamparameters.h", "sdk/include/cpp/owt/base/logging.h", + "sdk/include/cpp/owt/base/pluginmanager.h", "sdk/include/cpp/owt/base/stream.h", + "sdk/include/cpp/owt/base/videoframepostprocessor.h", "sdk/include/cpp/owt/base/videorendererinterface.h", + "sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h", ] if (is_win || is_linux) { sources += [ @@ -143,13 +145,13 @@ static_library("owt_sdk_base") { "sdk/base/customizedvideosource.h", "sdk/base/encodedvideoencoderfactory.cc", "sdk/base/encodedvideoencoderfactory.h", - "sdk/base/icmanager.cc", - "sdk/base/icmanager.h", "sdk/base/sharedobjectloader.h", + "sdk/base/sharedobjectpointer.h", "sdk/base/webrtcvideorendererimpl.cc", "sdk/base/webrtcvideorendererimpl.h", "sdk/base/windowcapturer.cc", "sdk/include/cpp/owt/base/videodecoderinterface.h", + "sdk/include/cpp/owt/ic/icmanagerinterface.h", ] } public_deps = [ diff --git a/talk/owt/sdk/base/cameravideocapturer.cc b/talk/owt/sdk/base/cameravideocapturer.cc index cb765331c..42d7d730e 100644 --- a/talk/owt/sdk/base/cameravideocapturer.cc +++ b/talk/owt/sdk/base/cameravideocapturer.cc @@ -83,8 +83,8 @@ void CameraVideoCapturer::RemoveSink( } void CameraVideoCapturer::AddVideoFramePostProcessor( - std::shared_ptr post_processing) { - video_frame_post_processors_.emplace_back(post_processing); + std::shared_ptr post_processor) { + video_frame_post_processors_.emplace_back(post_processor); } void CameraVideoCapturer::UpdateVideoAdapter() { diff --git a/talk/owt/sdk/base/cameravideocapturer.h b/talk/owt/sdk/base/cameravideocapturer.h index cad74ffa7..649eccfe5 100644 --- a/talk/owt/sdk/base/cameravideocapturer.h +++ b/talk/owt/sdk/base/cameravideocapturer.h @@ -20,7 +20,7 @@ #include "media/base/video_adapter.h" #include "media/base/video_broadcaster.h" -#include "videoframepostprocessing.h" +#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" // This file is borrowed from webrtc project namespace owt { @@ -36,7 +36,7 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface void RemoveSink(rtc::VideoSinkInterface* sink) override; void AddVideoFramePostProcessor( - const std::shared_ptr post_processing); + const std::shared_ptr post_processor); protected: void OnFrame(const webrtc::VideoFrame& frame); diff --git a/talk/owt/sdk/base/icmanager.cc b/talk/owt/sdk/base/icmanager.cc deleted file mode 100644 index ce876e0af..000000000 --- a/talk/owt/sdk/base/icmanager.cc +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#include "icmanager.h" - -#include - -#include "third_party/jsoncpp/source/include/json/json.h" -#include "webrtc/rtc_base/logging.h" - -namespace owt { -namespace base { - -ICManager* ICManager::GetInstance() { - static ICManager instance; - return &instance; -} - -bool ICManager::LoadConfig(const std::string& config_path) { - Json::Reader reader; - Json::Value config; - std::ifstream fin(config_path); - if (!fin) { - return false; - } - if (!reader.parse(fin, config)) { - return false; - } - Json::Value backgroundBlur = config.get("background_blur", Json::Value()); - Json::Value bins = backgroundBlur.get("bin", Json::Value()); - for (auto& bin : bins) { - std::string source = bin.get("source", "").asString(); - std::string target = bin.get("target", "").asString(); - // TODO WIP - } -} - -std::shared_ptr ICManager::CreatePostProcessor( - const char* name) { - return create_post_processing_ - ? std::shared_ptr( - create_post_processing_(name), - [](VideoFramePostProcessor* ptr) { ptr->Release(); }) - : nullptr; -} - -ICManager::ICManager() - : so_( -#ifdef WEBRTC_WIN - "owt_ic.dll" -#elif WEBRTC_LINUX - "owt_ic.so" -#endif - ) { - if (so_.IsLoaded()) { - RTC_LOG(INFO) << "owt_ic.dll is loaded."; - create_post_processing_ = reinterpret_cast( - so_.GetSymbol("CreatePostProcessor")); - LoadConfig("config/owt_ic.json"); - } else { - RTC_LOG(WARNING) << "owt_ic.dll is not loaded."; - } -} - -} // namespace base -} // namespace owt diff --git a/talk/owt/sdk/base/icmanager.h b/talk/owt/sdk/base/icmanager.h deleted file mode 100644 index 7d918e741..000000000 --- a/talk/owt/sdk/base/icmanager.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_BASE_ICMANAGER_H_ -#define OWT_BASE_ICMANAGER_H_ - -#include -#include - -#include "base/macros.h" -#include "talk/owt/sdk/base/sharedobjectloader.h" -#include "talk/owt/sdk/base/videoframepostprocessing.h" -#include "webrtc/api/scoped_refptr.h" - -namespace owt { -namespace base { - -class ICManager { - public: - static ICManager* GetInstance(); - - bool LoadConfig(const std::string& config_path); - std::shared_ptr CreatePostProcessor( - const char* name); - - private: - ICManager(); - ~ICManager() = default; - - typedef owt::base::VideoFramePostProcessor* (*CREATE_POST_PROCESSOR)( - const char* name); - SharedObjectLoader so_; - CREATE_POST_PROCESSOR create_post_processing_ = nullptr; - - DISALLOW_COPY_AND_ASSIGN(ICManager); -}; - -} // namespace base -} // namespace owt - -#endif // OWT_BASE_ICMANAGER_H_ diff --git a/talk/owt/sdk/base/intelligentcollaborationparameters.cc b/talk/owt/sdk/base/intelligentcollaborationparameters.cc index d55ac8858..1a0e9961d 100644 --- a/talk/owt/sdk/base/intelligentcollaborationparameters.cc +++ b/talk/owt/sdk/base/intelligentcollaborationparameters.cc @@ -1,23 +1,17 @@ -#include "talk/owt/sdk/include/cpp/owt/base/intelligentcollaborationparameters.h" +#include "talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h" namespace owt { -namespace base { +namespace ic { -void IntelligentCollaborationParameters::BackgroundBlur(bool enable) { - enable_background_blur_ = enable; +std::vector>& +IntelligentCollaborationParameters::PostProcessors() { + return post_processors_; } -void IntelligentCollaborationParameters::BlurRadius(int radius) { - blur_radius_ = radius; -} - -bool IntelligentCollaborationParameters::BackgroundBlur() const { - return enable_background_blur_; -} - -int IntelligentCollaborationParameters::BlurRadius() const { - return blur_radius_; +const std::vector>& +IntelligentCollaborationParameters::PostProcessors() const { + return post_processors_; } } // namespace base -} // namespace owt +} // namespace owt \ No newline at end of file diff --git a/talk/owt/sdk/base/linux/sharedobjectloader.cc b/talk/owt/sdk/base/linux/sharedobjectloader.cc index d12168f6a..024fad359 100644 --- a/talk/owt/sdk/base/linux/sharedobjectloader.cc +++ b/talk/owt/sdk/base/linux/sharedobjectloader.cc @@ -9,21 +9,17 @@ namespace owt { namespace base { -SharedObjectLoader::SharedObjectLoader(const char* name) - : shared_object(dlopen(name, RTLD_NOW)) {} +SharedObjectLoader::SharedObjectLoader(const char* path) + : shared_object(dlopen(path, RTLD_NOW), dlclose) {} -SharedObjectLoader::~SharedObjectLoader() { - if (shared_object) { - dlclose(shared_object); - } -} +SharedObjectLoader::~SharedObjectLoader() {} bool SharedObjectLoader::IsLoaded() const { - return shared_object_; + return shared_object_.get(); } -void* SharedObjectLoader::get_symbol(const char* name) { - return shared_object_ ? dlsym(shared_object, name) : nullptr; +void* SharedObjectLoader::get_symbol(const char* name) const { + return shared_object_ ? dlsym(shared_object.get(), name) : nullptr; } } // namespace base diff --git a/talk/owt/sdk/base/localcamerastreamparameters.cc b/talk/owt/sdk/base/localcamerastreamparameters.cc index 7e30b1f3f..05697ae95 100644 --- a/talk/owt/sdk/base/localcamerastreamparameters.cc +++ b/talk/owt/sdk/base/localcamerastreamparameters.cc @@ -1,8 +1,8 @@ // Copyright (C) <2018> Intel Corporation // // SPDX-License-Identifier: Apache-2.0 -#include "talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h" #include +#include "talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h" namespace owt { namespace base { LocalCameraStreamParameters::LocalCameraStreamParameters(bool audio_enabled, @@ -19,10 +19,11 @@ LocalCameraStreamParameters::LocalCameraStreamParameters(bool audio_enabled, void LocalCameraStreamParameters::Fps(int fps) { fps_ = fps; } -IntelligentCollaborationParameters& LocalCameraStreamParameters::ICParams() { +owt::ic::IntelligentCollaborationParameters& +LocalCameraStreamParameters::ICParams() { return ic_params_; } -const IntelligentCollaborationParameters& +const owt::ic::IntelligentCollaborationParameters& LocalCameraStreamParameters::ICParams() const { return ic_params_; } diff --git a/talk/owt/sdk/base/pluginmanager.cc b/talk/owt/sdk/base/pluginmanager.cc new file mode 100644 index 000000000..c18e903de --- /dev/null +++ b/talk/owt/sdk/base/pluginmanager.cc @@ -0,0 +1,27 @@ +#include "..\include\cpp\owt\base\pluginmanager.h" + +namespace owt { +namespace base { + +PluginManager* PluginManager::GetInstance() { + static PluginManager plugin_manager; + return &plugin_manager; +} + +owt::base::SharedObjectPointer& +PluginManager::ICPlugin() { + return ic_plugin; +} + +PluginManager::PluginManager() + : ic_plugin( +#ifdef WEBRTC_WIN + "owt_ic.dll" +#elif WEBRTC_LINUX + "owt_ic.so" +#endif + ) { +} + +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/base/sharedobjectloader.h b/talk/owt/sdk/base/sharedobjectloader.h index 5f83c1321..a33e61043 100644 --- a/talk/owt/sdk/base/sharedobjectloader.h +++ b/talk/owt/sdk/base/sharedobjectloader.h @@ -5,20 +5,26 @@ #ifndef OWT_BASE_SHAREDOBJECTLOADER_H_ #define OWT_BASE_SHAREDOBJECTLOADER_H_ +#include +#include + namespace owt { namespace base { +// This class is derived from openvino inference engine class SharedObjectLoader { - public: - explicit SharedObjectLoader(const char* name); + public: + explicit SharedObjectLoader(const char* path); ~SharedObjectLoader(); bool IsLoaded() const; - void* GetSymbol(const char* name); + void* GetSymbol(const char* name) const; + + void* GetSymbol(const std::string& name) const; private: - void* shared_object_; + std::shared_ptr shared_object_; }; } // namespace base diff --git a/talk/owt/sdk/base/sharedobjectpointer.h b/talk/owt/sdk/base/sharedobjectpointer.h new file mode 100644 index 000000000..f99241e23 --- /dev/null +++ b/talk/owt/sdk/base/sharedobjectpointer.h @@ -0,0 +1,61 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_SHAREDOBJECTPOINTER_H_ +#define OWT_BASE_SHAREDOBJECTPOINTER_H_ + +#include "talk/owt/sdk/base/sharedobjectloader.h" + +#include +#include + +namespace owt { +namespace base { + +// This class is derived from openvino inference engine +template +struct SOTrait {}; + +// This class is derived from openvino inference engine +template +class SharedObjectPointer { + public: + explicit SharedObjectPointer(const char* path) : so_(path) { Load(); } + + explicit SharedObjectPointer(const SharedObjectLoader& loader) : so(loader) { + Load(); + } + + explicit SharedObjectPointer(SharedObjectLoader&& loader) + : so(std::move(loader)) { + Load(); + } + + bool IsLoaded() const { return so_.IsLoaded() && ptr_.get(); } + + T* operator->() { return ptr_.get(); } + + protected: + void Load() { + if (so_.IsLoaded()) { + using Creator = T*(); + using Destroyer = void(T*); + Creator* creator = reinterpret_cast( + so_.GetSymbol(std::string("Create") + SOTrait::name)); + Destroyer* destroyer = reinterpret_cast( + so_.GetSymbol(std::string("Destroy") + SOTrait::name)); + if (creator && destroyer) { + ptr_ = std::shared_ptr(creator(), destroyer); + } + } + } + + SharedObjectLoader so_; + std::shared_ptr ptr_; +}; + +} // namespace base +} // namespace owt + +#endif // OWT_BASE_SHAREDOBJECTPOINTER_H_ diff --git a/talk/owt/sdk/base/stream.cc b/talk/owt/sdk/base/stream.cc index a7de19c38..218a277c9 100644 --- a/talk/owt/sdk/base/stream.cc +++ b/talk/owt/sdk/base/stream.cc @@ -34,9 +34,6 @@ #endif #include "talk/owt/sdk/include/cpp/owt/base/deviceutils.h" #include "talk/owt/sdk/include/cpp/owt/base/stream.h" -#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) -#include "talk/owt/sdk/base/icmanager.h" -#endif using namespace rtc; namespace owt { @@ -49,8 +46,8 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { const size_t height, const size_t fps, int capture_device_idx, - const IntelligentCollaborationParameters& ic_params = - IntelligentCollaborationParameters()) { + const owt::ic::IntelligentCollaborationParameters& ic_params = + owt::ic::IntelligentCollaborationParameters()) { std::unique_ptr capturer; std::unique_ptr info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -61,19 +58,8 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { for (int i = 0; i < num_devices; ++i) { capturer = absl::WrapUnique(owt::base::VcmCapturer::Create( width, height, fps, capture_device_idx)); - if (ic_params.BackgroundBlur()) { -#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) - auto post_processor = - ICManager::GetInstance()->CreatePostProcessor("background_blur"); - if (post_processor) { + for (auto post_processor : ic_params.PostProcessors()) { capturer->AddVideoFramePostProcessor(post_processor); - } else { - RTC_LOG(WARNING) << "Unable to create background blur instance."; - } -#else - RTC_LOG(WARNING) - << "Background blur is not supported on this platform."; -#endif } if (capturer) { return new rtc::RefCountedObject( diff --git a/talk/owt/sdk/base/win/sharedobjectloader.cc b/talk/owt/sdk/base/win/sharedobjectloader.cc index fafa505e4..5a5f51a85 100644 --- a/talk/owt/sdk/base/win/sharedobjectloader.cc +++ b/talk/owt/sdk/base/win/sharedobjectloader.cc @@ -9,23 +9,25 @@ namespace owt { namespace base { -SharedObjectLoader::SharedObjectLoader(const char* name) - : shared_object_(LoadLibraryA(name)) {} +SharedObjectLoader::SharedObjectLoader(const char* path) + : shared_object_(LoadLibraryA(path), FreeLibrary) {} -SharedObjectLoader::~SharedObjectLoader() { - if (shared_object_) { - FreeLibrary(reinterpret_cast(shared_object_)); - } -} +SharedObjectLoader::~SharedObjectLoader() {} bool SharedObjectLoader::IsLoaded() const { - return shared_object_; + return shared_object_.get(); +} + +void* SharedObjectLoader::GetSymbol(const char* name) const { + if (shared_object_) { + return GetProcAddress(reinterpret_cast(shared_object_.get()), + name); + } + return nullptr; } -void* SharedObjectLoader::GetSymbol(const char* name) { - return shared_object_ - ? GetProcAddress(reinterpret_cast(shared_object_), name) - : nullptr; +void* SharedObjectLoader::GetSymbol(const std::string& name) const { + return GetSymbol(name.c_str()); } } // namespace base diff --git a/talk/owt/sdk/config/owt_ic.json b/talk/owt/sdk/config/owt_ic.json deleted file mode 100644 index 9c8b98c2d..000000000 --- a/talk/owt/sdk/config/owt_ic.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "inference_engine": { - "xml_config_file": "ic/inference_engine/plugins.xml" - }, - "background_blur": { - "bin": [{ - "source": "https://raw.githubusercontent.com/open-webrtc-toolkit/owt-model-zoo/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.xml", - "target": "ic/model/owt-selfie-segmentation-144x256.xml" - }, { - "source": "https://raw.githubusercontent.com/open-webrtc-toolkit/owt-model-zoo/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.bin", - "target": "ic/model/owt-selfie-segmentation-144x256.bin" - }] - } -} diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index 805329c76..93468e2c4 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -44,8 +44,8 @@ shared_library("owt_ic") { sources = [ "backgroundblur.cc", "backgroundblur.h", - "postprocessing.cc", - "postprocessing.h", + "icmanager.cc", + "icmanager.h", "selfiesegmentation.cc", "selfiesegmentation.h", ] diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index c29cf4b70..82ed211d9 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -20,6 +20,17 @@ BackgroundBlur::BackgroundBlur() "C:/Users/wangzhib/Desktop/segmentation/contrib/" "owt-selfie-segmentation-144x256.xml")) {} +bool BackgroundBlur::SetParameter(const std::string& key, + const std::string& value) { + if (key == "model_path") { + + } else if (key == "blur_radius") { + blur_radius_ = stoi(value); + return true; + } + return false; +} + rtc::scoped_refptr BackgroundBlur::Process( const rtc::scoped_refptr& buffer) { auto i420 = buffer->GetI420(); @@ -30,25 +41,31 @@ rtc::scoped_refptr BackgroundBlur::Process( cv::Mat frame(buffer->height(), buffer->width(), CV_8UC3, rgb.data()); cv::Mat input; - frame.convertTo(input, CV_32FC3, - 1. / std::numeric_limits::max()); + frame.convertTo(input, CV_32FC3, 1. / UCHAR_MAX); cv::Mat mask = model->predict(input); // mask is of 8UC1 - cv::Mat resizedMask; - cv::resize(mask, resizedMask, {buffer->width(), buffer->height()}); - cv::Mat mask3; - std::vector masks{ - std::numeric_limits::max() - resizedMask, - std::numeric_limits::max() - resizedMask, - std::numeric_limits::max() - resizedMask}; - cv::merge(masks.data(), 3, mask3); + cv::resize(mask, mask, {buffer->width(), buffer->height()}); + cv::Mat foregroundMask; + cv::merge(std::vector{mask, mask, mask}, foregroundMask); + cv::Mat backgroundMask; + cv::merge(std::vector{UCHAR_MAX - mask, UCHAR_MAX - mask, + UCHAR_MAX - mask}, + backgroundMask); + // apply a masked blur on background + cv::Mat maskedFrame; + cv::multiply(frame, backgroundMask, maskedFrame, 1. / UCHAR_MAX); cv::Mat background; - cv::multiply(frame, mask3, background, - 1. / std::numeric_limits::max()); - frame -= background; - cv::GaussianBlur(background, background, {blur_radius_, blur_radius_}, 0); - cv::add(frame, background, frame); + cv::GaussianBlur(maskedFrame, background, {blur_radius_, blur_radius_}, 0); + cv::Mat blurredMask; + cv::GaussianBlur(backgroundMask, blurredMask, {blur_radius_, blur_radius_}, + 0); + cv::divide(background, blurredMask, background, UCHAR_MAX); + cv::multiply(background, backgroundMask, background, 1. / UCHAR_MAX); + + cv::Mat foreground; + cv::multiply(frame, foregroundMask, foreground, 1. / UCHAR_MAX); + frame = foreground + background; auto newBuffer = webrtc::I420Buffer::Create(buffer->width(), buffer->height()); diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h index 3b71539ae..b310b0e40 100644 --- a/talk/owt/sdk/ic/backgroundblur.h +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -7,17 +7,20 @@ #include -#include "talk/owt/sdk/base/videoframepostprocessing.h" +#include "rtc_base/logging.h" +#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" +#include "talk/owt/sdk/ic/selfiesegmentation.h" namespace owt { namespace ic { -class SelfieSegmentation; class BackgroundBlur : public owt::base::VideoFramePostProcessor { public: BackgroundBlur(); ~BackgroundBlur() override = default; + bool SetParameter(const std::string& key, const std::string& value) override; + rtc::scoped_refptr Process( const rtc::scoped_refptr& buffer) override; diff --git a/talk/owt/sdk/ic/icmanager.cc b/talk/owt/sdk/ic/icmanager.cc new file mode 100644 index 000000000..0b09b1952 --- /dev/null +++ b/talk/owt/sdk/ic/icmanager.cc @@ -0,0 +1,37 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "talk/owt/sdk/ic/icmanager.h" + +#include "talk/owt/sdk/ic/backgroundblur.h" + +namespace owt { +namespace ic { + +bool ICManager::InitializeInferenceEngineCore( + const std::string& plugins_xml_path) { + return false; +} + +std::shared_ptr +ICManager::CreatePostProcessor(ICPlugin plugin) { + switch (plugin) { + case ICPlugin::BACKGROUND_BLUR: + return std::shared_ptr( + new BackgroundBlur); + default:; + } + return nullptr; +} + +} // namespace ic +} // namespace owt + +owt::ic::ICManagerInterface* CreateICManager() { + return new owt::ic::ICManager; +} + +void DestroyICManager(owt::ic::ICManagerInterface* ic_manager) { + delete reinterpret_cast(ic_manager); +} diff --git a/talk/owt/sdk/ic/icmanager.h b/talk/owt/sdk/ic/icmanager.h new file mode 100644 index 000000000..86d2b4e58 --- /dev/null +++ b/talk/owt/sdk/ic/icmanager.h @@ -0,0 +1,37 @@ +#ifndef OWT_IC_ICMANAGER_H_ +#define OWT_IC_ICMANAGER_H_ + +#include "talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h" + +namespace owt { +namespace ic { + +class ICManager : public ICManagerInterface { + public: + ICManager() = default; + + bool InitializeInferenceEngineCore( + const std::string& plugins_xml_path) override; + + std::shared_ptr CreatePostProcessor( + ICPlugin plugin) override; +}; + +} // namespace ic +} // namespace owt + +#ifdef WEBRTC_WIN +#ifdef OWT_IC_IMPL +#define OWT_IC_EXPORT extern "C" __declspec(dllexport) +#else +#define OWT_IC_EXPORT extern "C" __declspec(dllimport) +#endif +#else +#define OWT_IC_EXPORT extern "C" __attribute__((visibility("default"))) +#endif + +OWT_IC_EXPORT owt::ic::ICManagerInterface* CreateICManager(); + +OWT_IC_EXPORT void DestroyICManager(owt::ic::ICManagerInterface* ic_manager); + +#endif // OWT_IC_ICMANAGER_H_ diff --git a/talk/owt/sdk/ic/postprocessing.cc b/talk/owt/sdk/ic/postprocessing.cc deleted file mode 100644 index b535b628d..000000000 --- a/talk/owt/sdk/ic/postprocessing.cc +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#include "postprocessing.h" - -#include - -#include "backgroundblur.h" - -owt::base::VideoFramePostProcessor* CreatePostProcessor(const char* name) { - if (name) { - if (strcmp(name, "background_blur") == 0) { - return new owt::ic::BackgroundBlur; - } - } - return nullptr; -} diff --git a/talk/owt/sdk/ic/postprocessing.h b/talk/owt/sdk/ic/postprocessing.h deleted file mode 100644 index c82ca2bde..000000000 --- a/talk/owt/sdk/ic/postprocessing.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_IC_POSTPROCESSING_H_ -#define OWT_IC_POSTPROCESSING_H_ - -#include "../base/videoframepostprocessing.h" - -#ifdef WEBRTC_WIN -#ifdef OWT_IC_IMPL -#define OWT_IC_EXPORT extern "C" __declspec(dllexport) -#else -#define OWT_IC_EXPORT extern "C" __declspec(dllimport) -#endif -#else -#define OWT_IC_EXPORT -#endif - -OWT_IC_EXPORT owt::base::VideoFramePostProcessor* CreatePostProcessor( - const char* name); - -#endif // OWT_IC_POSTPROCESSING_H_ diff --git a/talk/owt/sdk/ic/selfiesegmentation.cc b/talk/owt/sdk/ic/selfiesegmentation.cc index 93c5cef6c..6f769d8ce 100644 --- a/talk/owt/sdk/ic/selfiesegmentation.cc +++ b/talk/owt/sdk/ic/selfiesegmentation.cc @@ -18,16 +18,13 @@ SelfieSegmentation::SelfieSegmentation(const std::string& xmlPath, network(core.ReadNetwork(xmlPath)), executableNetwork(), request() { - for (auto& p : network.getInputsInfo()) { - inputName = p.first; - break; - } - for (auto& p : network.getOutputsInfo()) { - p.second->setPrecision(IE::Precision::U8); - outputName = p.first; - outputSize = p.second->getTensorDesc().getDims(); - break; - } + assert(!network.getInputsInfo().empty()); + assert(!network.getOutputsInfo().empty()); + inputName = network.getInputsInfo().begin()->first; + outputName = network.getOutputsInfo().begin()->first; + IE::DataPtr outputData = network.getOutputsInfo().begin()->second; + outputData->setPrecision(IE::Precision::U8); + outputSize = outputData->getTensorDesc().getDims(); IE::PreProcessInfo& preprocess = network.getInputsInfo()[inputName]->getPreProcess(); diff --git a/talk/owt/sdk/include/cpp/owt/base/icmanagerinterface.h b/talk/owt/sdk/include/cpp/owt/base/icmanagerinterface.h new file mode 100644 index 000000000..47bf0856c --- /dev/null +++ b/talk/owt/sdk/include/cpp/owt/base/icmanagerinterface.h @@ -0,0 +1,31 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_ICMANAGERINTERFACE_H_ +#define OWT_BASE_ICMANAGERINTERFACE_H_ + +#include +#include + +#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" + +namespace owt { +namespace base { + + enum class ICPlugin { + BACKGROUND_BLUR, + }; + +class ICManagerInterface { + public: + virtual void InitializeInferenceEngineCore(const std::string &plugin_xml_path); + + virtual std::shared_ptr CreatePostProcessor( + ICPlugin plugin); +}; + +} // namespace base +} // namespace owt + +#endif // OWT_BASE_ICMANAGERINTERFACE_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/intelligentcollaborationparameters.h b/talk/owt/sdk/include/cpp/owt/base/intelligentcollaborationparameters.h deleted file mode 100644 index fb3622b50..000000000 --- a/talk/owt/sdk/include/cpp/owt/base/intelligentcollaborationparameters.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_BASE_INTELLIGENTCOLLABORATIONPARAMETERS_H_ -#define OWT_BASE_INTELLIGENTCOLLABORATIONPARAMETERS_H_ - -namespace owt { -namespace base { - -class IntelligentCollaborationParameters final { - public: - IntelligentCollaborationParameters() = default; - ~IntelligentCollaborationParameters() = default; - - void BackgroundBlur(bool enable); - void BlurRadius(int radius); - - bool BackgroundBlur() const; - int BlurRadius() const; - - private: - bool enable_background_blur_ = false; - int blur_radius_ = 55; -}; - -} // namespace base -} // namespace owt - -#endif // OWT_BASE_INTELLIGENTCOLLABORATIONPARAMETERS_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h index c4b926df5..e79e93272 100644 --- a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h +++ b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h @@ -6,7 +6,7 @@ #define OWT_BASE_LOCALCAMERASTREAMPARAMETERS_H_ #include #include "owt/base/commontypes.h" -#include "owt/base/intelligentcollaborationparameters.h" +#include "owt/ic/intelligentcollaborationparameters.h" namespace owt { namespace base { /** @@ -55,8 +55,8 @@ class LocalCameraStreamParameters final { /** @brief Get the reference of intelligent collaboration parameters */ - IntelligentCollaborationParameters& ICParams(); - const IntelligentCollaborationParameters& ICParams() const; + owt::ic::IntelligentCollaborationParameters& ICParams(); + const owt::ic::IntelligentCollaborationParameters& ICParams() const; /** @cond */ std::string CameraId() const { return camera_id_; } std::string StreamName() const { return stream_name_; } @@ -74,7 +74,7 @@ class LocalCameraStreamParameters final { int fps_; bool video_enabled_; bool audio_enabled_; - IntelligentCollaborationParameters ic_params_; + owt::ic::IntelligentCollaborationParameters ic_params_; }; #ifdef OWT_ENABLE_QUIC diff --git a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h new file mode 100644 index 000000000..4c69c9c85 --- /dev/null +++ b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h @@ -0,0 +1,23 @@ +#ifndef OWT_BASE_PLUGINMANAGER_H_ +#define OWT_BASE_PLUGINMANAGER_H_ + +#include "talk/owt/sdk/base/sharedobjectpointer.h" +#include "talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h" + +namespace owt { +namespace base { +class PluginManager { + public: + static PluginManager* GetInstance(); + + owt::base::SharedObjectPointer& ICPlugin(); + + private: + PluginManager(); + + owt::base::SharedObjectPointer ic_plugin; +}; +} // namespace base +} // namespace owt + +#endif // OWT_BASE_PLUGINMANAGER_H_ diff --git a/talk/owt/sdk/base/videoframepostprocessing.h b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h similarity index 57% rename from talk/owt/sdk/base/videoframepostprocessing.h rename to talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h index 6b5f906da..1e4140e49 100644 --- a/talk/owt/sdk/base/videoframepostprocessing.h +++ b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h @@ -1,27 +1,36 @@ -// Copyright (C) <2018> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_ -#define OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_ - -#include "api/scoped_refptr.h" -#include "api/video/video_frame_buffer.h" - -namespace owt { -namespace base { - -class VideoFramePostProcessor { - public: - virtual ~VideoFramePostProcessor() = default; - - virtual rtc::scoped_refptr Process( - const rtc::scoped_refptr& buffer) = 0; - - virtual void Release() noexcept { delete this; } -}; - -} // namespace base -} // namespace owt - -#endif // OWT_IC_VIDEOFRAMEPOSTPROCESSING_H_ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_ +#define OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_ + +#include "rtc_base/logging.h" +#include "api/scoped_refptr.h" +#include "api/video/video_frame_buffer.h" +#include "talk/owt/sdk/base/sharedobjectpointer.h" + +namespace owt { +namespace base { + +class VideoFramePostProcessor { + public: + virtual ~VideoFramePostProcessor() {} + + virtual bool SetParameter(const std::string& key, const std::string& value) { + return false; + } + + virtual rtc::scoped_refptr Process( + const rtc::scoped_refptr& buffer) = 0; +}; + +template <> +struct SOTrait { + static constexpr auto name = "PostProcessor"; +}; + +} // namespace base +} // namespace owt + +#endif // OWT_IC_VIDEOFRAMEPOSTPROCESSING_H_ diff --git a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h new file mode 100644 index 000000000..8eaf41b4e --- /dev/null +++ b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h @@ -0,0 +1,40 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_IC_ICMANAGERINTERFACE_H_ +#define OWT_IC_ICMANAGERINTERFACE_H_ + +#include +#include + +#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" + +namespace owt { +namespace ic { + +enum class ICPlugin { BACKGROUND_BLUR }; + +class ICManagerInterface + : public std::enable_shared_from_this { + public: + virtual bool InitializeInferenceEngineCore( + const std::string& plugins_xml_path) = 0; + + virtual std::shared_ptr + CreatePostProcessor(ICPlugin plugin) = 0; + + protected: + ~ICManagerInterface() = default; +}; + +} // namespace ic + +template <> +struct base::SOTrait { + static constexpr auto name = "ICManager"; +}; + +} // namespace owt + +#endif // OWT_IC_ICMANAGERINTERFACE_H_ diff --git a/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h b/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h new file mode 100644 index 000000000..87c10beed --- /dev/null +++ b/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h @@ -0,0 +1,34 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_IC_INTELLIGENTCOLLABORATIONPARAMETERS_H_ +#define OWT_IC_INTELLIGENTCOLLABORATIONPARAMETERS_H_ + +#include +#include + +#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" + +namespace owt { +namespace ic { + +class IntelligentCollaborationParameters final { + public: + IntelligentCollaborationParameters() = default; + ~IntelligentCollaborationParameters() = default; + + std::vector>& + PostProcessors(); + const std::vector>& + PostProcessors() const; + + private: + std::vector> + post_processors_; +}; + +} // namespace ic +} // namespace owt + +#endif // OWT_IC_INTELLIGENTCOLLABORATIONPARAMETERS_H_ diff --git a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc index 3656ade87..19652d542 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc +++ b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc @@ -8,16 +8,17 @@ #include "owt/base/deviceutils.h" #include "owt/base/globalconfiguration.h" #include "owt/base/localcamerastreamparameters.h" +#include "owt/base/logging.h" +#include "owt/base/pluginmanager.h" #include "owt/base/stream.h" using namespace owt::base; class MainWindow : public VideoRenderWindow { - const wchar_t* class_name = L"MainWindow"; - const wchar_t* title = L"Camera"; + static constexpr const wchar_t* class_name = L"MainWindow"; public: - MainWindow(int width, int height) { + MainWindow(int width, int height, const wchar_t* title) { WNDCLASSEX cls = {}; cls.cbSize = sizeof(WNDCLASSEX); cls.lpfnWndProc = WindowProcedure; @@ -54,23 +55,43 @@ class MainWindow : public VideoRenderWindow { }; int main(int, char*[]) { + int id = 0; + std::string device_name = DeviceUtils::GetDeviceNameByIndex(id); + std::string camera_id = DeviceUtils::VideoCapturerIds()[id]; + VideoTrackCapabilities capability = + DeviceUtils::VideoCapturerSupportedCapabilities(camera_id)[id]; + LocalCameraStreamParameters param(false, true); - std::string default_id = DeviceUtils::VideoCapturerIds()[0]; - param.CameraId(default_id); - auto capability = - DeviceUtils::VideoCapturerSupportedCapabilities(default_id)[0]; + param.CameraId(camera_id); param.Resolution(capability.width, capability.height); param.Fps(capability.frameRate); - param.ICParams().BackgroundBlur(true); - param.ICParams().BlurRadius(55); + + auto &ic_plugin = PluginManager::GetInstance()->ICPlugin(); + if (ic_plugin.IsLoaded()) { + ic_plugin->InitializeInferenceEngineCore("plugins.xml"); + auto background_blur = + ic_plugin->CreatePostProcessor(owt::ic::ICPlugin::BACKGROUND_BLUR); + if (background_blur) { + background_blur->SetParameter("model_path", + "ic/model/background_blur.xml"); + background_blur->SetParameter("blur_radius", "55"); + param.ICParams().PostProcessors().push_back(background_blur); + } else { + std::cerr << "Create background blur failed." << std::endl; + } + } else { + std::cerr << "Unable to initialize IC plugin." << std::endl; + } + int error = 0; - auto stream = LocalStream::Create(param, error); + std::shared_ptr stream = LocalStream::Create(param, error); if (error == 0) { - MainWindow w(capability.width, capability.height); + MainWindow w(capability.width, capability.height, + std::wstring(device_name.begin(), device_name.end()).c_str()); stream->AttachVideoRenderer(w); return w.Exec(); } else { - std::cerr << "Create local stream failed, error code " << error + std::cerr << "Create local stream failed, error code " << error << "." << std::endl; return 0; } From 65ee09d3af53d175ff378c7d71c892ad0739b7a2 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Thu, 28 Oct 2021 16:49:08 +0800 Subject: [PATCH 09/30] Fixups --- talk/owt/sdk/ic/backgroundblur.cc | 18 +++-- talk/owt/sdk/ic/icmanager.cc | 23 +++++- talk/owt/sdk/ic/icmanager.h | 14 ++++ talk/owt/sdk/ic/selfiesegmentation.cc | 8 +- talk/owt/sdk/ic/selfiesegmentation.h | 1 - .../include/cpp/owt/ic/icmanagerinterface.h | 3 +- .../sample_background_blur.cc | 73 +++++++++++-------- 7 files changed, 98 insertions(+), 42 deletions(-) diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index 82ed211d9..f5868df43 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -4,6 +4,7 @@ #include "backgroundblur.h" +#include #include #include "api/video/i420_buffer.h" @@ -15,15 +16,17 @@ namespace owt { namespace ic { -BackgroundBlur::BackgroundBlur() - : model(new SelfieSegmentation( - "C:/Users/wangzhib/Desktop/segmentation/contrib/" - "owt-selfie-segmentation-144x256.xml")) {} +BackgroundBlur::BackgroundBlur() {} bool BackgroundBlur::SetParameter(const std::string& key, const std::string& value) { if (key == "model_path") { - + try { + model = std::make_unique(value); + return true; + } catch (std::exception &e) { + return false; + } } else if (key == "blur_radius") { blur_radius_ = stoi(value); return true; @@ -33,6 +36,11 @@ bool BackgroundBlur::SetParameter(const std::string& key, rtc::scoped_refptr BackgroundBlur::Process( const rtc::scoped_refptr& buffer) { + if (model == nullptr) { + RTC_LOG(WARNING) << "Background blur model is not initialized."; + return buffer; + } + auto i420 = buffer->GetI420(); std::vector rgb(3ll * i420->width() * i420->height()); libyuv::I420ToRAW(i420->DataY(), i420->StrideY(), i420->DataU(), diff --git a/talk/owt/sdk/ic/icmanager.cc b/talk/owt/sdk/ic/icmanager.cc index 0b09b1952..7687fc88c 100644 --- a/talk/owt/sdk/ic/icmanager.cc +++ b/talk/owt/sdk/ic/icmanager.cc @@ -4,6 +4,8 @@ #include "talk/owt/sdk/ic/icmanager.h" +#include "inference_engine.hpp" + #include "talk/owt/sdk/ic/backgroundblur.h" namespace owt { @@ -11,7 +13,7 @@ namespace ic { bool ICManager::InitializeInferenceEngineCore( const std::string& plugins_xml_path) { - return false; + return InitializeInferenceEngineCoreImpl(plugins_xml_path); } std::shared_ptr @@ -25,6 +27,25 @@ ICManager::CreatePostProcessor(ICPlugin plugin) { return nullptr; } +InferenceEngine::Core& ICManager::InferenceEngineCore() { + if (!core) { + InitializeInferenceEngineCoreImpl(""); + } + return *core; +} + +bool ICManager::InitializeInferenceEngineCoreImpl( + const std::string& plugins_xml_path) { + try { + core = std::make_shared(plugins_xml_path); + return true; + } catch (...) { + return false; + } +} + +std::shared_ptr ICManager::core; + } // namespace ic } // namespace owt diff --git a/talk/owt/sdk/ic/icmanager.h b/talk/owt/sdk/ic/icmanager.h index 86d2b4e58..f2174f8b1 100644 --- a/talk/owt/sdk/ic/icmanager.h +++ b/talk/owt/sdk/ic/icmanager.h @@ -3,6 +3,12 @@ #include "talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h" +#include + +namespace InferenceEngine { +class Core; +} + namespace owt { namespace ic { @@ -15,6 +21,14 @@ class ICManager : public ICManagerInterface { std::shared_ptr CreatePostProcessor( ICPlugin plugin) override; + + static InferenceEngine::Core& InferenceEngineCore(); + + private: + static bool InitializeInferenceEngineCoreImpl( + const std::string& plugins_xml_path); + + static std::shared_ptr core; }; } // namespace ic diff --git a/talk/owt/sdk/ic/selfiesegmentation.cc b/talk/owt/sdk/ic/selfiesegmentation.cc index 6f769d8ce..b5f043654 100644 --- a/talk/owt/sdk/ic/selfiesegmentation.cc +++ b/talk/owt/sdk/ic/selfiesegmentation.cc @@ -7,6 +7,8 @@ #include #include +#include "talk/owt/sdk/ic/icmanager.h" + namespace IE = InferenceEngine; namespace owt { @@ -14,8 +16,7 @@ namespace ic { SelfieSegmentation::SelfieSegmentation(const std::string& xmlPath, const std::string& device) - : core(), - network(core.ReadNetwork(xmlPath)), + : network(ICManager::InferenceEngineCore().ReadNetwork(xmlPath)), executableNetwork(), request() { assert(!network.getInputsInfo().empty()); @@ -30,7 +31,8 @@ SelfieSegmentation::SelfieSegmentation(const std::string& xmlPath, network.getInputsInfo()[inputName]->getPreProcess(); preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); - executableNetwork = core.LoadNetwork(network, device); + executableNetwork = + ICManager::InferenceEngineCore().LoadNetwork(network, device); request = executableNetwork.CreateInferRequest(); } diff --git a/talk/owt/sdk/ic/selfiesegmentation.h b/talk/owt/sdk/ic/selfiesegmentation.h index 6b9bb6f5c..8e4ba117e 100644 --- a/talk/owt/sdk/ic/selfiesegmentation.h +++ b/talk/owt/sdk/ic/selfiesegmentation.h @@ -24,7 +24,6 @@ class SelfieSegmentation { cv::Mat waitForFinished(); private: - InferenceEngine::Core core; InferenceEngine::CNNNetwork network; InferenceEngine::ExecutableNetwork executableNetwork; InferenceEngine::InferRequest request; diff --git a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h index 8eaf41b4e..199d3cc3d 100644 --- a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h +++ b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h @@ -15,8 +15,7 @@ namespace ic { enum class ICPlugin { BACKGROUND_BLUR }; -class ICManagerInterface - : public std::enable_shared_from_this { +class ICManagerInterface { public: virtual bool InitializeInferenceEngineCore( const std::string& plugins_xml_path) = 0; diff --git a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc index 19652d542..93fb257a9 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc +++ b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc @@ -12,9 +12,7 @@ #include "owt/base/pluginmanager.h" #include "owt/base/stream.h" -using namespace owt::base; - -class MainWindow : public VideoRenderWindow { +class MainWindow : public owt::base::VideoRenderWindow { static constexpr const wchar_t* class_name = L"MainWindow"; public: @@ -56,43 +54,58 @@ class MainWindow : public VideoRenderWindow { int main(int, char*[]) { int id = 0; - std::string device_name = DeviceUtils::GetDeviceNameByIndex(id); - std::string camera_id = DeviceUtils::VideoCapturerIds()[id]; - VideoTrackCapabilities capability = - DeviceUtils::VideoCapturerSupportedCapabilities(camera_id)[id]; + std::string device_name = owt::base::DeviceUtils::GetDeviceNameByIndex(id); + std::string camera_id = owt::base::DeviceUtils::VideoCapturerIds()[id]; + owt::base::VideoTrackCapabilities capability = + owt::base::DeviceUtils::VideoCapturerSupportedCapabilities(camera_id)[id]; - LocalCameraStreamParameters param(false, true); + owt::base::LocalCameraStreamParameters param(false, true); param.CameraId(camera_id); param.Resolution(capability.width, capability.height); param.Fps(capability.frameRate); - auto &ic_plugin = PluginManager::GetInstance()->ICPlugin(); - if (ic_plugin.IsLoaded()) { - ic_plugin->InitializeInferenceEngineCore("plugins.xml"); - auto background_blur = - ic_plugin->CreatePostProcessor(owt::ic::ICPlugin::BACKGROUND_BLUR); - if (background_blur) { - background_blur->SetParameter("model_path", - "ic/model/background_blur.xml"); - background_blur->SetParameter("blur_radius", "55"); - param.ICParams().PostProcessors().push_back(background_blur); - } else { - std::cerr << "Create background blur failed." << std::endl; - } - } else { + // Load the owt_ic.dll and initialize ICManager + auto& ic_plugin = owt::base::PluginManager::GetInstance()->ICPlugin(); + if (!ic_plugin.IsLoaded()) { std::cerr << "Unable to initialize IC plugin." << std::endl; + return 1; + } + + // Initialize global inference engine core, to prevent slow first time + // initialization of background blur post processor + if (!ic_plugin->InitializeInferenceEngineCore("plugins.xml")) { + std::cerr << "Unable to initialize inference engine core" << std::endl; + return 2; + } + + std::shared_ptr background_blur = + ic_plugin->CreatePostProcessor(owt::ic::ICPlugin::BACKGROUND_BLUR); + if (!background_blur) { + std::cerr << "Create background blur failed." << std::endl; + return 3; + } + if (!background_blur->SetParameter("model_path", + "data/ic_model/background_blur.xml")) { + std::cerr << "Failed to load model." << std::endl; + return 4; + } + if (!background_blur->SetParameter("blur_radius", "55")) { + std::cerr << "Failed to set blur radius." << std::endl; + return 5; } + param.ICParams().PostProcessors().push_back(background_blur); int error = 0; - std::shared_ptr stream = LocalStream::Create(param, error); - if (error == 0) { - MainWindow w(capability.width, capability.height, - std::wstring(device_name.begin(), device_name.end()).c_str()); - stream->AttachVideoRenderer(w); - return w.Exec(); - } else { + std::shared_ptr stream = + owt::base::LocalStream::Create(param, error); + if (error) { std::cerr << "Create local stream failed, error code " << error << "." << std::endl; - return 0; + return 6; } + + MainWindow w(capability.width, capability.height, + std::wstring(device_name.begin(), device_name.end()).c_str()); + stream->AttachVideoRenderer(w); + return w.Exec(); } From 8e12bed5154c133811be3c7168ae26f3682563ef Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Thu, 28 Oct 2021 17:02:39 +0800 Subject: [PATCH 10/30] Fixups --- talk/owt/sdk/ic/BUILD.gn | 31 +++++++++++++++---- .../win/sample_background_blur/BUILD.gn | 8 ++--- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index 93468e2c4..dacffadc6 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -24,9 +24,7 @@ config("openvino") { } else { lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Release" ] } - configs = [ - "//build/config/compiler:rtti", - ] + configs = [ "//build/config/compiler:rtti" ] } config("openvino_opencv") { @@ -37,8 +35,20 @@ config("openvino_opencv") { lib_dirs = [ owt_openvino_root + "/opencv/lib" ] } -openvino_modules = [ "ir_reader", "lp_transformations", "preproc", "transformations" ] -opencv_modules = ["core", "imgproc", "imgcodecs", "highgui", "videoio"] +openvino_modules = [ + "ir_reader", + "lp_transformations", + "preproc", + "transformations", +] + +opencv_modules = [ + "core", + "highgui", + "imgcodecs", + "imgproc", + "videoio", +] shared_library("owt_ic") { sources = [ @@ -72,12 +82,21 @@ shared_library("owt_ic") { } copy("openvino_library") { + sources = [] if (is_debug) { bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Debug" + sources += [ + owt_openvino_root + "/inference_engine/external/tbb/bin/tbb_debug.dll", + owt_openvino_root + "/deployment_tools/ngraph/lib/ngraphd.dll", + ] } else { bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Release" + sources += [ + owt_openvino_root + "/inference_engine/external/tbb/bin/tbb.dll", + owt_openvino_root + "/deployment_tools/ngraph/lib/ngraph.dll", + ] } - sources = [ bin_dir + "/inference_engine" + dll_suffix ] + sources += [ bin_dir + "/inference_engine" + dll_suffix ] foreach(module, openvino_modules) { sources += [ bin_dir + "/inference_engine_" + module + dll_suffix ] } diff --git a/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn index d22e60ce4..03ab7f537 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn +++ b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn @@ -13,10 +13,6 @@ executable("sample_background_blur") { "//talk/owt/sdk/ic:openvino", "//talk/owt/sdk/ic:openvino_opencv", ] - deps = [ - "//talk/owt:owt_sdk_base", - ] - data_deps = [ - "//talk/owt/sdk/ic:owt_ic", - ] + deps = [ "//talk/owt:owt_sdk_base" ] + data_deps = [ "//talk/owt/sdk/ic:owt_ic" ] } From 51311bc5166a8625b3365b17138f00c1da0a7c0a Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Fri, 29 Oct 2021 10:34:42 +0800 Subject: [PATCH 11/30] Fixups --- talk/owt/sdk/base/pluginmanager.cc | 7 ++++++- talk/owt/sdk/base/sharedobjectpointer.h | 4 ++-- talk/owt/sdk/ic/BUILD.gn | 5 ++--- talk/owt/sdk/include/cpp/owt/base/pluginmanager.h | 4 ++++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/talk/owt/sdk/base/pluginmanager.cc b/talk/owt/sdk/base/pluginmanager.cc index c18e903de..565c9d610 100644 --- a/talk/owt/sdk/base/pluginmanager.cc +++ b/talk/owt/sdk/base/pluginmanager.cc @@ -8,19 +8,24 @@ PluginManager* PluginManager::GetInstance() { return &plugin_manager; } +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) owt::base::SharedObjectPointer& PluginManager::ICPlugin() { return ic_plugin; } +#endif PluginManager::PluginManager() +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) : ic_plugin( #ifdef WEBRTC_WIN "owt_ic.dll" #elif WEBRTC_LINUX "owt_ic.so" #endif - ) { + ) +#endif +{ } } // namespace base diff --git a/talk/owt/sdk/base/sharedobjectpointer.h b/talk/owt/sdk/base/sharedobjectpointer.h index f99241e23..ec4fbc6a5 100644 --- a/talk/owt/sdk/base/sharedobjectpointer.h +++ b/talk/owt/sdk/base/sharedobjectpointer.h @@ -23,12 +23,12 @@ class SharedObjectPointer { public: explicit SharedObjectPointer(const char* path) : so_(path) { Load(); } - explicit SharedObjectPointer(const SharedObjectLoader& loader) : so(loader) { + explicit SharedObjectPointer(const SharedObjectLoader& loader) : so_(loader) { Load(); } explicit SharedObjectPointer(SharedObjectLoader&& loader) - : so(std::move(loader)) { + : so_(std::move(loader)) { Load(); } diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index dacffadc6..8900c7f22 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -82,16 +82,15 @@ shared_library("owt_ic") { } copy("openvino_library") { - sources = [] if (is_debug) { bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Debug" - sources += [ + sources = [ owt_openvino_root + "/inference_engine/external/tbb/bin/tbb_debug.dll", owt_openvino_root + "/deployment_tools/ngraph/lib/ngraphd.dll", ] } else { bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Release" - sources += [ + sources = [ owt_openvino_root + "/inference_engine/external/tbb/bin/tbb.dll", owt_openvino_root + "/deployment_tools/ngraph/lib/ngraph.dll", ] diff --git a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h index 4c69c9c85..bddf0e080 100644 --- a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h +++ b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h @@ -10,12 +10,16 @@ class PluginManager { public: static PluginManager* GetInstance(); +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) owt::base::SharedObjectPointer& ICPlugin(); +#endif private: PluginManager(); +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) owt::base::SharedObjectPointer ic_plugin; +#endif }; } // namespace base } // namespace owt From 09d1d4d32961d2272c20184a4908c2e25a4654a9 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Fri, 29 Oct 2021 10:55:41 +0800 Subject: [PATCH 12/30] Fixups --- .gitignore | 1 - talk/owt/BUILD.gn | 1 + talk/owt/sdk/base/cameravideocapturer.cc | 4 +- .../intelligentcollaborationparameters.cc | 7 ++- .../sdk/base/localcamerastreamparameters.cc | 11 ++-- talk/owt/sdk/base/pluginmanager.cc | 17 ++++--- talk/owt/sdk/base/sharedobjectloader.h | 2 - talk/owt/sdk/base/stream.cc | 50 ++++++------------- talk/owt/sdk/ic/icmanager.h | 4 ++ .../owt/base/localcamerastreamparameters.h | 41 +++++++++------ .../sdk/include/cpp/owt/base/pluginmanager.h | 4 ++ 11 files changed, 75 insertions(+), 67 deletions(-) diff --git a/.gitignore b/.gitignore index 32a92ce49..bcbbdd677 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ /tools /dist /talk/owt/docs/ios/html -/.vscode/settings.json diff --git a/talk/owt/BUILD.gn b/talk/owt/BUILD.gn index 258c6fbf0..b09779599 100644 --- a/talk/owt/BUILD.gn +++ b/talk/owt/BUILD.gn @@ -290,6 +290,7 @@ static_library("owt_sdk_base") { "sdk/base/linux/msdkvideodecoderfactory.h", "sdk/base/linux/msdkvideodecoder.cc", "sdk/base/linux/msdkvideodecoder.h", + ] } sources += [ diff --git a/talk/owt/sdk/base/cameravideocapturer.cc b/talk/owt/sdk/base/cameravideocapturer.cc index 42d7d730e..4f333df27 100644 --- a/talk/owt/sdk/base/cameravideocapturer.cc +++ b/talk/owt/sdk/base/cameravideocapturer.cc @@ -91,5 +91,5 @@ void CameraVideoCapturer::UpdateVideoAdapter() { video_adapter_.OnSinkWants(broadcaster_.wants()); } -} // namespace base -} // namespace owt +} // namespace test +} // namespace webrtc diff --git a/talk/owt/sdk/base/intelligentcollaborationparameters.cc b/talk/owt/sdk/base/intelligentcollaborationparameters.cc index 1a0e9961d..ed04e775f 100644 --- a/talk/owt/sdk/base/intelligentcollaborationparameters.cc +++ b/talk/owt/sdk/base/intelligentcollaborationparameters.cc @@ -1,5 +1,8 @@ -#include "talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h" +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +#include "talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h" namespace owt { namespace ic { @@ -14,4 +17,4 @@ IntelligentCollaborationParameters::PostProcessors() const { } } // namespace base -} // namespace owt \ No newline at end of file +} // namespace owt diff --git a/talk/owt/sdk/base/localcamerastreamparameters.cc b/talk/owt/sdk/base/localcamerastreamparameters.cc index 05697ae95..07507a305 100644 --- a/talk/owt/sdk/base/localcamerastreamparameters.cc +++ b/talk/owt/sdk/base/localcamerastreamparameters.cc @@ -34,11 +34,12 @@ void LocalCameraStreamParameters::Resolution(int width, int height) { resolution_width_ = width; resolution_height_ = height; } -void LocalCameraStreamParameters::StreamName(const std::string& stream_name) { +void LocalCameraStreamParameters::StreamName(const std::string& stream_name){ stream_name_ = stream_name; } -LocalDesktopStreamParameters::LocalDesktopStreamParameters(bool audio_enabled, - bool video_enabled) +LocalDesktopStreamParameters::LocalDesktopStreamParameters( + bool audio_enabled, + bool video_enabled) : video_enabled_(video_enabled), audio_enabled_(audio_enabled), fps_(30), @@ -47,5 +48,5 @@ LocalDesktopStreamParameters::LocalDesktopStreamParameters(bool audio_enabled, void LocalDesktopStreamParameters::Fps(int fps) { fps_ = fps; } -} // namespace base -} // namespace owt +} +} diff --git a/talk/owt/sdk/base/pluginmanager.cc b/talk/owt/sdk/base/pluginmanager.cc index 565c9d610..841e07573 100644 --- a/talk/owt/sdk/base/pluginmanager.cc +++ b/talk/owt/sdk/base/pluginmanager.cc @@ -1,3 +1,7 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #include "..\include\cpp\owt\base\pluginmanager.h" namespace owt { @@ -15,18 +19,19 @@ PluginManager::ICPlugin() { } #endif -PluginManager::PluginManager() #if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) +PluginManager::PluginManager() : ic_plugin( -#ifdef WEBRTC_WIN +#if defined(WEBRTC_WIN) "owt_ic.dll" -#elif WEBRTC_LINUX +#elif defined(WEBRTC_LINUX) "owt_ic.so" #endif - ) -#endif -{ + ) { } +#else +PluginManager::PluginManager() {} +#endif } // namespace base } // namespace owt diff --git a/talk/owt/sdk/base/sharedobjectloader.h b/talk/owt/sdk/base/sharedobjectloader.h index a33e61043..5452bde1b 100644 --- a/talk/owt/sdk/base/sharedobjectloader.h +++ b/talk/owt/sdk/base/sharedobjectloader.h @@ -18,9 +18,7 @@ class SharedObjectLoader { ~SharedObjectLoader(); bool IsLoaded() const; - void* GetSymbol(const char* name) const; - void* GetSymbol(const std::string& name) const; private: diff --git a/talk/owt/sdk/base/stream.cc b/talk/owt/sdk/base/stream.cc index 218a277c9..34d6b4ff1 100644 --- a/talk/owt/sdk/base/stream.cc +++ b/talk/owt/sdk/base/stream.cc @@ -58,7 +58,7 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { for (int i = 0; i < num_devices; ++i) { capturer = absl::WrapUnique(owt::base::VcmCapturer::Create( width, height, fps, capture_device_idx)); - for (auto post_processor : ic_params.PostProcessors()) { + for (auto& post_processor : ic_params.PostProcessors()) { capturer->AddVideoFramePostProcessor(post_processor); } if (capturer) { @@ -95,8 +95,7 @@ Stream::Stream() source_(AudioSourceInfo::kUnknown, VideoSourceInfo::kUnknown), #endif ended_(false), - id_("") { -} + id_("") {} Stream::Stream(const std::string& id) : media_stream_(nullptr), renderer_impl_(nullptr), @@ -110,44 +109,25 @@ Stream::Stream(const std::string& id) source_(AudioSourceInfo::kUnknown, VideoSourceInfo::kUnknown), #endif ended_(false), - id_(id) { -} + id_(id) {} #elif defined(WEBRTC_LINUX) Stream::Stream() - : media_stream_(nullptr), - renderer_impl_(nullptr), - audio_renderer_impl_(nullptr), - va_renderer_impl_(nullptr), - ended_(false), - id_("") {} + : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), va_renderer_impl_(nullptr), ended_(false), id_("") {} Stream::Stream(MediaStreamInterface* media_stream, StreamSourceInfo source) - : media_stream_(nullptr), va_renderer_impl_(nullptr), source_(source) { + : media_stream_(nullptr), va_renderer_impl_(nullptr),source_(source) { MediaStream(media_stream); } Stream::Stream(const std::string& id) - : media_stream_(nullptr), - renderer_impl_(nullptr), - audio_renderer_impl_(nullptr), - va_renderer_impl_(nullptr), - ended_(false), - id_(id) {} + : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), va_renderer_impl_(nullptr), ended_(false), id_(id) {} #else Stream::Stream() - : media_stream_(nullptr), - renderer_impl_(nullptr), - audio_renderer_impl_(nullptr), - ended_(false), - id_("") {} + : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), ended_(false), id_("") {} Stream::Stream(MediaStreamInterface* media_stream, StreamSourceInfo source) : media_stream_(nullptr), source_(source) { MediaStream(media_stream); } Stream::Stream(const std::string& id) - : media_stream_(nullptr), - renderer_impl_(nullptr), - audio_renderer_impl_(nullptr), - ended_(false), - id_(id) {} + : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), ended_(false), id_(id) {} #endif MediaStreamInterface* Stream::MediaStream() const { @@ -734,7 +714,9 @@ std::string RemoteStream::Origin() { } RemoteStream::RemoteStream(const std::string& id, const std::string& from) - : Stream(id), origin_(from) {} + :Stream(id), + origin_(from) { +} void RemoteStream::MediaStream(MediaStreamInterface* media_stream) { Stream::MediaStream(media_stream); @@ -745,12 +727,10 @@ MediaStreamInterface* RemoteStream::MediaStream() { #ifdef OWT_ENABLE_QUIC QuicStream::QuicStream(owt::quic::WebTransportStreamInterface* quic_stream, - const std::string& session_id) - : quic_stream_(quic_stream), - session_id_(session_id), - can_read_(true), - can_write_(true), - fin_read_(false) {} + const std::string& session_id) + : quic_stream_(quic_stream), session_id_(session_id), can_read_(true), + can_write_(true), fin_read_(false) { +} QuicStream::~QuicStream() {} diff --git a/talk/owt/sdk/ic/icmanager.h b/talk/owt/sdk/ic/icmanager.h index f2174f8b1..96688c6da 100644 --- a/talk/owt/sdk/ic/icmanager.h +++ b/talk/owt/sdk/ic/icmanager.h @@ -1,3 +1,7 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #ifndef OWT_IC_ICMANAGER_H_ #define OWT_IC_ICMANAGER_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h index e79e93272..f438bc12c 100644 --- a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h +++ b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h @@ -8,7 +8,7 @@ #include "owt/base/commontypes.h" #include "owt/ic/intelligentcollaborationparameters.h" namespace owt { -namespace base { +namespace base{ /** @brief This class contains parameters and methods that needed for creating a local camera stream. @@ -77,6 +77,7 @@ class LocalCameraStreamParameters final { owt::ic::IntelligentCollaborationParameters ic_params_; }; + #ifdef OWT_ENABLE_QUIC /** @brief This class contains parameters and methods needed for creating @@ -86,11 +87,15 @@ class LocalCameraStreamParameters final { */ class LocalDataStreamParameters final { public: - LocalDataStreamParameters(bool data_enabled) { data_enabled_ = data_enabled; } + LocalDataStreamParameters(bool data_enabled) { + data_enabled_ = data_enabled; + } ~LocalDataStreamParameters() {} /** @cond */ - bool DataEnabled() const { return data_enabled_; } + bool DataEnabled() const { + return data_enabled_; + } /** @endcond */ private: @@ -112,11 +117,11 @@ class LocalCustomizedStreamParameters final { @param video_anabled Indicates if video is enabled for this stream. */ LocalCustomizedStreamParameters(bool audio_enabled, bool video_enabled) { - video_enabled_ = video_enabled; - audio_enabled_ = audio_enabled; - fps_ = 0; - bitrate_kbps_ = 0; - resolution_width_ = resolution_height_ = 0; + video_enabled_ = video_enabled; + audio_enabled_ = audio_enabled; + fps_ = 0; + bitrate_kbps_ = 0; + resolution_width_ = resolution_height_ = 0; } ~LocalCustomizedStreamParameters() {} /** @@ -136,12 +141,16 @@ class LocalCustomizedStreamParameters final { will failed. @param fps The frame rate of the video. */ - void Fps(int fps) { fps_ = fps; } + void Fps(int fps) { + fps_ = fps; + } /** @brief Set the bitrate of encoded frame. @param bitrate_kbps The bitrate expected for the encoded stream. */ - void Bitrate(int bitrate_kbps) { bitrate_kbps_ = bitrate_kbps; } + void Bitrate(int bitrate_kbps) { + bitrate_kbps_ = bitrate_kbps; + } /** @cond */ int ResolutionWidth() const { return resolution_width_; } int ResolutionHeight() const { return resolution_height_; } @@ -202,7 +211,9 @@ class LocalDesktopStreamParameters final { @param soruce_type Indicates if capture from screen or an app. @param capture_policy the OR of any of the DesktopCapturePolicy options. */ - LocalDesktopStreamParameters(bool audio_enabled, bool video_enabled); + LocalDesktopStreamParameters( + bool audio_enabled, + bool video_enabled); ~LocalDesktopStreamParameters() {} /** @brief Get video is enabled or not for this stream. @@ -219,7 +230,9 @@ class LocalDesktopStreamParameters final { @param source_type Indicate if capturing the full screen or an application. */ - void SourceType(DesktopSourceType source_type) { source_type_ = source_type; } + void SourceType(DesktopSourceType source_type) { + source_type_ = source_type; + } /** @brief Set the capturer features @params capture_policy Indicate the feature used by the capturer. @@ -245,6 +258,6 @@ class LocalDesktopStreamParameters final { DesktopSourceType source_type_; DesktopCapturePolicy capture_policy_; }; -} // namespace base -} // namespace owt +} +} #endif // OWT_BASE_LOCALCAMERASTREAMPARAMETERS_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h index bddf0e080..14c55a50c 100644 --- a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h +++ b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h @@ -1,3 +1,7 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + #ifndef OWT_BASE_PLUGINMANAGER_H_ #define OWT_BASE_PLUGINMANAGER_H_ From 59891e20a70cecc0aea0874d165f1d70f74cfdd7 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Fri, 29 Oct 2021 11:12:35 +0800 Subject: [PATCH 13/30] Fixups --- talk/owt/BUILD.gn | 2 +- .../intelligentcollaborationparameters.cc | 1 + talk/owt/sdk/base/pluginmanager.cc | 74 +++++++++---------- talk/owt/sdk/base/stream.cc | 9 ++- talk/owt/sdk/ic/icmanager.cc | 5 +- .../include/cpp/owt/base/icmanagerinterface.h | 31 -------- .../cpp/owt/base/videoframepostprocessor.h | 72 +++++++++--------- 7 files changed, 82 insertions(+), 112 deletions(-) delete mode 100644 talk/owt/sdk/include/cpp/owt/base/icmanagerinterface.h diff --git a/talk/owt/BUILD.gn b/talk/owt/BUILD.gn index b09779599..c74693124 100644 --- a/talk/owt/BUILD.gn +++ b/talk/owt/BUILD.gn @@ -290,7 +290,7 @@ static_library("owt_sdk_base") { "sdk/base/linux/msdkvideodecoderfactory.h", "sdk/base/linux/msdkvideodecoder.cc", "sdk/base/linux/msdkvideodecoder.h", - + ] } sources += [ diff --git a/talk/owt/sdk/base/intelligentcollaborationparameters.cc b/talk/owt/sdk/base/intelligentcollaborationparameters.cc index ed04e775f..c757bb11b 100644 --- a/talk/owt/sdk/base/intelligentcollaborationparameters.cc +++ b/talk/owt/sdk/base/intelligentcollaborationparameters.cc @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h" + namespace owt { namespace ic { diff --git a/talk/owt/sdk/base/pluginmanager.cc b/talk/owt/sdk/base/pluginmanager.cc index 841e07573..1f5cef17e 100644 --- a/talk/owt/sdk/base/pluginmanager.cc +++ b/talk/owt/sdk/base/pluginmanager.cc @@ -1,37 +1,37 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#include "..\include\cpp\owt\base\pluginmanager.h" - -namespace owt { -namespace base { - -PluginManager* PluginManager::GetInstance() { - static PluginManager plugin_manager; - return &plugin_manager; -} - -#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) -owt::base::SharedObjectPointer& -PluginManager::ICPlugin() { - return ic_plugin; -} -#endif - -#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) -PluginManager::PluginManager() - : ic_plugin( -#if defined(WEBRTC_WIN) - "owt_ic.dll" -#elif defined(WEBRTC_LINUX) - "owt_ic.so" -#endif - ) { -} -#else -PluginManager::PluginManager() {} -#endif - -} // namespace base -} // namespace owt +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "talk/owt/sdk/include/cpp/owt/base/pluginmanager.h" + +namespace owt { +namespace base { + +PluginManager* PluginManager::GetInstance() { + static PluginManager plugin_manager; + return &plugin_manager; +} + +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) +owt::base::SharedObjectPointer& +PluginManager::ICPlugin() { + return ic_plugin; +} +#endif + +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) +PluginManager::PluginManager() + : ic_plugin( +#if defined(WEBRTC_WIN) + "owt_ic.dll" +#elif defined(WEBRTC_LINUX) + "owt_ic.so" +#endif + ) { +} +#else +PluginManager::PluginManager() {} +#endif + +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/base/stream.cc b/talk/owt/sdk/base/stream.cc index 34d6b4ff1..a7d314779 100644 --- a/talk/owt/sdk/base/stream.cc +++ b/talk/owt/sdk/base/stream.cc @@ -118,7 +118,7 @@ Stream::Stream(MediaStreamInterface* media_stream, StreamSourceInfo source) MediaStream(media_stream); } Stream::Stream(const std::string& id) - : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), va_renderer_impl_(nullptr), ended_(false), id_(id) {} + : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), va_renderer_impl_(nullptr), ended_(false), id_(id) {} #else Stream::Stream() : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), ended_(false), id_("") {} @@ -127,7 +127,7 @@ Stream::Stream(MediaStreamInterface* media_stream, StreamSourceInfo source) MediaStream(media_stream); } Stream::Stream(const std::string& id) - : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), ended_(false), id_(id) {} + : media_stream_(nullptr), renderer_impl_(nullptr), audio_renderer_impl_(nullptr), ended_(false), id_(id) {} #endif MediaStreamInterface* Stream::MediaStream() const { @@ -714,7 +714,7 @@ std::string RemoteStream::Origin() { } RemoteStream::RemoteStream(const std::string& id, const std::string& from) - :Stream(id), + :Stream(id), origin_(from) { } @@ -732,7 +732,8 @@ QuicStream::QuicStream(owt::quic::WebTransportStreamInterface* quic_stream, can_write_(true), fin_read_(false) { } -QuicStream::~QuicStream() {} +QuicStream::~QuicStream() { +} size_t QuicStream::Write(uint8_t* data, size_t length) { if (quic_stream_ && data != nullptr && length > 0) { diff --git a/talk/owt/sdk/ic/icmanager.cc b/talk/owt/sdk/ic/icmanager.cc index 7687fc88c..c1edba74c 100644 --- a/talk/owt/sdk/ic/icmanager.cc +++ b/talk/owt/sdk/ic/icmanager.cc @@ -20,8 +20,7 @@ std::shared_ptr ICManager::CreatePostProcessor(ICPlugin plugin) { switch (plugin) { case ICPlugin::BACKGROUND_BLUR: - return std::shared_ptr( - new BackgroundBlur); + return std::make_shared(); default:; } return nullptr; @@ -29,7 +28,7 @@ ICManager::CreatePostProcessor(ICPlugin plugin) { InferenceEngine::Core& ICManager::InferenceEngineCore() { if (!core) { - InitializeInferenceEngineCoreImpl(""); + InitializeInferenceEngineCoreImpl({}); } return *core; } diff --git a/talk/owt/sdk/include/cpp/owt/base/icmanagerinterface.h b/talk/owt/sdk/include/cpp/owt/base/icmanagerinterface.h deleted file mode 100644 index 47bf0856c..000000000 --- a/talk/owt/sdk/include/cpp/owt/base/icmanagerinterface.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_BASE_ICMANAGERINTERFACE_H_ -#define OWT_BASE_ICMANAGERINTERFACE_H_ - -#include -#include - -#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" - -namespace owt { -namespace base { - - enum class ICPlugin { - BACKGROUND_BLUR, - }; - -class ICManagerInterface { - public: - virtual void InitializeInferenceEngineCore(const std::string &plugin_xml_path); - - virtual std::shared_ptr CreatePostProcessor( - ICPlugin plugin); -}; - -} // namespace base -} // namespace owt - -#endif // OWT_BASE_ICMANAGERINTERFACE_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h index 1e4140e49..7327c492e 100644 --- a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h +++ b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h @@ -1,36 +1,36 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_ -#define OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_ - -#include "rtc_base/logging.h" -#include "api/scoped_refptr.h" -#include "api/video/video_frame_buffer.h" -#include "talk/owt/sdk/base/sharedobjectpointer.h" - -namespace owt { -namespace base { - -class VideoFramePostProcessor { - public: - virtual ~VideoFramePostProcessor() {} - - virtual bool SetParameter(const std::string& key, const std::string& value) { - return false; - } - - virtual rtc::scoped_refptr Process( - const rtc::scoped_refptr& buffer) = 0; -}; - -template <> -struct SOTrait { - static constexpr auto name = "PostProcessor"; -}; - -} // namespace base -} // namespace owt - -#endif // OWT_IC_VIDEOFRAMEPOSTPROCESSING_H_ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ +#define OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ + +#include "rtc_base/logging.h" +#include "api/scoped_refptr.h" +#include "api/video/video_frame_buffer.h" +#include "talk/owt/sdk/base/sharedobjectpointer.h" + +namespace owt { +namespace base { + +class VideoFramePostProcessor { + public: + virtual ~VideoFramePostProcessor() {} + + virtual bool SetParameter(const std::string& key, const std::string& value) { + return false; + } + + virtual rtc::scoped_refptr Process( + const rtc::scoped_refptr& buffer) = 0; +}; + +template <> +struct SOTrait { + static constexpr auto name = "PostProcessor"; +}; + +} // namespace base +} // namespace owt + +#endif // OWT_IC_VIDEOFRAMEPOSTPROCESSOR_H_ From 9c7a7ce58be2bedd6175c632610350b13fae5862 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Fri, 29 Oct 2021 15:15:06 +0800 Subject: [PATCH 14/30] Fixups --- talk/owt/sdk/base/pluginmanager.cc | 30 +++++++--------- talk/owt/sdk/base/sharedobjectpointer.h | 34 ++++++++++++------- talk/owt/sdk/ic/BUILD.gn | 1 + talk/owt/sdk/ic/backgroundblur.cc | 2 +- .../sdk/include/cpp/owt/base/pluginmanager.h | 12 ++----- .../cpp/owt/base/videoframepostprocessor.h | 7 ---- .../include/cpp/owt/ic/icmanagerinterface.h | 8 +---- .../ic/intelligentcollaborationparameters.h | 2 +- .../sample_background_blur.cc | 7 ++-- 9 files changed, 43 insertions(+), 60 deletions(-) diff --git a/talk/owt/sdk/base/pluginmanager.cc b/talk/owt/sdk/base/pluginmanager.cc index 1f5cef17e..dffaee3fc 100644 --- a/talk/owt/sdk/base/pluginmanager.cc +++ b/talk/owt/sdk/base/pluginmanager.cc @@ -4,33 +4,27 @@ #include "talk/owt/sdk/include/cpp/owt/base/pluginmanager.h" +#include "talk/owt/sdk/base/sharedobjectpointer.h" + namespace owt { namespace base { -PluginManager* PluginManager::GetInstance() { - static PluginManager plugin_manager; - return &plugin_manager; -} - -#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) -owt::base::SharedObjectPointer& -PluginManager::ICPlugin() { - return ic_plugin; -} -#endif +template <> +struct SOTrait { + static constexpr auto name = "ICManager"; +}; #if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) -PluginManager::PluginManager() - : ic_plugin( +owt::ic::ICManagerInterface* PluginManager::ICPlugin() { + static owt::base::SharedObjectPointer ic_plugin( #if defined(WEBRTC_WIN) - "owt_ic.dll" + "owt_ic.dll" #elif defined(WEBRTC_LINUX) - "owt_ic.so" + "owt_ic.so" #endif - ) { + ); + return ic_plugin.Get(); } -#else -PluginManager::PluginManager() {} #endif } // namespace base diff --git a/talk/owt/sdk/base/sharedobjectpointer.h b/talk/owt/sdk/base/sharedobjectpointer.h index ec4fbc6a5..d3b7c58c1 100644 --- a/talk/owt/sdk/base/sharedobjectpointer.h +++ b/talk/owt/sdk/base/sharedobjectpointer.h @@ -10,6 +10,8 @@ #include #include +#include "third_party/webrtc/rtc_base/logging.h" + namespace owt { namespace base { @@ -32,22 +34,30 @@ class SharedObjectPointer { Load(); } - bool IsLoaded() const { return so_.IsLoaded() && ptr_.get(); } + bool IsLoaded() const { return so_.IsLoaded(); } - T* operator->() { return ptr_.get(); } + T* Get() { return ptr_.get(); } protected: void Load() { - if (so_.IsLoaded()) { - using Creator = T*(); - using Destroyer = void(T*); - Creator* creator = reinterpret_cast( - so_.GetSymbol(std::string("Create") + SOTrait::name)); - Destroyer* destroyer = reinterpret_cast( - so_.GetSymbol(std::string("Destroy") + SOTrait::name)); - if (creator && destroyer) { - ptr_ = std::shared_ptr(creator(), destroyer); - } + if (!so_.IsLoaded()) { + RTC_LOG(LS_WARNING) << "Shared object is not loaded."; + return; + } + + using Creator = T*(); + using Destroyer = void(T*); + Creator* creator = reinterpret_cast( + so_.GetSymbol(std::string("Create") + SOTrait::name)); + Destroyer* destroyer = reinterpret_cast( + so_.GetSymbol(std::string("Destroy") + SOTrait::name)); + + if (creator && destroyer) { + ptr_ = std::shared_ptr(creator(), destroyer); + } else { + RTC_LOG(LS_WARNING) << "Create" << SOTrait::name // + << " or Destroy" << SOTrait::name + << " is missing in the shared object."; } } diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index 8900c7f22..9048a5c40 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -64,6 +64,7 @@ shared_library("owt_ic") { ":openvino", ":openvino_opencv", ] + include_dirs = [ "//talk/owt/sdk/include/cpp" ] libs = [ "inference_engine" + lib_suffix, "inference_engine_transformations" + lib_suffix, diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index f5868df43..c549958d6 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -37,7 +37,7 @@ bool BackgroundBlur::SetParameter(const std::string& key, rtc::scoped_refptr BackgroundBlur::Process( const rtc::scoped_refptr& buffer) { if (model == nullptr) { - RTC_LOG(WARNING) << "Background blur model is not initialized."; + RTC_LOG(LS_WARNING) << "Background blur model is not initialized."; return buffer; } diff --git a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h index 14c55a50c..c2ec86bf4 100644 --- a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h +++ b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h @@ -5,25 +5,19 @@ #ifndef OWT_BASE_PLUGINMANAGER_H_ #define OWT_BASE_PLUGINMANAGER_H_ -#include "talk/owt/sdk/base/sharedobjectpointer.h" -#include "talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h" +#include "owt/ic/icmanagerinterface.h" namespace owt { namespace base { + class PluginManager { public: - static PluginManager* GetInstance(); - #if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) - owt::base::SharedObjectPointer& ICPlugin(); + static owt::ic::ICManagerInterface* ICPlugin(); #endif private: PluginManager(); - -#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) - owt::base::SharedObjectPointer ic_plugin; -#endif }; } // namespace base } // namespace owt diff --git a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h index 7327c492e..9e63cb57f 100644 --- a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h +++ b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h @@ -5,10 +5,8 @@ #ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ #define OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ -#include "rtc_base/logging.h" #include "api/scoped_refptr.h" #include "api/video/video_frame_buffer.h" -#include "talk/owt/sdk/base/sharedobjectpointer.h" namespace owt { namespace base { @@ -25,11 +23,6 @@ class VideoFramePostProcessor { const rtc::scoped_refptr& buffer) = 0; }; -template <> -struct SOTrait { - static constexpr auto name = "PostProcessor"; -}; - } // namespace base } // namespace owt diff --git a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h index 199d3cc3d..d8b1bf0b9 100644 --- a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h +++ b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h @@ -8,7 +8,7 @@ #include #include -#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" +#include "owt/base/videoframepostprocessor.h" namespace owt { namespace ic { @@ -28,12 +28,6 @@ class ICManagerInterface { }; } // namespace ic - -template <> -struct base::SOTrait { - static constexpr auto name = "ICManager"; -}; - } // namespace owt #endif // OWT_IC_ICMANAGERINTERFACE_H_ diff --git a/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h b/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h index 87c10beed..9259bc9d2 100644 --- a/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h +++ b/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h @@ -8,7 +8,7 @@ #include #include -#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" +#include "owt/base/videoframepostprocessor.h" namespace owt { namespace ic { diff --git a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc index 93fb257a9..a830b74d3 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc +++ b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc @@ -6,9 +6,6 @@ #include #include "owt/base/deviceutils.h" -#include "owt/base/globalconfiguration.h" -#include "owt/base/localcamerastreamparameters.h" -#include "owt/base/logging.h" #include "owt/base/pluginmanager.h" #include "owt/base/stream.h" @@ -65,8 +62,8 @@ int main(int, char*[]) { param.Fps(capability.frameRate); // Load the owt_ic.dll and initialize ICManager - auto& ic_plugin = owt::base::PluginManager::GetInstance()->ICPlugin(); - if (!ic_plugin.IsLoaded()) { + auto ic_plugin = owt::base::PluginManager::ICPlugin(); + if (!ic_plugin) { std::cerr << "Unable to initialize IC plugin." << std::endl; return 1; } From 6ef5289ad583778dfa35d9d0b8c8b1d767b4b030 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Tue, 2 Nov 2021 13:30:31 +0800 Subject: [PATCH 15/30] Fix FreeLibrary deadlocks --- talk/owt/sdk/base/pluginmanager.cc | 15 +- talk/owt/sdk/base/sharedobjectpointer.h | 10 -- talk/owt/sdk/base/win/sharedobjectloader.cc | 8 +- talk/owt/sdk/ic/BUILD.gn | 10 +- talk/owt/sdk/ic/backgroundblur.cc | 18 +-- talk/owt/sdk/ic/backgroundblur.h | 7 +- talk/owt/sdk/ic/icmanager.cc | 33 ++--- talk/owt/sdk/ic/icmanager.h | 11 +- ...entation.cc => selfiesegmentationmodel.cc} | 137 +++++++++--------- ...gmentation.h => selfiesegmentationmodel.h} | 77 +++++----- .../cpp/owt/base/videoframepostprocessor.h | 4 +- .../include/cpp/owt/ic/icmanagerinterface.h | 5 +- .../win/sample_background_blur/BUILD.gn | 4 - .../sample_background_blur.cc | 37 ++--- 14 files changed, 179 insertions(+), 197 deletions(-) rename talk/owt/sdk/ic/{selfiesegmentation.cc => selfiesegmentationmodel.cc} (60%) rename talk/owt/sdk/ic/{selfiesegmentation.h => selfiesegmentationmodel.h} (57%) diff --git a/talk/owt/sdk/base/pluginmanager.cc b/talk/owt/sdk/base/pluginmanager.cc index dffaee3fc..51ef9f1b5 100644 --- a/talk/owt/sdk/base/pluginmanager.cc +++ b/talk/owt/sdk/base/pluginmanager.cc @@ -6,23 +6,24 @@ #include "talk/owt/sdk/base/sharedobjectpointer.h" +#if defined(WEBRTC_WIN) +#define DLL_SUFFIX ".dll" +#elif defined(WEBRTC_LINUX) +#define DLL_SUFFIX ".so" +#endif + namespace owt { namespace base { +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) template <> struct SOTrait { static constexpr auto name = "ICManager"; }; -#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) owt::ic::ICManagerInterface* PluginManager::ICPlugin() { static owt::base::SharedObjectPointer ic_plugin( -#if defined(WEBRTC_WIN) - "owt_ic.dll" -#elif defined(WEBRTC_LINUX) - "owt_ic.so" -#endif - ); + "owt_ic" DLL_SUFFIX); return ic_plugin.Get(); } #endif diff --git a/talk/owt/sdk/base/sharedobjectpointer.h b/talk/owt/sdk/base/sharedobjectpointer.h index d3b7c58c1..c6f0d0b5c 100644 --- a/talk/owt/sdk/base/sharedobjectpointer.h +++ b/talk/owt/sdk/base/sharedobjectpointer.h @@ -25,15 +25,6 @@ class SharedObjectPointer { public: explicit SharedObjectPointer(const char* path) : so_(path) { Load(); } - explicit SharedObjectPointer(const SharedObjectLoader& loader) : so_(loader) { - Load(); - } - - explicit SharedObjectPointer(SharedObjectLoader&& loader) - : so_(std::move(loader)) { - Load(); - } - bool IsLoaded() const { return so_.IsLoaded(); } T* Get() { return ptr_.get(); } @@ -41,7 +32,6 @@ class SharedObjectPointer { protected: void Load() { if (!so_.IsLoaded()) { - RTC_LOG(LS_WARNING) << "Shared object is not loaded."; return; } diff --git a/talk/owt/sdk/base/win/sharedobjectloader.cc b/talk/owt/sdk/base/win/sharedobjectloader.cc index 5a5f51a85..d129a6efd 100644 --- a/talk/owt/sdk/base/win/sharedobjectloader.cc +++ b/talk/owt/sdk/base/win/sharedobjectloader.cc @@ -6,11 +6,17 @@ #include +#include "third_party/webrtc/rtc_base/logging.h" + namespace owt { namespace base { SharedObjectLoader::SharedObjectLoader(const char* path) - : shared_object_(LoadLibraryA(path), FreeLibrary) {} + : shared_object_(LoadLibraryA(path), FreeLibrary) { + if (!shared_object_) { + RTC_LOG(LS_WARNING) << "Shared object " << path << " is not loaded."; + } +} SharedObjectLoader::~SharedObjectLoader() {} diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index 9048a5c40..d0b4a19fc 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -56,8 +56,8 @@ shared_library("owt_ic") { "backgroundblur.h", "icmanager.cc", "icmanager.h", - "selfiesegmentation.cc", - "selfiesegmentation.h", + "selfiesegmentationmodel.cc", + "selfiesegmentationmodel.h", ] defines = [ "OWT_IC_IMPL" ] configs += [ @@ -96,7 +96,11 @@ copy("openvino_library") { owt_openvino_root + "/deployment_tools/ngraph/lib/ngraph.dll", ] } - sources += [ bin_dir + "/inference_engine" + dll_suffix ] + sources += [ + bin_dir + "/inference_engine" + dll_suffix, + bin_dir + "/plugins.xml", + bin_dir + "/MKLDNNPlugin" + dll_suffix, + ] foreach(module, openvino_modules) { sources += [ bin_dir + "/inference_engine_" + module + dll_suffix ] } diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index c549958d6..6e87168e2 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -7,26 +7,22 @@ #include #include -#include "api/video/i420_buffer.h" #include "libyuv/convert.h" #include "opencv2/imgproc.hpp" +#include "third_party/webrtc/api/video/i420_buffer.h" +#include "third_party/webrtc/rtc_base/logging.h" -#include "selfiesegmentation.h" +#include "selfiesegmentationmodel.h" namespace owt { namespace ic { -BackgroundBlur::BackgroundBlur() {} +BackgroundBlur::BackgroundBlur(InferenceEngine::Core& core) : model(core) {} bool BackgroundBlur::SetParameter(const std::string& key, const std::string& value) { if (key == "model_path") { - try { - model = std::make_unique(value); - return true; - } catch (std::exception &e) { - return false; - } + return model.LoadModel(value); } else if (key == "blur_radius") { blur_radius_ = stoi(value); return true; @@ -36,7 +32,7 @@ bool BackgroundBlur::SetParameter(const std::string& key, rtc::scoped_refptr BackgroundBlur::Process( const rtc::scoped_refptr& buffer) { - if (model == nullptr) { + if (!model.IsLoaded()) { RTC_LOG(LS_WARNING) << "Background blur model is not initialized."; return buffer; } @@ -50,7 +46,7 @@ rtc::scoped_refptr BackgroundBlur::Process( cv::Mat input; frame.convertTo(input, CV_32FC3, 1. / UCHAR_MAX); - cv::Mat mask = model->predict(input); // mask is of 8UC1 + cv::Mat mask = model.predict(input); // mask is of 8UC1 cv::resize(mask, mask, {buffer->width(), buffer->height()}); cv::Mat foregroundMask; diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h index b310b0e40..508e1ff68 100644 --- a/talk/owt/sdk/ic/backgroundblur.h +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -7,16 +7,15 @@ #include -#include "rtc_base/logging.h" #include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" -#include "talk/owt/sdk/ic/selfiesegmentation.h" +#include "talk/owt/sdk/ic/selfiesegmentationmodel.h" namespace owt { namespace ic { class BackgroundBlur : public owt::base::VideoFramePostProcessor { public: - BackgroundBlur(); + BackgroundBlur(InferenceEngine::Core& core); ~BackgroundBlur() override = default; bool SetParameter(const std::string& key, const std::string& value) override; @@ -25,7 +24,7 @@ class BackgroundBlur : public owt::base::VideoFramePostProcessor { const rtc::scoped_refptr& buffer) override; private: - std::unique_ptr model; + SelfieSegmentationModel model; int blur_radius_ = 55; }; diff --git a/talk/owt/sdk/ic/icmanager.cc b/talk/owt/sdk/ic/icmanager.cc index c1edba74c..ff5b4fdb4 100644 --- a/talk/owt/sdk/ic/icmanager.cc +++ b/talk/owt/sdk/ic/icmanager.cc @@ -5,46 +5,33 @@ #include "talk/owt/sdk/ic/icmanager.h" #include "inference_engine.hpp" - #include "talk/owt/sdk/ic/backgroundblur.h" namespace owt { namespace ic { +ICManager::ICManager() : core(new InferenceEngine::Core()) { +} bool ICManager::InitializeInferenceEngineCore( const std::string& plugins_xml_path) { - return InitializeInferenceEngineCoreImpl(plugins_xml_path); + try { + core->RegisterPlugins(plugins_xml_path); + return true; + } catch (std::exception &) { + return false; + } } std::shared_ptr ICManager::CreatePostProcessor(ICPlugin plugin) { switch (plugin) { case ICPlugin::BACKGROUND_BLUR: - return std::make_shared(); + return std::make_shared(*core); default:; } return nullptr; } -InferenceEngine::Core& ICManager::InferenceEngineCore() { - if (!core) { - InitializeInferenceEngineCoreImpl({}); - } - return *core; -} - -bool ICManager::InitializeInferenceEngineCoreImpl( - const std::string& plugins_xml_path) { - try { - core = std::make_shared(plugins_xml_path); - return true; - } catch (...) { - return false; - } -} - -std::shared_ptr ICManager::core; - } // namespace ic } // namespace owt @@ -53,5 +40,5 @@ owt::ic::ICManagerInterface* CreateICManager() { } void DestroyICManager(owt::ic::ICManagerInterface* ic_manager) { - delete reinterpret_cast(ic_manager); + delete ic_manager; } diff --git a/talk/owt/sdk/ic/icmanager.h b/talk/owt/sdk/ic/icmanager.h index 96688c6da..c9cb02803 100644 --- a/talk/owt/sdk/ic/icmanager.h +++ b/talk/owt/sdk/ic/icmanager.h @@ -18,7 +18,7 @@ namespace ic { class ICManager : public ICManagerInterface { public: - ICManager() = default; + ICManager(); bool InitializeInferenceEngineCore( const std::string& plugins_xml_path) override; @@ -26,13 +26,8 @@ class ICManager : public ICManagerInterface { std::shared_ptr CreatePostProcessor( ICPlugin plugin) override; - static InferenceEngine::Core& InferenceEngineCore(); - - private: - static bool InitializeInferenceEngineCoreImpl( - const std::string& plugins_xml_path); - - static std::shared_ptr core; + protected: + std::shared_ptr core; }; } // namespace ic diff --git a/talk/owt/sdk/ic/selfiesegmentation.cc b/talk/owt/sdk/ic/selfiesegmentationmodel.cc similarity index 60% rename from talk/owt/sdk/ic/selfiesegmentation.cc rename to talk/owt/sdk/ic/selfiesegmentationmodel.cc index b5f043654..1376078ea 100644 --- a/talk/owt/sdk/ic/selfiesegmentation.cc +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.cc @@ -1,65 +1,72 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#include "selfiesegmentation.h" - -#include -#include - -#include "talk/owt/sdk/ic/icmanager.h" - -namespace IE = InferenceEngine; - -namespace owt { -namespace ic { - -SelfieSegmentation::SelfieSegmentation(const std::string& xmlPath, - const std::string& device) - : network(ICManager::InferenceEngineCore().ReadNetwork(xmlPath)), - executableNetwork(), - request() { - assert(!network.getInputsInfo().empty()); - assert(!network.getOutputsInfo().empty()); - inputName = network.getInputsInfo().begin()->first; - outputName = network.getOutputsInfo().begin()->first; - IE::DataPtr outputData = network.getOutputsInfo().begin()->second; - outputData->setPrecision(IE::Precision::U8); - outputSize = outputData->getTensorDesc().getDims(); - - IE::PreProcessInfo& preprocess = - network.getInputsInfo()[inputName]->getPreProcess(); - preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); - - executableNetwork = - ICManager::InferenceEngineCore().LoadNetwork(network, device); - request = executableNetwork.CreateInferRequest(); -} - -cv::Mat SelfieSegmentation::predict(const cv::Mat& frame) { - predictAsync(frame); - return waitForFinished(); -} - -void SelfieSegmentation::predictAsync(const cv::Mat& frame) { - CV_DbgAssert(frame.type() == CV_32FC3); - size_t height = frame.rows; - size_t width = frame.cols; - auto blob = IE::make_shared_blob( - IE::TensorDesc(IE::Precision::FP32, {1, 3, height, width}, - IE::Layout::NHWC), - (float*)frame.data); - request.SetBlob(inputName, blob); - request.StartAsync(); -} - -cv::Mat SelfieSegmentation::waitForFinished() { - request.Wait(IE::InferRequest::WaitMode::RESULT_READY); - const unsigned char* data = request.GetBlob(outputName)->buffer(); - int height = outputSize[1]; - int width = outputSize[2]; - return cv::Mat(height, width, CV_8U, (void*)data); -} - -} // namespace ic -} // namespace owt +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "selfiesegmentationmodel.h" + +#include +#include + +#include "talk/owt/sdk/ic/icmanager.h" +#include "third_party/webrtc/rtc_base/logging.h" + +namespace IE = InferenceEngine; + +namespace owt { +namespace ic { +SelfieSegmentationModel::SelfieSegmentationModel(InferenceEngine::Core& core) + : core(core) {} + +bool SelfieSegmentationModel::LoadModel(const std::string& xmlPath, + const std::string& device) { + InferenceEngine::CNNNetwork network = core.ReadNetwork(xmlPath); + if (network.getInputsInfo().empty() || network.getOutputsInfo().empty()) { + RTC_LOG(LS_WARNING) << "Bad segmentation model: no input / output"; + return false; + } + inputName = network.getInputsInfo().begin()->first; + outputName = network.getOutputsInfo().begin()->first; + IE::DataPtr outputData = network.getOutputsInfo().begin()->second; + outputData->setPrecision(IE::Precision::U8); + outputSize = outputData->getTensorDesc().getDims(); + + IE::PreProcessInfo& preprocess = + network.getInputsInfo()[inputName]->getPreProcess(); + preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); + + InferenceEngine::ExecutableNetwork executableNetwork = + core.LoadNetwork(network, device); + request = executableNetwork.CreateInferRequest(); +} + +bool SelfieSegmentationModel::IsLoaded() const { + return static_cast(request); +} + +cv::Mat SelfieSegmentationModel::predict(const cv::Mat& frame) { + predictAsync(frame); + return waitForFinished(); +} + +void SelfieSegmentationModel::predictAsync(const cv::Mat& frame) { + CV_DbgAssert(frame.type() == CV_32FC3); + size_t height = frame.rows; + size_t width = frame.cols; + auto blob = IE::make_shared_blob( + IE::TensorDesc(IE::Precision::FP32, {1, 3, height, width}, + IE::Layout::NHWC), + (float*)frame.data); + request.SetBlob(inputName, blob); + request.StartAsync(); +} + +cv::Mat SelfieSegmentationModel::waitForFinished() { + request.Wait(IE::InferRequest::WaitMode::RESULT_READY); + const unsigned char* data = request.GetBlob(outputName)->buffer(); + int height = outputSize[1]; + int width = outputSize[2]; + return cv::Mat(height, width, CV_8U, (void*)data); +} + +} // namespace ic +} // namespace owt diff --git a/talk/owt/sdk/ic/selfiesegmentation.h b/talk/owt/sdk/ic/selfiesegmentationmodel.h similarity index 57% rename from talk/owt/sdk/ic/selfiesegmentation.h rename to talk/owt/sdk/ic/selfiesegmentationmodel.h index 8e4ba117e..23684fe8f 100644 --- a/talk/owt/sdk/ic/selfiesegmentation.h +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.h @@ -1,38 +1,39 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_IC_SELFIESEGMENTATION_H_ -#define OWT_IC_SELFIESEGMENTATION_H_ - -#include -#include - -#include "inference_engine.hpp" -#include "opencv2/core/mat.hpp" - -namespace owt { -namespace ic { - -class SelfieSegmentation { - public: - explicit SelfieSegmentation(const std::string& xmlPath, - const std::string& device = "CPU"); - - cv::Mat predict(const cv::Mat& frame); - void predictAsync(const cv::Mat& frame); - cv::Mat waitForFinished(); - - private: - InferenceEngine::CNNNetwork network; - InferenceEngine::ExecutableNetwork executableNetwork; - InferenceEngine::InferRequest request; - std::string inputName; - std::string outputName; - InferenceEngine::SizeVector outputSize; -}; - -} // namespace ic -} // namespace owt - -#endif // OWT_IC_SELFIE_SEGMENTATION_H_ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_IC_SELFIESEGMENTATIONMODEL_H_ +#define OWT_IC_SELFIESEGMENTATIONMODEL_H_ + +#include +#include + +#include "inference_engine.hpp" +#include "opencv2/core/mat.hpp" + +namespace owt { +namespace ic { + +class SelfieSegmentationModel { + public: + explicit SelfieSegmentationModel(InferenceEngine::Core& core); + + bool LoadModel(const std::string& xmlPath, const std::string& device = "CPU"); + bool IsLoaded() const; + + cv::Mat predict(const cv::Mat& frame); + void predictAsync(const cv::Mat& frame); + cv::Mat waitForFinished(); + + private: + InferenceEngine::Core &core; + InferenceEngine::InferRequest request; + std::string inputName; + std::string outputName; + InferenceEngine::SizeVector outputSize; +}; + +} // namespace ic +} // namespace owt + +#endif // OWT_IC_SELFIESEGMENTATIONMODEL_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h index 9e63cb57f..d57471c01 100644 --- a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h +++ b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h @@ -5,8 +5,8 @@ #ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ #define OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ -#include "api/scoped_refptr.h" -#include "api/video/video_frame_buffer.h" +#include "third_party/webrtc/api/scoped_refptr.h" +#include "third_party/webrtc/api/video/video_frame_buffer.h" namespace owt { namespace base { diff --git a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h index d8b1bf0b9..b940b6c98 100644 --- a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h +++ b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h @@ -17,14 +17,13 @@ enum class ICPlugin { BACKGROUND_BLUR }; class ICManagerInterface { public: + virtual ~ICManagerInterface() = default; + virtual bool InitializeInferenceEngineCore( const std::string& plugins_xml_path) = 0; virtual std::shared_ptr CreatePostProcessor(ICPlugin plugin) = 0; - - protected: - ~ICManagerInterface() = default; }; } // namespace ic diff --git a/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn index 03ab7f537..87bbc98ac 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn +++ b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn @@ -9,10 +9,6 @@ import("//testing/test.gni") executable("sample_background_blur") { sources = [ "sample_background_blur.cc" ] include_dirs = [ "//talk/owt/sdk/include/cpp" ] - configs += [ - "//talk/owt/sdk/ic:openvino", - "//talk/owt/sdk/ic:openvino_opencv", - ] deps = [ "//talk/owt:owt_sdk_base" ] data_deps = [ "//talk/owt/sdk/ic:owt_ic" ] } diff --git a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc index a830b74d3..a426a114b 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc +++ b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc @@ -10,21 +10,7 @@ #include "owt/base/stream.h" class MainWindow : public owt::base::VideoRenderWindow { - static constexpr const wchar_t* class_name = L"MainWindow"; - - public: - MainWindow(int width, int height, const wchar_t* title) { - WNDCLASSEX cls = {}; - cls.cbSize = sizeof(WNDCLASSEX); - cls.lpfnWndProc = WindowProcedure; - cls.hInstance = GetModuleHandle(nullptr); - cls.lpszClassName = class_name; - RegisterClassEx(&cls); - HWND handle = CreateWindow(class_name, title, WS_OVERLAPPEDWINDOW, - CW_USEDEFAULT, CW_USEDEFAULT, width, height, - nullptr, nullptr, cls.hInstance, nullptr); - SetWindowHandle(handle); - } + static constexpr PCTSTR class_name = TEXT("MainWindow"); static LRESULT WindowProcedure(HWND window, unsigned int msg, @@ -39,8 +25,22 @@ class MainWindow : public owt::base::VideoRenderWindow { } } + public: + MainWindow(int width, int height, PCTSTR title) { + WNDCLASS cls = {}; + cls.lpfnWndProc = WindowProcedure; + cls.hInstance = GetModuleHandle(nullptr); + cls.lpszClassName = class_name; + RegisterClass(&cls); + HWND handle = CreateWindow(class_name, title, WS_OVERLAPPEDWINDOW, + CW_USEDEFAULT, CW_USEDEFAULT, width, height, + nullptr, nullptr, cls.hInstance, nullptr); + SetWindowHandle(handle); + } + + void Show() { ShowWindow(GetWindowHandle(), SW_SHOW); } + int Exec() { - ShowWindow(GetWindowHandle(), SW_SHOW); MSG msg; while (GetMessage(&msg, 0, 0, 0)) { DispatchMessage(&msg); @@ -101,8 +101,9 @@ int main(int, char*[]) { return 6; } - MainWindow w(capability.width, capability.height, - std::wstring(device_name.begin(), device_name.end()).c_str()); + std::basic_string title(device_name.begin(), device_name.end()); + MainWindow w(capability.width, capability.height, title.c_str()); stream->AttachVideoRenderer(w); + w.Show(); return w.Exec(); } From f9c7381d8da0f33c44c3f630480ce83c0bc21115 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Wed, 3 Nov 2021 13:47:23 +0800 Subject: [PATCH 16/30] Fixups --- .../include/cpp/owt/base/videoframepostprocessor.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h index d57471c01..fc6438ace 100644 --- a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h +++ b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h @@ -5,8 +5,16 @@ #ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ #define OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ -#include "third_party/webrtc/api/scoped_refptr.h" -#include "third_party/webrtc/api/video/video_frame_buffer.h" +#include + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class VideoFrameBuffer; +} namespace owt { namespace base { From a474a598a84f7810ecafb5bd3c42664b46eabc12 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Thu, 4 Nov 2021 13:23:05 +0800 Subject: [PATCH 17/30] Naming fixups and etc. --- .../intelligentcollaborationparameters.cc | 2 +- .../sdk/base/localcamerastreamparameters.cc | 4 +- talk/owt/sdk/base/stream.cc | 4 +- talk/owt/sdk/ic/backgroundblur.cc | 48 +++++++++---------- talk/owt/sdk/ic/backgroundblur.h | 2 +- talk/owt/sdk/ic/icmanager.cc | 6 +-- talk/owt/sdk/ic/icmanager.h | 2 +- talk/owt/sdk/ic/selfiesegmentationmodel.cc | 32 ++++++------- talk/owt/sdk/ic/selfiesegmentationmodel.h | 10 ++-- .../owt/base/localcamerastreamparameters.h | 6 +-- .../ic/intelligentcollaborationparameters.h | 2 +- 11 files changed, 59 insertions(+), 59 deletions(-) diff --git a/talk/owt/sdk/base/intelligentcollaborationparameters.cc b/talk/owt/sdk/base/intelligentcollaborationparameters.cc index c757bb11b..6cb699ea8 100644 --- a/talk/owt/sdk/base/intelligentcollaborationparameters.cc +++ b/talk/owt/sdk/base/intelligentcollaborationparameters.cc @@ -5,7 +5,7 @@ #include "talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h" namespace owt { -namespace ic { +namespace base { std::vector>& IntelligentCollaborationParameters::PostProcessors() { diff --git a/talk/owt/sdk/base/localcamerastreamparameters.cc b/talk/owt/sdk/base/localcamerastreamparameters.cc index 07507a305..5064f072d 100644 --- a/talk/owt/sdk/base/localcamerastreamparameters.cc +++ b/talk/owt/sdk/base/localcamerastreamparameters.cc @@ -19,11 +19,11 @@ LocalCameraStreamParameters::LocalCameraStreamParameters(bool audio_enabled, void LocalCameraStreamParameters::Fps(int fps) { fps_ = fps; } -owt::ic::IntelligentCollaborationParameters& +IntelligentCollaborationParameters& LocalCameraStreamParameters::ICParams() { return ic_params_; } -const owt::ic::IntelligentCollaborationParameters& +const IntelligentCollaborationParameters& LocalCameraStreamParameters::ICParams() const { return ic_params_; } diff --git a/talk/owt/sdk/base/stream.cc b/talk/owt/sdk/base/stream.cc index a7d314779..dc410d91e 100644 --- a/talk/owt/sdk/base/stream.cc +++ b/talk/owt/sdk/base/stream.cc @@ -46,8 +46,8 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { const size_t height, const size_t fps, int capture_device_idx, - const owt::ic::IntelligentCollaborationParameters& ic_params = - owt::ic::IntelligentCollaborationParameters()) { + const IntelligentCollaborationParameters& ic_params = + IntelligentCollaborationParameters()) { std::unique_ptr capturer; std::unique_ptr info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index 6e87168e2..dfe71554d 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -17,12 +17,12 @@ namespace owt { namespace ic { -BackgroundBlur::BackgroundBlur(InferenceEngine::Core& core) : model(core) {} +BackgroundBlur::BackgroundBlur(InferenceEngine::Core& core) : model_(core) {} bool BackgroundBlur::SetParameter(const std::string& key, const std::string& value) { if (key == "model_path") { - return model.LoadModel(value); + return model_.LoadModel(value); } else if (key == "blur_radius") { blur_radius_ = stoi(value); return true; @@ -32,8 +32,8 @@ bool BackgroundBlur::SetParameter(const std::string& key, rtc::scoped_refptr BackgroundBlur::Process( const rtc::scoped_refptr& buffer) { - if (!model.IsLoaded()) { - RTC_LOG(LS_WARNING) << "Background blur model is not initialized."; + if (!model_.IsLoaded()) { + RTC_LOG(LS_WARNING) << "Background blur model_ is not initialized."; return buffer; } @@ -46,39 +46,39 @@ rtc::scoped_refptr BackgroundBlur::Process( cv::Mat input; frame.convertTo(input, CV_32FC3, 1. / UCHAR_MAX); - cv::Mat mask = model.predict(input); // mask is of 8UC1 + cv::Mat mask = model_.predict(input); // mask is of 8UC1 cv::resize(mask, mask, {buffer->width(), buffer->height()}); - cv::Mat foregroundMask; - cv::merge(std::vector{mask, mask, mask}, foregroundMask); - cv::Mat backgroundMask; + cv::Mat foreground_mask; + cv::merge(std::vector{mask, mask, mask}, foreground_mask); + cv::Mat background_mask; cv::merge(std::vector{UCHAR_MAX - mask, UCHAR_MAX - mask, UCHAR_MAX - mask}, - backgroundMask); + background_mask); - // apply a masked blur on background - cv::Mat maskedFrame; - cv::multiply(frame, backgroundMask, maskedFrame, 1. / UCHAR_MAX); + // Apply a masked blur on background + cv::Mat masked_frame; + cv::multiply(frame, background_mask, masked_frame, 1. / UCHAR_MAX); cv::Mat background; - cv::GaussianBlur(maskedFrame, background, {blur_radius_, blur_radius_}, 0); - cv::Mat blurredMask; - cv::GaussianBlur(backgroundMask, blurredMask, {blur_radius_, blur_radius_}, + cv::GaussianBlur(masked_frame, background, {blur_radius_, blur_radius_}, 0); + cv::Mat blurred_mask; + cv::GaussianBlur(background_mask, blurred_mask, {blur_radius_, blur_radius_}, 0); - cv::divide(background, blurredMask, background, UCHAR_MAX); - cv::multiply(background, backgroundMask, background, 1. / UCHAR_MAX); + cv::divide(background, blurred_mask, background, UCHAR_MAX); + cv::multiply(background, background_mask, background, 1. / UCHAR_MAX); cv::Mat foreground; - cv::multiply(frame, foregroundMask, foreground, 1. / UCHAR_MAX); + cv::multiply(frame, foreground_mask, foreground, 1. / UCHAR_MAX); frame = foreground + background; - auto newBuffer = + auto new_buffer = webrtc::I420Buffer::Create(buffer->width(), buffer->height()); - libyuv::RAWToI420(frame.data, frame.cols * 3, newBuffer->MutableDataY(), - newBuffer->StrideY(), newBuffer->MutableDataU(), - newBuffer->StrideU(), newBuffer->MutableDataV(), - newBuffer->StrideV(), buffer->width(), buffer->height()); + libyuv::RAWToI420(frame.data, frame.cols * 3, new_buffer->MutableDataY(), + new_buffer->StrideY(), new_buffer->MutableDataU(), + new_buffer->StrideU(), new_buffer->MutableDataV(), + new_buffer->StrideV(), buffer->width(), buffer->height()); - return newBuffer; + return new_buffer; } } // namespace ic diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h index 508e1ff68..6d8826a14 100644 --- a/talk/owt/sdk/ic/backgroundblur.h +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -24,7 +24,7 @@ class BackgroundBlur : public owt::base::VideoFramePostProcessor { const rtc::scoped_refptr& buffer) override; private: - SelfieSegmentationModel model; + SelfieSegmentationModel model_; int blur_radius_ = 55; }; diff --git a/talk/owt/sdk/ic/icmanager.cc b/talk/owt/sdk/ic/icmanager.cc index ff5b4fdb4..c0a04240f 100644 --- a/talk/owt/sdk/ic/icmanager.cc +++ b/talk/owt/sdk/ic/icmanager.cc @@ -9,13 +9,13 @@ namespace owt { namespace ic { -ICManager::ICManager() : core(new InferenceEngine::Core()) { +ICManager::ICManager() : core_(new InferenceEngine::Core()) { } bool ICManager::InitializeInferenceEngineCore( const std::string& plugins_xml_path) { try { - core->RegisterPlugins(plugins_xml_path); + core_->RegisterPlugins(plugins_xml_path); return true; } catch (std::exception &) { return false; @@ -26,7 +26,7 @@ std::shared_ptr ICManager::CreatePostProcessor(ICPlugin plugin) { switch (plugin) { case ICPlugin::BACKGROUND_BLUR: - return std::make_shared(*core); + return std::make_shared(*core_); default:; } return nullptr; diff --git a/talk/owt/sdk/ic/icmanager.h b/talk/owt/sdk/ic/icmanager.h index c9cb02803..48018eef2 100644 --- a/talk/owt/sdk/ic/icmanager.h +++ b/talk/owt/sdk/ic/icmanager.h @@ -27,7 +27,7 @@ class ICManager : public ICManagerInterface { ICPlugin plugin) override; protected: - std::shared_ptr core; + std::shared_ptr core_; }; } // namespace ic diff --git a/talk/owt/sdk/ic/selfiesegmentationmodel.cc b/talk/owt/sdk/ic/selfiesegmentationmodel.cc index 1376078ea..0ee404384 100644 --- a/talk/owt/sdk/ic/selfiesegmentationmodel.cc +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.cc @@ -15,32 +15,32 @@ namespace IE = InferenceEngine; namespace owt { namespace ic { SelfieSegmentationModel::SelfieSegmentationModel(InferenceEngine::Core& core) - : core(core) {} + : core_(core) {} bool SelfieSegmentationModel::LoadModel(const std::string& xmlPath, const std::string& device) { - InferenceEngine::CNNNetwork network = core.ReadNetwork(xmlPath); + InferenceEngine::CNNNetwork network = core_.ReadNetwork(xmlPath); if (network.getInputsInfo().empty() || network.getOutputsInfo().empty()) { - RTC_LOG(LS_WARNING) << "Bad segmentation model: no input / output"; + RTC_LOG(LS_WARNING) << "Bad segmentation model_: no input / output"; return false; } - inputName = network.getInputsInfo().begin()->first; - outputName = network.getOutputsInfo().begin()->first; + input_name_ = network.getInputsInfo().begin()->first; + output_name_ = network.getOutputsInfo().begin()->first; IE::DataPtr outputData = network.getOutputsInfo().begin()->second; outputData->setPrecision(IE::Precision::U8); - outputSize = outputData->getTensorDesc().getDims(); + output_size_ = outputData->getTensorDesc().getDims(); IE::PreProcessInfo& preprocess = - network.getInputsInfo()[inputName]->getPreProcess(); + network.getInputsInfo()[input_name_]->getPreProcess(); preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); InferenceEngine::ExecutableNetwork executableNetwork = - core.LoadNetwork(network, device); - request = executableNetwork.CreateInferRequest(); + core_.LoadNetwork(network, device); + request_ = executableNetwork.CreateInferRequest(); } bool SelfieSegmentationModel::IsLoaded() const { - return static_cast(request); + return static_cast(request_); } cv::Mat SelfieSegmentationModel::predict(const cv::Mat& frame) { @@ -56,15 +56,15 @@ void SelfieSegmentationModel::predictAsync(const cv::Mat& frame) { IE::TensorDesc(IE::Precision::FP32, {1, 3, height, width}, IE::Layout::NHWC), (float*)frame.data); - request.SetBlob(inputName, blob); - request.StartAsync(); + request_.SetBlob(input_name_, blob); + request_.StartAsync(); } cv::Mat SelfieSegmentationModel::waitForFinished() { - request.Wait(IE::InferRequest::WaitMode::RESULT_READY); - const unsigned char* data = request.GetBlob(outputName)->buffer(); - int height = outputSize[1]; - int width = outputSize[2]; + request_.Wait(IE::InferRequest::WaitMode::RESULT_READY); + const unsigned char* data = request_.GetBlob(output_name_)->buffer(); + int height = output_size_[1]; + int width = output_size_[2]; return cv::Mat(height, width, CV_8U, (void*)data); } diff --git a/talk/owt/sdk/ic/selfiesegmentationmodel.h b/talk/owt/sdk/ic/selfiesegmentationmodel.h index 23684fe8f..7a65eb4b3 100644 --- a/talk/owt/sdk/ic/selfiesegmentationmodel.h +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.h @@ -26,11 +26,11 @@ class SelfieSegmentationModel { cv::Mat waitForFinished(); private: - InferenceEngine::Core &core; - InferenceEngine::InferRequest request; - std::string inputName; - std::string outputName; - InferenceEngine::SizeVector outputSize; + InferenceEngine::Core &core_; + InferenceEngine::InferRequest request_; + std::string input_name_; + std::string output_name_; + InferenceEngine::SizeVector output_size_; }; } // namespace ic diff --git a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h index f438bc12c..5618b70e7 100644 --- a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h +++ b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h @@ -55,8 +55,8 @@ class LocalCameraStreamParameters final { /** @brief Get the reference of intelligent collaboration parameters */ - owt::ic::IntelligentCollaborationParameters& ICParams(); - const owt::ic::IntelligentCollaborationParameters& ICParams() const; + IntelligentCollaborationParameters& ICParams(); + const IntelligentCollaborationParameters& ICParams() const; /** @cond */ std::string CameraId() const { return camera_id_; } std::string StreamName() const { return stream_name_; } @@ -74,7 +74,7 @@ class LocalCameraStreamParameters final { int fps_; bool video_enabled_; bool audio_enabled_; - owt::ic::IntelligentCollaborationParameters ic_params_; + IntelligentCollaborationParameters ic_params_; }; diff --git a/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h b/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h index 9259bc9d2..5a957aa66 100644 --- a/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h +++ b/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h @@ -11,7 +11,7 @@ #include "owt/base/videoframepostprocessor.h" namespace owt { -namespace ic { +namespace base { class IntelligentCollaborationParameters final { public: From 13d8027c00b991eea7891ad20209258f58148e6e Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Fri, 5 Nov 2021 13:17:50 +0800 Subject: [PATCH 18/30] Fixups --- talk/owt/BUILD.gn | 2 - .../intelligentcollaborationparameters.cc | 21 --- .../sdk/base/localcamerastreamparameters.cc | 12 +- talk/owt/sdk/base/stream.cc | 10 +- talk/owt/sdk/ic/BUILD.gn | 5 +- talk/owt/sdk/ic/backgroundblur.cc | 9 +- talk/owt/sdk/ic/backgroundblur.h | 4 +- talk/owt/sdk/ic/icmanager.cc | 38 +++-- talk/owt/sdk/ic/icmanager.h | 6 +- talk/owt/sdk/ic/selfiesegmentationmodel.cc | 149 +++++++++--------- talk/owt/sdk/ic/selfiesegmentationmodel.h | 78 ++++----- .../owt/base/localcamerastreamparameters.h | 18 ++- .../sdk/include/cpp/owt/base/pluginmanager.h | 55 ++++--- .../cpp/owt/base/videoframepostprocessor.h | 9 ++ .../include/cpp/owt/ic/icmanagerinterface.h | 18 ++- .../ic/intelligentcollaborationparameters.h | 34 ---- .../sample_background_blur.cc | 8 +- 17 files changed, 238 insertions(+), 238 deletions(-) delete mode 100644 talk/owt/sdk/base/intelligentcollaborationparameters.cc delete mode 100644 talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h diff --git a/talk/owt/BUILD.gn b/talk/owt/BUILD.gn index c74693124..4c594f30c 100644 --- a/talk/owt/BUILD.gn +++ b/talk/owt/BUILD.gn @@ -98,7 +98,6 @@ static_library("owt_sdk_base") { "sdk/base/functionalobserver.cc", "sdk/base/functionalobserver.h", "sdk/base/globalconfiguration.cc", - "sdk/base/intelligentcollaborationparameters.cc", "sdk/base/localcamerastreamparameters.cc", "sdk/base/logging.cc", "sdk/base/logsinks.cc", @@ -133,7 +132,6 @@ static_library("owt_sdk_base") { "sdk/include/cpp/owt/base/stream.h", "sdk/include/cpp/owt/base/videoframepostprocessor.h", "sdk/include/cpp/owt/base/videorendererinterface.h", - "sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h", ] if (is_win || is_linux) { sources += [ diff --git a/talk/owt/sdk/base/intelligentcollaborationparameters.cc b/talk/owt/sdk/base/intelligentcollaborationparameters.cc deleted file mode 100644 index 6cb699ea8..000000000 --- a/talk/owt/sdk/base/intelligentcollaborationparameters.cc +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#include "talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h" - -namespace owt { -namespace base { - -std::vector>& -IntelligentCollaborationParameters::PostProcessors() { - return post_processors_; -} - -const std::vector>& -IntelligentCollaborationParameters::PostProcessors() const { - return post_processors_; -} - -} // namespace base -} // namespace owt diff --git a/talk/owt/sdk/base/localcamerastreamparameters.cc b/talk/owt/sdk/base/localcamerastreamparameters.cc index 5064f072d..41be59a21 100644 --- a/talk/owt/sdk/base/localcamerastreamparameters.cc +++ b/talk/owt/sdk/base/localcamerastreamparameters.cc @@ -19,14 +19,6 @@ LocalCameraStreamParameters::LocalCameraStreamParameters(bool audio_enabled, void LocalCameraStreamParameters::Fps(int fps) { fps_ = fps; } -IntelligentCollaborationParameters& -LocalCameraStreamParameters::ICParams() { - return ic_params_; -} -const IntelligentCollaborationParameters& -LocalCameraStreamParameters::ICParams() const { - return ic_params_; -} void LocalCameraStreamParameters::CameraId(const std::string& camera_id) { camera_id_ = camera_id; } @@ -37,6 +29,10 @@ void LocalCameraStreamParameters::Resolution(int width, int height) { void LocalCameraStreamParameters::StreamName(const std::string& stream_name){ stream_name_ = stream_name; } +std::vector>& +LocalCameraStreamParameters::PostProcessors() { + return post_processors_; +} LocalDesktopStreamParameters::LocalDesktopStreamParameters( bool audio_enabled, bool video_enabled) diff --git a/talk/owt/sdk/base/stream.cc b/talk/owt/sdk/base/stream.cc index dc410d91e..f795a7fd2 100644 --- a/talk/owt/sdk/base/stream.cc +++ b/talk/owt/sdk/base/stream.cc @@ -46,8 +46,8 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { const size_t height, const size_t fps, int capture_device_idx, - const IntelligentCollaborationParameters& ic_params = - IntelligentCollaborationParameters()) { + const std::vector>& + post_processors) { std::unique_ptr capturer; std::unique_ptr info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -58,8 +58,8 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { for (int i = 0; i < num_devices; ++i) { capturer = absl::WrapUnique(owt::base::VcmCapturer::Create( width, height, fps, capture_device_idx)); - for (auto& post_processor : ic_params.PostProcessors()) { - capturer->AddVideoFramePostProcessor(post_processor); + for (auto& post_processor : post_processors) { + capturer->AddVideoFramePostProcessor(post_processor); } if (capturer) { return new rtc::RefCountedObject( @@ -519,7 +519,7 @@ LocalStream::LocalStream(const LocalCameraStreamParameters& parameters, parameters.ResolutionWidth(), parameters.ResolutionHeight(), parameters.Fps(), DeviceUtils::GetVideoCaptureDeviceIndex(parameters.CameraId()), - parameters.ICParams()); + parameters.PostProcessors()); #else capturer_ = ObjcVideoCapturerFactory::Create(parameters); if (!capturer_) { diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index d0b4a19fc..76e6a1822 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -24,7 +24,10 @@ config("openvino") { } else { lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Release" ] } - configs = [ "//build/config/compiler:rtti" ] + configs = [ + "//build/config/compiler:exceptions", + "//build/config/compiler:rtti", + ] } config("openvino_opencv") { diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index dfe71554d..c5f52c552 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -33,7 +33,12 @@ bool BackgroundBlur::SetParameter(const std::string& key, rtc::scoped_refptr BackgroundBlur::Process( const rtc::scoped_refptr& buffer) { if (!model_.IsLoaded()) { - RTC_LOG(LS_WARNING) << "Background blur model_ is not initialized."; + RTC_LOG(LS_WARNING) << "Background blur model is not initialized."; + return buffer; + } + + if (buffer->type() == webrtc::VideoFrameBuffer::Type::kNative) { + RTC_LOG(LS_WARNING) << "Native video frame buffer is not supported."; return buffer; } @@ -46,7 +51,7 @@ rtc::scoped_refptr BackgroundBlur::Process( cv::Mat input; frame.convertTo(input, CV_32FC3, 1. / UCHAR_MAX); - cv::Mat mask = model_.predict(input); // mask is of 8UC1 + cv::Mat mask = model_.Predict(input); // mask is of 8UC1 cv::resize(mask, mask, {buffer->width(), buffer->height()}); cv::Mat foreground_mask; diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h index 6d8826a14..e6146d36d 100644 --- a/talk/owt/sdk/ic/backgroundblur.h +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -7,13 +7,13 @@ #include -#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" #include "talk/owt/sdk/ic/selfiesegmentationmodel.h" +#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" namespace owt { namespace ic { -class BackgroundBlur : public owt::base::VideoFramePostProcessor { +class BackgroundBlur final : public owt::base::VideoFramePostProcessor { public: BackgroundBlur(InferenceEngine::Core& core); ~BackgroundBlur() override = default; diff --git a/talk/owt/sdk/ic/icmanager.cc b/talk/owt/sdk/ic/icmanager.cc index c0a04240f..7482ff1a6 100644 --- a/talk/owt/sdk/ic/icmanager.cc +++ b/talk/owt/sdk/ic/icmanager.cc @@ -6,28 +6,42 @@ #include "inference_engine.hpp" #include "talk/owt/sdk/ic/backgroundblur.h" +#include "third_party/webrtc/rtc_base/logging.h" namespace owt { namespace ic { -ICManager::ICManager() : core_(new InferenceEngine::Core()) { + +ICManager::ICManager() : core_() { + try { + core_.reset(new InferenceEngine::Core); + } catch (std::exception& e) { + RTC_LOG(LS_ERROR) << "Failed to initialize inference engine core: " + << e.what(); + } } -bool ICManager::InitializeInferenceEngineCore( +bool ICManager::RegisterInferenceEnginePlugins( const std::string& plugins_xml_path) { - try { - core_->RegisterPlugins(plugins_xml_path); - return true; - } catch (std::exception &) { - return false; + if (core_) { + try { + core_->RegisterPlugins(plugins_xml_path); + return true; + } catch (std::exception& e) { + RTC_LOG(LS_ERROR) << "Failed to register inference engine plugins: " + << e.what(); + } } + return false; } std::shared_ptr -ICManager::CreatePostProcessor(ICPlugin plugin) { - switch (plugin) { - case ICPlugin::BACKGROUND_BLUR: - return std::make_shared(*core_); - default:; +ICManager::CreatePostProcessor(ICPostProcessor processor) { + if (core_) { + switch (processor) { + case ICPostProcessor::BACKGROUND_BLUR: + return std::make_shared(*core_); + default:; + } } return nullptr; } diff --git a/talk/owt/sdk/ic/icmanager.h b/talk/owt/sdk/ic/icmanager.h index 48018eef2..b3cf3d64f 100644 --- a/talk/owt/sdk/ic/icmanager.h +++ b/talk/owt/sdk/ic/icmanager.h @@ -16,15 +16,15 @@ class Core; namespace owt { namespace ic { -class ICManager : public ICManagerInterface { +class ICManager final : public ICManagerInterface { public: ICManager(); - bool InitializeInferenceEngineCore( + bool RegisterInferenceEnginePlugins( const std::string& plugins_xml_path) override; std::shared_ptr CreatePostProcessor( - ICPlugin plugin) override; + ICPostProcessor processor) override; protected: std::shared_ptr core_; diff --git a/talk/owt/sdk/ic/selfiesegmentationmodel.cc b/talk/owt/sdk/ic/selfiesegmentationmodel.cc index 0ee404384..df4077293 100644 --- a/talk/owt/sdk/ic/selfiesegmentationmodel.cc +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.cc @@ -1,72 +1,77 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#include "selfiesegmentationmodel.h" - -#include -#include - -#include "talk/owt/sdk/ic/icmanager.h" -#include "third_party/webrtc/rtc_base/logging.h" - -namespace IE = InferenceEngine; - -namespace owt { -namespace ic { -SelfieSegmentationModel::SelfieSegmentationModel(InferenceEngine::Core& core) - : core_(core) {} - -bool SelfieSegmentationModel::LoadModel(const std::string& xmlPath, - const std::string& device) { - InferenceEngine::CNNNetwork network = core_.ReadNetwork(xmlPath); - if (network.getInputsInfo().empty() || network.getOutputsInfo().empty()) { - RTC_LOG(LS_WARNING) << "Bad segmentation model_: no input / output"; - return false; - } - input_name_ = network.getInputsInfo().begin()->first; - output_name_ = network.getOutputsInfo().begin()->first; - IE::DataPtr outputData = network.getOutputsInfo().begin()->second; - outputData->setPrecision(IE::Precision::U8); - output_size_ = outputData->getTensorDesc().getDims(); - - IE::PreProcessInfo& preprocess = - network.getInputsInfo()[input_name_]->getPreProcess(); - preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); - - InferenceEngine::ExecutableNetwork executableNetwork = - core_.LoadNetwork(network, device); - request_ = executableNetwork.CreateInferRequest(); -} - -bool SelfieSegmentationModel::IsLoaded() const { - return static_cast(request_); -} - -cv::Mat SelfieSegmentationModel::predict(const cv::Mat& frame) { - predictAsync(frame); - return waitForFinished(); -} - -void SelfieSegmentationModel::predictAsync(const cv::Mat& frame) { - CV_DbgAssert(frame.type() == CV_32FC3); - size_t height = frame.rows; - size_t width = frame.cols; - auto blob = IE::make_shared_blob( - IE::TensorDesc(IE::Precision::FP32, {1, 3, height, width}, - IE::Layout::NHWC), - (float*)frame.data); - request_.SetBlob(input_name_, blob); - request_.StartAsync(); -} - -cv::Mat SelfieSegmentationModel::waitForFinished() { - request_.Wait(IE::InferRequest::WaitMode::RESULT_READY); - const unsigned char* data = request_.GetBlob(output_name_)->buffer(); - int height = output_size_[1]; - int width = output_size_[2]; - return cv::Mat(height, width, CV_8U, (void*)data); -} - -} // namespace ic -} // namespace owt +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "selfiesegmentationmodel.h" + +#include +#include + +#include "talk/owt/sdk/ic/icmanager.h" +#include "third_party/webrtc/rtc_base/logging.h" + +namespace IE = InferenceEngine; + +namespace owt { +namespace ic { +SelfieSegmentationModel::SelfieSegmentationModel(InferenceEngine::Core& core) + : core_(core) {} + +bool SelfieSegmentationModel::LoadModel(const std::string& xmlPath, + const std::string& device) { + try { + InferenceEngine::CNNNetwork network = core_.ReadNetwork(xmlPath); + if (network.getInputsInfo().empty() || network.getOutputsInfo().empty()) { + RTC_LOG(LS_WARNING) << "Bad segmentation model: no input / output"; + return false; + } + input_name_ = network.getInputsInfo().begin()->first; + output_name_ = network.getOutputsInfo().begin()->first; + IE::DataPtr outputData = network.getOutputsInfo().begin()->second; + outputData->setPrecision(IE::Precision::U8); + output_size_ = outputData->getTensorDesc().getDims(); + + IE::PreProcessInfo& preprocess = + network.getInputsInfo()[input_name_]->getPreProcess(); + preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); + + InferenceEngine::ExecutableNetwork executableNetwork = + core_.LoadNetwork(network, device); + request_ = executableNetwork.CreateInferRequest(); + } catch (std::exception& e) { + RTC_LOG(LS_ERROR) << "Load model failed: " << e.what(); + return false; + } +} + +bool SelfieSegmentationModel::IsLoaded() const { + return static_cast(request_); +} + +cv::Mat SelfieSegmentationModel::Predict(const cv::Mat& frame) { + PredictAsync(frame); + return WaitForFinished(); +} + +void SelfieSegmentationModel::PredictAsync(const cv::Mat& frame) { + CV_DbgAssert(frame.type() == CV_32FC3); + size_t height = frame.rows; + size_t width = frame.cols; + auto blob = IE::make_shared_blob( + IE::TensorDesc(IE::Precision::FP32, {1, 3, height, width}, + IE::Layout::NHWC), + (float*)frame.data); + request_.SetBlob(input_name_, blob); + request_.StartAsync(); +} + +cv::Mat SelfieSegmentationModel::WaitForFinished() { + request_.Wait(IE::InferRequest::WaitMode::RESULT_READY); + const unsigned char* data = request_.GetBlob(output_name_)->buffer(); + int height = output_size_[1]; + int width = output_size_[2]; + return cv::Mat(height, width, CV_8U, (void*)data); +} + +} // namespace ic +} // namespace owt diff --git a/talk/owt/sdk/ic/selfiesegmentationmodel.h b/talk/owt/sdk/ic/selfiesegmentationmodel.h index 7a65eb4b3..fe657d9b0 100644 --- a/talk/owt/sdk/ic/selfiesegmentationmodel.h +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.h @@ -1,39 +1,39 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_IC_SELFIESEGMENTATIONMODEL_H_ -#define OWT_IC_SELFIESEGMENTATIONMODEL_H_ - -#include -#include - -#include "inference_engine.hpp" -#include "opencv2/core/mat.hpp" - -namespace owt { -namespace ic { - -class SelfieSegmentationModel { - public: - explicit SelfieSegmentationModel(InferenceEngine::Core& core); - - bool LoadModel(const std::string& xmlPath, const std::string& device = "CPU"); - bool IsLoaded() const; - - cv::Mat predict(const cv::Mat& frame); - void predictAsync(const cv::Mat& frame); - cv::Mat waitForFinished(); - - private: - InferenceEngine::Core &core_; - InferenceEngine::InferRequest request_; - std::string input_name_; - std::string output_name_; - InferenceEngine::SizeVector output_size_; -}; - -} // namespace ic -} // namespace owt - -#endif // OWT_IC_SELFIESEGMENTATIONMODEL_H_ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_IC_SELFIESEGMENTATIONMODEL_H_ +#define OWT_IC_SELFIESEGMENTATIONMODEL_H_ + +#include +#include + +#include "inference_engine.hpp" +#include "opencv2/core/mat.hpp" + +namespace owt { +namespace ic { + +class SelfieSegmentationModel { + public: + explicit SelfieSegmentationModel(InferenceEngine::Core& core); + + bool LoadModel(const std::string& xmlPath, const std::string& device = "CPU"); + bool IsLoaded() const; + + cv::Mat Predict(const cv::Mat& frame); + void PredictAsync(const cv::Mat& frame); + cv::Mat WaitForFinished(); + + private: + InferenceEngine::Core &core_; + InferenceEngine::InferRequest request_; + std::string input_name_; + std::string output_name_; + InferenceEngine::SizeVector output_size_; +}; + +} // namespace ic +} // namespace owt + +#endif // OWT_IC_SELFIESEGMENTATIONMODEL_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h index 5618b70e7..e86f25b98 100644 --- a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h +++ b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h @@ -4,9 +4,11 @@ #ifndef OWT_BASE_LOCALCAMERASTREAMPARAMETERS_H_ #define OWT_BASE_LOCALCAMERASTREAMPARAMETERS_H_ +#include #include +#include #include "owt/base/commontypes.h" -#include "owt/ic/intelligentcollaborationparameters.h" +#include "owt/base/videoframepostprocessor.h" namespace owt { namespace base{ /** @@ -53,10 +55,11 @@ class LocalCameraStreamParameters final { */ void Fps(int fps); /** - @brief Get the reference of intelligent collaboration parameters + @brief Get the reference of video frame post processors, to be applied in + onFrame method. */ - IntelligentCollaborationParameters& ICParams(); - const IntelligentCollaborationParameters& ICParams() const; + std::vector>& + PostProcessors(); /** @cond */ std::string CameraId() const { return camera_id_; } std::string StreamName() const { return stream_name_; } @@ -65,6 +68,10 @@ class LocalCameraStreamParameters final { int Fps() const { return fps_; } bool VideoEnabled() const { return video_enabled_; } bool AudioEnabled() const { return audio_enabled_; } + const std::vector>& + PostProcessors() const { + return post_processors_; + } /** @endcond */ private: std::string camera_id_; @@ -74,7 +81,8 @@ class LocalCameraStreamParameters final { int fps_; bool video_enabled_; bool audio_enabled_; - IntelligentCollaborationParameters ic_params_; + std::vector> + post_processors_; }; diff --git a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h index c2ec86bf4..012254e34 100644 --- a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h +++ b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h @@ -1,25 +1,30 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_BASE_PLUGINMANAGER_H_ -#define OWT_BASE_PLUGINMANAGER_H_ - -#include "owt/ic/icmanagerinterface.h" - -namespace owt { -namespace base { - -class PluginManager { - public: -#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) - static owt::ic::ICManagerInterface* ICPlugin(); -#endif - - private: - PluginManager(); -}; -} // namespace base -} // namespace owt - -#endif // OWT_BASE_PLUGINMANAGER_H_ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_PLUGINMANAGER_H_ +#define OWT_BASE_PLUGINMANAGER_H_ + +#include "owt/ic/icmanagerinterface.h" + +namespace owt { +namespace base { + +/// Static class of managing dll plugins. +class PluginManager { + public: +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) + /** + @brief Get the pointer of IC plugin. If loading owt_ic.dll is failed, a + nullptr will be returned. + */ + static owt::ic::ICManagerInterface* ICPlugin(); +#endif + + private: + PluginManager(); +}; +} // namespace base +} // namespace owt + +#endif // OWT_BASE_PLUGINMANAGER_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h index fc6438ace..376665b76 100644 --- a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h +++ b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h @@ -19,14 +19,23 @@ class VideoFrameBuffer; namespace owt { namespace base { +/// Post processor that will be applied on the video frame. This is the abstract +/// base class of the post processors provided by OWT plugins. class VideoFramePostProcessor { public: virtual ~VideoFramePostProcessor() {} + /** + @brief Set the parameter of the processor. The content of key and value + should follow the definition in the sub-class. + */ virtual bool SetParameter(const std::string& key, const std::string& value) { return false; } + /** + @brief Process the VideoFrameBuffer. Implemented by OWT plugins. + */ virtual rtc::scoped_refptr Process( const rtc::scoped_refptr& buffer) = 0; }; diff --git a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h index b940b6c98..5f5958414 100644 --- a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h +++ b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h @@ -13,17 +13,29 @@ namespace owt { namespace ic { -enum class ICPlugin { BACKGROUND_BLUR }; +/// The type of IC post processor +enum class ICPostProcessor { BACKGROUND_BLUR }; +/// The IC plugin manager, for creating the post processor instance. class ICManagerInterface { public: virtual ~ICManagerInterface() = default; - virtual bool InitializeInferenceEngineCore( + /** + @brief Register inference engine plugins. + @param plugins_xml_path The path to plugins.xml of OpenVINO Inference + Engine. + */ + virtual bool RegisterInferenceEnginePlugins( const std::string& plugins_xml_path) = 0; + /** + @brief Create and get the IC post processor. + @param processor The type of processor to be created. + @return The shared pointer of the post processor. + */ virtual std::shared_ptr - CreatePostProcessor(ICPlugin plugin) = 0; + CreatePostProcessor(ICPostProcessor processor) = 0; }; } // namespace ic diff --git a/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h b/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h deleted file mode 100644 index 5a957aa66..000000000 --- a/talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_IC_INTELLIGENTCOLLABORATIONPARAMETERS_H_ -#define OWT_IC_INTELLIGENTCOLLABORATIONPARAMETERS_H_ - -#include -#include - -#include "owt/base/videoframepostprocessor.h" - -namespace owt { -namespace base { - -class IntelligentCollaborationParameters final { - public: - IntelligentCollaborationParameters() = default; - ~IntelligentCollaborationParameters() = default; - - std::vector>& - PostProcessors(); - const std::vector>& - PostProcessors() const; - - private: - std::vector> - post_processors_; -}; - -} // namespace ic -} // namespace owt - -#endif // OWT_IC_INTELLIGENTCOLLABORATIONPARAMETERS_H_ diff --git a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc index a426a114b..50bacd190 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc +++ b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc @@ -70,13 +70,13 @@ int main(int, char*[]) { // Initialize global inference engine core, to prevent slow first time // initialization of background blur post processor - if (!ic_plugin->InitializeInferenceEngineCore("plugins.xml")) { - std::cerr << "Unable to initialize inference engine core" << std::endl; + if (!ic_plugin->RegisterInferenceEnginePlugins("plugins.xml")) { + std::cerr << "Unable to register inference engine plugins" << std::endl; return 2; } std::shared_ptr background_blur = - ic_plugin->CreatePostProcessor(owt::ic::ICPlugin::BACKGROUND_BLUR); + ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR); if (!background_blur) { std::cerr << "Create background blur failed." << std::endl; return 3; @@ -90,7 +90,7 @@ int main(int, char*[]) { std::cerr << "Failed to set blur radius." << std::endl; return 5; } - param.ICParams().PostProcessors().push_back(background_blur); + param.PostProcessors().push_back(background_blur); int error = 0; std::shared_ptr stream = From 4db61163e481bbb3d39ecb5c129854573e958fb0 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Mon, 8 Nov 2021 13:43:45 +0800 Subject: [PATCH 19/30] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 4778e30e4..84cae5b54 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,9 @@ The following dependencies are for Windows only: - [Visual Studio](https://chromium.googlesource.com/chromium/src/+/refs/heads/main/docs/windows_build_instructions.md#visual-studio). - [Intel Media SDK for Windows, version 2020 R1 or higher](https://software.intel.com/en-us/media-sdk/choose-download/client). +If you want intelligent collaboration video post-processor features, you need: +- [OpenVINO Toolkit](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit-download.html). + ### Get the code - Make sure you clone the source code to a directory named `src`. - Create a file named .gclient in the directory above the `src` dir, with these contents: @@ -54,6 +57,7 @@ target_os = [] - The optional `msdk_root` should be set to the directory of your Intel MediaSDK for Windows, version 2020 R1 or higher. This is typically `C:\Program Files (x86)\IntelSWTools\Intel(R) Media SDK 2020 R1\Software Development Kit`. If specified, will enable hardware accelerated video codecs for most of the video codecs. - The optional `--sdk` is to inform the build script to use `lib.exe` that is part of Visual Studio toolchain for merging owt libraries with external openssl libraries. + - To enable intelligent collaboration video post-processors (background blur, etc.), set the `openvino_root` option, which would typically be `--openvino_root "C:\Program Files (x86)\Intel\openvino_2021"`. #### Linux From 2f919563acf337efe45666f3c9a28341ebbcd219 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Thu, 11 Nov 2021 10:54:02 +0800 Subject: [PATCH 20/30] Fixups (bugs) --- .../features/plugins/ic/ic-developer-guide.md | 31 +++ .../docs/features/plugins/ic/ic-user-guide.md | 75 ++++++ .../plugins/plugin-developer-guide.md | 0 talk/owt/sdk/base/sharedobjectpointer.h | 122 ++++----- talk/owt/sdk/base/win/sharedobjectloader.cc | 20 +- talk/owt/sdk/ic/BUILD.gn | 240 +++++++++--------- talk/owt/sdk/ic/backgroundblur.cc | 37 ++- talk/owt/sdk/ic/backgroundblur.h | 5 +- talk/owt/sdk/ic/icmanager.cc | 44 ++-- talk/owt/sdk/ic/icmanager.h | 2 +- talk/owt/sdk/ic/selfiesegmentationmodel.cc | 66 ++--- talk/owt/sdk/ic/selfiesegmentationmodel.h | 6 +- .../cpp/owt/base/videoframepostprocessor.h | 11 +- talk/owt/sdk/sample/BUILD.gn | 26 +- talk/owt/sdk/sample/win/BUILD.gn | 20 +- .../win/sample_background_blur/BUILD.gn | 26 +- .../sample_background_blur.cc | 17 +- 17 files changed, 445 insertions(+), 303 deletions(-) create mode 100644 talk/owt/docs/features/plugins/ic/ic-developer-guide.md create mode 100644 talk/owt/docs/features/plugins/ic/ic-user-guide.md create mode 100644 talk/owt/docs/features/plugins/plugin-developer-guide.md diff --git a/talk/owt/docs/features/plugins/ic/ic-developer-guide.md b/talk/owt/docs/features/plugins/ic/ic-developer-guide.md new file mode 100644 index 000000000..c63f9f7ec --- /dev/null +++ b/talk/owt/docs/features/plugins/ic/ic-developer-guide.md @@ -0,0 +1,31 @@ +# IC Developer Guide + +This guide is to introduce how to add a post processor into IC plugin. + +The IC plugin is to support Intelligent Collaboration features in the OWT, which +are background blur, face detection, face framing, eye contact correction and +etc. video frame post processing algorithms using deep learning neural networks. +In this plugin, OpenVINO inference engine is used to do the neural network +inference. + +The IC plugin is compiled to an individual shared library, whose entry is defined +in `talk/owt/sdk/ic/icmanager.h`. The ICManager will be initialized as a global +singleton instance and will be used to create the post processor instances. + +To introduce a new post processor, you need to create a class in the +`talk/owt/sdk/ic` directory, inheriting the `owt::ic::VideoFramePostProcessor` +interface and implement its virtual methods. The post processor's main logic +should be in the `Process` function, which returns the processed frame buffer. +You may use inference engine C++ API for neural network inferencing, but be sure +to catch and handle all exceptions. Exceptions are not expected in the main OWT. + +To make it possible for creating your post processor by `CreatePostProcessor` +method., change the code in `talk/owt/sdk/ic/icmanager.cc`. You may get the +inference engine core instance from the `ICManager`. You also need to add new +enumeration item of `owt::ic::ICPostProcessor` into +`talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h`. + +Finally, update the `ic-user-guide.md` to introduce the parameters' usage to the +user. Also edit the `.gn` files to make the new codes compiled. + +See the background blur's code for more detail. diff --git a/talk/owt/docs/features/plugins/ic/ic-user-guide.md b/talk/owt/docs/features/plugins/ic/ic-user-guide.md new file mode 100644 index 000000000..3efdbc623 --- /dev/null +++ b/talk/owt/docs/features/plugins/ic/ic-user-guide.md @@ -0,0 +1,75 @@ +# IC User Guide + +The IC plugin is to support Intelligent Collaboration features in the OWT, which +are background blur, face detection, face framing, eye contact correction and +etc. video frame post processing algorithms using deep learning neural networks. + +To use the IC plugin, first initialize the global plugin instance in plugin +manager. +```cpp +owt::ic::ICManagerInterface* ic_plugin = owt::base::PluginManager::ICPlugin(); +``` +If the `owt::ic::ICManagerInterface* ic_plugin` is not `nullptr`, the ic plugin +is succesfully initialized. Otherwise, check whether `owt_ic.dll` is in +your path. The pointer is managed by `PluginManager` so you do not delete it. + +You may get a `std::shared_ptr` by +using `ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor)`. Then configure +the post-processor, and finally add it into the +`LocalCameraStreamParameters.PostProcessors()`, which is used for creating +`LocalStream`. + +The post-processors will be applied to the produced frame in the order you add +them. + +```cpp +std::shared_ptr post_processor = + ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR); +owt::base::LocalCameraStreamParameters param(true, true); +param.PostProcessors().push_back(post_processor); +``` + +See below sections for each post processor's usage. + +## Background Blur +The background blur post processor uses a selfie segmentation neural network +model to detect the area of human in frames. The background part will be applied +with a Gaussian filter to make it blurred. You can customize its blur radius to +change the blurring degree. +Use `ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR)` +to get a background blur post processor instance. + +### Model +This post-processor use a neural network model, which can be found at +[owt-selfie-segmentation-144x256.xml](https://github.com/open-webrtc-toolkit/owt-model-zoo/raw/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.xml) and +[owt-selfie-segmentation-144x256.bin](https://github.com/open-webrtc-toolkit/owt-model-zoo/raw/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.bin). Download them and save to your project, then call `SetParameter` to load +the model. You only need to specify the path to `.xml` file, and the `.bin` file +should be in the same directory to be automatically loaded by the framework. + +Note that the loading process will taken place in the function, so this may be +time-consuming depends on the model's size. Make sure you call this in a proper +time. + +The function will return false if there is any error, and the error +message will be printed to the log output. + +```cpp +background_blur->LoadModel("/path/to/model.xml", "CPU"); +``` + +### Parameters +**blur_radius**: A positive integer representing the blur radius used in +Gaussian blur processing to the background. The larger the blur radius is, the +more blurred will the frame be, and vise versa. Zero or negative blur_radius +will making the Gaussian blur taking no effect. Default: 55. + +### Sample +```cpp +owt::ic::ICManagerInterface* ic_plugin = owt::base::PluginManager::ICPlugin(); +std::shared_ptr background_blur = + ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR); +background_blur->SetParameter("blur_radius", 55); +``` + +There is a sample program about using background blur, which can be found at +`talk/owt/sdk/sample/win/sample_background_blur`. diff --git a/talk/owt/docs/features/plugins/plugin-developer-guide.md b/talk/owt/docs/features/plugins/plugin-developer-guide.md new file mode 100644 index 000000000..e69de29bb diff --git a/talk/owt/sdk/base/sharedobjectpointer.h b/talk/owt/sdk/base/sharedobjectpointer.h index c6f0d0b5c..75b4ae236 100644 --- a/talk/owt/sdk/base/sharedobjectpointer.h +++ b/talk/owt/sdk/base/sharedobjectpointer.h @@ -1,61 +1,61 @@ -// Copyright (C) <2021> Intel Corporation -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef OWT_BASE_SHAREDOBJECTPOINTER_H_ -#define OWT_BASE_SHAREDOBJECTPOINTER_H_ - -#include "talk/owt/sdk/base/sharedobjectloader.h" - -#include -#include - -#include "third_party/webrtc/rtc_base/logging.h" - -namespace owt { -namespace base { - -// This class is derived from openvino inference engine -template -struct SOTrait {}; - -// This class is derived from openvino inference engine -template -class SharedObjectPointer { - public: - explicit SharedObjectPointer(const char* path) : so_(path) { Load(); } - - bool IsLoaded() const { return so_.IsLoaded(); } - - T* Get() { return ptr_.get(); } - - protected: - void Load() { - if (!so_.IsLoaded()) { - return; - } - - using Creator = T*(); - using Destroyer = void(T*); - Creator* creator = reinterpret_cast( - so_.GetSymbol(std::string("Create") + SOTrait::name)); - Destroyer* destroyer = reinterpret_cast( - so_.GetSymbol(std::string("Destroy") + SOTrait::name)); - - if (creator && destroyer) { - ptr_ = std::shared_ptr(creator(), destroyer); - } else { - RTC_LOG(LS_WARNING) << "Create" << SOTrait::name // - << " or Destroy" << SOTrait::name - << " is missing in the shared object."; - } - } - - SharedObjectLoader so_; - std::shared_ptr ptr_; -}; - -} // namespace base -} // namespace owt - -#endif // OWT_BASE_SHAREDOBJECTPOINTER_H_ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_SHAREDOBJECTPOINTER_H_ +#define OWT_BASE_SHAREDOBJECTPOINTER_H_ + +#include "talk/owt/sdk/base/sharedobjectloader.h" + +#include +#include + +#include "third_party/webrtc/rtc_base/logging.h" + +namespace owt { +namespace base { + +// This class is derived from openvino inference engine +template +struct SOTrait {}; + +// This class is derived from openvino inference engine +template +class SharedObjectPointer { + public: + explicit SharedObjectPointer(const char* path) : so_(path) { Load(); } + + bool IsLoaded() const { return so_.IsLoaded(); } + + T* Get() { return ptr_.get(); } + + protected: + void Load() { + if (!so_.IsLoaded()) { + return; + } + + using Creator = T*(); + using Destroyer = void(T*); + Creator* creator = reinterpret_cast( + so_.GetSymbol(std::string("Create") + SOTrait::name)); + Destroyer* destroyer = reinterpret_cast( + so_.GetSymbol(std::string("Destroy") + SOTrait::name)); + + if (creator && destroyer) { + ptr_ = std::shared_ptr(creator(), destroyer); + } else { + RTC_LOG(LS_WARNING) << "Create" << SOTrait::name // + << " or Destroy" << SOTrait::name + << " is missing in the shared object."; + } + } + + SharedObjectLoader so_; + std::shared_ptr ptr_; +}; + +} // namespace base +} // namespace owt + +#endif // OWT_BASE_SHAREDOBJECTPOINTER_H_ diff --git a/talk/owt/sdk/base/win/sharedobjectloader.cc b/talk/owt/sdk/base/win/sharedobjectloader.cc index d129a6efd..73588192d 100644 --- a/talk/owt/sdk/base/win/sharedobjectloader.cc +++ b/talk/owt/sdk/base/win/sharedobjectloader.cc @@ -5,16 +5,34 @@ #include "talk/owt/sdk/base/sharedobjectloader.h" #include +#include #include "third_party/webrtc/rtc_base/logging.h" namespace owt { namespace base { +namespace { + +std::string GetErrorAsString(DWORD message_id) { + LPSTR buffer = NULL; + size_t size = FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, + message_id, MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), (LPSTR)&buffer, + 0, NULL); + std::string message(buffer, size); + LocalFree(buffer); + return message; +} + +} // namespace SharedObjectLoader::SharedObjectLoader(const char* path) : shared_object_(LoadLibraryA(path), FreeLibrary) { if (!shared_object_) { - RTC_LOG(LS_WARNING) << "Shared object " << path << " is not loaded."; + DWORD message_id = GetLastError(); + RTC_LOG(LS_WARNING) << "Shared object " << path << " is not loaded. " + << "Error code " << message_id << ": " + << GetErrorAsString(message_id); } } diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index 76e6a1822..c725a82c1 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -1,120 +1,120 @@ -# Copyright (C) <2021> Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 - -import("//build_overrides/webrtc.gni") -import("//build_overrides/owt.gni") -import("//testing/test.gni") - -declare_args() { - owt_openvino_root = "" - owt_openvino_opencv_version = "" - lib_suffix = "" - dll_suffix = "" -} - -config("openvino") { - assert(owt_openvino_root != "", "owt_openvino_root not set") - defines = [ "OWT_USE_OPENVINO" ] - include_dirs = [ - owt_openvino_root + "/inference_engine/include", - ] - if (is_debug) { - lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Debug" ] - } else { - lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Release" ] - } - configs = [ - "//build/config/compiler:exceptions", - "//build/config/compiler:rtti", - ] -} - -config("openvino_opencv") { - assert(owt_openvino_root != "", "owt_openvino_root not set") - assert(owt_openvino_opencv_version != "", "owt_openvino_opencv_version not set") - defines = [ "OWT_USE_OPENCV" ] - include_dirs = [ owt_openvino_root + "/opencv/include" ] - lib_dirs = [ owt_openvino_root + "/opencv/lib" ] -} - -openvino_modules = [ - "ir_reader", - "lp_transformations", - "preproc", - "transformations", -] - -opencv_modules = [ - "core", - "highgui", - "imgcodecs", - "imgproc", - "videoio", -] - -shared_library("owt_ic") { - sources = [ - "backgroundblur.cc", - "backgroundblur.h", - "icmanager.cc", - "icmanager.h", - "selfiesegmentationmodel.cc", - "selfiesegmentationmodel.h", - ] - defines = [ "OWT_IC_IMPL" ] - configs += [ - ":openvino", - ":openvino_opencv", - ] - include_dirs = [ "//talk/owt/sdk/include/cpp" ] - libs = [ - "inference_engine" + lib_suffix, - "inference_engine_transformations" + lib_suffix, - ] - foreach(module, opencv_modules) { - libs += [ "opencv_" + module + owt_openvino_opencv_version + lib_suffix ] - } - deps = [ - "//third_party/webrtc/api/video:video_frame", - "//third_party/libyuv:libyuv", - ] - data_deps = [ - ":openvino_library", - ":openvino_opencv_library", - ] -} - -copy("openvino_library") { - if (is_debug) { - bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Debug" - sources = [ - owt_openvino_root + "/inference_engine/external/tbb/bin/tbb_debug.dll", - owt_openvino_root + "/deployment_tools/ngraph/lib/ngraphd.dll", - ] - } else { - bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Release" - sources = [ - owt_openvino_root + "/inference_engine/external/tbb/bin/tbb.dll", - owt_openvino_root + "/deployment_tools/ngraph/lib/ngraph.dll", - ] - } - sources += [ - bin_dir + "/inference_engine" + dll_suffix, - bin_dir + "/plugins.xml", - bin_dir + "/MKLDNNPlugin" + dll_suffix, - ] - foreach(module, openvino_modules) { - sources += [ bin_dir + "/inference_engine_" + module + dll_suffix ] - } - outputs = [ "$target_out_dir/{{source_file_part}}" ] -} - -copy("openvino_opencv_library") { - bin_dir = owt_openvino_root + "/opencv/bin" - sources = [] - foreach(module, opencv_modules) { - sources += [ bin_dir + "/opencv_" + module + owt_openvino_opencv_version + dll_suffix ] - } - outputs = [ "$target_out_dir/{{source_file_part}}" ] -} +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import("//build_overrides/owt.gni") +import("//build_overrides/webrtc.gni") +import("//testing/test.gni") + +declare_args() { + owt_openvino_root = "" + owt_openvino_opencv_version = "" + lib_suffix = "" + dll_suffix = "" +} + +config("openvino") { + assert(owt_openvino_root != "", "owt_openvino_root not set") + defines = [ "OWT_USE_OPENVINO" ] + include_dirs = [ owt_openvino_root + "/inference_engine/include" ] + if (is_debug) { + lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Debug" ] + } else { + lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Release" ] + } + configs = [ + "//build/config/compiler:exceptions", + "//build/config/compiler:rtti", + ] +} + +config("openvino_opencv") { + assert(owt_openvino_root != "", "owt_openvino_root not set") + assert(owt_openvino_opencv_version != "", + "owt_openvino_opencv_version not set") + defines = [ "OWT_USE_OPENCV" ] + include_dirs = [ owt_openvino_root + "/opencv/include" ] + lib_dirs = [ owt_openvino_root + "/opencv/lib" ] +} + +openvino_modules = [ + "ir_reader", + "lp_transformations", + "preproc", + "transformations", +] + +opencv_modules = [ + "core", + "highgui", + "imgcodecs", + "imgproc", + "videoio", +] + +shared_library("owt_ic") { + sources = [ + "backgroundblur.cc", + "backgroundblur.h", + "icmanager.cc", + "icmanager.h", + "selfiesegmentationmodel.cc", + "selfiesegmentationmodel.h", + ] + defines = [ "OWT_IC_IMPL" ] + configs += [ + ":openvino", + ":openvino_opencv", + ] + include_dirs = [ "//talk/owt/sdk/include/cpp" ] + libs = [ + "inference_engine" + lib_suffix, + "inference_engine_transformations" + lib_suffix, + ] + foreach(module, opencv_modules) { + libs += [ "opencv_" + module + owt_openvino_opencv_version + lib_suffix ] + } + deps = [ + "//third_party/libyuv:libyuv", + "//third_party/webrtc/api/video:video_frame", + ] + data_deps = [ + # ":openvino_library", + # ":openvino_opencv_library", + ] +} + +copy("openvino_library") { + if (is_debug) { + bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Debug" + sources = [ + owt_openvino_root + "/inference_engine/external/tbb/bin/tbb_debug.dll", + owt_openvino_root + "/deployment_tools/ngraph/lib/ngraphd.dll", + ] + } else { + bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Release" + sources = [ + owt_openvino_root + "/inference_engine/external/tbb/bin/tbb.dll", + owt_openvino_root + "/deployment_tools/ngraph/lib/ngraph.dll", + ] + } + sources += [ + bin_dir + "/inference_engine" + dll_suffix, + bin_dir + "/plugins.xml", + bin_dir + "/MKLDNNPlugin" + dll_suffix, + ] + foreach(module, openvino_modules) { + sources += [ bin_dir + "/inference_engine_" + module + dll_suffix ] + } + outputs = [ "$root_build_dir/{{source_file_part}}" ] +} + +copy("openvino_opencv_library") { + bin_dir = owt_openvino_root + "/opencv/bin" + sources = [] + foreach(module, opencv_modules) { + sources += [ bin_dir + "/opencv_" + module + owt_openvino_opencv_version + + dll_suffix ] + } + outputs = [ "$root_build_dir/{{source_file_part}}" ] +} diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index c5f52c552..ed4382ea8 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -19,21 +19,34 @@ namespace ic { BackgroundBlur::BackgroundBlur(InferenceEngine::Core& core) : model_(core) {} -bool BackgroundBlur::SetParameter(const std::string& key, - const std::string& value) { - if (key == "model_path") { - return model_.LoadModel(value); - } else if (key == "blur_radius") { - blur_radius_ = stoi(value); +bool BackgroundBlur::LoadModel(const std::string& modelXmlPath, + const std::string& device) { + try { + model_.LoadModel(modelXmlPath, device); + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << "Load model failed: " << e.what(); + return false; + } + return true; +} + +bool BackgroundBlur::SetParameter(const std::string& key, int value) { + if (key == "blur_radius") { + try { + blur_radius_ = value; + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << "Set blur_radius failed: " << e.what(); + return false; + } return true; } - return false; + return owt::base::VideoFramePostProcessor::SetParameter(key, value); } rtc::scoped_refptr BackgroundBlur::Process( const rtc::scoped_refptr& buffer) { if (!model_.IsLoaded()) { - RTC_LOG(LS_WARNING) << "Background blur model is not initialized."; + RTC_LOG(LS_WARNING) << "Background blur model is not loaded."; return buffer; } @@ -51,7 +64,13 @@ rtc::scoped_refptr BackgroundBlur::Process( cv::Mat input; frame.convertTo(input, CV_32FC3, 1. / UCHAR_MAX); - cv::Mat mask = model_.Predict(input); // mask is of 8UC1 + cv::Mat mask; + try { + mask = model_.Predict(input); // mask is of 8UC1 + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << e.what(); + return buffer; + } cv::resize(mask, mask, {buffer->width(), buffer->height()}); cv::Mat foreground_mask; diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h index e6146d36d..5d7442a28 100644 --- a/talk/owt/sdk/ic/backgroundblur.h +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -18,7 +18,10 @@ class BackgroundBlur final : public owt::base::VideoFramePostProcessor { BackgroundBlur(InferenceEngine::Core& core); ~BackgroundBlur() override = default; - bool SetParameter(const std::string& key, const std::string& value) override; + bool LoadModel(const std::string& modelXmlPath, + const std::string& device) override; + + bool SetParameter(const std::string& key, int value) override; rtc::scoped_refptr Process( const rtc::scoped_refptr& buffer) override; diff --git a/talk/owt/sdk/ic/icmanager.cc b/talk/owt/sdk/ic/icmanager.cc index 7482ff1a6..2d4a41b7e 100644 --- a/talk/owt/sdk/ic/icmanager.cc +++ b/talk/owt/sdk/ic/icmanager.cc @@ -11,37 +11,37 @@ namespace owt { namespace ic { -ICManager::ICManager() : core_() { - try { - core_.reset(new InferenceEngine::Core); - } catch (std::exception& e) { - RTC_LOG(LS_ERROR) << "Failed to initialize inference engine core: " - << e.what(); - } -} - bool ICManager::RegisterInferenceEnginePlugins( const std::string& plugins_xml_path) { - if (core_) { + if (!core_) { try { - core_->RegisterPlugins(plugins_xml_path); - return true; - } catch (std::exception& e) { - RTC_LOG(LS_ERROR) << "Failed to register inference engine plugins: " - << e.what(); + core_.reset(new InferenceEngine::Core(plugins_xml_path)); + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << "Cannot initialize inference engine core: " << e.what(); + return false; } } - return false; + try { + core_->RegisterPlugins(plugins_xml_path); + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << "Failed to register inference engine plugins: " + << e.what(); + return false; + } + return true; } std::shared_ptr ICManager::CreatePostProcessor(ICPostProcessor processor) { - if (core_) { - switch (processor) { - case ICPostProcessor::BACKGROUND_BLUR: - return std::make_shared(*core_); - default:; - } + if (!core_ && !RegisterInferenceEnginePlugins({})) { + return nullptr; + } + switch (processor) { + case ICPostProcessor::BACKGROUND_BLUR: + return std::make_shared(*core_); + default: + RTC_LOG(LS_WARNING) << "CreatePostProcessor(" << processor + << ") is not implemented."; } return nullptr; } diff --git a/talk/owt/sdk/ic/icmanager.h b/talk/owt/sdk/ic/icmanager.h index b3cf3d64f..b6ff879cb 100644 --- a/talk/owt/sdk/ic/icmanager.h +++ b/talk/owt/sdk/ic/icmanager.h @@ -18,7 +18,7 @@ namespace ic { class ICManager final : public ICManagerInterface { public: - ICManager(); + ICManager() = default; bool RegisterInferenceEnginePlugins( const std::string& plugins_xml_path) override; diff --git a/talk/owt/sdk/ic/selfiesegmentationmodel.cc b/talk/owt/sdk/ic/selfiesegmentationmodel.cc index df4077293..1f8da6ee3 100644 --- a/talk/owt/sdk/ic/selfiesegmentationmodel.cc +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.cc @@ -14,34 +14,28 @@ namespace IE = InferenceEngine; namespace owt { namespace ic { + SelfieSegmentationModel::SelfieSegmentationModel(InferenceEngine::Core& core) : core_(core) {} -bool SelfieSegmentationModel::LoadModel(const std::string& xmlPath, +void SelfieSegmentationModel::LoadModel(const std::string& xmlPath, const std::string& device) { - try { - InferenceEngine::CNNNetwork network = core_.ReadNetwork(xmlPath); - if (network.getInputsInfo().empty() || network.getOutputsInfo().empty()) { - RTC_LOG(LS_WARNING) << "Bad segmentation model: no input / output"; - return false; - } - input_name_ = network.getInputsInfo().begin()->first; - output_name_ = network.getOutputsInfo().begin()->first; - IE::DataPtr outputData = network.getOutputsInfo().begin()->second; - outputData->setPrecision(IE::Precision::U8); - output_size_ = outputData->getTensorDesc().getDims(); - - IE::PreProcessInfo& preprocess = - network.getInputsInfo()[input_name_]->getPreProcess(); - preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); - - InferenceEngine::ExecutableNetwork executableNetwork = - core_.LoadNetwork(network, device); - request_ = executableNetwork.CreateInferRequest(); - } catch (std::exception& e) { - RTC_LOG(LS_ERROR) << "Load model failed: " << e.what(); - return false; - } + InferenceEngine::CNNNetwork network = core_.ReadNetwork(xmlPath); + IE_ASSERT(network.getInputsInfo().empty() || + network.getOutputsInfo().empty()); + input_name_ = network.getInputsInfo().begin()->first; + output_name_ = network.getOutputsInfo().begin()->first; + IE::DataPtr outputData = network.getOutputsInfo().begin()->second; + outputData->setPrecision(IE::Precision::U8); + output_shape_ = outputData->getTensorDesc().getDims(); + + IE::PreProcessInfo& preprocess = + network.getInputsInfo()[input_name_]->getPreProcess(); + preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); + + InferenceEngine::ExecutableNetwork executableNetwork = + core_.LoadNetwork(network, device); + request_ = executableNetwork.CreateInferRequest(); } bool SelfieSegmentationModel::IsLoaded() const { @@ -49,28 +43,18 @@ bool SelfieSegmentationModel::IsLoaded() const { } cv::Mat SelfieSegmentationModel::Predict(const cv::Mat& frame) { - PredictAsync(frame); - return WaitForFinished(); -} - -void SelfieSegmentationModel::PredictAsync(const cv::Mat& frame) { CV_DbgAssert(frame.type() == CV_32FC3); - size_t height = frame.rows; - size_t width = frame.cols; + IE::SizeVector inputShape = {1, 3, static_cast(frame.rows), + static_cast(frame.cols)}; auto blob = IE::make_shared_blob( - IE::TensorDesc(IE::Precision::FP32, {1, 3, height, width}, - IE::Layout::NHWC), + IE::TensorDesc(IE::Precision::FP32, inputShape, IE::Layout::NHWC), (float*)frame.data); request_.SetBlob(input_name_, blob); - request_.StartAsync(); -} - -cv::Mat SelfieSegmentationModel::WaitForFinished() { - request_.Wait(IE::InferRequest::WaitMode::RESULT_READY); + request_.Infer(); const unsigned char* data = request_.GetBlob(output_name_)->buffer(); - int height = output_size_[1]; - int width = output_size_[2]; - return cv::Mat(height, width, CV_8U, (void*)data); + size_t output_dims = output_shape_.size(); + return cv::Mat(output_shape_[output_dims - 2], output_shape_[output_dims - 1], + CV_8U, (void*)data); } } // namespace ic diff --git a/talk/owt/sdk/ic/selfiesegmentationmodel.h b/talk/owt/sdk/ic/selfiesegmentationmodel.h index fe657d9b0..8b9b2383b 100644 --- a/talk/owt/sdk/ic/selfiesegmentationmodel.h +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.h @@ -18,19 +18,17 @@ class SelfieSegmentationModel { public: explicit SelfieSegmentationModel(InferenceEngine::Core& core); - bool LoadModel(const std::string& xmlPath, const std::string& device = "CPU"); + void LoadModel(const std::string& xmlPath, const std::string& device = "CPU"); bool IsLoaded() const; cv::Mat Predict(const cv::Mat& frame); - void PredictAsync(const cv::Mat& frame); - cv::Mat WaitForFinished(); private: InferenceEngine::Core &core_; InferenceEngine::InferRequest request_; std::string input_name_; std::string output_name_; - InferenceEngine::SizeVector output_size_; + InferenceEngine::SizeVector output_shape_; }; } // namespace ic diff --git a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h index 376665b76..33da6ccf6 100644 --- a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h +++ b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h @@ -7,6 +7,8 @@ #include +#include "third_party/webrtc/rtc_base/logging.h" + namespace rtc { template class scoped_refptr; @@ -25,11 +27,18 @@ class VideoFramePostProcessor { public: virtual ~VideoFramePostProcessor() {} + virtual bool LoadModel(const std::string& modelXmlPath, + const std::string& device) { + RTC_LOG(LS_ERROR) << "LoadModel is not implemented"; + return false; + } + /** @brief Set the parameter of the processor. The content of key and value should follow the definition in the sub-class. */ - virtual bool SetParameter(const std::string& key, const std::string& value) { + virtual bool SetParameter(const std::string& key, int value) { + RTC_LOG(LS_ERROR) << "Set parameter failed: Unknown key " << key; return false; } diff --git a/talk/owt/sdk/sample/BUILD.gn b/talk/owt/sdk/sample/BUILD.gn index 8816d474b..bbf558718 100644 --- a/talk/owt/sdk/sample/BUILD.gn +++ b/talk/owt/sdk/sample/BUILD.gn @@ -1,14 +1,14 @@ -# Copyright (C) <2021> Intel Corporation -# +# Copyright (C) <2021> Intel Corporation +# # SPDX-License-Identifier: Apache-2.0 - -import("//build_overrides/webrtc.gni") -import("//build_overrides/owt.gni") -import("//testing/test.gni") - -group("sample") { - deps = [] - if (is_win) { - deps += [ "win:sample_win" ] - } -} + +import("//build_overrides/owt.gni") +import("//build_overrides/webrtc.gni") +import("//testing/test.gni") + +group("sample") { + deps = [] + if (is_win) { + deps += [ "win:sample_win" ] + } +} diff --git a/talk/owt/sdk/sample/win/BUILD.gn b/talk/owt/sdk/sample/win/BUILD.gn index 2f045b9d6..de1e99af6 100644 --- a/talk/owt/sdk/sample/win/BUILD.gn +++ b/talk/owt/sdk/sample/win/BUILD.gn @@ -1,11 +1,11 @@ -# Copyright (C) <2021> Intel Corporation -# +# Copyright (C) <2021> Intel Corporation +# # SPDX-License-Identifier: Apache-2.0 - -import("//build_overrides/webrtc.gni") -import("//build_overrides/owt.gni") -import("//testing/test.gni") - -group("sample_win") { - deps = [ "sample_background_blur" ] -} + +import("//build_overrides/owt.gni") +import("//build_overrides/webrtc.gni") +import("//testing/test.gni") + +group("sample_win") { + deps = [ "sample_background_blur" ] +} diff --git a/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn index 87bbc98ac..2610b7ab7 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn +++ b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn @@ -1,14 +1,14 @@ -# Copyright (C) <2021> Intel Corporation -# +# Copyright (C) <2021> Intel Corporation +# # SPDX-License-Identifier: Apache-2.0 - -import("//build_overrides/webrtc.gni") -import("//build_overrides/owt.gni") -import("//testing/test.gni") - -executable("sample_background_blur") { - sources = [ "sample_background_blur.cc" ] - include_dirs = [ "//talk/owt/sdk/include/cpp" ] - deps = [ "//talk/owt:owt_sdk_base" ] - data_deps = [ "//talk/owt/sdk/ic:owt_ic" ] -} + +import("//build_overrides/owt.gni") +import("//build_overrides/webrtc.gni") +import("//testing/test.gni") + +executable("sample_background_blur") { + sources = [ "sample_background_blur.cc" ] + include_dirs = [ "//talk/owt/sdk/include/cpp" ] + deps = [ "//talk/owt:owt_sdk_base" ] + data_deps = [ "//talk/owt/sdk/ic:owt_ic" ] +} diff --git a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc index 50bacd190..4f96dd599 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc +++ b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc @@ -49,7 +49,13 @@ class MainWindow : public owt::base::VideoRenderWindow { } }; -int main(int, char*[]) { +int main(int argc, char* argv[]) { + if (argc != 2) { + std::cout << "Usage: sample_background_blur " << std::endl; + return 0; + } + std::string model_path = argv[1]; + int id = 0; std::string device_name = owt::base::DeviceUtils::GetDeviceNameByIndex(id); std::string camera_id = owt::base::DeviceUtils::VideoCapturerIds()[id]; @@ -62,7 +68,7 @@ int main(int, char*[]) { param.Fps(capability.frameRate); // Load the owt_ic.dll and initialize ICManager - auto ic_plugin = owt::base::PluginManager::ICPlugin(); + owt::ic::ICManagerInterface* ic_plugin = owt::base::PluginManager::ICPlugin(); if (!ic_plugin) { std::cerr << "Unable to initialize IC plugin." << std::endl; return 1; @@ -81,12 +87,11 @@ int main(int, char*[]) { std::cerr << "Create background blur failed." << std::endl; return 3; } - if (!background_blur->SetParameter("model_path", - "data/ic_model/background_blur.xml")) { - std::cerr << "Failed to load model." << std::endl; + if (!background_blur->LoadModel(model_path, "CPU")) { + std::cerr << "Failed to load the model." << std::endl; return 4; } - if (!background_blur->SetParameter("blur_radius", "55")) { + if (!background_blur->SetParameter("blur_radius", 55)) { std::cerr << "Failed to set blur radius." << std::endl; return 5; } From e8c02e5c1a0585aefea36d1f0e5603568e25321e Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Wed, 17 Nov 2021 15:40:18 +0800 Subject: [PATCH 21/30] Update docs --- .../features/plugins/ic/ic-developer-guide.md | 2 +- .../docs/features/plugins/ic/ic-user-guide.md | 20 ++-- .../plugins/plugin-developer-guide.md | 94 +++++++++++++++++++ talk/owt/sdk/base/win/sharedobjectloader.cc | 2 +- talk/owt/sdk/ic/BUILD.gn | 4 +- talk/owt/sdk/ic/backgroundblur.cc | 22 +++-- talk/owt/sdk/ic/backgroundblur.h | 5 +- talk/owt/sdk/ic/selfiesegmentationmodel.cc | 36 +++---- talk/owt/sdk/ic/selfiesegmentationmodel.h | 4 +- .../sdk/include/cpp/owt/base/pluginmanager.h | 7 +- .../cpp/owt/base/videoframepostprocessor.h | 22 ++++- .../include/cpp/owt/ic/icmanagerinterface.h | 4 +- .../sample_background_blur.cc | 15 ++- 13 files changed, 179 insertions(+), 58 deletions(-) diff --git a/talk/owt/docs/features/plugins/ic/ic-developer-guide.md b/talk/owt/docs/features/plugins/ic/ic-developer-guide.md index c63f9f7ec..6236a77fa 100644 --- a/talk/owt/docs/features/plugins/ic/ic-developer-guide.md +++ b/talk/owt/docs/features/plugins/ic/ic-developer-guide.md @@ -22,7 +22,7 @@ to catch and handle all exceptions. Exceptions are not expected in the main OWT. To make it possible for creating your post processor by `CreatePostProcessor` method., change the code in `talk/owt/sdk/ic/icmanager.cc`. You may get the inference engine core instance from the `ICManager`. You also need to add new -enumeration item of `owt::ic::ICPostProcessor` into +enumeration item of `owt::ic::ICPostProcessor` in `talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h`. Finally, update the `ic-user-guide.md` to introduce the parameters' usage to the diff --git a/talk/owt/docs/features/plugins/ic/ic-user-guide.md b/talk/owt/docs/features/plugins/ic/ic-user-guide.md index 3efdbc623..c3e407041 100644 --- a/talk/owt/docs/features/plugins/ic/ic-user-guide.md +++ b/talk/owt/docs/features/plugins/ic/ic-user-guide.md @@ -25,6 +25,7 @@ them. ```cpp std::shared_ptr post_processor = ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR); +// do some preparation, see below owt::base::LocalCameraStreamParameters param(true, true); param.PostProcessors().push_back(post_processor); ``` @@ -41,10 +42,13 @@ to get a background blur post processor instance. ### Model This post-processor use a neural network model, which can be found at -[owt-selfie-segmentation-144x256.xml](https://github.com/open-webrtc-toolkit/owt-model-zoo/raw/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.xml) and -[owt-selfie-segmentation-144x256.bin](https://github.com/open-webrtc-toolkit/owt-model-zoo/raw/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.bin). Download them and save to your project, then call `SetParameter` to load -the model. You only need to specify the path to `.xml` file, and the `.bin` file -should be in the same directory to be automatically loaded by the framework. +[owt-selfie-segmentation-144x256.xml](https://github.com/open-webrtc-toolkit/owt-model-zoo/raw/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.xml) +and +[owt-selfie-segmentation-144x256.bin](https://github.com/open-webrtc-toolkit/owt-model-zoo/raw/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.bin). +Download and save them to your project, then call `ReadModel` and `LoadModel` to +prepare the model. When reading model, you only need to specify the path to +`.xml` file, and the `.bin` file should be in the same directory, which will be +automatically loaded by the framework. Note that the loading process will taken place in the function, so this may be time-consuming depends on the model's size. Make sure you call this in a proper @@ -54,20 +58,18 @@ The function will return false if there is any error, and the error message will be printed to the log output. ```cpp -background_blur->LoadModel("/path/to/model.xml", "CPU"); +background_blur->ReadModel("/path/to/model.xml"); +background_blur->LoadModel("CPU"); ``` ### Parameters **blur_radius**: A positive integer representing the blur radius used in Gaussian blur processing to the background. The larger the blur radius is, the more blurred will the frame be, and vise versa. Zero or negative blur_radius -will making the Gaussian blur taking no effect. Default: 55. +will lead to the Gaussian blur taking no effect. Default: 55. ### Sample ```cpp -owt::ic::ICManagerInterface* ic_plugin = owt::base::PluginManager::ICPlugin(); -std::shared_ptr background_blur = - ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR); background_blur->SetParameter("blur_radius", 55); ``` diff --git a/talk/owt/docs/features/plugins/plugin-developer-guide.md b/talk/owt/docs/features/plugins/plugin-developer-guide.md index e69de29bb..f0cf424d3 100644 --- a/talk/owt/docs/features/plugins/plugin-developer-guide.md +++ b/talk/owt/docs/features/plugins/plugin-developer-guide.md @@ -0,0 +1,94 @@ +# Plugin Developer Guide + +There is an `owt::base::PluginManager` managing all DLL plugins. For new dynamic +linked library, you may use `PluginManager` to manage its loading and unloading +process, and get accessed from the `PluginManager` singleton instance. + +## OWT side +To avoid resource release issues, the DLL is expected to be loaded before the +first time it is used, and to be unloaded when the program exits. The lifecycle +of the DLL will be managed by `owt::base::SharedObjectPointer`. + +In `pluginmanager.h`, add one method to access the singleton instance of your +plugin +```cpp +static owt::your_namespace::YourPluginInterface* YourPlugin(); +``` +In `pluginmanager.cc`, implement a template specialization of `SOTrait` to +assign the suffix of the name of C function exported in the DLL. Then implement +a singleton getter. +```cpp +template <> +struct SOTrait { + static constexpr auto name = "YourPlugin"; +}; + +owt::your_namespace::YourPluginInterface* PluginManager::YourPlugin() { + static owt::base::SharedObjectPointer + your_plugin("dll_name" DLL_SUFFIX); + return your_plugin.Get(); +} +``` + +## DLL side +There should be a `YourPlugin` entrance class inheriting the +`YourPluginInterface`. As is designed in the `PluginManager` the `YourPlugin` +will have only one instance. Do the necessary resource acquisition in the +constructor and release them in the destructor. Using static members or +variables is not recommended. + +Create your class instance as a shared pointer, making it easy to be destructed +using the destructor defined in the DLL. + +Finally, create a creator and destroyer of your plugin entrance class, naming +them the name you set in the `SOTrait` template specialization with a `Create` +or `Destroy` prefix. + +In `yourplugin.h`: +```cpp +#include + +namespace owt { +namespace your_namespace { + +class YourPlugin : public YourPluginInterface { +public: + YourPlugin() = default; + ~YourPlugin() override = default; + + std::shared_ptr CreateYourClass() override; +}; + +} // namespace your_namespace +} // namespace owt + +YourPluginInterface* CreateYourPlugin(); + +void DestroyYourPlugin(YourPluginInterface* ptr); +``` + +In `yourplugin.cc`: +```cpp +#include "path/to/yourplugin.h" + +namespace owt { +namespace your_namespace { + +std::shared_ptr YourPlugin::CreateYourClass() override { + return std::make_shared(); +} + +} // namespace your_namespace +} // namespace owt + +YourPluginInterface* CreateYourPlugin() { + return new YourPlugin; +} + +void DestroyYourPlugin(YourPluginInterface* ptr) { + delete ptr; +} +``` + +Add your files into a `shared_library` target named `dll_name` (the name you +used in the `pluginmanager.cc`) in the `BUILD.gn` file. diff --git a/talk/owt/sdk/base/win/sharedobjectloader.cc b/talk/owt/sdk/base/win/sharedobjectloader.cc index 73588192d..7143d11a2 100644 --- a/talk/owt/sdk/base/win/sharedobjectloader.cc +++ b/talk/owt/sdk/base/win/sharedobjectloader.cc @@ -30,7 +30,7 @@ SharedObjectLoader::SharedObjectLoader(const char* path) : shared_object_(LoadLibraryA(path), FreeLibrary) { if (!shared_object_) { DWORD message_id = GetLastError(); - RTC_LOG(LS_WARNING) << "Shared object " << path << " is not loaded. " + RTC_LOG(LS_WARNING) << "Load shared library " << path << " failed. " << "Error code " << message_id << ": " << GetErrorAsString(message_id); } diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index c725a82c1..e20d63108 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -79,8 +79,8 @@ shared_library("owt_ic") { "//third_party/webrtc/api/video:video_frame", ] data_deps = [ - # ":openvino_library", - # ":openvino_opencv_library", + ":openvino_library", + ":openvino_opencv_library", ] } diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc index ed4382ea8..1e995a0dd 100644 --- a/talk/owt/sdk/ic/backgroundblur.cc +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -19,10 +19,19 @@ namespace ic { BackgroundBlur::BackgroundBlur(InferenceEngine::Core& core) : model_(core) {} -bool BackgroundBlur::LoadModel(const std::string& modelXmlPath, - const std::string& device) { +bool BackgroundBlur::ReadModel(const std::string& modelXmlPath) { try { - model_.LoadModel(modelXmlPath, device); + model_.ReadModel(modelXmlPath); + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << "Read model failed: " << e.what(); + return false; + } + return true; +} + +bool BackgroundBlur::LoadModel(const std::string& device) { + try { + model_.LoadModel(device); } catch (const std::exception& e) { RTC_LOG(LS_ERROR) << "Load model failed: " << e.what(); return false; @@ -32,12 +41,7 @@ bool BackgroundBlur::LoadModel(const std::string& modelXmlPath, bool BackgroundBlur::SetParameter(const std::string& key, int value) { if (key == "blur_radius") { - try { - blur_radius_ = value; - } catch (const std::exception& e) { - RTC_LOG(LS_ERROR) << "Set blur_radius failed: " << e.what(); - return false; - } + blur_radius_ = value; return true; } return owt::base::VideoFramePostProcessor::SetParameter(key, value); diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h index 5d7442a28..7d0e688e5 100644 --- a/talk/owt/sdk/ic/backgroundblur.h +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -18,9 +18,8 @@ class BackgroundBlur final : public owt::base::VideoFramePostProcessor { BackgroundBlur(InferenceEngine::Core& core); ~BackgroundBlur() override = default; - bool LoadModel(const std::string& modelXmlPath, - const std::string& device) override; - + bool ReadModel(const std::string& modelXmlPath) override; + bool LoadModel(const std::string& device) override; bool SetParameter(const std::string& key, int value) override; rtc::scoped_refptr Process( diff --git a/talk/owt/sdk/ic/selfiesegmentationmodel.cc b/talk/owt/sdk/ic/selfiesegmentationmodel.cc index 1f8da6ee3..37cbca42f 100644 --- a/talk/owt/sdk/ic/selfiesegmentationmodel.cc +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.cc @@ -4,37 +4,31 @@ #include "selfiesegmentationmodel.h" -#include -#include - -#include "talk/owt/sdk/ic/icmanager.h" -#include "third_party/webrtc/rtc_base/logging.h" - namespace IE = InferenceEngine; namespace owt { namespace ic { -SelfieSegmentationModel::SelfieSegmentationModel(InferenceEngine::Core& core) +SelfieSegmentationModel::SelfieSegmentationModel(IE::Core& core) : core_(core) {} -void SelfieSegmentationModel::LoadModel(const std::string& xmlPath, - const std::string& device) { - InferenceEngine::CNNNetwork network = core_.ReadNetwork(xmlPath); - IE_ASSERT(network.getInputsInfo().empty() || - network.getOutputsInfo().empty()); - input_name_ = network.getInputsInfo().begin()->first; - output_name_ = network.getOutputsInfo().begin()->first; - IE::DataPtr outputData = network.getOutputsInfo().begin()->second; +void SelfieSegmentationModel::ReadModel(const std::string& modelXmlPath) { + network_ = core_.ReadNetwork(modelXmlPath); + auto input_info = network_.getInputsInfo(); + auto output_info = network_.getOutputsInfo(); + IE_ASSERT(input_info.size() == 1); + IE_ASSERT(output_info.size() == 1); + input_name_ = input_info.begin()->first; + output_name_ = output_info.begin()->first; + IE::DataPtr outputData = output_info.begin()->second; outputData->setPrecision(IE::Precision::U8); output_shape_ = outputData->getTensorDesc().getDims(); - - IE::PreProcessInfo& preprocess = - network.getInputsInfo()[input_name_]->getPreProcess(); + IE::PreProcessInfo& preprocess = input_info[input_name_]->getPreProcess(); preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); +} - InferenceEngine::ExecutableNetwork executableNetwork = - core_.LoadNetwork(network, device); +void SelfieSegmentationModel::LoadModel(const std::string& device) { + IE::ExecutableNetwork executableNetwork = core_.LoadNetwork(network_, device); request_ = executableNetwork.CreateInferRequest(); } @@ -43,7 +37,7 @@ bool SelfieSegmentationModel::IsLoaded() const { } cv::Mat SelfieSegmentationModel::Predict(const cv::Mat& frame) { - CV_DbgAssert(frame.type() == CV_32FC3); + IE_ASSERT(frame.type() == CV_32FC3); IE::SizeVector inputShape = {1, 3, static_cast(frame.rows), static_cast(frame.cols)}; auto blob = IE::make_shared_blob( diff --git a/talk/owt/sdk/ic/selfiesegmentationmodel.h b/talk/owt/sdk/ic/selfiesegmentationmodel.h index 8b9b2383b..3a0b9272c 100644 --- a/talk/owt/sdk/ic/selfiesegmentationmodel.h +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.h @@ -18,13 +18,15 @@ class SelfieSegmentationModel { public: explicit SelfieSegmentationModel(InferenceEngine::Core& core); - void LoadModel(const std::string& xmlPath, const std::string& device = "CPU"); + void ReadModel(const std::string& modelXmlPath); + void LoadModel(const std::string& device); bool IsLoaded() const; cv::Mat Predict(const cv::Mat& frame); private: InferenceEngine::Core &core_; + InferenceEngine::CNNNetwork network_; InferenceEngine::InferRequest request_; std::string input_name_; std::string output_name_; diff --git a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h index 012254e34..7761ca818 100644 --- a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h +++ b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h @@ -10,13 +10,14 @@ namespace owt { namespace base { -/// Static class of managing dll plugins. +/// Static class managing dll plugins. class PluginManager { public: #if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) /** - @brief Get the pointer of IC plugin. If loading owt_ic.dll is failed, a - nullptr will be returned. + @brief Get the pointer of IC plugin. + @return The pointer to the instance of owt::ic::ICManagerInterface. If + loading owt_ic.dll failed, a nullptr will be returned. */ static owt::ic::ICManagerInterface* ICPlugin(); #endif diff --git a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h index 33da6ccf6..aed5971db 100644 --- a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h +++ b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h @@ -27,15 +27,31 @@ class VideoFramePostProcessor { public: virtual ~VideoFramePostProcessor() {} - virtual bool LoadModel(const std::string& modelXmlPath, - const std::string& device) { + /** + @brief Read the model from the specific file path. + @param modelXmlPath The path to IR model description .xml file. + @return Whether the process succeeds. + */ + virtual bool ReadModel(const std::string& modelXmlPath) { + RTC_LOG(LS_ERROR) << "ReadModel is not implemented"; + return false; + } + + /** + @brief Read the model to specific device. + @param device The device name. Must be "CPU" for now. + @return Whether the process succeeds. + */ + virtual bool LoadModel(const std::string& device) { RTC_LOG(LS_ERROR) << "LoadModel is not implemented"; return false; } /** @brief Set the parameter of the processor. The content of key and value - should follow the definition in the sub-class. + should follow the definition of the sub-class, see documents for more + detail. + @return Whether the process succeeds. */ virtual bool SetParameter(const std::string& key, int value) { RTC_LOG(LS_ERROR) << "Set parameter failed: Unknown key " << key; diff --git a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h index 5f5958414..501771c85 100644 --- a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h +++ b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h @@ -16,7 +16,7 @@ namespace ic { /// The type of IC post processor enum class ICPostProcessor { BACKGROUND_BLUR }; -/// The IC plugin manager, for creating the post processor instance. +/// The IC plugin manager, which creates the post processor instance. class ICManagerInterface { public: virtual ~ICManagerInterface() = default; @@ -25,6 +25,8 @@ class ICManagerInterface { @brief Register inference engine plugins. @param plugins_xml_path The path to plugins.xml of OpenVINO Inference Engine. + @return Whether the process succeeds. In case of failure, the error message + will be printed to RTC log. */ virtual bool RegisterInferenceEnginePlugins( const std::string& plugins_xml_path) = 0; diff --git a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc index 4f96dd599..f6721b95a 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc +++ b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc @@ -6,6 +6,7 @@ #include #include "owt/base/deviceutils.h" +#include "owt/base/logging.h" #include "owt/base/pluginmanager.h" #include "owt/base/stream.h" @@ -56,6 +57,8 @@ int main(int argc, char* argv[]) { } std::string model_path = argv[1]; + owt::base::Logging::Severity(owt::base::LoggingSeverity::kWarning); + int id = 0; std::string device_name = owt::base::DeviceUtils::GetDeviceNameByIndex(id); std::string camera_id = owt::base::DeviceUtils::VideoCapturerIds()[id]; @@ -87,13 +90,17 @@ int main(int argc, char* argv[]) { std::cerr << "Create background blur failed." << std::endl; return 3; } - if (!background_blur->LoadModel(model_path, "CPU")) { - std::cerr << "Failed to load the model." << std::endl; + if (!background_blur->ReadModel(model_path)) { + std::cerr << "Failed to read the model." << std::endl; return 4; } + if (!background_blur->LoadModel("CPU")) { + std::cerr << "Failed to load the model." << std::endl; + return 5; + } if (!background_blur->SetParameter("blur_radius", 55)) { std::cerr << "Failed to set blur radius." << std::endl; - return 5; + return 6; } param.PostProcessors().push_back(background_blur); @@ -103,7 +110,7 @@ int main(int argc, char* argv[]) { if (error) { std::cerr << "Create local stream failed, error code " << error << "." << std::endl; - return 6; + return 7; } std::basic_string title(device_name.begin(), device_name.end()); From 7980fcfeb00b1eb50956bfa9f97f4fa0792c8557 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Fri, 19 Nov 2021 15:12:58 +0800 Subject: [PATCH 22/30] Fix debug build --- build_overrides/owt.gni | 1 + scripts/build-win.py | 13 +-- scripts/prepare_dev.py | 3 +- .../patches/0018-Enable-msvcrt-switch.patch | 33 ++++++ talk/owt/sdk/ic/BUILD.gn | 102 +++++++++++------- talk/owt/sdk/ic/icmanager.cc | 5 +- .../win/sample_background_blur/BUILD.gn | 2 +- 7 files changed, 109 insertions(+), 50 deletions(-) create mode 100644 talk/owt/patches/0018-Enable-msvcrt-switch.patch diff --git a/build_overrides/owt.gni b/build_overrides/owt.gni index c73fb8d36..d43743147 100644 --- a/build_overrides/owt.gni +++ b/build_overrides/owt.gni @@ -6,4 +6,5 @@ declare_args() { owt_msdk_lib_root = "" owt_msdk_header_root = "" owt_build_ic = false + owt_msvcrt = false } diff --git a/scripts/build-win.py b/scripts/build-win.py index 0f4cbb4f5..f6b7a6522 100644 --- a/scripts/build-win.py +++ b/scripts/build-win.py @@ -50,15 +50,9 @@ def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests): gn_args.append('use_custom_libcxx=false') if scheme == 'release': gn_args.append('is_debug=false') - if openvino_root: - gn_args.append('lib_suffix=".lib"') - gn_args.append('dll_suffix=".dll"') else: gn_args.append('is_debug=true') gn_args.append('enable_iterator_debugging=true') - if openvino_root: - gn_args.append('lib_suffix="d.lib"') - gn_args.append('dll_suffix="d.dll"') if ssl_root: gn_args.append('owt_use_openssl=true') gn_args.append('owt_openssl_header_root="%s"' % (ssl_root + r'\include')) @@ -85,12 +79,15 @@ def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests): gn_args.append('owt_use_quic=true') if openvino_root: gn_args.append('owt_build_ic=true') + gn_args.append('owt_msvcrt=true') gn_args.append('owt_openvino_root="{}"'.format(openvino_root)) - opencv_version_bin = os.path.join(openvino_root, 'opencv', 'bin', 'opencv_version.exe') + opencv_root = os.path.join(openvino_root, 'opencv') + opencv_version_bin = os.path.join(opencv_root, 'bin', 'opencv_version.exe') try: output = subprocess.check_output([opencv_version_bin]) opencv_version = ''.join(re.findall('\d', output.decode())) - gn_args.append('owt_openvino_opencv_version={}'.format(opencv_version)) + gn_args.append('owt_opencv_root={}'.format(opencv_root)) + gn_args.append('owt_opencv_version={}'.format(opencv_version)) except FileNotFoundError as e: print('File not found: {}'.format(opencv_version_bin)) return False diff --git a/scripts/prepare_dev.py b/scripts/prepare_dev.py index d6ed6a8fe..f34c9f1ce 100644 --- a/scripts/prepare_dev.py +++ b/scripts/prepare_dev.py @@ -45,7 +45,8 @@ ('0014-Fix-missing-ffmpeg-configure-item-for-msvc-build.patch', FFMPEG_PATH), ('0015-Remove-custom-d8-dependency.patch', BUILD_PATH), ('0016-Remove-deprecated-create_srcjar-property.patch', THIRD_PARTY_PATH), - ('0017-Build-libvpx-with-RTC-rate-control-impl-included.patch', THIRD_PARTY_PATH) + ('0017-Build-libvpx-with-RTC-rate-control-impl-included.patch', THIRD_PARTY_PATH), + ('0018-Enable-mscvrt-switch.patch', BUILD_PATH), ] def _patch(ignoreFailures=False): diff --git a/talk/owt/patches/0018-Enable-msvcrt-switch.patch b/talk/owt/patches/0018-Enable-msvcrt-switch.patch new file mode 100644 index 000000000..69d0c1d62 --- /dev/null +++ b/talk/owt/patches/0018-Enable-msvcrt-switch.patch @@ -0,0 +1,33 @@ +From 0105cb7e2fe398e7707d19089d063d2d87418018 Mon Sep 17 00:00:00 2001 +From: Zhibo Wang +Date: Fri, 19 Nov 2021 15:04:19 +0800 +Subject: [PATCH] Enable msvcrt switch + +--- + config/win/BUILD.gn | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/config/win/BUILD.gn b/config/win/BUILD.gn +index 51437c66c..1d02e0600 100644 +--- a/config/win/BUILD.gn ++++ b/config/win/BUILD.gn +@@ -11,6 +11,7 @@ import("//build/config/win/visual_studio_version.gni") + import("//build/timestamp.gni") + import("//build/toolchain/goma.gni") + import("//build/toolchain/toolchain.gni") ++import("//build_overrides/owt.gni") + + assert(is_win) + +@@ -473,7 +474,7 @@ config("delayloads_not_for_child_dll") { + # See https://msdn.microsoft.com/en-us/library/2kzt1wy3.aspx for a reference of + # what each value does. + config("default_crt") { +- if (is_component_build) { ++ if (is_component_build || owt_msvcrt) { + # Component mode: dynamic CRT. Since the library is shared, it requires + # exceptions or will give errors about things not matching, so keep + # exceptions on. +-- +2.33.0 + diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index e20d63108..bd9040fbd 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -8,33 +8,43 @@ import("//testing/test.gni") declare_args() { owt_openvino_root = "" - owt_openvino_opencv_version = "" - lib_suffix = "" - dll_suffix = "" + owt_openvino_source_code = false + owt_opencv_root = "" + owt_opencv_version = 0 } -config("openvino") { - assert(owt_openvino_root != "", "owt_openvino_root not set") - defines = [ "OWT_USE_OPENVINO" ] - include_dirs = [ owt_openvino_root + "/inference_engine/include" ] - if (is_debug) { - lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Debug" ] - } else { - lib_dirs = [ owt_openvino_root + "/inference_engine/lib/intel64/Release" ] - } - configs = [ - "//build/config/compiler:exceptions", - "//build/config/compiler:rtti", - ] +# Currently OpenVINO is used only in IC plugin, so check args in file scope. +assert(owt_openvino_root != "", "owt_openvino_root is not set") +assert(owt_opencv_root != "" && owt_opencv_version != 0, + "owt_opencv_root or owt_opencv_version is not set") + +if (is_debug) { + capitalized_scheme = "Debug" + lib_suffix = "d.lib" + dll_suffix = "d.dll" +} else { + capitalized_scheme = "Release" + lib_suffix = ".lib" + dll_suffix = ".dll" } -config("openvino_opencv") { - assert(owt_openvino_root != "", "owt_openvino_root not set") - assert(owt_openvino_opencv_version != "", - "owt_openvino_opencv_version not set") - defines = [ "OWT_USE_OPENCV" ] - include_dirs = [ owt_openvino_root + "/opencv/include" ] - lib_dirs = [ owt_openvino_root + "/opencv/lib" ] +if (owt_openvino_source_code) { + inference_engine_root = owt_openvino_root + "/inference-engine" + bin_dir = owt_openvino_root + "/bin/intel64/" + capitalized_scheme + lib_dir = bin_dir + ngraph_bin_dir = bin_dir + tbb_bin_dir = inference_engine_root + "/temp/tbb/bin" + openvino_include_dirs = [ + inference_engine_root + "/src/inference_engine/include/ie", + owt_openvino_root + "/ngraph/core/include", + ] +} else { + inference_engine_root = owt_openvino_root + "/inference_engine" + bin_dir = inference_engine_root + "/bin/intel64/" + capitalized_scheme + lib_dir = inference_engine_root + "/lib/intel64/" + capitalized_scheme + ngraph_bin_dir = owt_openvino_root + "/deployment_tools/ngraph/lib" + tbb_bin_dir = inference_engine_root + "/external/tbb/bin" + openvino_include_dirs = [ inference_engine_root + "/include" ] } openvino_modules = [ @@ -52,6 +62,21 @@ opencv_modules = [ "videoio", ] +config("openvino") { + defines = [ + "OWT_USE_OPENVINO", + "_ITERATOR_DEBUG_LEVEL=2", + ] + include_dirs = openvino_include_dirs + lib_dirs = [ lib_dir ] +} + +config("openvino_opencv") { + defines = [ "OWT_USE_OPENCV" ] + include_dirs = [ owt_opencv_root + "/include" ] + lib_dirs = [ owt_opencv_root + "/lib" ] +} + shared_library("owt_ic") { sources = [ "backgroundblur.cc", @@ -62,6 +87,10 @@ shared_library("owt_ic") { "selfiesegmentationmodel.h", ] defines = [ "OWT_IC_IMPL" ] + configs -= [ + "//build/config/compiler:no_exceptions", + "//build/config/compiler:no_rtti", + ] configs += [ ":openvino", ":openvino_opencv", @@ -72,7 +101,7 @@ shared_library("owt_ic") { "inference_engine_transformations" + lib_suffix, ] foreach(module, opencv_modules) { - libs += [ "opencv_" + module + owt_openvino_opencv_version + lib_suffix ] + libs += [ "opencv_" + module + owt_opencv_version + lib_suffix ] } deps = [ "//third_party/libyuv:libyuv", @@ -85,36 +114,33 @@ shared_library("owt_ic") { } copy("openvino_library") { + sources = [ ngraph_bin_dir + "/ngraph" + dll_suffix ] if (is_debug) { - bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Debug" - sources = [ - owt_openvino_root + "/inference_engine/external/tbb/bin/tbb_debug.dll", - owt_openvino_root + "/deployment_tools/ngraph/lib/ngraphd.dll", - ] + sources += [ tbb_bin_dir + "/tbb_debug.dll" ] } else { - bin_dir = owt_openvino_root + "/inference_engine/bin/intel64/Release" - sources = [ - owt_openvino_root + "/inference_engine/external/tbb/bin/tbb.dll", - owt_openvino_root + "/deployment_tools/ngraph/lib/ngraph.dll", - ] + sources += [ tbb_bin_dir + "/tbb.dll" ] } sources += [ bin_dir + "/inference_engine" + dll_suffix, - bin_dir + "/plugins.xml", bin_dir + "/MKLDNNPlugin" + dll_suffix, ] foreach(module, openvino_modules) { sources += [ bin_dir + "/inference_engine_" + module + dll_suffix ] } + if (owt_openvino_source_code && is_debug) { + foreach(dll_file, sources) { + sources += [ string_replace(dll_file, ".dll", ".pdb") ] + } + } + sources += [ bin_dir + "/plugins.xml" ] outputs = [ "$root_build_dir/{{source_file_part}}" ] } copy("openvino_opencv_library") { - bin_dir = owt_openvino_root + "/opencv/bin" sources = [] foreach(module, opencv_modules) { - sources += [ bin_dir + "/opencv_" + module + owt_openvino_opencv_version + - dll_suffix ] + sources += [ owt_opencv_root + "/bin/opencv_" + module + + owt_opencv_version + dll_suffix ] } outputs = [ "$root_build_dir/{{source_file_part}}" ] } diff --git a/talk/owt/sdk/ic/icmanager.cc b/talk/owt/sdk/ic/icmanager.cc index 2d4a41b7e..2be20cd81 100644 --- a/talk/owt/sdk/ic/icmanager.cc +++ b/talk/owt/sdk/ic/icmanager.cc @@ -17,14 +17,15 @@ bool ICManager::RegisterInferenceEnginePlugins( try { core_.reset(new InferenceEngine::Core(plugins_xml_path)); } catch (const std::exception& e) { - RTC_LOG(LS_ERROR) << "Cannot initialize inference engine core: " << e.what(); + RTC_LOG(LS_ERROR) << "Cannot initialize inference engine core: " + << e.what(); return false; } } try { core_->RegisterPlugins(plugins_xml_path); } catch (const std::exception& e) { - RTC_LOG(LS_ERROR) << "Failed to register inference engine plugins: " + RTC_LOG(LS_ERROR) << "Cannot register inference engine plugins: " << e.what(); return false; } diff --git a/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn index 2610b7ab7..52ad233b6 100644 --- a/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn +++ b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn @@ -1,6 +1,6 @@ # Copyright (C) <2021> Intel Corporation # -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 import("//build_overrides/owt.gni") import("//build_overrides/webrtc.gni") From 705860c3a411a5bb1a9a3ff317a5b472d56c4878 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Mon, 22 Nov 2021 12:29:33 +0800 Subject: [PATCH 23/30] Fixups --- scripts/prepare_dev.py | 2 +- .../cpp/owt/base/videoframepostprocessor.h | 21 +++++-------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/scripts/prepare_dev.py b/scripts/prepare_dev.py index f34c9f1ce..df791cd6e 100644 --- a/scripts/prepare_dev.py +++ b/scripts/prepare_dev.py @@ -46,7 +46,7 @@ ('0015-Remove-custom-d8-dependency.patch', BUILD_PATH), ('0016-Remove-deprecated-create_srcjar-property.patch', THIRD_PARTY_PATH), ('0017-Build-libvpx-with-RTC-rate-control-impl-included.patch', THIRD_PARTY_PATH), - ('0018-Enable-mscvrt-switch.patch', BUILD_PATH), + ('0018-Enable-msvcrt-switch.patch', BUILD_PATH), ] def _patch(ignoreFailures=False): diff --git a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h index aed5971db..291b6625d 100644 --- a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h +++ b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h @@ -7,8 +7,6 @@ #include -#include "third_party/webrtc/rtc_base/logging.h" - namespace rtc { template class scoped_refptr; @@ -25,27 +23,21 @@ namespace base { /// base class of the post processors provided by OWT plugins. class VideoFramePostProcessor { public: - virtual ~VideoFramePostProcessor() {} + virtual ~VideoFramePostProcessor() = default; /** @brief Read the model from the specific file path. @param modelXmlPath The path to IR model description .xml file. @return Whether the process succeeds. */ - virtual bool ReadModel(const std::string& modelXmlPath) { - RTC_LOG(LS_ERROR) << "ReadModel is not implemented"; - return false; - } + virtual bool ReadModel(const std::string& modelXmlPath) { return false; } /** @brief Read the model to specific device. @param device The device name. Must be "CPU" for now. @return Whether the process succeeds. */ - virtual bool LoadModel(const std::string& device) { - RTC_LOG(LS_ERROR) << "LoadModel is not implemented"; - return false; - } + virtual bool LoadModel(const std::string& device) { return false; } /** @brief Set the parameter of the processor. The content of key and value @@ -53,10 +45,7 @@ class VideoFramePostProcessor { detail. @return Whether the process succeeds. */ - virtual bool SetParameter(const std::string& key, int value) { - RTC_LOG(LS_ERROR) << "Set parameter failed: Unknown key " << key; - return false; - } + virtual bool SetParameter(const std::string& key, int value) { return false; } /** @brief Process the VideoFrameBuffer. Implemented by OWT plugins. @@ -68,4 +57,4 @@ class VideoFramePostProcessor { } // namespace base } // namespace owt -#endif // OWT_IC_VIDEOFRAMEPOSTPROCESSOR_H_ +#endif // OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ From ab80773ccee6037a28f66e4941dcca092a621a54 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Wed, 24 Nov 2021 17:10:56 +0800 Subject: [PATCH 24/30] Update windows.jenkinsfile --- jenkins/windows.jenkinsfile | 96 +++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/jenkins/windows.jenkinsfile b/jenkins/windows.jenkinsfile index eef26e373..62645b714 100644 --- a/jenkins/windows.jenkinsfile +++ b/jenkins/windows.jenkinsfile @@ -15,7 +15,7 @@ pipeline { timeout(time: 40, unit: "MINUTES") } stages { - stage("init paramters"){ + stage("InitParamters"){ steps { script { if (env.CHANGE_TARGET == null){ @@ -35,7 +35,7 @@ pipeline { } steps { echo "$GIT_COMMIT" - bat "%windowsCIPathBranch%_%CHANGE_TARGET%/buildSdk.bat %GIT_COMMIT%" + bat "%windowsCIPathBranch%_%CHANGE_TARGET%/buildSdkOpenVINO.bat %GIT_COMMIT%" } } stage("startMcu") { @@ -65,49 +65,61 @@ pipeline { } } } - stage("BulidConference") { - agent{ - node { - label "windows" - } - } - steps { - echo "Buliding.." - bat "%windowsConferenceCasePath%/build.bat" - } - } - stage("BulidTestP2P") { - agent{ - node { - label "windows" - } - } - steps { - echo "Buliding.." - bat "%windowsP2PCasePath%/build.bat" - } - } - stage("TestConference") { - agent{ - node { - label "windows" + stage("Test") { + parallel { + stage("Conference") { + stages { + stage("BulidConference") { + agent{ + node { + label "windows" + } + } + steps { + echo "Buliding.." + bat "%windowsConferenceCasePath%/build.bat" + } + } + stage("TestConference") { + agent{ + node { + label "windows" + } + } + steps { + echo "Testing.." + bat "%windowsConferenceCasePath%/runCase.bat" + } + } + } } - } - steps { - echo "Testing.." - bat "%windowsConferenceCasePath%/runCase.bat" - } - } - stage("TestP2P") { - agent{ - node { - label "windows" + stage("P2P") { + stages { + stage("BulidP2P") { + agent{ + node { + label "windows" + } + } + steps { + echo "Buliding.." + bat "%windowsP2PCasePath%/build.bat" + } + } + stage("TestP2P") { + agent{ + node { + label "windows" + } + } + steps { + echo "Testing.." + bat "%windowsP2PCasePath%/runCase.bat" + } + } + } } } - steps { - echo "Testing.." - bat "%windowsP2PCasePath%/runCase.bat" - } } } post { From 6287a533f6e90d39b24da4d0e1ae435dca6eaae0 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Thu, 25 Nov 2021 10:57:14 +0800 Subject: [PATCH 25/30] Fixups --- jenkins/windows.jenkinsfile | 8 ++++---- scripts/build-win.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/windows.jenkinsfile b/jenkins/windows.jenkinsfile index 62645b714..30652d708 100644 --- a/jenkins/windows.jenkinsfile +++ b/jenkins/windows.jenkinsfile @@ -26,7 +26,7 @@ pipeline { } stage("BuildAndEnv"){ parallel { - stage("build") { + stage("Build") { agent{ node { label "windows" @@ -38,7 +38,7 @@ pipeline { bat "%windowsCIPathBranch%_%CHANGE_TARGET%/buildSdkOpenVINO.bat %GIT_COMMIT%" } } - stage("startMcu") { + stage("StartMcu") { agent{ node { label "$env.windowsServer" @@ -51,7 +51,7 @@ pipeline { } } } - stage("startP2PServer") { + stage("StartP2PServer") { agent{ node { label "$env.windowsServer" @@ -66,7 +66,7 @@ pipeline { } } stage("Test") { - parallel { + stages { stage("Conference") { stages { stage("BulidConference") { diff --git a/scripts/build-win.py b/scripts/build-win.py index f6b7a6522..86351c054 100644 --- a/scripts/build-win.py +++ b/scripts/build-win.py @@ -86,7 +86,7 @@ def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests): try: output = subprocess.check_output([opencv_version_bin]) opencv_version = ''.join(re.findall('\d', output.decode())) - gn_args.append('owt_opencv_root={}'.format(opencv_root)) + gn_args.append('owt_opencv_root="{}"'.format(opencv_root)) gn_args.append('owt_opencv_version={}'.format(opencv_version)) except FileNotFoundError as e: print('File not found: {}'.format(opencv_version_bin)) From f23c70c26d30fea40e44edb35b6c749241f79527 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Thu, 25 Nov 2021 11:24:54 +0800 Subject: [PATCH 26/30] Fixup --- talk/owt/sdk/ic/BUILD.gn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn index bd9040fbd..b73baecd8 100644 --- a/talk/owt/sdk/ic/BUILD.gn +++ b/talk/owt/sdk/ic/BUILD.gn @@ -63,10 +63,10 @@ opencv_modules = [ ] config("openvino") { - defines = [ - "OWT_USE_OPENVINO", - "_ITERATOR_DEBUG_LEVEL=2", - ] + defines = [ "OWT_USE_OPENVINO" ] + if (is_debug) { + defines += [ "_ITERATOR_DEBUG_LEVEL=2" ] + } include_dirs = openvino_include_dirs lib_dirs = [ lib_dir ] } From 3e5dd15b0e61d083d6f238f9080b35e092856b2c Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Fri, 26 Nov 2021 13:08:45 +0800 Subject: [PATCH 27/30] Update windows.jenkinsfile --- jenkins/windows.jenkinsfile | 92 ++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 52 deletions(-) diff --git a/jenkins/windows.jenkinsfile b/jenkins/windows.jenkinsfile index 30652d708..9fc0872d6 100644 --- a/jenkins/windows.jenkinsfile +++ b/jenkins/windows.jenkinsfile @@ -65,61 +65,49 @@ pipeline { } } } - stage("Test") { - stages { - stage("Conference") { - stages { - stage("BulidConference") { - agent{ - node { - label "windows" - } - } - steps { - echo "Buliding.." - bat "%windowsConferenceCasePath%/build.bat" - } - } - stage("TestConference") { - agent{ - node { - label "windows" - } - } - steps { - echo "Testing.." - bat "%windowsConferenceCasePath%/runCase.bat" - } - } - } + stage("BulidConference") { + agent{ + node { + label "windows" } - stage("P2P") { - stages { - stage("BulidP2P") { - agent{ - node { - label "windows" - } - } - steps { - echo "Buliding.." - bat "%windowsP2PCasePath%/build.bat" - } - } - stage("TestP2P") { - agent{ - node { - label "windows" - } - } - steps { - echo "Testing.." - bat "%windowsP2PCasePath%/runCase.bat" - } - } - } + } + steps { + echo "Buliding.." + bat "%windowsConferenceCasePath%/buildMD.bat" + } + } + stage("BulidP2P") { + agent{ + node { + label "windows" } } + steps { + echo "Buliding.." + bat "%windowsP2PCasePath%/buildMD.bat" + } + } + stage("TestConference") { + agent{ + node { + label "windows" + } + } + steps { + echo "Testing.." + bat "%windowsConferenceCasePath%/runCase.bat" + } + } + stage("TestP2P") { + agent{ + node { + label "windows" + } + } + steps { + echo "Testing.." + bat "%windowsP2PCasePath%/runCase.bat" + } } } post { From 16de4f9d458784bcdd993d1fb60951d28c9650c3 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Mon, 29 Nov 2021 16:48:40 +0800 Subject: [PATCH 28/30] Update windows.jenkinsfile --- jenkins/windows.jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/windows.jenkinsfile b/jenkins/windows.jenkinsfile index 9fc0872d6..f5e42ac13 100644 --- a/jenkins/windows.jenkinsfile +++ b/jenkins/windows.jenkinsfile @@ -95,7 +95,7 @@ pipeline { } steps { echo "Testing.." - bat "%windowsConferenceCasePath%/runCase.bat" + bat "%windowsConferenceCasePath%/runCase.bat --configuration Release-MD" } } stage("TestP2P") { @@ -106,7 +106,7 @@ pipeline { } steps { echo "Testing.." - bat "%windowsP2PCasePath%/runCase.bat" + bat "%windowsP2PCasePath%/runCase.bat --configuration Release-MD" } } } From d00bb7e85bd40ee4f858f8c47a3fc9755bd33757 Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Mon, 13 Dec 2021 15:38:19 +0800 Subject: [PATCH 29/30] Update windows.jenkinsfile --- jenkins/windows.jenkinsfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jenkins/windows.jenkinsfile b/jenkins/windows.jenkinsfile index f5e42ac13..b00cdf4f5 100644 --- a/jenkins/windows.jenkinsfile +++ b/jenkins/windows.jenkinsfile @@ -15,7 +15,7 @@ pipeline { timeout(time: 40, unit: "MINUTES") } stages { - stage("InitParamters"){ + stage("init paramters"){ steps { script { if (env.CHANGE_TARGET == null){ @@ -26,7 +26,7 @@ pipeline { } stage("BuildAndEnv"){ parallel { - stage("Build") { + stage("build") { agent{ node { label "windows" @@ -38,7 +38,7 @@ pipeline { bat "%windowsCIPathBranch%_%CHANGE_TARGET%/buildSdkOpenVINO.bat %GIT_COMMIT%" } } - stage("StartMcu") { + stage("startMcu") { agent{ node { label "$env.windowsServer" @@ -51,7 +51,7 @@ pipeline { } } } - stage("StartP2PServer") { + stage("startP2PServer") { agent{ node { label "$env.windowsServer" @@ -73,7 +73,7 @@ pipeline { } steps { echo "Buliding.." - bat "%windowsConferenceCasePath%/buildMD.bat" + bat "%windowsConferenceCasePath%/build.bat --configuration Release-MD" } } stage("BulidP2P") { @@ -84,7 +84,7 @@ pipeline { } steps { echo "Buliding.." - bat "%windowsP2PCasePath%/buildMD.bat" + bat "%windowsP2PCasePath%/build.bat --configuration Release-MD" } } stage("TestConference") { From 32a61d73303157ab7fb50475e1e8530c9e7969fc Mon Sep 17 00:00:00 2001 From: Zhibo Wang Date: Thu, 16 Dec 2021 12:48:06 +0800 Subject: [PATCH 30/30] Update windows.jenkinsfile --- jenkins/windows.jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/windows.jenkinsfile b/jenkins/windows.jenkinsfile index b00cdf4f5..d103c7b40 100644 --- a/jenkins/windows.jenkinsfile +++ b/jenkins/windows.jenkinsfile @@ -35,7 +35,7 @@ pipeline { } steps { echo "$GIT_COMMIT" - bat "%windowsCIPathBranch%_%CHANGE_TARGET%/buildSdkOpenVINO.bat %GIT_COMMIT%" + bat "%windowsCIPathBranch%_%CHANGE_TARGET%/buildSdk.bat %GIT_COMMIT% --with-openvino" } } stage("startMcu") { @@ -75,7 +75,7 @@ pipeline { echo "Buliding.." bat "%windowsConferenceCasePath%/build.bat --configuration Release-MD" } - } + } stage("BulidP2P") { agent{ node {