From 1f9a7b8fd30f19e42fd29d1b8102d8ac466c0414 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Wed, 20 Oct 2021 12:43:32 +0300 Subject: [PATCH] Merge pull request #20738 from sivanov-work:merge_master_vpl_dev_select G-API: oneVPL - Implement IDeviceSelector & default cfg_param-based selector * Initial commit * Add MACRO undef * Change IDeviceSelector, Change Inf sample for choose external device * Fix compilation * Address some comments * Fix compilation * Add missing header * Add EXPORT to dev selector * Add guard * Remove enum type attr * Fix compilation without VPL * Add HAVE_INFER guard in sample * Remove unusable include from tests * Remove unusable include from sample * Remove cl_d3d11 header from unit test --- modules/gapi/CMakeLists.txt | 15 + .../onevpl/device_selector_interface.hpp | 102 ++++++ .../opencv2/gapi/streaming/onevpl/source.hpp | 24 ++ .../gapi/samples/onevpl_infer_single_roi.cpp | 138 +++++++- .../onevpl/accelerators/accel_policy_dx11.hpp | 1 + .../onevpl/cfg_param_device_selector.cpp | 314 ++++++++++++++++++ .../onevpl/cfg_param_device_selector.hpp | 44 +++ .../onevpl/device_selector_interface.cpp | 87 +++++ modules/gapi/src/streaming/onevpl/source.cpp | 63 +++- .../gapi/src/streaming/onevpl/source_priv.cpp | 6 +- .../gapi/src/streaming/onevpl/source_priv.hpp | 3 +- .../gapi_streaming_vpl_device_selector.cpp | 229 +++++++++++++ 12 files changed, 1018 insertions(+), 8 deletions(-) create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp create mode 100644 modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp create mode 100644 modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp create mode 100644 modules/gapi/src/streaming/onevpl/device_selector_interface.cpp create mode 100644 modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 917d1e0814..61ab5397d7 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -181,6 +181,9 @@ set(gapi_srcs src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp src/streaming/onevpl/engine/decode/decode_session.cpp + src/streaming/onevpl/cfg_param_device_selector.cpp + src/streaming/onevpl/device_selector_interface.cpp + # Utils (ITT tracing) src/utils/itt.cpp ) @@ -283,3 +286,15 @@ endif() ocv_add_perf_tests() ocv_add_samples() + + +# Required for sample with inference on host +if (TARGET example_gapi_onevpl_infer_single_roi) + if(OPENCV_GAPI_INF_ENGINE) + ocv_target_link_libraries(example_gapi_onevpl_infer_single_roi PRIVATE ${INF_ENGINE_TARGET}) + ocv_target_compile_definitions(example_gapi_onevpl_infer_single_roi PRIVATE -DHAVE_INF_ENGINE) + endif() + if(HAVE_D3D11 AND HAVE_OPENCL) + ocv_target_include_directories(example_gapi_onevpl_infer_single_roi SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + endif() +endif() diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp new file mode 100644 index 0000000000..ca19849d72 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp @@ -0,0 +1,102 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_DEVICE_SELECTOR_INTERFACE_HPP +#define GAPI_STREAMING_ONEVPL_DEVICE_SELECTOR_INTERFACE_HPP + +#include +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +enum class AccelType : uint8_t { + HOST, + DX11, + + LAST_VALUE = std::numeric_limits::max() +}; + +GAPI_EXPORTS const char* to_cstring(AccelType type); + +struct IDeviceSelector; +struct GAPI_EXPORTS Device { + friend struct IDeviceSelector; + using Ptr = void*; + + ~Device(); + const std::string& get_name() const; + Ptr get_ptr() const; + AccelType get_type() const; +private: + Device(Ptr device_ptr, const std::string& device_name, + AccelType device_type); + + std::string name; + Ptr ptr; + AccelType type; +}; + +struct GAPI_EXPORTS Context { + friend struct IDeviceSelector; + using Ptr = void*; + + ~Context(); + Ptr get_ptr() const; + AccelType get_type() const; +private: + Context(Ptr ctx_ptr, AccelType ctx_type); + Ptr ptr; + AccelType type; +}; + +struct GAPI_EXPORTS IDeviceSelector { + using Ptr = std::shared_ptr; + + struct GAPI_EXPORTS Score { + friend struct IDeviceSelector; + using Type = int16_t; + static constexpr Type MaxActivePriority = std::numeric_limits::max(); + static constexpr Type MinActivePriority = 0; + static constexpr Type MaxPassivePriority = MinActivePriority - 1; + static constexpr Type MinPassivePriority = std::numeric_limits::min(); + + Score(Type val); + ~Score(); + + operator Type () const; + Type get() const; + friend bool operator< (Score lhs, Score rhs) { + return lhs.get() < rhs.get(); + } + private: + Type value; + }; + + using DeviceScoreTable = std::map; + using DeviceContexts = std::vector; + + virtual ~IDeviceSelector(); + virtual DeviceScoreTable select_devices() const = 0; + virtual DeviceContexts select_context() = 0; +protected: + template + static Entity create(Args &&...args) { + return Entity(std::forward(args)...); + } +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_DEVICE_SELECTOR_INTERFACE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp index a8dbefdf50..6334480c1b 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace cv { namespace gapi { @@ -38,8 +39,31 @@ public: GSource(const std::string& filePath, const CfgParams& cfg_params = CfgParams{}); + + GSource(const std::string& filePath, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr); + + GSource(const std::string& filePath, + const CfgParams& cfg_params, + std::shared_ptr selector); + + GSource(std::shared_ptr source, const CfgParams& cfg_params = CfgParams{}); + + GSource(std::shared_ptr source, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr); + + GSource(std::shared_ptr source, + const CfgParams& cfg_params, + std::shared_ptr selector); + ~GSource() override; bool pull(cv::gapi::wip::Data& data) override; diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index f3aee09d42..deca86f1ca 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -10,8 +11,30 @@ #include #include #include +#include #include // CommandLineParser +#ifdef HAVE_INF_ENGINE +#include // ParamMap + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#include +#include +#pragma comment(lib, "dxgi") +#undef NOMINMAX +#undef D3D11_NO_HELPERS + +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_INF_ENGINE + const std::string about = "This is an OpenCV-based version of oneVPLSource decoder example"; const std::string keys = @@ -36,6 +59,37 @@ std::string get_weights_path(const std::string &model_path) { CV_Assert(ext == ".xml"); return model_path.substr(0u, sz - EXT_LEN) + ".bin"; } + +#ifdef HAVE_INF_ENGINE +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + +using AccelParamsType = std::tuple, CComPtr>; + +AccelParamsType create_device_with_ctx(CComPtr adapter) { + UINT flags = 0; + D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL featureLevel; + ID3D11Device* ret_device_ptr = nullptr; + ID3D11DeviceContext* ret_ctx_ptr = nullptr; + HRESULT err = D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN, + nullptr, flags, + feature_levels, + ARRAYSIZE(feature_levels), + D3D11_SDK_VERSION, &ret_device_ptr, + &featureLevel, &ret_ctx_ptr); + if (FAILED(err)) { + throw std::runtime_error("Cannot create D3D11CreateDevice, error: " + + std::to_string(HRESULT_CODE(err))); + } + + return std::make_tuple(ret_device_ptr, ret_ctx_ptr); +} +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_INF_ENGINE } // anonymous namespace namespace custom { @@ -197,11 +251,84 @@ int main(int argc, char *argv[]) { return -1; } + const std::string& device_id = cmd.get("faced"); auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR get_weights_path(face_model_path), // path to weights - cmd.get("faced"), // device specifier + device_id }; + + // Create device_ptr & context_ptr using graphic API + // InferenceEngine requires such device & context to create its own + // remote shared context through InferenceEngine::ParamMap in + // GAPI InferenceEngine backend to provide interoperability with onevpl::GSource + // So GAPI InferenceEngine backend and onevpl::GSource MUST share the same + // device and context + void* accel_device_ptr = nullptr; + void* accel_ctx_ptr = nullptr; + +#ifdef HAVE_INF_ENGINE +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + CComPtr dx11_dev; + CComPtr dx11_ctx; + + if (device_id.find("GPU") != std::string::npos) { + CComPtr adapter_factory; + CComPtr intel_adapters; + { + IDXGIFactory* out_factory = nullptr; + HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), + reinterpret_cast(&out_factory)); + if (FAILED(err)) { + std::cerr << "Cannot create CreateDXGIFactory, error: " << HRESULT_CODE(err) << std::endl; + return -1; + } + adapter_factory.Attach(out_factory); + } + + CComPtr intel_adapter; + UINT adapter_index = 0; + const unsigned int refIntelVendorID = 0x8086; + IDXGIAdapter* out_adapter = nullptr; + + while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) { + DXGI_ADAPTER_DESC desc{}; + out_adapter->GetDesc(&desc); + if (desc.VendorId == refIntelVendorID) { + intel_adapter.Attach(out_adapter); + break; + } + ++adapter_index; + } + + if (!intel_adapter) { + std::cerr << "No Intel GPU adapter on aboard. Exit" << std::endl; + return -1; + } + + std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter); + accel_device_ptr = reinterpret_cast(dx11_dev.p); + accel_ctx_ptr = reinterpret_cast(dx11_ctx.p); + + // put accel type description for VPL source + source_cfgs.push_back(cfg::create_from_string( + "mfxImplDescription.AccelerationMode" + ":" + "MFX_ACCEL_MODE_VIA_D3D11")); + } + +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + // set ctx_config for GPU device only - no need in case of CPU device type + if (device_id.find("GPU") != std::string::npos) { + InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"}, + {"VA_DEVICE", accel_device_ptr} }); + + face_net.cfgContextParams(ctx_config); + } +#endif // HAVE_INF_ENGINE + auto kernels = cv::gapi::kernels < custom::OCVLocateROI , custom::OCVParseSSD @@ -211,7 +338,14 @@ int main(int argc, char *argv[]) { // Create source cv::Ptr cap; try { - cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); + if (device_id.find("GPU") != std::string::npos) { + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs, + device_id, + accel_device_ptr, + accel_ctx_ptr); + } else { + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); + } std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl; } catch (const std::exception& ex) { std::cerr << "Cannot create source: " << ex.what() << std::endl; diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp index a875f57085..946640d886 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp @@ -22,6 +22,7 @@ #ifdef HAVE_DIRECTX #ifdef HAVE_D3D11 #define D3D11_NO_HELPERS +#define NOMINMAX #include #include #include "opencv2/core/directx.hpp" diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp new file mode 100644 index 0000000000..c8fd49c1ad --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp @@ -0,0 +1,314 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL +#include +#include +#include + +#include "streaming/onevpl/cfg_param_device_selector.hpp" +#include "logger.hpp" + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#include +#include +#pragma comment(lib, "dxgi") +#undef D3D11_NO_HELPERS +#undef NOMINMAX + +#include +#include "opencv2/core/directx.hpp" +#ifdef HAVE_OPENCL +#include +#endif + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +// TODO Will be changed on generic function from `onevpl_param_parser` as soons as feature merges +static mfxVariant cfg_param_to_mfx_variant(const CfgParam& accel_param) { + mfxVariant ret; + const CfgParam::value_t& accel_val = accel_param.get_value(); + if (!cv::util::holds_alternative(accel_val)) { + // expected string or uint32_t as value + if (!cv::util::holds_alternative(accel_val)) { + throw std::logic_error("Incorrect value type of \"mfxImplDescription.AccelerationMode\" " + " std::string is expected" ); + } + ret.Type = MFX_VARIANT_TYPE_U32; + ret.Data.U32 = cv::util::get(accel_val); + return ret; + } + + const std::string& accel_val_str = cv::util::get(accel_val); + ret.Type = MFX_VARIANT_TYPE_U32; + if (accel_val_str == "MFX_ACCEL_MODE_NA") { + ret.Data.U32 = MFX_ACCEL_MODE_NA; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_D3D9") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_D3D9; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_D3D11") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_D3D11; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_GLX") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_GLX; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_X11") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_X11; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND; + } else if (accel_val_str == "MFX_ACCEL_MODE_VIA_HDDLUNITE") { + ret.Data.U32 = MFX_ACCEL_MODE_VIA_HDDLUNITE; + } + return ret; +} + +CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : + suggested_device(IDeviceSelector::create(nullptr, "CPU", AccelType::HOST)), + suggested_context(IDeviceSelector::create(nullptr, AccelType::HOST)) { + + auto accel_mode_it = + std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.AccelerationMode"; + }); + if (accel_mode_it == cfg_params.end()) + { + GAPI_LOG_DEBUG(nullptr, "No HW Accel requested. Use default CPU"); + return; + } + + GAPI_LOG_DEBUG(nullptr, "Add HW acceleration support"); + mfxVariant accel_mode = cfg_param_to_mfx_variant(*accel_mode_it); + + switch(accel_mode.Data.U32) { + case MFX_ACCEL_MODE_VIA_D3D11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + ID3D11Device *hw_handle = nullptr; + ID3D11DeviceContext* device_context = nullptr; + + //Create device + UINT creationFlags = 0;//D3D11_CREATE_DEVICE_BGRA_SUPPORT; + +#if defined _DEBUG || defined CV_STATIC_ANALYSIS + // If the project is in a debug build, enable debugging via SDK Layers with this flag. + creationFlags |= D3D11_CREATE_DEVICE_DEBUG; +#endif + + D3D_FEATURE_LEVEL featureLevels[] = { D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL featureLevel; + + CComPtr adapter_factory; + CComPtr intel_adapters; + { + IDXGIFactory* out_factory = nullptr; + HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), + reinterpret_cast(&out_factory)); + if (FAILED(err)) { + throw std::runtime_error("Cannot create CreateDXGIFactory, error: " + std::to_string(HRESULT_CODE(err))); + } + adapter_factory.Attach(out_factory); + } + + CComPtr intel_adapter; + UINT adapter_index = 0; + const unsigned int refIntelVendorID = 0x8086; + IDXGIAdapter* out_adapter = nullptr; + + while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) { + DXGI_ADAPTER_DESC desc{}; + out_adapter->GetDesc(&desc); + if (desc.VendorId == refIntelVendorID) { + intel_adapter.Attach(out_adapter); + break; + } + ++adapter_index; + } + + if (!intel_adapter) { + throw std::runtime_error("No Intel GPU adapter on aboard"); + } + + // Create the Direct3D 11 API device object and a corresponding context. + HRESULT err = D3D11CreateDevice(intel_adapter, + D3D_DRIVER_TYPE_UNKNOWN, + nullptr, creationFlags, + featureLevels, ARRAYSIZE(featureLevels), + D3D11_SDK_VERSION, + &hw_handle, &featureLevel, + &device_context); + if(FAILED(err)) { + throw std::logic_error("Cannot create D3D11CreateDevice, error: " + std::to_string(HRESULT_CODE(err))); + } + + // oneVPL recommendation + { + ID3D11Multithread *pD11Multithread = nullptr; + device_context->QueryInterface(IID_PPV_ARGS(&pD11Multithread)); + pD11Multithread->SetMultithreadProtected(true); + pD11Multithread->Release(); + } + + suggested_device = IDeviceSelector::create(hw_handle, "GPU", AccelType::DX11); + suggested_context = IDeviceSelector::create(device_context, AccelType::DX11); +#else + GAPI_LOG_WARNING(nullptr, "Unavailable \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\"" + "was chosen for current project configuration"); + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\""); +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + case MFX_ACCEL_MODE_NA: { + // nothing to do + break; + } + default: + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode\" requested: " + + std::to_string(accel_mode.Data.U32)); + break; + } +} + +CfgParamDeviceSelector::CfgParamDeviceSelector(Device::Ptr device_ptr, + const std::string& device_id, + Context::Ptr ctx_ptr, + const CfgParams& cfg_params) : + suggested_device(IDeviceSelector::create(nullptr, "CPU", AccelType::HOST)), + suggested_context(IDeviceSelector::create(nullptr, AccelType::HOST)) { + auto accel_mode_it = + std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.AccelerationMode"; + }); + if (accel_mode_it == cfg_params.end()) { + GAPI_LOG_WARNING(nullptr, "Cannot deternime \"device_ptr\" type. " + "Make sure a param \"mfxImplDescription.AccelerationMode\" " + "presents in configurations and has correct value according to " + "\"device_ptr\" type"); + throw std::logic_error("Missing \"mfxImplDescription.AccelerationMode\" param"); + } + + GAPI_LOG_DEBUG(nullptr, "Turn on HW acceleration support for device: " << + device_ptr << + ", context: " << ctx_ptr); + if (!device_ptr) { + GAPI_LOG_WARNING(nullptr, "Empty \"device_ptr\" is not allowed when " + "param \"mfxImplDescription.AccelerationMode\" existed"); + throw std::logic_error("Invalid param: \"device_ptr\""); + } + + if (!ctx_ptr) { + GAPI_LOG_WARNING(nullptr, "Empty \"ctx_ptr\" is not allowed"); + throw std::logic_error("Invalid param: \"ctx_ptr\""); + } + mfxVariant accel_mode = cfg_param_to_mfx_variant(*accel_mode_it); + + switch(accel_mode.Data.U32) { + case MFX_ACCEL_MODE_VIA_D3D11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + suggested_device = IDeviceSelector::create(device_ptr, device_id, AccelType::DX11); + ID3D11Device* dx_device_ptr = + reinterpret_cast(suggested_device.get_ptr()); + dx_device_ptr->AddRef(); + + suggested_context = IDeviceSelector::create(ctx_ptr, AccelType::DX11); + ID3D11DeviceContext* dx_ctx_ptr = + reinterpret_cast(suggested_context.get_ptr()); + dx_ctx_ptr->AddRef(); +#else + GAPI_LOG_WARNING(nullptr, "Unavailable \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\"" + "was chosen for current project configuration"); + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\""); +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + case MFX_ACCEL_MODE_NA: { + GAPI_LOG_WARNING(nullptr, "Incompatible \"mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_NA\" with " + "\"device_ptr\" and \"ctx_ptr\" arguments. " + "You should not clarify these arguments with \"MFX_ACCEL_MODE_NA\" mode"); + throw std::logic_error("Incompatible param: MFX_ACCEL_MODE_NA"); + } + default: + throw std::logic_error("Unsupported \"mfxImplDescription.AccelerationMode\" requested: " + + std::to_string(accel_mode.Data.U32)); + break; + } +} + +CfgParamDeviceSelector::~CfgParamDeviceSelector() { + GAPI_LOG_INFO(nullptr, "release context: " << suggested_context.get_ptr()); + AccelType ctype = suggested_context.get_type(); + switch(ctype) { + case AccelType::HOST: + //nothing to do + break; + case AccelType::DX11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + ID3D11DeviceContext* device_ctx_ptr = + reinterpret_cast(suggested_context.get_ptr()); + device_ctx_ptr->Release(); + device_ctx_ptr = nullptr; +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + default: + break; + } + + GAPI_LOG_INFO(nullptr, "release device by name: " << + suggested_device.get_name() << + ", ptr: " << suggested_device.get_ptr()); + AccelType dtype = suggested_device.get_type(); + switch(dtype) { + case AccelType::HOST: + //nothing to do + break; + case AccelType::DX11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + ID3D11Device* device_ptr = reinterpret_cast(suggested_device.get_ptr()); + device_ptr->Release(); + device_ptr = nullptr; +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + break; + } + default: + break; + } +} + +CfgParamDeviceSelector::DeviceScoreTable CfgParamDeviceSelector::select_devices() const { + return {std::make_pair(Score::MaxActivePriority, suggested_device)}; +} + +CfgParamDeviceSelector::DeviceContexts CfgParamDeviceSelector::select_context() { + return {suggested_context}; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp new file mode 100644 index 0000000000..2a55fb09cf --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp @@ -0,0 +1,44 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP +#define GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP + +#ifdef HAVE_ONEVPL + +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct GAPI_EXPORTS CfgParamDeviceSelector final: public IDeviceSelector { + CfgParamDeviceSelector(const CfgParams& params = {}); + CfgParamDeviceSelector(Device::Ptr device_ptr, + const std::string& device_id, + Context::Ptr ctx_ptr, + const CfgParams& params); + ~CfgParamDeviceSelector(); + + DeviceScoreTable select_devices() const override; + DeviceContexts select_context() override; + +private: + Device suggested_device; + Context suggested_context; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif //HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP diff --git a/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp b/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp new file mode 100644 index 0000000000..1ac88bd807 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp @@ -0,0 +1,87 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +const char* to_cstring(AccelType type) { + + switch(type) { + case AccelType::HOST: + return "HOST"; + case AccelType::DX11: + return "DX11"; + default: + GAPI_DbgAssert(false && "Unexpected AccelType"); + break; + } + return "UNKNOWN"; +} + +Device::Device(Ptr device_ptr, const std::string& device_name, AccelType device_type) : + name(device_name), + ptr(device_ptr), + type(device_type) { +} + +Device::~Device() { +} + +const std::string& Device::get_name() const { + return name; +} + +Device::Ptr Device::get_ptr() const { + return ptr; +} + +AccelType Device::get_type() const { + return type; +} + +Context::Context(Ptr ctx_ptr, AccelType ctx_type) : + ptr(ctx_ptr), + type(ctx_type) { +} + +Context::~Context() { +} + +Context::Ptr Context::get_ptr() const { + return ptr; +} + +AccelType Context::get_type() const { + return type; +} + +IDeviceSelector::Score::Score(Type val) : + value(val) { +} + +IDeviceSelector::Score::~Score() { +} + +IDeviceSelector::Score::operator Type () const { + return value; +} +IDeviceSelector::Score::Type IDeviceSelector::Score::get() const { + return value; +} + +IDeviceSelector::~IDeviceSelector() { +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/source.cpp b/modules/gapi/src/streaming/onevpl/source.cpp index 0decb1358b..806017a90d 100644 --- a/modules/gapi/src/streaming/onevpl/source.cpp +++ b/modules/gapi/src/streaming/onevpl/source.cpp @@ -8,6 +8,8 @@ #include "streaming/onevpl/source_priv.hpp" #include "streaming/onevpl/file_data_provider.hpp" +#include "streaming/onevpl/cfg_param_device_selector.hpp" + namespace cv { namespace gapi { namespace wip { @@ -15,27 +17,82 @@ namespace onevpl { #ifdef HAVE_ONEVPL GSource::GSource(const std::string& filePath, const CfgParams& cfg_params) : - GSource(std::unique_ptr(new GSource::Priv(std::make_shared(filePath), - cfg_params))) { + GSource(filePath, cfg_params, std::make_shared(cfg_params)) { + if (filePath.empty()) { + util::throw_error(std::logic_error("Cannot create 'GSource' on empty source file name")); + } +} + +GSource::GSource(const std::string& filePath, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr) : + GSource(filePath, cfg_params, + std::make_shared(accel_device_ptr, device_id, + accel_ctx_ptr, cfg_params)) { +} +GSource::GSource(const std::string& filePath, + const CfgParams& cfg_params, + std::shared_ptr selector) : + GSource(std::make_shared(filePath), cfg_params, selector) { if (filePath.empty()) { util::throw_error(std::logic_error("Cannot create 'GSource' on empty source file name")); } } GSource::GSource(std::shared_ptr source, const CfgParams& cfg_params) : - GSource(std::unique_ptr(new GSource::Priv(source, cfg_params))) { + GSource(source, cfg_params, + std::make_shared(cfg_params)) { +} + +GSource::GSource(std::shared_ptr source, + const CfgParams& cfg_params, + const std::string& device_id, + void* accel_device_ptr, + void* accel_ctx_ptr) : + GSource(source, cfg_params, + std::make_shared(accel_device_ptr, device_id, + accel_ctx_ptr, cfg_params)) { } + +// common delegating parameters c-tor +GSource::GSource(std::shared_ptr source, + const CfgParams& cfg_params, + std::shared_ptr selector) : + GSource(std::unique_ptr(new GSource::Priv(source, cfg_params, selector))) { +} + #else GSource::GSource(const std::string&, const CfgParams&) { GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); } +GSource::GSource(const std::string&, const CfgParams&, const std::string&, + void*, void*) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +GSource::GSource(const std::string&, const CfgParams&, std::shared_ptr) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + GSource::GSource(std::shared_ptr, const CfgParams&) { GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); } + +GSource::GSource(std::shared_ptr, const CfgParams&, + const std::string&, void*, void*) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +GSource::GSource(std::shared_ptr, const CfgParams&, std::shared_ptr) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} #endif +// final delegating c-tor GSource::GSource(std::unique_ptr&& impl) : IStreamSource(), m_priv(std::move(impl)) { diff --git a/modules/gapi/src/streaming/onevpl/source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp index 28d438a947..d00074925d 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.cpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -58,8 +58,10 @@ GSource::Priv::Priv() : GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); } -GSource::Priv::Priv(std::shared_ptr provider, const std::vector& params) : - GSource::Priv() +GSource::Priv::Priv(std::shared_ptr provider, + const std::vector& params, + std::shared_ptr) : + GSource::Priv() { // Enable Config if (params.empty()) diff --git a/modules/gapi/src/streaming/onevpl/source_priv.hpp b/modules/gapi/src/streaming/onevpl/source_priv.hpp index cdaab4eb6a..955184c05c 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.hpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.hpp @@ -38,7 +38,8 @@ class ProcessingEngineBase; struct GSource::Priv { explicit Priv(std::shared_ptr provider, - const std::vector& params); + const std::vector& params, + std::shared_ptr selector); ~Priv(); static const std::vector& getDefaultCfgParams(); diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp new file mode 100644 index 0000000000..2f42742b88 --- /dev/null +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_device_selector.cpp @@ -0,0 +1,229 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + + +#include "../test_precomp.hpp" + +#include "../common/gapi_tests_common.hpp" + +#include +#include + +#include + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#include +#include "opencv2/core/directx.hpp" +#undef D3D11_NO_HELPERS +#undef NOMINMAX +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#ifdef HAVE_ONEVPL +#include +#include "streaming/onevpl/cfg_param_device_selector.hpp" + +namespace opencv_test +{ +namespace +{ + +void test_dev_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceScoreTable::value_type &scored_device, + cv::gapi::wip::onevpl::IDeviceSelector::Score expected_score, + cv::gapi::wip::onevpl::AccelType expected_type, + cv::gapi::wip::onevpl::Device::Ptr expected_ptr) { + EXPECT_EQ(std::get<0>(scored_device), expected_score); + EXPECT_EQ(std::get<1>(scored_device).get_type(), expected_type); + EXPECT_EQ(std::get<1>(scored_device).get_ptr(), expected_ptr); +} + +void test_ctx_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceContexts::value_type &ctx, + cv::gapi::wip::onevpl::AccelType expected_type, + cv::gapi::wip::onevpl::Context::Ptr expected_ptr) { + EXPECT_EQ(ctx.get_type(), expected_type); + EXPECT_EQ(ctx.get_ptr(), expected_ptr); +} + +void test_host_dev_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceScoreTable::value_type &scored_device, + cv::gapi::wip::onevpl::IDeviceSelector::Score expected_score) { + test_dev_eq(scored_device, expected_score, + cv::gapi::wip::onevpl::AccelType::HOST, nullptr); +} + +void test_host_ctx_eq(const typename cv::gapi::wip::onevpl::IDeviceSelector::DeviceContexts::value_type &ctx) { + test_ctx_eq(ctx, cv::gapi::wip::onevpl::AccelType::HOST, nullptr); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDevice) +{ + using namespace cv::gapi::wip::onevpl; + CfgParamDeviceSelector selector; + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithEmptyCfgParam) +{ + using namespace cv::gapi::wip::onevpl; + std::vector empty_params; + CfgParamDeviceSelector selector(empty_params); + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithAccelNACfgParam) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_no_accel; + cfg_params_w_no_accel.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_NA)); + CfgParamDeviceSelector selector(cfg_params_w_no_accel); + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithEmptyCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + std::vector empty_params; + CfgParamDeviceSelector selector(empty_params); + IDeviceSelector::DeviceScoreTable devs = selector.select_devices(); + EXPECT_EQ(devs.size(), 1); + test_host_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority); + + IDeviceSelector::DeviceContexts ctxs = selector.select_context(); + EXPECT_EQ(ctxs.size(), 1); + test_host_ctx_eq(*ctxs.begin()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, DefaultDeviceWithDX11AccelCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + std::unique_ptr selector_ptr; + EXPECT_NO_THROW(selector_ptr.reset(new CfgParamDeviceSelector(cfg_params_w_dx11))); + IDeviceSelector::DeviceScoreTable devs = selector_ptr->select_devices(); + + EXPECT_EQ(devs.size(), 1); + test_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority, + AccelType::DX11, + std::get<1>(*devs.begin()).get_ptr() /* compare just type */); + + IDeviceSelector::DeviceContexts ctxs = selector_ptr->select_context(); + EXPECT_EQ(ctxs.size(), 1); + EXPECT_TRUE(ctxs.begin()->get_ptr()); +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, NULLDeviceWithDX11AccelCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + Device::Ptr empty_device_ptr = nullptr; + Context::Ptr empty_ctx_ptr = nullptr; + EXPECT_THROW(CfgParamDeviceSelector sel(empty_device_ptr, "GPU", + empty_ctx_ptr, + cfg_params_w_dx11), + std::logic_error); // empty_device_ptr must be invalid +} + +TEST(OneVPL_Source_Device_Selector_CfgParam, ExternalDeviceWithDX11AccelCfgParam_DX11_ENABLED) +{ + using namespace cv::gapi::wip::onevpl; + ID3D11Device *device = nullptr; + ID3D11DeviceContext* device_context = nullptr; + { + UINT flags = 0; + D3D_FEATURE_LEVEL features[] = { D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL feature_level; + + // Create the Direct3D 11 API device object and a corresponding context. + HRESULT err = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, + nullptr, flags, + features, + ARRAYSIZE(features), D3D11_SDK_VERSION, + &device, &feature_level, &device_context); + EXPECT_FALSE(FAILED(err)); + } + + std::unique_ptr selector_ptr; + std::vector cfg_params_w_dx11; + cfg_params_w_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + EXPECT_NO_THROW(selector_ptr.reset(new CfgParamDeviceSelector(device, "GPU", + device_context, + cfg_params_w_dx11))); + IDeviceSelector::DeviceScoreTable devs = selector_ptr->select_devices(); + + EXPECT_EQ(devs.size(), 1); + test_dev_eq(*devs.begin(), IDeviceSelector::Score::MaxActivePriority, + AccelType::DX11, device); + + IDeviceSelector::DeviceContexts ctxs = selector_ptr->select_context(); + EXPECT_EQ(ctxs.size(), 1); + EXPECT_EQ(reinterpret_cast(ctxs.begin()->get_ptr()), + device_context); +} + +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#ifndef HAVE_DIRECTX +#ifndef HAVE_D3D11 +TEST(OneVPL_Source_Device_Selector_CfgParam, DX11DeviceFromCfgParamWithDX11Disabled) +{ + using namespace cv::gapi::wip::onevpl; + std::vector cfg_params_w_non_existed_dx11; + cfg_params_w_not_existed_dx11.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11)); + EXPECT_THROW(CfgParamDeviceSelector{cfg_params_w_non_existed_dx11}, + std::logic_error); +} +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +TEST(OneVPL_Source_Device_Selector_CfgParam, UnknownPtrDeviceFromCfgParam) +{ + using namespace cv::gapi::wip::onevpl; + std::vector empty_params; + Device::Ptr empty_device_ptr = nullptr; + Context::Ptr empty_ctx_ptr = nullptr; + EXPECT_THROW(CfgParamDeviceSelector sel(empty_device_ptr, "", + empty_ctx_ptr, + empty_params), + std::logic_error); // params must describe device_ptr explicitly +} +} +} // namespace opencv_test +#endif // HAVE_ONEVPL