From f3945fbddb4202f1de423c5404ce69aa00b4abf9 Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Fri, 1 Apr 2022 13:06:47 +0300 Subject: [PATCH] Merge pull request #21688 from sivanov-work:vpp_ie_integration G-API: VPP preprocessing GIEBackend integration * Add ROI in VPP prepro * Apply comments * Integration to IE * Removed extra invocations * Fix no-vpl compilation * Fix compilations * Apply comments * Use thin wrapper for device & ctx * Implement Device/Context constructor functions * Apply samples comment * Fix compilation * Fix compilation * Fix build * Separate Device&Context from selector, apply other comments * Fix typo * Intercept OV exception with pretty print out --- modules/gapi/CMakeLists.txt | 1 + .../gapi/include/opencv2/gapi/infer/ie.hpp | 19 +- .../gapi/streaming/onevpl/accel_types.hpp | 72 +++++ .../onevpl/device_selector_interface.hpp | 43 +-- .../opencv2/gapi/streaming/onevpl/source.hpp | 4 + .../gapi/samples/onevpl_infer_single_roi.cpp | 205 +++++++++----- .../gapi/samples/pipeline_modeling_tool.cpp | 2 + modules/gapi/src/backends/ie/giebackend.cpp | 254 +++++++++++++++++- .../onevpl/cfg_param_device_selector.cpp | 60 +++++ .../onevpl/cfg_param_device_selector.hpp | 7 +- .../onevpl/device_selector_interface.cpp | 36 +++ .../engine/preproc/preproc_dispatcher.cpp | 32 ++- .../engine/preproc/preproc_dispatcher.hpp | 5 - .../onevpl/engine/preproc/preproc_engine.cpp | 4 +- .../onevpl/engine/preproc_defines.hpp | 4 +- .../engine/preproc_engine_interface.cpp | 83 ++++++ .../engine/preproc_engine_interface.hpp | 10 + modules/gapi/src/streaming/onevpl/source.cpp | 11 + 18 files changed, 716 insertions(+), 136 deletions(-) create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/accel_types.hpp create mode 100644 modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 04e1906c75..29036c4e26 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -204,6 +204,7 @@ set(gapi_srcs src/streaming/onevpl/engine/preproc/preproc_engine.cpp src/streaming/onevpl/engine/preproc/preproc_session.cpp src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp + src/streaming/onevpl/engine/preproc_engine_interface.cpp src/streaming/onevpl/demux/async_mfp_demux_data_provider.cpp src/streaming/onevpl/data_provider_dispatcher.cpp diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index e6b7be58ad..204bd8f266 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -20,6 +20,7 @@ #include // GAPI_EXPORTS #include // GKernelPackage #include // Generic +#include // Preproc Dev & Ctx namespace cv { namespace gapi { @@ -84,6 +85,9 @@ struct ParamDesc { // have 2D (Layout::NC) input and if the first dimension not equal to 1 // net.setBatchSize(1) will overwrite it. cv::optional batch_size; + + cv::optional vpl_preproc_device; + cv::optional vpl_preproc_ctx; }; } // namespace detail @@ -126,6 +130,8 @@ public: , {} , 1u , {} + , {} + , {} , {}} { }; @@ -148,6 +154,8 @@ public: , {} , 1u , {} + , {} + , {} , {}} { }; @@ -336,6 +344,13 @@ public: return *this; } + Params& cfgPreprocessingParams(const cv::gapi::wip::onevpl::Device &device, + const cv::gapi::wip::onevpl::Context &ctx) { + desc.vpl_preproc_device = cv::util::make_optional(device); + desc.vpl_preproc_ctx = cv::util::make_optional(ctx); + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::ie::backend(); } std::string tag() const { return Net::tag(); } @@ -370,7 +385,7 @@ public: const std::string &device) : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u, - {}, {}}, + {}, {}, {}, {}}, m_tag(tag) { }; @@ -388,7 +403,7 @@ public: const std::string &device) : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u, - {}, {}}, + {}, {}, {}, {}}, m_tag(tag) { }; diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/accel_types.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/accel_types.hpp new file mode 100644 index 0000000000..421b592aae --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/accel_types.hpp @@ -0,0 +1,72 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2022 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCEL_TYPES_HPP +#define GAPI_STREAMING_ONEVPL_ACCEL_TYPES_HPP + +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +enum class AccelType: uint8_t { + HOST, + DX11, + + LAST_VALUE = std::numeric_limits::max() +}; + +GAPI_EXPORTS const char* to_cstring(AccelType type); + +struct IDeviceSelector; +struct GAPI_EXPORTS Device { + friend struct IDeviceSelector; + using Ptr = void*; + + ~Device(); + const std::string& get_name() const; + Ptr get_ptr() const; + AccelType get_type() const; +private: + Device(Ptr device_ptr, const std::string& device_name, + AccelType device_type); + + std::string name; + Ptr ptr; + AccelType type; +}; + +struct GAPI_EXPORTS Context { + friend struct IDeviceSelector; + using Ptr = void*; + + ~Context(); + Ptr get_ptr() const; + AccelType get_type() const; +private: + Context(Ptr ctx_ptr, AccelType ctx_type); + Ptr ptr; + AccelType type; +}; + +GAPI_EXPORTS Device create_host_device(); +GAPI_EXPORTS Context create_host_context(); + +GAPI_EXPORTS Device create_dx11_device(Device::Ptr device_ptr, + const std::string& device_name); +GAPI_EXPORTS Context create_dx11_context(Context::Ptr ctx_ptr); + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ACCEL_TYPES_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp index 04f8cae02a..2e2d879fba 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp @@ -12,53 +12,12 @@ #include #include -#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS +#include namespace cv { namespace gapi { namespace wip { namespace onevpl { - -enum class AccelType: uint8_t { - HOST, - DX11, - - LAST_VALUE = std::numeric_limits::max() -}; - -GAPI_EXPORTS const char* to_cstring(AccelType type); - -struct IDeviceSelector; -struct GAPI_EXPORTS Device { - friend struct IDeviceSelector; - using Ptr = void*; - - ~Device(); - const std::string& get_name() const; - Ptr get_ptr() const; - AccelType get_type() const; -private: - Device(Ptr device_ptr, const std::string& device_name, - AccelType device_type); - - std::string name; - Ptr ptr; - AccelType type; -}; - -struct GAPI_EXPORTS Context { - friend struct IDeviceSelector; - using Ptr = void*; - - ~Context(); - Ptr get_ptr() const; - AccelType get_type() const; -private: - Context(Ptr ctx_ptr, AccelType ctx_type); - Ptr ptr; - AccelType type; -}; - struct GAPI_EXPORTS IDeviceSelector { using Ptr = std::shared_ptr; diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp index 6334480c1b..04dc2e246d 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -46,6 +46,10 @@ public: void* accel_device_ptr, void* accel_ctx_ptr); + GSource(const std::string& filePath, + const CfgParams& cfg_params, + const Device &device, const Context &ctx); + GSource(const std::string& filePath, const CfgParams& cfg_params, std::shared_ptr selector); diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 6935cbb709..7f0da6070c 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include // CommandLineParser #include @@ -46,26 +45,45 @@ const std::string keys = "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }" "{ streaming_queue_capacity | 1 | Streaming executor queue capacity. Calculated automaticaly if 0 }" "{ frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size}" - "{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}"; + "{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}" + "{ roi | -1,-1,-1,-1 | Region of interest (ROI) to use for inference. Identified automatically when not set }"; namespace { -bool is_gpu(const std::string &device_name) { - return device_name.find("GPU") != std::string::npos; -} - std::string get_weights_path(const std::string &model_path) { const auto EXT_LEN = 4u; const auto sz = model_path.size(); - CV_Assert(sz > EXT_LEN); + GAPI_Assert(sz > EXT_LEN); auto ext = model_path.substr(sz - EXT_LEN); std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ return static_cast(std::tolower(c)); }); - CV_Assert(ext == ".xml"); + GAPI_Assert(ext == ".xml"); return model_path.substr(0u, sz - EXT_LEN) + ".bin"; } +// TODO: It duplicates infer_single_roi sample +cv::util::optional parse_roi(const std::string &rc) { + cv::Rect rv; + char delim[3]; + + std::stringstream is(rc); + is >> rv.x >> delim[0] >> rv.y >> delim[1] >> rv.width >> delim[2] >> rv.height; + if (is.bad()) { + return cv::util::optional(); // empty value + } + const auto is_delim = [](char c) { + return c == ','; + }; + if (!std::all_of(std::begin(delim), std::end(delim), is_delim)) { + return cv::util::optional(); // empty value + } + if (rv.x < 0 || rv.y < 0 || rv.width <= 0 || rv.height <= 0) { + return cv::util::optional(); // empty value + } + return cv::util::make_optional(std::move(rv)); +} + #ifdef HAVE_INF_ENGINE #ifdef HAVE_DIRECTX #ifdef HAVE_D3D11 @@ -127,9 +145,15 @@ using GRect = cv::GOpaque; using GSize = cv::GOpaque; using GPrims = cv::GArray; -G_API_OP(LocateROI, )>, "sample.custom.locate-roi") { - static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &, - std::reference_wrapper) { +G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +// TODO: It duplicates infer_single_roi sample +G_API_OP(LocateROI, , "sample.custom.locate-roi") { + static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) { return cv::empty_gopaque_desc(); } }; @@ -151,29 +175,18 @@ GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { // the most convenient aspect ratio for detectors to use) static void run(const cv::Size& in_size, - std::reference_wrapper device_id_ref, cv::Rect &out_rect) { // Identify the central point & square size (- some padding) - // NB: GPU plugin in InferenceEngine doesn't support ROI at now - if (!is_gpu(device_id_ref.get())) { - const auto center = cv::Point{in_size.width/2, in_size.height/2}; - auto sqside = std::min(in_size.width, in_size.height); - - // Now build the central square ROI - out_rect = cv::Rect{ center.x - sqside/2 - , center.y - sqside/2 - , sqside - , sqside - }; - } else { - // use whole frame for GPU device - out_rect = cv::Rect{ 0 - , 0 - , in_size.width - , in_size.height - }; - } + const auto center = cv::Point{in_size.width/2, in_size.height/2}; + auto sqside = std::min(in_size.width, in_size.height); + + // Now build the central square ROI + out_rect = cv::Rect{ center.x - sqside/2 + , center.y - sqside/2 + , sqside + , sqside + }; } }; @@ -194,6 +207,55 @@ GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { } }; +GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { + static void run(const cv::Mat &in_ssd_result, + const cv::Rect &in_roi, + const cv::Size &in_parent_size, + std::vector &out_objects) { + const auto &in_ssd_dims = in_ssd_result.size; + GAPI_Assert(in_ssd_dims.dims() == 4u); + + const int MAX_PROPOSALS = in_ssd_dims[2]; + const int OBJECT_SIZE = in_ssd_dims[3]; + GAPI_Assert(OBJECT_SIZE == 7); // fixed SSD object size + + const cv::Size up_roi = in_roi.size(); + const cv::Rect surface({0,0}, in_parent_size); + + out_objects.clear(); + + const float *data = in_ssd_result.ptr(); + for (int i = 0; i < MAX_PROPOSALS; i++) { + const float image_id = data[i * OBJECT_SIZE + 0]; + const float label = data[i * OBJECT_SIZE + 1]; + const float confidence = data[i * OBJECT_SIZE + 2]; + const float rc_left = data[i * OBJECT_SIZE + 3]; + const float rc_top = data[i * OBJECT_SIZE + 4]; + const float rc_right = data[i * OBJECT_SIZE + 5]; + const float rc_bottom = data[i * OBJECT_SIZE + 6]; + (void) label; // unused + + if (image_id < 0.f) { + break; // marks end-of-detections + } + if (confidence < 0.5f) { + continue; // skip objects with low confidence + } + + // map relative coordinates to the original image scale + // taking the ROI into account + cv::Rect rc; + rc.x = static_cast(rc_left * up_roi.width); + rc.y = static_cast(rc_top * up_roi.height); + rc.width = static_cast(rc_right * up_roi.width) - rc.x; + rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; + rc.x += in_roi.x; + rc.y += in_roi.y; + out_objects.emplace_back(rc & surface); + } + } +}; + } // namespace custom namespace cfg { @@ -212,6 +274,7 @@ int main(int argc, char *argv[]) { // get file name const auto file_path = cmd.get("input"); const auto output = cmd.get("output"); + const auto opt_roi = parse_roi(cmd.get("roi")); const auto face_model_path = cmd.get("facem"); const auto streaming_queue_capacity = cmd.get("streaming_queue_capacity"); const auto source_decode_queue_capacity = cmd.get("frames_pool_size"); @@ -259,8 +322,8 @@ int main(int argc, char *argv[]) { // GAPI InferenceEngine backend to provide interoperability with onevpl::GSource // So GAPI InferenceEngine backend and onevpl::GSource MUST share the same // device and context - void* accel_device_ptr = nullptr; - void* accel_ctx_ptr = nullptr; + cv::util::optional accel_device; + cv::util::optional accel_ctx; #ifdef HAVE_INF_ENGINE #ifdef HAVE_DIRECTX @@ -268,7 +331,7 @@ int main(int argc, char *argv[]) { auto dx11_dev = createCOMPtrGuard(); auto dx11_ctx = createCOMPtrGuard(); - if (is_gpu(device_id)) { + if (device_id.find("GPU") != std::string::npos) { auto adapter_factory = createCOMPtrGuard(); { IDXGIFactory* out_factory = nullptr; @@ -302,8 +365,13 @@ int main(int argc, char *argv[]) { } std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter.get()); - accel_device_ptr = reinterpret_cast(dx11_dev.get()); - accel_ctx_ptr = reinterpret_cast(dx11_ctx.get()); + accel_device = cv::util::make_optional( + cv::gapi::wip::onevpl::create_dx11_device( + reinterpret_cast(dx11_dev.get()), + device_id)); + accel_ctx = cv::util::make_optional( + cv::gapi::wip::onevpl::create_dx11_context( + reinterpret_cast(dx11_ctx.get()))); // put accel type description for VPL source source_cfgs.push_back(cfg::create_from_string( @@ -315,9 +383,10 @@ int main(int argc, char *argv[]) { #endif // HAVE_D3D11 #endif // HAVE_DIRECTX // set ctx_config for GPU device only - no need in case of CPU device type - if (is_gpu(device_id)) { + if (accel_device.has_value() && + accel_device.value().get_name().find("GPU") != std::string::npos) { InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"}, - {"VA_DEVICE", accel_device_ptr} }); + {"VA_DEVICE", accel_device.value().get_ptr()} }); face_net.cfgContextParams(ctx_config); // NB: consider NV12 surface because it's one of native GPU image format @@ -325,8 +394,16 @@ int main(int argc, char *argv[]) { } #endif // HAVE_INF_ENGINE + // turn on preproc + if (accel_device.has_value() && accel_ctx.has_value()) { + face_net.cfgPreprocessingParams(accel_device.value(), + accel_ctx.value()); + std::cout << "enforce VPP preprocessing on " << device_id << std::endl; + } + auto kernels = cv::gapi::kernels < custom::OCVLocateROI + , custom::OCVParseSSD , custom::OCVBBoxes>(); auto networks = cv::gapi::networks(face_net); auto face_detection_args = cv::compile_args(networks, kernels); @@ -335,13 +412,12 @@ int main(int argc, char *argv[]) { } // Create source - cv::Ptr cap; + cv::gapi::wip::IStreamSource::Ptr cap; try { - if (is_gpu(device_id)) { + if (accel_device.has_value() && accel_ctx.has_value()) { cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs, - device_id, - accel_device_ptr, - accel_ctx_ptr); + accel_device.value(), + accel_ctx.value()); } else { cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); } @@ -353,29 +429,35 @@ int main(int argc, char *argv[]) { cv::GMetaArg descr = cap->descr_of(); auto frame_descr = cv::util::get(descr); + cv::GOpaque in_roi; + auto inputs = cv::gin(cap); // Now build the graph cv::GFrame in; auto size = cv::gapi::streaming::size(in); - auto roi = custom::LocateROI::on(size, std::cref(device_id)); - auto blob = cv::gapi::infer(in); - cv::GArray rcs = cv::gapi::parseSSD(blob, size, 0.5f, true, true); - auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi)); - auto out = cv::gapi::streaming::BGR(out_frame); - - cv::GStreamingCompiled pipeline; - try { - pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) - .compileStreaming(std::move(face_detection_args)); - } catch (const std::exception& ex) { - std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl; - return -1; + auto graph_inputs = cv::GIn(in); + if (!opt_roi.has_value()) { + // Automatically detect ROI to infer. Make it output parameter + std::cout << "ROI is not set or invalid. Locating it automatically" + << std::endl; + in_roi = custom::LocateROI::on(size); + } else { + // Use the value provided by user + std::cout << "Will run inference for static region " + << opt_roi.value() + << " only" + << std::endl; + graph_inputs += cv::GIn(in_roi); + inputs += cv::gin(opt_roi.value()); } + auto blob = cv::gapi::infer(in_roi, in); + cv::GArray rcs = custom::ParseSSD::on(blob, in_roi, size); + auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, in_roi)); + auto out = cv::gapi::streaming::BGR(out_frame); + cv::GStreamingCompiled pipeline = cv::GComputation(std::move(graph_inputs), cv::GOut(out)) // and move here + .compileStreaming(std::move(face_detection_args)); // The execution part - - // TODO USE may set pool size from outside and set queue_capacity size, - // compile arg: cv::gapi::streaming::queue_capacity - pipeline.setSource(std::move(cap)); + pipeline.setSource(std::move(inputs)); pipeline.start(); size_t frames = 0u; @@ -384,7 +466,7 @@ int main(int argc, char *argv[]) { if (!output.empty() && !writer.isOpened()) { const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height}; writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); - CV_Assert(writer.isOpened()); + GAPI_Assert(writer.isOpened()); } cv::Mat outMat; @@ -399,6 +481,7 @@ int main(int argc, char *argv[]) { } tm.stop(); std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; + return 0; } diff --git a/modules/gapi/samples/pipeline_modeling_tool.cpp b/modules/gapi/samples/pipeline_modeling_tool.cpp index 2ed9642256..7a0f94655c 100644 --- a/modules/gapi/samples/pipeline_modeling_tool.cpp +++ b/modules/gapi/samples/pipeline_modeling_tool.cpp @@ -10,7 +10,9 @@ #include #if defined(_WIN32) +#define NOMINMAX #include +#undef NOMINMAX #endif #include "pipeline_modeling_tool/dummy_source.hpp" diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 52c60c1f0b..4969b79860 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -64,6 +64,9 @@ template using QueueClass = cv::gapi::own::concurrent_bounded_queue< #include "utils/itt.hpp" +#include "streaming/onevpl/engine/preproc_engine_interface.hpp" +#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp" + namespace IE = InferenceEngine; namespace { @@ -261,12 +264,36 @@ struct IEUnit { InferenceEngine::RemoteContext::Ptr rctx = nullptr; + std::shared_ptr preproc_engine_impl; + // FIXME: Unlike loadNetwork case, importNetwork requires that preprocessing // should be passed as ExecutableNetwork::SetBlob method, so need to collect // and store this information at the graph compilation stage (outMeta) and use in runtime. using PreProcMap = std::unordered_map; PreProcMap preproc_map; + // NEW FIXME: Need to aggregate getInputInfo & GetInputInfo from network + // into generic wrapper and invoke it at once in single place instead of + // analyzing ParamDesc::Kind::Load/Import every time when we need to get access + // for network info. + // In term of introducing custom VPP/VPL preprocessing functionality + // It was decided to use GFrameDesc as such aggregated network info with limitation + // that VPP/VPL produces cv::MediaFrame only. But it should be not considered as + // final solution + class InputFramesDesc { + using input_name_type = std::string; + using description_type = cv::GFrameDesc; + std::map map; + public: + static bool is_applicable(const cv::GMetaArg &mm); + const description_type &get_param(const input_name_type &input) const; + + void set_param(const input_name_type &input, + const IE::TensorDesc& desc); + }; + + InputFramesDesc net_input_params; + explicit IEUnit(const cv::gapi::ie::detail::ParamDesc &pp) : params(pp) { InferenceEngine::ParamMap* ctx_params = @@ -336,6 +363,17 @@ struct IEUnit { } else { cv::util::throw_error(std::logic_error("Unsupported ParamDesc::Kind")); } + + using namespace cv::gapi::wip::onevpl; + if (params.vpl_preproc_device.has_value() && params.vpl_preproc_ctx.has_value()) { + using namespace cv::gapi::wip; + GAPI_LOG_INFO(nullptr, "VPP preproc creation requested"); + preproc_engine_impl = + IPreprocEngine::create_preproc_engine( + params.vpl_preproc_device.value(), + params.vpl_preproc_ctx.value()); + GAPI_LOG_INFO(nullptr, "VPP preproc created successfuly"); + } } // This method is [supposed to be] called at Island compilation stage @@ -354,6 +392,39 @@ struct IEUnit { } }; +bool IEUnit::InputFramesDesc::is_applicable(const cv::GMetaArg &mm) { + return cv::util::holds_alternative(mm); +} + +const IEUnit::InputFramesDesc::description_type & +IEUnit::InputFramesDesc::get_param(const input_name_type &input) const { + auto it = map.find(input); + GAPI_Assert(it != map.end() && "No appropriate input is found in InputFramesDesc"); + return it->second; +} + +void IEUnit::InputFramesDesc::set_param(const input_name_type &input, + const IE::TensorDesc& desc) { + description_type ret; + ret.fmt = cv::MediaFormat::NV12; + const InferenceEngine::SizeVector& inDims = desc.getDims(); + auto layout = desc.getLayout(); + GAPI_LOG_DEBUG(nullptr, "network input: " << input << + ", tensor dims: " << inDims[0] << ", " << inDims[1] << + ", " << inDims[2] << ", " << inDims[3]); + if (layout != InferenceEngine::NHWC && layout != InferenceEngine::NCHW) { + GAPI_LOG_WARNING(nullptr, "Unsupported layout for VPP preproc: " << layout << + ", input name: " << input); + GAPI_Assert(false && "Unsupported layout for VPP preproc"); + } + GAPI_Assert(inDims.size() == 4u); + ret.size.width = static_cast(inDims[3]); + ret.size.height = static_cast(inDims[2]); + + auto res = map.emplace(input, ret); + GAPI_Assert(res.second && "Duplicated input info in InputFramesDesc are not allowable"); +} + class IECallContext { public: @@ -396,6 +467,9 @@ public: // To store exception appeared in callback. std::exception_ptr eptr; + using req_key_t = void*; + cv::MediaFrame* prepareKeepAliveFrameSlot(req_key_t key); + size_t releaseKeepAliveFrame(req_key_t key); private: cv::detail::VectorRef& outVecRef(std::size_t idx); @@ -417,6 +491,10 @@ private: // Input parameters passed to an inference operation. cv::GArgs m_args; cv::GShapes m_in_shapes; + + // keep alive preprocessed frames + std::mutex keep_alive_frames_mutex; + std::unordered_map keep_alive_pp_frames; }; IECallContext::IECallContext(const IEUnit & unit, @@ -516,6 +594,35 @@ cv::GArg IECallContext::packArg(const cv::GArg &arg) { } } +cv::MediaFrame* IECallContext::prepareKeepAliveFrameSlot(req_key_t key) { + std::lock_guard lock(keep_alive_frames_mutex); + return &keep_alive_pp_frames[key]; +} + +size_t IECallContext::releaseKeepAliveFrame(req_key_t key) { + size_t elapsed_count = 0; + void *prev_slot = nullptr; + // NB: release MediaFrame previously captured by prepareKeepAliveFrameSlot + // We must capture it to keep a reference counter on inner media adapter + // to ensure that frame resource would be locked until inference done. + // Otherwise decoder could seized this frame resource as free/unlocked resource + // from resource pool + // Current function just take a unique frame `key` and overwrite stored + // actual frame by empty frame + { + std::lock_guard lock(keep_alive_frames_mutex); + auto ka_frame_it = keep_alive_pp_frames.find(key); + if (ka_frame_it != keep_alive_pp_frames.end()) { + prev_slot = &ka_frame_it->second; + ka_frame_it->second = cv::MediaFrame(); + } + elapsed_count = keep_alive_pp_frames.size(); + } + GAPI_LOG_DEBUG(nullptr, "Release keep alive frame, slot: " << prev_slot << + ", reserved frames count: " << elapsed_count); + return elapsed_count; +} + struct IECallable { static const char *name() { return "IERequestCallable"; } using Run = std::function, cv::gimpl::ie::RequestPool&)>; @@ -552,11 +659,65 @@ using GConstGIEModel = ade::ConstTypedGraph , IECallable >; -inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) { +cv::MediaFrame preprocess_frame_impl(cv::MediaFrame &&in_frame, const std::string &layer_name, + IECallContext& ctx, + const cv::util::optional &opt_roi, + cv::MediaFrame* out_keep_alive_frame, + bool* out_is_preprocessed) { + cv::util::optional param = + ctx.uu.preproc_engine_impl->is_applicable(in_frame); + if (param.has_value()) { + GAPI_LOG_DEBUG(nullptr, "VPP preprocessing for decoded remote frame will be used"); + cv::GFrameDesc expected_net_input_descr = + ctx.uu.net_input_params.get_param(layer_name); + + // TODO: Find a better place to configure media format for GPU + // adjust color conversion to NV12 according to OV GPU limitation + if(ctx.uu.params.device_id.find("GPU") != std::string::npos && + ctx.uu.rctx) { + auto it = ctx.uu.params.config.find(std::string("GPU_NV12_TWO_INPUTS")); + if (it != ctx.uu.params.config.end()) { + if (it->second == "YES") { + GAPI_LOG_DEBUG(nullptr, "Adjust preprocessing GPU media format to NV12"); + expected_net_input_descr.fmt = cv::MediaFormat::NV12; + } + } + } + + cv::gapi::wip::pp_session pp_sess = + ctx.uu.preproc_engine_impl->initialize_preproc(param.value(), + expected_net_input_descr); + + in_frame = ctx.uu.preproc_engine_impl->run_sync(pp_sess, in_frame, opt_roi); + + if (out_keep_alive_frame != nullptr) { + GAPI_LOG_DEBUG(nullptr, "remember preprocessed remote frame to keep it busy from reuse, slot: " << + out_keep_alive_frame); + *out_keep_alive_frame = in_frame; + } + if (out_is_preprocessed) { + *out_is_preprocessed = true; + } + } // otherwise it is not suitable frame, then check on other preproc backend or rely on IE plugin + return std::move(in_frame); +} + +inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i, + const std::string &layer_name, + const cv::util::optional &opt_roi, + cv::MediaFrame* out_keep_alive_frame, + bool* out_is_preprocessed) { GAPI_Assert(ctx.inShape(i) == cv::GShape::GFRAME && "Remote blob is supported for MediaFrame only"); + cv::MediaFrame frame = ctx.inFrame(i); + if (ctx.uu.preproc_engine_impl) { + GAPI_LOG_DEBUG(nullptr, "Try to use preprocessing for decoded remote frame in remote ctx"); + frame = preprocess_frame_impl(std::move(frame), layer_name, ctx, opt_roi, + out_keep_alive_frame, out_is_preprocessed); + } - cv::util::any any_blob_params = ctx.inFrame(i).blobParams(); + // Request params for result frame whatever it got preprocessed or not + cv::util::any any_blob_params = frame.blobParams(); using ParamType = std::pair; using NV12ParamType = std::pair; @@ -582,14 +743,24 @@ inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) { inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i, - cv::gapi::ie::TraitAs hint) { + cv::gapi::ie::TraitAs hint, + const std::string& layer_name, + const cv::util::optional &opt_roi, + cv::MediaFrame* out_keep_alive_frame = nullptr, + bool* out_is_preprocessed = nullptr) { if (ctx.uu.rctx != nullptr) { - return extractRemoteBlob(ctx, i); + return extractRemoteBlob(ctx, i, layer_name, opt_roi, + out_keep_alive_frame, out_is_preprocessed); } switch (ctx.inShape(i)) { case cv::GShape::GFRAME: { - const auto& frame = ctx.inFrame(i); + auto frame = ctx.inFrame(i); + if (ctx.uu.preproc_engine_impl) { + GAPI_LOG_DEBUG(nullptr, "Try to use preprocessing for decoded frame in local ctx"); + frame = preprocess_frame_impl(std::move(frame), layer_name, ctx, opt_roi, + out_keep_alive_frame, out_is_preprocessed); + } ctx.views.emplace_back(new cv::MediaFrame::View(frame.access(cv::MediaFrame::Access::R))); return wrapIE(*(ctx.views.back()), frame.desc()); } @@ -626,10 +797,20 @@ static void setROIBlob(InferenceEngine::InferRequest& req, const IECallContext& ctx) { if (ctx.uu.params.device_id.find("GPU") != std::string::npos && ctx.uu.rctx) { - GAPI_LOG_WARNING(nullptr, "ROI blob creation for device_id: " << - ctx.uu.params.device_id << ", layer: " << layer_name << - "is not supported yet"); - GAPI_Assert(false && "Unsupported ROI blob creation for GPU remote context"); + try { + // NB: make_shared_blob() cannot work with GPU NV12 & ROI at the moment. + // OpenVINO produces exception with unsupported status. + // To do not encounter with silent crash situation we should catch OV exception + // and suggest to avoid this problem by using inner preprocessing feature. + // VPP/VPL proprocessing are supported at the moment + setBlob(req, layer_name, IE::make_shared_blob(blob, toIE(roi)), ctx); + } catch (const std::exception &ex) { + GAPI_LOG_WARNING(nullptr, "cannot set ROI blob for layer: " << layer_name << + ", reason:\n" << ex.what() << + "\nTry using self GAPI preprocessing feature: " + " Check method `cfgPreprocessingParams` in `cv::gapi::ie::Params`"); + throw; + } } else { setBlob(req, layer_name, IE::make_shared_blob(blob, toIE(roi)), ctx); } @@ -975,6 +1156,8 @@ static void PostOutputs(InferenceEngine::InferRequest &request, ctx->out.meta(output, ctx->input(0).meta); ctx->out.post(std::move(output), ctx->eptr); } + + ctx->releaseKeepAliveFrame(&request); } class PostOutputsList { @@ -1088,6 +1271,12 @@ struct Infer: public cv::detail::KernelTag { if (isApplicableForResize(ii->getTensorDesc())) { ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } + + // NB: configure input param for further preproc + if (uu.net_input_params.is_applicable(mm)) { + const_cast(uu.net_input_params) + .set_param(input_name, ii->getTensorDesc()); + } } // FIXME: This isn't the best place to call reshape function. @@ -1107,6 +1296,12 @@ struct Infer: public cv::detail::KernelTag { auto ii = inputs.at(input_name); const auto & mm = std::get<1>(it); non_const_prepm->emplace(input_name, configurePreProcInfo(ii, mm)); + + // NB: configure input param for further preproc + if (uu.net_input_params.is_applicable(mm)) { + const_cast(uu.net_input_params) + .set_param(input_name, ii->getTensorDesc()); + } } } @@ -1145,7 +1340,9 @@ struct Infer: public cv::detail::KernelTag { (layout == IE::Layout::NCHW || layout == IE::Layout::NHWC) ? cv::gapi::ie::TraitAs::IMAGE : cv::gapi::ie::TraitAs::TENSOR; - IE::Blob::Ptr this_blob = extractBlob(*ctx, i, hint); + IE::Blob::Ptr this_blob = extractBlob(*ctx, i, hint, + layer_name, + cv::util::optional{}); setBlob(req, layer_name, this_blob, *ctx); } // FIXME: Should it be done by kernel ? @@ -1200,6 +1397,12 @@ struct InferROI: public cv::detail::KernelTag { if (!input_reshape_table.empty()) { const_cast(&uu.net)->reshape(input_reshape_table); } + + // NB: configure input param for further preproc + if (uu.net_input_params.is_applicable(mm)) { + const_cast(uu.net_input_params) + .set_param(input_name, ii->getTensorDesc()); + } } else { GAPI_Assert(uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import); auto inputs = uu.this_network.GetInputsInfo(); @@ -1207,6 +1410,12 @@ struct InferROI: public cv::detail::KernelTag { auto* non_const_prepm = const_cast(&uu.preproc_map); auto ii = inputs.at(input_name); non_const_prepm->emplace(input_name, configurePreProcInfo(ii, mm)); + + // NB: configure intput param for further preproc + if (uu.net_input_params.is_applicable(mm)) { + const_cast(uu.net_input_params) + .set_param(input_name, ii->getTensorDesc()); + } } // FIXME: It would be nice here to have an exact number of network's @@ -1236,13 +1445,26 @@ struct InferROI: public cv::detail::KernelTag { GAPI_Assert(ctx->uu.params.num_in == 1); auto&& this_roi = ctx->inArg(0).rref(); + // reserve unique slot for keep alive preprocessed frame + cv::MediaFrame* slot_ptr = ctx->prepareKeepAliveFrameSlot(&req); + // NB: This blob will be used to make roi from its, so // it should be treated as image + bool preprocessed = false; IE::Blob::Ptr this_blob = - extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE); - setROIBlob(req, + extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE, + *(ctx->uu.params.input_names.begin()), + cv::util::make_optional(this_roi), + slot_ptr, &preprocessed); + if (!preprocessed) { + setROIBlob(req, *(ctx->uu.params.input_names.begin()), this_blob, this_roi, *ctx); + } else { + setBlob(req, + *(ctx->uu.params.input_names.begin()), + this_blob, *ctx); + } // FIXME: Should it be done by kernel ? // What about to do that in RequestPool ? req.StartAsync(); @@ -1336,7 +1558,9 @@ struct InferList: public cv::detail::KernelTag { // NB: This blob will be used to make roi from its, so // it should be treated as image - IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE); + IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE, + ctx->uu.params.input_names[0u], + cv::util::optional{}); std::vector> cached_dims(ctx->uu.params.num_out); for (auto i : ade::util::iota(ctx->uu.params.num_out)) { @@ -1483,7 +1707,9 @@ struct InferList2: public cv::detail::KernelTag { && "This operation must have at least two arguments"); // NB: This blob will be used to make roi from its, so // it should be treated as image - IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0, cv::gapi::ie::TraitAs::IMAGE); + IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0, cv::gapi::ie::TraitAs::IMAGE, + ctx->uu.params.input_names[0u], + cv::util::optional{}); const auto list_size = ctx->inArg(1u).size(); if (list_size == 0u) { for (auto i : ade::util::iota(ctx->uu.params.num_out)) { diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp index a4d85f2598..b3beb71fb1 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp @@ -38,6 +38,20 @@ namespace gapi { namespace wip { namespace onevpl { +std::vector insertCfgparam(std::vector &¶m_array, AccelType type) { + switch (type) { + case AccelType::HOST: + break; + case AccelType::DX11: + param_array.push_back(CfgParam::create_acceleration_mode(MFX_ACCEL_MODE_VIA_D3D11)); + break; + default: + GAPI_DbgAssert(false && "Unexpected AccelType"); + break; + } + return std::move(param_array); +} + CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) : suggested_device(IDeviceSelector::create(nullptr, "CPU", AccelType::HOST)), suggested_context(IDeviceSelector::create(nullptr, AccelType::HOST)) { @@ -231,6 +245,52 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(Device::Ptr device_ptr, } } +CfgParamDeviceSelector::CfgParamDeviceSelector(const Device &device, + const Context &ctx, + CfgParams) : + suggested_device(device), + suggested_context(ctx) { + + switch(device.get_type()) { + case AccelType::DX11: { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + ID3D11Device* dx_device_ptr = + reinterpret_cast(suggested_device.get_ptr()); + dx_device_ptr->AddRef(); + + ID3D11DeviceContext* dx_ctx_ptr = + reinterpret_cast(suggested_context.get_ptr()); + + // oneVPL recommendation + { + ID3D11Multithread *pD11Multithread = nullptr; + dx_ctx_ptr->QueryInterface(IID_PPV_ARGS(&pD11Multithread)); + pD11Multithread->SetMultithreadProtected(true); + pD11Multithread->Release(); + } + + dx_ctx_ptr->AddRef(); + break; +#else + GAPI_LOG_WARNING(nullptr, "Unavailable \"" << CfgParam::acceleration_mode_name() << + ": MFX_ACCEL_MODE_VIA_D3D11\"" + "was chosen for current project configuration"); + throw std::logic_error(std::string("Unsupported \"") + + CfgParam::acceleration_mode_name() + ": MFX_ACCEL_MODE_VIA_D3D11\""); +#endif // HAVE_DIRECTX +#endif // HAVE_D3D11 + } + case AccelType::HOST: + break; + default: + throw std::logic_error(std::string("Unsupported \"") + CfgParam::acceleration_mode_name() + + "\" requested: " + + to_cstring(device.get_type())); + break; + } +} + CfgParamDeviceSelector::~CfgParamDeviceSelector() { GAPI_LOG_INFO(nullptr, "release context: " << suggested_context.get_ptr()); AccelType ctype = suggested_context.get_type(); diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp index 2a55fb09cf..5dae1c508d 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp @@ -7,14 +7,14 @@ #ifndef GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP #define GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP -#ifdef HAVE_ONEVPL - #include #include #include #include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS +#ifdef HAVE_ONEVPL + namespace cv { namespace gapi { namespace wip { @@ -26,6 +26,9 @@ struct GAPI_EXPORTS CfgParamDeviceSelector final: public IDeviceSelector { const std::string& device_id, Context::Ptr ctx_ptr, const CfgParams& params); + CfgParamDeviceSelector(const Device &device_ptr, + const Context &ctx_ptr, + CfgParams params); ~CfgParamDeviceSelector(); DeviceScoreTable select_devices() const override; diff --git a/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp b/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp index 1ac88bd807..404b2f3872 100644 --- a/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp +++ b/modules/gapi/src/streaming/onevpl/device_selector_interface.cpp @@ -81,6 +81,42 @@ IDeviceSelector::Score::Type IDeviceSelector::Score::get() const { IDeviceSelector::~IDeviceSelector() { } +namespace detail +{ +struct DeviceContextCreator : public IDeviceSelector { + DeviceScoreTable select_devices() const override { return {};} + DeviceContexts select_context() override { return {};} + + template + static Entity create_entity(Args &&...args) { + return IDeviceSelector::create(std::forward(args)...); + } +}; +} + +Device create_host_device() { + return detail::DeviceContextCreator::create_entity(nullptr, + "CPU", + AccelType::HOST); +} + +Context create_host_context() { + return detail::DeviceContextCreator::create_entity(nullptr, + AccelType::HOST); +} + +Device create_dx11_device(Device::Ptr device_ptr, + const std::string& device_name) { + return detail::DeviceContextCreator::create_entity(device_ptr, + device_name, + AccelType::DX11); +} + +Context create_dx11_context(Context::Ptr ctx_ptr) { + return detail::DeviceContextCreator::create_entity(ctx_ptr, + AccelType::DX11); +} + } // namespace onevpl } // namespace wip } // namespace gapi diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp index 23ad385b51..5a08f2bd09 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp @@ -4,30 +4,33 @@ // // Copyright (C) 2022 Intel Corporation -#ifdef HAVE_ONEVPL - #include #include #include +#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp" +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/onevpl_export.hpp" #include "streaming/onevpl/engine/preproc/preproc_engine.hpp" #include "streaming/onevpl/engine/preproc/preproc_session.hpp" -#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp" #include "streaming/onevpl/accelerators/accel_policy_interface.hpp" #include "streaming/onevpl/accelerators/surface/surface.hpp" #include "streaming/onevpl/cfg_params_parser.hpp" -#include "logger.hpp" +#endif // HAVE_ONEVPL +#include "logger.hpp" namespace cv { namespace gapi { namespace wip { namespace onevpl { +#ifdef HAVE_ONEVPL cv::util::optional VPPPreprocDispatcher::is_applicable(const cv::MediaFrame& in_frame) { cv::util::optional param; GAPI_LOG_DEBUG(nullptr, "workers: " << workers.size()); + bool worker_found = false; for (const auto &w : workers) { param = w->is_applicable(in_frame); if (param.has_value()) { @@ -42,11 +45,12 @@ cv::util::optional VPPPreprocDispatcher::is_applicable(const cv::Medi if (worker_accel_type == adapter->accel_type()){ vpp_param.reserved = reinterpret_cast(w.get()); GAPI_LOG_DEBUG(nullptr, "selected worker: " << vpp_param.reserved); + worker_found = true; break; } } } - return param; + return worker_found ? param : cv::util::optional{}; } pp_session VPPPreprocDispatcher::initialize_preproc(const pp_params& initial_frame_param, @@ -78,8 +82,24 @@ cv::MediaFrame VPPPreprocDispatcher::run_sync(const pp_session &session_handle, } GAPI_Assert(false && "Cannot invoke VPP preproc in dispatcher, no suitable worker"); } + +#else // HAVE_ONEVPL +cv::util::optional VPPPreprocDispatcher::is_applicable(const cv::MediaFrame&) { + return cv::util::optional{}; +} + +pp_session VPPPreprocDispatcher::initialize_preproc(const pp_params&, + const GFrameDesc&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +cv::MediaFrame VPPPreprocDispatcher::run_sync(const pp_session &, + const cv::MediaFrame&, + const cv::util::optional &) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} +#endif // HAVE_ONEVPL } // namespace onevpl } // namespace wip } // namespace gapi } // namespace cv -#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp index 6e2ebc81f9..ea808bd542 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp @@ -11,10 +11,6 @@ #include #include "streaming/onevpl/engine/preproc_engine_interface.hpp" -#include "streaming/onevpl/engine/preproc_defines.hpp" - -#ifdef HAVE_ONEVPL -#include "streaming/onevpl/onevpl_export.hpp" namespace cv { namespace gapi { @@ -49,5 +45,4 @@ private: } // namespace wip } // namespace gapi } // namespace cv -#endif // HAVE_ONEVPL #endif // GAPI_STREAMING_ONEVPL_PREPROC_DISPATCHER_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp index d205211903..ec27a6422d 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp @@ -51,7 +51,7 @@ void apply_roi(mfxFrameSurface1* surface_handle, VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr&& accel) : ProcessingEngineBase(std::move(accel)) { - GAPI_LOG_INFO(nullptr, "Create VPP preprocessing engine"); + GAPI_LOG_DEBUG(nullptr, "Create VPP preprocessing engine"); preprocessed_frames_count = 0; create_pipeline( // 0) preproc decoded surface with VPP params @@ -455,7 +455,7 @@ ProcessingEngineBase::ExecutionStatus VPPPreprocEngine::process_error(mfxStatus "MFX_ERR_REALLOC_SURFACE is not processed"); break; case MFX_WRN_IN_EXECUTION: - GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] got MFX_WRN_IN_EXECUTION"); + GAPI_LOG_DEBUG(nullptr, "[" << sess.session << "] got MFX_WRN_IN_EXECUTION"); return ExecutionStatus::Continue; default: GAPI_LOG_WARNING(nullptr, "Unknown status code: " << mfxstatus_to_string(status) << diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc_defines.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc_defines.hpp index 5f68d9c4f7..be215fec74 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc_defines.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc_defines.hpp @@ -23,8 +23,8 @@ namespace wip { #else // VPP_PREPROC_ENGINE struct empty_pp_params {}; struct empty_pp_session {}; -#define GAPI_BACKEND_PP_PARAMS cv::gapi::wip::empty_pp_params; -#define GAPI_BACKEND_PP_SESSIONS cv::gapi::wip::empty_pp_session; +#define GAPI_BACKEND_PP_PARAMS cv::gapi::wip::empty_pp_params +#define GAPI_BACKEND_PP_SESSIONS cv::gapi::wip::empty_pp_session #endif // VPP_PREPROC_ENGINE struct pp_params { diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp new file mode 100644 index 0000000000..ff9f103b5a --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp @@ -0,0 +1,83 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2022 Intel Corporation + +#include +#include "streaming/onevpl/engine/preproc_engine_interface.hpp" +#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp" + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/onevpl_export.hpp" +#include "streaming/onevpl/engine/preproc/preproc_engine.hpp" + +#include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/cfg_param_device_selector.hpp" +#include "streaming/onevpl/cfg_params_parser.hpp" + +#endif //HAVE_ONEVPL + +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { + +template +std::unique_ptr +IPreprocEngine::create_preproc_engine_impl(const PreprocEngineArgs& ...) { + GAPI_Assert(false && "Unsupported "); +} + +template <> +std::unique_ptr +IPreprocEngine::create_preproc_engine_impl(const onevpl::Device &device, + const onevpl::Context &context) { + using namespace onevpl; + cv::util::suppress_unused_warning(device); + cv::util::suppress_unused_warning(context); + std::unique_ptr dispatcher(new VPPPreprocDispatcher); +#ifdef HAVE_ONEVPL + if (device.get_type() == onevpl::AccelType::DX11) { + bool gpu_pp_is_created = false; +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + GAPI_LOG_INFO(nullptr, "Creating DX11 VPP preprocessing engine"); + // create GPU VPP preproc engine + dispatcher->insert_worker( + std::unique_ptr{ + new VPLDX11AccelerationPolicy( + std::make_shared( + device, context, CfgParams{})) + }); + GAPI_LOG_INFO(nullptr, "DX11 VPP preprocessing engine created"); + gpu_pp_is_created = true; +#endif +#endif + GAPI_Assert(gpu_pp_is_created && "VPP preproc for GPU is requested, but it is avaiable only for DX11 at now"); + } else { + GAPI_LOG_INFO(nullptr, "Creating CPU VPP preprocessing engine"); + dispatcher->insert_worker( + std::unique_ptr{ + new VPLCPUAccelerationPolicy( + std::make_shared(CfgParams{}))}); + GAPI_LOG_INFO(nullptr, "CPU VPP preprocessing engine created"); + } +#endif // HAVE_ONEVPL + return dispatcher; +} + + +// Force instantiation +template +std::unique_ptr +IPreprocEngine::create_preproc_engine_impl + (const onevpl::Device &device, + const onevpl::Context &ctx); +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp index be347a258f..72c1dbd0a7 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp @@ -29,6 +29,16 @@ struct IPreprocEngine { virtual cv::MediaFrame run_sync(const pp_session &sess, const cv::MediaFrame& in_frame, const cv::util::optional &opt_roi = {}) = 0; + + template + static std::unique_ptr create_preproc_engine(const PreprocEngineArgs& ...args) { + static_assert(std::is_base_of::value, + "SpecificPreprocEngine must have reachable ancessor IPreprocEngine"); + return create_preproc_engine_impl(args...); + } +private: + template + static std::unique_ptr create_preproc_engine_impl(const PreprocEngineArgs &...args); }; } // namespace wip } // namespace gapi diff --git a/modules/gapi/src/streaming/onevpl/source.cpp b/modules/gapi/src/streaming/onevpl/source.cpp index e5b045188d..3bad463e41 100644 --- a/modules/gapi/src/streaming/onevpl/source.cpp +++ b/modules/gapi/src/streaming/onevpl/source.cpp @@ -33,6 +33,13 @@ GSource::GSource(const std::string& filePath, accel_ctx_ptr, cfg_params)) { } +GSource::GSource(const std::string& filePath, + const CfgParams& cfg_params, + const Device &device, const Context &ctx) : + GSource(filePath, cfg_params, + std::make_shared(device, ctx, cfg_params)) { +} + GSource::GSource(const std::string& filePath, const CfgParams& cfg_params, std::shared_ptr selector) : @@ -74,6 +81,10 @@ GSource::GSource(const std::string&, const CfgParams&, const std::string&, GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); } +GSource::GSource(const std::string&, const CfgParams&, const Device &, const Context &) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + GSource::GSource(const std::string&, const CfgParams&, std::shared_ptr) { GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); }