From 46fb88c76fbe8d4b7ff3cc8eb53e5842753f6c3e Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Tue, 17 Aug 2021 20:11:22 +0300 Subject: [PATCH] Merge pull request #20546 from sivanov-work:initial_vpl_source G-API: oneVPL (simplification) source base commit * oneVPL source initial * Fix compilation * Fix compilation path * Fix NO VPL compile * Fix unused vars * Fix unused vars in example * Simplify oneVPL search: no custom path & download * Fix standalone GAPI * Apply comments --- modules/gapi/CMakeLists.txt | 15 ++ modules/gapi/cmake/init.cmake | 7 + modules/gapi/cmake/standalone.cmake | 7 + .../gapi/streaming/onevpl/onevpl_source.hpp | 44 +++ .../gapi/samples/onevpl_infer_single_roi.cpp | 254 ++++++++++++++++++ .../src/streaming/onevpl/onevpl_source.cpp | 48 ++++ .../streaming/onevpl/onevpl_source_priv.cpp | 63 +++++ .../streaming/onevpl/onevpl_source_priv.hpp | 62 +++++ 8 files changed, 500 insertions(+) create mode 100644 modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp create mode 100644 modules/gapi/samples/onevpl_infer_single_roi.cpp create mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source.cpp create mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp create mode 100644 modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index c5046e8be6..69c0aaaae8 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -163,6 +163,10 @@ set(gapi_srcs src/backends/ie/bindings_ie.cpp src/backends/python/gpythonbackend.cpp + # Streaming source + src/streaming/onevpl/onevpl_source.cpp + src/streaming/onevpl/onevpl_source_priv.cpp + # Utils (ITT tracing) src/utils/itt.cpp ) @@ -234,6 +238,17 @@ if(HAVE_PLAIDML) ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${PLAIDML_INCLUDE_DIRS}) endif() +if(HAVE_GAPI_ONEVPL) + if(TARGET opencv_test_gapi) + ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_ONEVPL) + ocv_target_link_libraries(opencv_test_gapi PRIVATE ${VPL_IMPORTED_TARGETS}) + endif() + ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_ONEVPL) + ocv_target_link_libraries(${the_module} PRIVATE ${VPL_IMPORTED_TARGETS}) + if(HAVE_D3D11 AND HAVE_OPENCL) + ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + endif() +endif() if(WIN32) # Required for htonl/ntohl on Windows diff --git a/modules/gapi/cmake/init.cmake b/modules/gapi/cmake/init.cmake index 4c25c75f55..1c464328ca 100644 --- a/modules/gapi/cmake/init.cmake +++ b/modules/gapi/cmake/init.cmake @@ -32,3 +32,10 @@ if(WITH_PLAIDML) set(HAVE_PLAIDML TRUE) endif() endif() + +if(WITH_GAPI_ONEVPL) + find_package(VPL) + if(VPL_FOUND) + set(HAVE_GAPI_ONEVPL TRUE) + endif() +endif() diff --git a/modules/gapi/cmake/standalone.cmake b/modules/gapi/cmake/standalone.cmake index d08eda1be5..f81c1c8a85 100644 --- a/modules/gapi/cmake/standalone.cmake +++ b/modules/gapi/cmake/standalone.cmake @@ -6,6 +6,13 @@ if (NOT TARGET ade ) find_package(ade 0.1.0 REQUIRED) endif() +if (WITH_GAPI_ONEVPL) + find_package(VPL) + if(VPL_FOUND) + set(HAVE_GAPI_ONEVPL TRUE) + endif() +endif() + set(FLUID_TARGET fluid) set(FLUID_ROOT "${CMAKE_CURRENT_LIST_DIR}/../") diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp new file mode 100644 index 0000000000..fec8c73dff --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/onevpl_source.hpp @@ -0,0 +1,44 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP + +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { + +class GAPI_EXPORTS OneVPLSource : public IStreamSource +{ +public: + struct Priv; + + explicit OneVPLSource(const std::string& filePath); + ~OneVPLSource() override; + + bool pull(cv::gapi::wip::Data& data) override; + GMetaArg descr_of() const override; + +private: + explicit OneVPLSource(std::unique_ptr&& impl); + std::unique_ptr m_priv; +}; + +template +GAPI_EXPORTS_W cv::Ptr inline make_vpl_src(const std::string& filePath, Args&&... args) +{ + return make_src(filePath, std::forward(args)...); +} + +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp new file mode 100644 index 0000000000..8a7efafabf --- /dev/null +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -0,0 +1,254 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include // CommandLineParser + +const std::string about = + "This is an OpenCV-based version of oneVPLSource decoder example"; +const std::string keys = + "{ h help | | Print this help message }" + "{ input | | Path to the input demultiplexed video file }" + "{ output | | Path to the output RAW video file. Use .avi extension }" + "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" + "{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }"; + +namespace { +std::string get_weights_path(const std::string &model_path) { + const auto EXT_LEN = 4u; + const auto sz = model_path.size(); + CV_Assert(sz > EXT_LEN); + + auto ext = model_path.substr(sz - EXT_LEN); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ + return static_cast(std::tolower(c)); + }); + CV_Assert(ext == ".xml"); + return model_path.substr(0u, sz - EXT_LEN) + ".bin"; +} +} // anonymous namespace + +namespace custom { +G_API_NET(FaceDetector, , "face-detector"); + +using GDetections = cv::GArray; +using GRect = cv::GOpaque; +using GSize = cv::GOpaque; +using GPrims = cv::GArray; + +G_API_OP(LocateROI, , "sample.custom.locate-roi") { + static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) { + return cv::empty_gopaque_desc(); + } +}; + +G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(BBoxes, , "sample.custom.b-boxes") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { + // This is the place where we can run extra analytics + // on the input image frame and select the ROI (region + // of interest) where we want to detect our objects (or + // run any other inference). + // + // Currently it doesn't do anything intelligent, + // but only crops the input image to square (this is + // the most convenient aspect ratio for detectors to use) + + static void run(const cv::Size& in_size, cv::Rect &out_rect) { + + // Identify the central point & square size (- some padding) + const auto center = cv::Point{in_size.width/2, in_size.height/2}; + auto sqside = std::min(in_size.width, in_size.height); + + // Now build the central square ROI + out_rect = cv::Rect{ center.x - sqside/2 + , center.y - sqside/2 + , sqside + , sqside + }; + } +}; + +GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { + static void run(const cv::Mat &in_ssd_result, + const cv::Rect &in_roi, + const cv::Size &in_parent_size, + std::vector &out_objects) { + const auto &in_ssd_dims = in_ssd_result.size; + CV_Assert(in_ssd_dims.dims() == 4u); + + const int MAX_PROPOSALS = in_ssd_dims[2]; + const int OBJECT_SIZE = in_ssd_dims[3]; + CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size + + const cv::Size up_roi = in_roi.size(); + const cv::Rect surface({0,0}, in_parent_size); + + out_objects.clear(); + + const float *data = in_ssd_result.ptr(); + for (int i = 0; i < MAX_PROPOSALS; i++) { + const float image_id = data[i * OBJECT_SIZE + 0]; + const float label = data[i * OBJECT_SIZE + 1]; + const float confidence = data[i * OBJECT_SIZE + 2]; + const float rc_left = data[i * OBJECT_SIZE + 3]; + const float rc_top = data[i * OBJECT_SIZE + 4]; + const float rc_right = data[i * OBJECT_SIZE + 5]; + const float rc_bottom = data[i * OBJECT_SIZE + 6]; + (void) label; // unused + + if (image_id < 0.f) { + break; // marks end-of-detections + } + if (confidence < 0.5f) { + continue; // skip objects with low confidence + } + + // map relative coordinates to the original image scale + // taking the ROI into account + cv::Rect rc; + rc.x = static_cast(rc_left * up_roi.width); + rc.y = static_cast(rc_top * up_roi.height); + rc.width = static_cast(rc_right * up_roi.width) - rc.x; + rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; + rc.x += in_roi.x; + rc.y += in_roi.y; + out_objects.emplace_back(rc & surface); + } + } +}; + +GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { + // This kernel converts the rectangles into G-API's + // rendering primitives + static void run(const std::vector &in_face_rcs, + const cv::Rect &in_roi, + std::vector &out_prims) { + out_prims.clear(); + const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) { + return cv::gapi::wip::draw::Rect(rc, clr, 2); + }; + out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan + for (auto &&rc : in_face_rcs) { + out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green + } + } +}; + +} // namespace custom + +int main(int argc, char *argv[]) { + + cv::CommandLineParser cmd(argc, argv, keys); + cmd.about(about); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + + // get file name + std::string file_path = cmd.get("input"); + const std::string output = cmd.get("output"); + const auto face_model_path = cmd.get("facem"); + + // check ouput file extension + if (!output.empty()) { + auto ext = output.find_last_of("."); + if (ext == std::string::npos || (output.substr(ext + 1) != "avi")) { + std::cerr << "Output file should have *.avi extension for output video" << std::endl; + return -1; + } + } + + auto face_net = cv::gapi::ie::Params { + face_model_path, // path to topology IR + get_weights_path(face_model_path), // path to weights + cmd.get("faced"), // device specifier + }; + auto kernels = cv::gapi::kernels + < custom::OCVLocateROI + , custom::OCVParseSSD + , custom::OCVBBoxes>(); + auto networks = cv::gapi::networks(face_net); + + // Create source + cv::Ptr cap; + try { + cap = cv::gapi::wip::make_vpl_src(file_path); + std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl; + } catch (const std::exception& ex) { + std::cerr << "Cannot create source: " << ex.what() << std::endl; + return -1; + } + + cv::GMetaArg descr = cap->descr_of(); + auto frame_descr = cv::util::get(descr); + + // Now build the graph + cv::GFrame in; + auto size = cv::gapi::streaming::size(in); + auto roi = custom::LocateROI::on(size); + auto blob = cv::gapi::infer(roi, in); + auto rcs = custom::ParseSSD::on(blob, roi, size); + auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi)); + auto out = cv::gapi::streaming::BGR(out_frame); + + cv::GStreamingCompiled pipeline; + try { + pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + .compileStreaming(cv::compile_args(kernels, networks)); + } catch (const std::exception& ex) { + std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl; + return -1; + } + // The execution part + + // TODO USE may set pool size from outside and set queue_capacity size, + // compile arg: cv::gapi::streaming::queue_capacity + pipeline.setSource(std::move(cap)); + pipeline.start(); + + int framesCount = 0; + cv::TickMeter t; + cv::VideoWriter writer; + if (!output.empty() && !writer.isOpened()) { + const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height}; + writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); + CV_Assert(writer.isOpened()); + } + + cv::Mat outMat; + t.start(); + while (pipeline.pull(cv::gout(outMat))) { + cv::imshow("Out", outMat); + cv::waitKey(1); + if (!output.empty()) { + writer << outMat; + } + framesCount++; + } + t.stop(); + std::cout << "Elapsed time: " << t.getTimeSec() << std::endl; + std::cout << "FPS: " << framesCount / t.getTimeSec() << std::endl; + std::cout << "framesCount: " << framesCount << std::endl; + + return 0; +} diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source.cpp b/modules/gapi/src/streaming/onevpl/onevpl_source.cpp new file mode 100644 index 0000000000..988986f6d9 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/onevpl_source.cpp @@ -0,0 +1,48 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include "streaming/onevpl/onevpl_source_priv.hpp" + +namespace cv { +namespace gapi { +namespace wip { + +#ifdef HAVE_ONEVPL +OneVPLSource::OneVPLSource(const std::string& filePath) : + OneVPLSource(std::unique_ptr(new OneVPLSource::Priv(filePath))) { + + if (filePath.empty()) { + util::throw_error(std::logic_error("Cannot create 'OneVPLSource' on empty source file name")); + } +} +#else +OneVPLSource::OneVPLSource(const std::string&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} +#endif + +OneVPLSource::OneVPLSource(std::unique_ptr&& impl) : + IStreamSource(), + m_priv(std::move(impl)) { +} + +OneVPLSource::~OneVPLSource() { +} + +bool OneVPLSource::pull(cv::gapi::wip::Data& data) +{ + return m_priv->pull(data); +} + +GMetaArg OneVPLSource::descr_of() const +{ + return m_priv->descr_of(); +} +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp new file mode 100644 index 0000000000..5c4e8e6941 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.cpp @@ -0,0 +1,63 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include +#include + +#include "streaming/onevpl/onevpl_source_priv.hpp" +#include "logger.hpp" + +#ifndef HAVE_ONEVPL +namespace cv { +namespace gapi { +namespace wip { +bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) { + return true; +} +GMetaArg OneVPLSource::Priv::descr_of() const { + return {}; +} +} // namespace wip +} // namespace gapi +} // namespace cv + +#else // HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +OneVPLSource::Priv::Priv() : + mfx_handle(MFXLoad()) +{ + GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); + description_is_valid = false; +} + +OneVPLSource::Priv::Priv(const std::string&) : + OneVPLSource::Priv() +{ +} + +OneVPLSource::Priv::~Priv() +{ + GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle); + MFXUnload(mfx_handle); +} + +bool OneVPLSource::Priv::pull(cv::gapi::wip::Data&) +{ + return false; +} + +GMetaArg OneVPLSource::Priv::descr_of() const +{ + return {}; +} +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp new file mode 100644 index 0000000000..b139add993 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/onevpl_source_priv.hpp @@ -0,0 +1,62 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP + +#include + +#include +#include + +#include +#include +#include + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { + +struct OneVPLSource::Priv +{ + explicit Priv(const std::string& file_path); + ~Priv(); + + bool pull(cv::gapi::wip::Data& data); + GMetaArg descr_of() const; +private: + Priv(); + mfxLoader mfx_handle; + bool description_is_valid; +}; +} // namespace wip +} // namespace gapi +} // namespace cv + +#else // HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +struct OneVPLSource::Priv final +{ + bool pull(cv::gapi::wip::Data&); + GMetaArg descr_of() const; +}; +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP