diff --git a/modules/gapi/include/opencv2/gapi/infer.hpp b/modules/gapi/include/opencv2/gapi/infer.hpp index f20f638017..8e3baedb79 100644 --- a/modules/gapi/include/opencv2/gapi/infer.hpp +++ b/modules/gapi/include/opencv2/gapi/infer.hpp @@ -16,6 +16,7 @@ #include // tuple #include // is_same, false_type +#include // all_satisfy #include // any<> #include // GKernelType[M], GBackend #include // GArg @@ -27,40 +28,54 @@ namespace cv { template class GNetworkType; namespace detail { - template - struct valid_infer2_types; - - // Terminal case 1 (50/50 success) - template - struct valid_infer2_types< std::tuple, std::tuple > { - // By default, Nets are limited to GMat argument types only - // for infer2, every GMat argument may translate to either - // GArray or GArray. GArray<> part is stripped - // already at this point. - static constexpr const auto value = - std::is_same::type, cv::GMat>::value - || std::is_same::type, cv::Rect>::value; - }; - - // Terminal case 2 (100% failure) - template - struct valid_infer2_types< std::tuple<>, std::tuple > - : public std::false_type { - }; - - // Terminal case 3 (100% failure) - template - struct valid_infer2_types< std::tuple, std::tuple<> > - : public std::false_type { - }; - - // Recursion -- generic - template - struct valid_infer2_types< std::tuple, std::tuple > { - static constexpr const auto value = - valid_infer2_types< std::tuple, std::tuple >::value - && valid_infer2_types< std::tuple, std::tuple >::value; - }; + +// Infer /////////////////////////////////////////////////////////////////////// +template +struct accepted_infer_types { + static constexpr const auto value = + std::is_same::type, cv::GMat>::value + || std::is_same::type, cv::GFrame>::value; +}; + +template +using valid_infer_types = all_satisfy; + +// Infer2 ////////////////////////////////////////////////////////////////////// + +template +struct valid_infer2_types; + +// Terminal case 1 (50/50 success) +template +struct valid_infer2_types< std::tuple, std::tuple > { + // By default, Nets are limited to GMat argument types only + // for infer2, every GMat argument may translate to either + // GArray or GArray. GArray<> part is stripped + // already at this point. + static constexpr const auto value = + std::is_same::type, cv::GMat>::value + || std::is_same::type, cv::Rect>::value; +}; + +// Terminal case 2 (100% failure) +template +struct valid_infer2_types< std::tuple<>, std::tuple > + : public std::false_type { +}; + +// Terminal case 3 (100% failure) +template +struct valid_infer2_types< std::tuple, std::tuple<> > + : public std::false_type { +}; + +// Recursion -- generic +template +struct valid_infer2_types< std::tuple, std::tuple > { + static constexpr const auto value = + valid_infer2_types< std::tuple, std::tuple >::value + && valid_infer2_types< std::tuple, std::tuple >::value; +}; } // namespace detail // TODO: maybe tuple_wrap_helper from util.hpp may help with this. @@ -76,10 +91,6 @@ public: using API = std::function; using ResultL = std::tuple< cv::GArray... >; - using APIList = std::function, Args...)>; - - // FIXME: Args... must be limited to a single GMat - using APIRoi = std::function, Args...)>; }; // Single-return-value network definition (specialized base class) @@ -94,20 +105,48 @@ public: using API = std::function; using ResultL = cv::GArray; - using APIList = std::function, Args...)>; +}; - // FIXME: Args... must be limited to a single GMat - using APIRoi = std::function, Args...)>; +// InferAPI: Accepts either GMat or GFrame for very individual network's input +template +struct InferAPI { + using type = typename std::enable_if + < detail::valid_infer_types::value + && std::tuple_size::value == sizeof...(Ts) + , std::function + >::type; +}; + +// InferAPIRoi: Accepts a rectangle and either GMat or GFrame +template +struct InferAPIRoi { + using type = typename std::enable_if + < detail::valid_infer_types::value + && std::tuple_size::value == 1u + , std::function, T)> + >::type; +}; + +// InferAPIList: Accepts a list of rectangles and list of GMat/GFrames; +// crops every input. +template +struct InferAPIList { + using type = typename std::enable_if + < detail::valid_infer_types::value + && std::tuple_size::value == sizeof...(Ts) + , std::function, Ts...)> + >::type; }; // APIList2 is also template to allow different calling options // (GArray vs GArray per input) -template +template struct InferAPIList2 { using type = typename std::enable_if - < cv::detail::valid_infer2_types< typename Net::InArgs + < detail::valid_infer_types::value && + cv::detail::valid_infer2_types< typename Net::InArgs , std::tuple >::value, - std::function...)> + std::function...)> >::type; }; @@ -206,11 +245,11 @@ struct GInferList2Base { // A generic inference kernel. API (::on()) is fully defined by the Net // template parameter. // Acts as a regular kernel in graph (via KernelTypeMedium). -template +template struct GInfer final : public GInferBase - , public detail::KernelTypeMedium< GInfer - , typename Net::API > { + , public detail::KernelTypeMedium< GInfer + , typename InferAPI::type > { using GInferBase::getOutMeta; // FIXME: name lookup conflict workaround? static constexpr const char* tag() { return Net::tag(); } @@ -218,11 +257,11 @@ struct GInfer final // A specific roi-inference kernel. API (::on()) is fixed here and // verified against Net. -template +template struct GInferROI final : public GInferROIBase - , public detail::KernelTypeMedium< GInferROI - , typename Net::APIRoi > { + , public detail::KernelTypeMedium< GInferROI + , typename InferAPIRoi::type > { using GInferROIBase::getOutMeta; // FIXME: name lookup conflict workaround? static constexpr const char* tag() { return Net::tag(); } @@ -231,11 +270,11 @@ struct GInferROI final // A generic roi-list inference kernel. API (::on()) is derived from // the Net template parameter (see more in infer<> overload). -template +template struct GInferList final : public GInferListBase - , public detail::KernelTypeMedium< GInferList - , typename Net::APIList > { + , public detail::KernelTypeMedium< GInferList + , typename InferAPIList::type > { using GInferListBase::getOutMeta; // FIXME: name lookup conflict workaround? static constexpr const char* tag() { return Net::tag(); } @@ -246,11 +285,11 @@ struct GInferList final // overload). // Takes an extra variadic template list to reflect how this network // was called (with Rects or GMats as array parameters) -template +template struct GInferList2 final : public GInferList2Base - , public detail::KernelTypeMedium< GInferList2 - , typename InferAPIList2::type > { + , public detail::KernelTypeMedium< GInferList2 + , typename InferAPIList2::type > { using GInferList2Base::getOutMeta; // FIXME: name lookup conflict workaround? static constexpr const char* tag() { return Net::tag(); } @@ -280,9 +319,9 @@ namespace gapi { * objects of appropriate type is returned. * @sa G_API_NET() */ -template -typename Net::Result infer(cv::GOpaque roi, cv::GMat in) { - return GInferROI::on(roi, in); +template +typename Net::Result infer(cv::GOpaque roi, T in) { + return GInferROI::on(roi, in); } /** @brief Calculates responses for the specified network (template @@ -300,7 +339,7 @@ typename Net::Result infer(cv::GOpaque roi, cv::GMat in) { */ template typename Net::ResultL infer(cv::GArray roi, Args&&... args) { - return GInferList::on(roi, std::forward(args)...); + return GInferList::on(roi, std::forward(args)...); } /** @brief Calculates responses for the specified network (template @@ -320,11 +359,12 @@ typename Net::ResultL infer(cv::GArray roi, Args&&... args) { * GArray<> objects is returned with the appropriate types inside. * @sa G_API_NET() */ -template -typename Net::ResultL infer2(cv::GMat image, cv::GArray... args) { + +template +typename Net::ResultL infer2(T image, cv::GArray... args) { // FIXME: Declared as "2" because in the current form it steals // overloads from the regular infer - return GInferList2::on(image, args...); + return GInferList2::on(image, args...); } /** @@ -340,7 +380,7 @@ typename Net::ResultL infer2(cv::GMat image, cv::GArray... args) { */ template typename Net::Result infer(Args&&... args) { - return GInfer::on(std::forward(args)...); + return GInfer::on(std::forward(args)...); } /** diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 85c0236ff1..3cbe24364a 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include "compiler/gobjref.hpp" #include "compiler/gmodel.hpp" @@ -45,6 +46,10 @@ #include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK! +#if INF_ENGINE_RELEASE < 2021010000 +#include "ie_compound_blob.h" +#endif + namespace IE = InferenceEngine; namespace { @@ -151,6 +156,25 @@ inline IE::Blob::Ptr wrapIE(const cv::Mat &mat, cv::gapi::ie::TraitAs hint) { return IE::Blob::Ptr{}; } +inline IE::Blob::Ptr wrapIE(const cv::MediaFrame::View& view, + const cv::GFrameDesc& desc) { + + switch (desc.fmt) { + case cv::MediaFormat::BGR: { + auto bgr = cv::Mat(desc.size, CV_8UC3, view.ptr[0], view.stride[0]); + return wrapIE(bgr, cv::gapi::ie::TraitAs::IMAGE); + } + case cv::MediaFormat::NV12: { + auto y_plane = cv::Mat(desc.size, CV_8UC1, view.ptr[0], view.stride[0]); + auto uv_plane = cv::Mat(desc.size / 2, CV_8UC2, view.ptr[1], view.stride[1]); + return cv::gapi::ie::util::to_ie(y_plane, uv_plane); + } + default: + GAPI_Assert(false && "Unsupported media format for IE backend"); + } + GAPI_Assert(false); +} + template inline void copyFromIE(const IE::Blob::Ptr &blob, MatType &mat) { switch (blob->getTensorDesc().getPrecision()) { @@ -256,6 +280,7 @@ struct IECallContext { // Input parameters passed to an inference operation. std::vector args; + cv::GShapes in_shapes; //FIXME: avoid conversion of arguments from internal representation to OpenCV one on each call //to OCV kernel. (This can be achieved by a two single time conversions in GCPUExecutable::run, @@ -267,6 +292,10 @@ struct IECallContext template const T& inArg(std::size_t input) { return args.at(input).get(); } + const cv::MediaFrame& inFrame(std::size_t input) { + return inArg(input); + } + // Syntax sugar const cv::Mat& inMat(std::size_t input) { return inArg(input); @@ -319,6 +348,24 @@ using GConstGIEModel = ade::ConstTypedGraph , IEUnit , IECallable >; + +using Views = std::vector>; + +inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i, Views& views) { + switch (ctx.in_shapes[i]) { + case cv::GShape::GFRAME: { + const auto& frame = ctx.inFrame(i); + views.emplace_back(new cv::MediaFrame::View(frame.access(cv::MediaFrame::Access::R))); + return wrapIE(*views.back(), frame.desc()); + } + case cv::GShape::GMAT: { + return wrapIE(ctx.inMat(i), cv::gapi::ie::TraitAs::IMAGE); + } + default: + GAPI_Assert("Unsupported input shape for IE backend"); + } + GAPI_Assert(false); +} } // anonymous namespace // GCPUExcecutable implementation ////////////////////////////////////////////// @@ -384,6 +431,8 @@ cv::GArg cv::gimpl::ie::GIEExecutable::packArg(const cv::GArg &arg) { // (and constructed by either bindIn/Out or resetInternal) case GShape::GOPAQUE: return GArg(m_res.slot().at(ref.id)); + case GShape::GFRAME: return GArg(m_res.slot().at(ref.id)); + default: util::throw_error(std::logic_error("Unsupported GShape type")); break; @@ -413,6 +462,12 @@ void cv::gimpl::ie::GIEExecutable::run(std::vector &&input_objs, std::back_inserter(context.args), std::bind(&GIEExecutable::packArg, this, _1)); + // NB: Need to store inputs shape to recognize GFrame/GMat + ade::util::transform(op.args, + std::back_inserter(context.in_shapes), + [](const cv::GArg& arg) { + return arg.get().shape; + }); // - Output parameters. for (const auto &out_it : ade::util::indexed(op.outs)) { // FIXME: Can the same GArg type resolution mechanism be reused here? @@ -438,6 +493,34 @@ namespace cv { namespace gimpl { namespace ie { +static void configureInputInfo(const IE::InputInfo::Ptr& ii, const cv::GMetaArg mm) { + switch (mm.index()) { + case cv::GMetaArg::index_of(): + { + ii->setPrecision(toIE(util::get(mm).depth)); + break; + } + case cv::GMetaArg::index_of(): + { + const auto &meta = util::get(mm); + switch (meta.fmt) { + case cv::MediaFormat::NV12: + ii->getPreProcess().setColorFormat(IE::ColorFormat::NV12); + break; + case cv::MediaFormat::BGR: + // NB: Do nothing + break; + default: + GAPI_Assert(false && "Unsupported media format for IE backend"); + } + ii->setPrecision(toIE(CV_8U)); + break; + } + default: + util::throw_error(std::runtime_error("Unsupported input meta for IE backend")); + } +} + struct Infer: public cv::detail::KernelTag { using API = cv::GInferBase; static cv::gapi::GBackend backend() { return cv::gapi::ie::backend(); } @@ -468,11 +551,7 @@ struct Infer: public cv::detail::KernelTag { auto &&ii = uu.inputs.at(std::get<0>(it)); const auto & mm = std::get<1>(it); - GAPI_Assert(util::holds_alternative(mm) - && "Non-GMat inputs are not supported"); - - const auto &meta = util::get(mm); - ii->setPrecision(toIE(meta.depth)); + configureInputInfo(ii, mm); ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } @@ -495,15 +574,12 @@ struct Infer: public cv::detail::KernelTag { static void run(IECompiled &iec, const IEUnit &uu, IECallContext &ctx) { // non-generic version for now: // - assumes all inputs/outputs are always Mats + Views views; for (auto i : ade::util::iota(uu.params.num_in)) { // TODO: Ideally we shouldn't do SetBlob() but GetBlob() instead, // and redirect our data producers to this memory // (A memory dialog comes to the picture again) - - const cv::Mat this_mat = ctx.inMat(i); - // FIXME: By default here we trait our inputs as images. - // May be we need to make some more intelligence here about it - IE::Blob::Ptr this_blob = wrapIE(this_mat, cv::gapi::ie::TraitAs::IMAGE); + IE::Blob::Ptr this_blob = extractBlob(ctx, i, views); iec.this_request.SetBlob(uu.params.input_names[i], this_blob); } iec.this_request.Infer(); @@ -540,10 +616,10 @@ struct InferROI: public cv::detail::KernelTag { GAPI_Assert(1u == uu.params.input_names.size()); GAPI_Assert(2u == in_metas.size()); - // 0th is ROI, 1st is in0put image - auto &&ii = uu.inputs.at(uu.params.input_names.at(0)); - const auto &meta = util::get(in_metas.at(1)); - ii->setPrecision(toIE(meta.depth)); + // 0th is ROI, 1st is input image + auto &&ii = uu.inputs.at(uu.params.input_names.at(0)); + auto &&mm = in_metas.at(1u); + configureInputInfo(ii, mm); ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); // FIXME: It would be nice here to have an exact number of network's @@ -566,10 +642,12 @@ struct InferROI: public cv::detail::KernelTag { // non-generic version for now, per the InferROI's definition GAPI_Assert(uu.params.num_in == 1); const auto& this_roi = ctx.inArg(0).rref(); - const auto this_mat = ctx.inMat(1); - IE::Blob::Ptr this_blob = wrapIE(this_mat, cv::gapi::ie::TraitAs::IMAGE); - IE::Blob::Ptr roi_blob = IE::make_shared_blob(this_blob, toIE(this_roi)); - iec.this_request.SetBlob(*uu.params.input_names.begin(), roi_blob); + + Views views; + IE::Blob::Ptr this_blob = extractBlob(ctx, 1, views); + + iec.this_request.SetBlob(*uu.params.input_names.begin(), + IE::make_shared_blob(this_blob, toIE(this_roi))); iec.this_request.Infer(); for (auto i : ade::util::iota(uu.params.num_out)) { cv::Mat& out_mat = ctx.outMatR(i); @@ -606,12 +684,7 @@ struct InferList: public cv::detail::KernelTag { for (auto &&input_name : uu.params.input_names) { auto &&ii = uu.inputs.at(input_name); const auto & mm = in_metas[idx++]; - - GAPI_Assert(util::holds_alternative(mm) - && "Non-GMat inputs are not supported"); - - const auto &meta = util::get(mm); - ii->setPrecision(toIE(meta.depth)); + configureInputInfo(ii, mm); ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } @@ -630,9 +703,9 @@ struct InferList: public cv::detail::KernelTag { GAPI_Assert(uu.params.num_in == 1); // roi list is not counted in net's inputs const auto& in_roi_vec = ctx.inArg(0u).rref(); - const cv::Mat this_mat = ctx.inMat(1u); - // Since we do a ROI list inference, always assume our input buffer is image - IE::Blob::Ptr this_blob = wrapIE(this_mat, cv::gapi::ie::TraitAs::IMAGE); + + Views views; + IE::Blob::Ptr this_blob = extractBlob(ctx, 1, views); // FIXME: This could be done ONCE at graph compile stage! std::vector< std::vector > cached_dims(uu.params.num_out); @@ -696,11 +769,30 @@ struct InferList2: public cv::detail::KernelTag { // "blob"-based ones) // FIXME: this is filtering not done, actually! GArrayDesc has // no hint for its underlying type! - const auto &mm_0 = in_metas[0u]; - const auto &meta_0 = util::get(mm_0); - GAPI_Assert( !meta_0.isND() + const auto &mm_0 = in_metas[0u]; + switch (in_metas[0u].index()) { + case cv::GMetaArg::index_of(): { + const auto &meta_0 = util::get(mm_0); + GAPI_Assert( !meta_0.isND() + && !meta_0.planar + && "Only images are supported as the 0th argument"); + break; + } + case cv::GMetaArg::index_of(): { + // FIXME: Is there any validation for GFrame ? + break; + } + default: + util::throw_error(std::runtime_error("Unsupported input meta for IE backend")); + } + + if (util::holds_alternative(mm_0)) { + const auto &meta_0 = util::get(mm_0); + GAPI_Assert( !meta_0.isND() && !meta_0.planar && "Only images are supported as the 0th argument"); + } + std::size_t idx = 1u; for (auto &&input_name : uu.params.input_names) { auto &ii = uu.inputs.at(input_name); @@ -710,7 +802,7 @@ struct InferList2: public cv::detail::KernelTag { if (op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_RECT) { // This is a cv::Rect -- configure the IE preprocessing - ii->setPrecision(toIE(meta_0.depth)); + configureInputInfo(ii, mm_0); ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } else { // This is a cv::GMat (equals to: cv::Mat) @@ -733,9 +825,8 @@ struct InferList2: public cv::detail::KernelTag { GAPI_Assert(ctx.args.size() > 1u && "This operation must have at least two arguments"); - // Since we do a ROI list inference, always assume our input buffer is image - const cv::Mat mat_0 = ctx.inMat(0u); - IE::Blob::Ptr blob_0 = wrapIE(mat_0, cv::gapi::ie::TraitAs::IMAGE); + Views views; + IE::Blob::Ptr blob_0 = extractBlob(ctx, 0, views); // Take the next argument, which must be vector (of any kind). // Use it only to obtain the ROI list size (sizes of all other @@ -869,6 +960,16 @@ IE::Blob::Ptr cv::gapi::ie::util::to_ie(cv::Mat &blob) { return wrapIE(blob, cv::gapi::ie::TraitAs::IMAGE); } +IE::Blob::Ptr cv::gapi::ie::util::to_ie(cv::Mat &y_plane, cv::Mat &uv_plane) { + auto y_blob = wrapIE(y_plane, cv::gapi::ie::TraitAs::IMAGE); + auto uv_blob = wrapIE(uv_plane, cv::gapi::ie::TraitAs::IMAGE); +#if INF_ENGINE_RELEASE >= 2021010000 + return IE::make_shared_blob(y_blob, uv_blob); +#else + return IE::make_shared_blob(y_blob, uv_blob); +#endif +} + #else // HAVE_INF_ENGINE cv::gapi::GBackend cv::gapi::ie::backend() { diff --git a/modules/gapi/src/backends/ie/util.hpp b/modules/gapi/src/backends/ie/util.hpp index b16ccbe0ce..080c88498f 100644 --- a/modules/gapi/src/backends/ie/util.hpp +++ b/modules/gapi/src/backends/ie/util.hpp @@ -28,6 +28,7 @@ namespace util { GAPI_EXPORTS std::vector to_ocv(const InferenceEngine::SizeVector &dims); GAPI_EXPORTS cv::Mat to_ocv(InferenceEngine::Blob::Ptr blob); GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(cv::Mat &blob); +GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(cv::Mat &y_plane, cv::Mat &uv_plane); }}}} diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp index 547c7c7d33..7b6f761b1c 100644 --- a/modules/gapi/test/infer/gapi_infer_ie_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_ie_test.cpp @@ -23,6 +23,45 @@ namespace opencv_test { namespace { +class TestMediaBGR final: public cv::MediaFrame::IAdapter { + cv::Mat m_mat; + using Cb = cv::MediaFrame::View::Callback; + Cb m_cb; + +public: + explicit TestMediaBGR(cv::Mat m, Cb cb = [](){}) + : m_mat(m), m_cb(cb) { + } + cv::GFrameDesc meta() const override { + return cv::GFrameDesc{cv::MediaFormat::BGR, cv::Size(m_mat.cols, m_mat.rows)}; + } + cv::MediaFrame::View access(cv::MediaFrame::Access) override { + cv::MediaFrame::View::Ptrs pp = { m_mat.ptr(), nullptr, nullptr, nullptr }; + cv::MediaFrame::View::Strides ss = { m_mat.step, 0u, 0u, 0u }; + return cv::MediaFrame::View(std::move(pp), std::move(ss), Cb{m_cb}); + } +}; + +class TestMediaNV12 final: public cv::MediaFrame::IAdapter { + cv::Mat m_y; + cv::Mat m_uv; +public: + TestMediaNV12(cv::Mat y, cv::Mat uv) : m_y(y), m_uv(uv) { + } + cv::GFrameDesc meta() const override { + return cv::GFrameDesc{cv::MediaFormat::NV12, cv::Size(m_y.cols, m_y.rows)}; + } + cv::MediaFrame::View access(cv::MediaFrame::Access) override { + cv::MediaFrame::View::Ptrs pp = { + m_y.ptr(), m_uv.ptr(), nullptr, nullptr + }; + cv::MediaFrame::View::Strides ss = { + m_y.step, m_uv.step, 0u, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } +}; + // FIXME: taken from DNN module static void initDLDTDataPath() { @@ -64,11 +103,15 @@ void normAssert(cv::InputArray ref, cv::InputArray test, namespace IE = InferenceEngine; -void setNetParameters(IE::CNNNetwork& net) { - auto &ii = net.getInputsInfo().at("data"); +void setNetParameters(IE::CNNNetwork& net, bool is_nv12 = false) { + auto ii = net.getInputsInfo().at("data"); ii->setPrecision(IE::Precision::U8); ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + if (is_nv12) { + ii->getPreProcess().setColorFormat(IE::ColorFormat::NV12); + } } + } // anonymous namespace // TODO: Probably DNN/IE part can be further parametrized with a template @@ -246,6 +289,81 @@ struct ROIList: public ::testing::Test { } }; // ROIList +struct ROIListNV12: public ::testing::Test { + cv::gapi::ie::detail::ParamDesc params; + + cv::Mat m_in_uv; + cv::Mat m_in_y; + std::vector m_roi_list; + + std::vector m_out_ie_ages; + std::vector m_out_ie_genders; + + std::vector m_out_gapi_ages; + std::vector m_out_gapi_genders; + + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + void SetUp() { + initDLDTDataPath(); + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + params.device_id = "CPU"; + + cv::Size sz{320, 240}; + m_in_y = cv::Mat{sz, CV_8UC1}; + cv::randu(m_in_y, 0, 255); + m_in_uv = cv::Mat{sz / 2, CV_8UC2}; + cv::randu(m_in_uv, 0, 255); + + // both ROIs point to the same face, with a slightly changed geometry + m_roi_list = { + cv::Rect(cv::Point{64, 60}, cv::Size{ 96, 96}), + cv::Rect(cv::Point{50, 32}, cv::Size{128, 160}), + }; + + // Load & run IE network + { + auto plugin = cv::gimpl::ie::wrap::getPlugin(params); + auto net = cv::gimpl::ie::wrap::readNetwork(params); + setNetParameters(net, true); + auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params); + auto infer_request = this_network.CreateInferRequest(); + auto frame_blob = cv::gapi::ie::util::to_ie(m_in_y, m_in_uv); + + for (auto &&rc : m_roi_list) { + const auto ie_rc = IE::ROI { + 0u + , static_cast(rc.x) + , static_cast(rc.y) + , static_cast(rc.width) + , static_cast(rc.height) + }; + infer_request.SetBlob("data", IE::make_shared_blob(frame_blob, ie_rc)); + infer_request.Infer(); + + using namespace cv::gapi::ie::util; + m_out_ie_ages.push_back(to_ocv(infer_request.GetBlob("age_conv3")).clone()); + m_out_ie_genders.push_back(to_ocv(infer_request.GetBlob("prob")).clone()); + } + } // namespace IE = .. + } // ROIList() + + void validate() { + // Validate with IE itself (avoid DNN module dependency here) + ASSERT_EQ(2u, m_out_ie_ages.size()); + ASSERT_EQ(2u, m_out_ie_genders.size()); + ASSERT_EQ(2u, m_out_gapi_ages.size()); + ASSERT_EQ(2u, m_out_gapi_genders.size()); + + normAssert(m_out_ie_ages [0], m_out_gapi_ages [0], "0: Test age output"); + normAssert(m_out_ie_genders[0], m_out_gapi_genders[0], "0: Test gender output"); + normAssert(m_out_ie_ages [1], m_out_gapi_ages [1], "1: Test age output"); + normAssert(m_out_ie_genders[1], m_out_gapi_genders[1], "1: Test gender output"); + } +}; + TEST_F(ROIList, TestInfer) { cv::GArray rr; @@ -505,6 +623,320 @@ TEST(TestAgeGenderIE, CPUConfig) cv::compile_args(cv::gapi::networks(pp)))); } +TEST_F(ROIList, MediaInputBGR) +{ + initDLDTDataPath(); + + cv::GFrame in; + cv::GArray rr; + cv::GArray age, gender; + std::tie(age, gender) = cv::gapi::infer(rr, in); + cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender)); + + auto frame = MediaFrame::Create(m_in_mat); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }); + comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp))); + + validate(); +} + +TEST_F(ROIListNV12, MediaInputNV12) +{ + initDLDTDataPath(); + + cv::GFrame in; + cv::GArray rr; + cv::GArray age, gender; + std::tie(age, gender) = cv::gapi::infer(rr, in); + cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender)); + + auto frame = MediaFrame::Create(m_in_y, m_in_uv); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }); + comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp))); + + validate(); +} + +TEST(TestAgeGenderIE, MediaInputNV12) +{ + initDLDTDataPath(); + + cv::gapi::ie::detail::ParamDesc params; + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + params.device_id = "CPU"; + + cv::Size sz{320, 240}; + cv::Mat in_y_mat(sz, CV_8UC1); + cv::randu(in_y_mat, 0, 255); + cv::Mat in_uv_mat(sz / 2, CV_8UC2); + cv::randu(in_uv_mat, 0, 255); + + cv::Mat gapi_age, gapi_gender; + + // Load & run IE network + IE::Blob::Ptr ie_age, ie_gender; + { + auto plugin = cv::gimpl::ie::wrap::getPlugin(params); + auto net = cv::gimpl::ie::wrap::readNetwork(params); + setNetParameters(net, true); + auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params); + auto infer_request = this_network.CreateInferRequest(); + infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in_y_mat, in_uv_mat)); + infer_request.Infer(); + ie_age = infer_request.GetBlob("age_conv3"); + ie_gender = infer_request.GetBlob("prob"); + } + + // Configure & run G-API + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + cv::GFrame in; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(in); + cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender)); + + auto frame = MediaFrame::Create(in_y_mat, in_uv_mat); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }); + comp.apply(cv::gin(frame), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp))); + + + // Validate with IE itself (avoid DNN module dependency here) + normAssert(cv::gapi::ie::util::to_ocv(ie_age), gapi_age, "Test age output" ); + normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output"); +} + +TEST(TestAgeGenderIE, MediaInputBGR) +{ + initDLDTDataPath(); + + cv::gapi::ie::detail::ParamDesc params; + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + params.device_id = "CPU"; + + cv::Size sz{320, 240}; + cv::Mat in_mat(sz, CV_8UC3); + cv::randu(in_mat, 0, 255); + + cv::Mat gapi_age, gapi_gender; + + // Load & run IE network + IE::Blob::Ptr ie_age, ie_gender; + { + auto plugin = cv::gimpl::ie::wrap::getPlugin(params); + auto net = cv::gimpl::ie::wrap::readNetwork(params); + setNetParameters(net); + auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params); + auto infer_request = this_network.CreateInferRequest(); + infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in_mat)); + infer_request.Infer(); + ie_age = infer_request.GetBlob("age_conv3"); + ie_gender = infer_request.GetBlob("prob"); + } + + // Configure & run G-API + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + cv::GFrame in; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(in); + cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender)); + + auto frame = MediaFrame::Create(in_mat); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }); + comp.apply(cv::gin(frame), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp))); + + + // Validate with IE itself (avoid DNN module dependency here) + normAssert(cv::gapi::ie::util::to_ocv(ie_age), gapi_age, "Test age output" ); + normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output"); +} + +TEST(InferROI, MediaInputBGR) +{ + initDLDTDataPath(); + + cv::gapi::ie::detail::ParamDesc params; + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + params.device_id = "CPU"; + + cv::Size sz{320, 240}; + cv::Mat in_mat(sz, CV_8UC3); + cv::randu(in_mat, 0, 255); + + cv::Mat gapi_age, gapi_gender; + cv::Rect rect(cv::Point{64, 60}, cv::Size{96, 96}); + + // Load & run IE network + IE::Blob::Ptr ie_age, ie_gender; + { + auto plugin = cv::gimpl::ie::wrap::getPlugin(params); + auto net = cv::gimpl::ie::wrap::readNetwork(params); + setNetParameters(net); + auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params); + auto infer_request = this_network.CreateInferRequest(); + const auto ie_rc = IE::ROI { + 0u + , static_cast(rect.x) + , static_cast(rect.y) + , static_cast(rect.width) + , static_cast(rect.height) + }; + IE::Blob::Ptr roi_blob = IE::make_shared_blob(cv::gapi::ie::util::to_ie(in_mat), ie_rc); + infer_request.SetBlob("data", roi_blob); + infer_request.Infer(); + ie_age = infer_request.GetBlob("age_conv3"); + ie_gender = infer_request.GetBlob("prob"); + } + + // Configure & run G-API + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + cv::GFrame in; + cv::GOpaque roi; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(roi, in); + cv::GComputation comp(cv::GIn(in, roi), cv::GOut(age, gender)); + + auto frame = MediaFrame::Create(in_mat); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }); + comp.apply(cv::gin(frame, rect), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp))); + + + // Validate with IE itself (avoid DNN module dependency here) + normAssert(cv::gapi::ie::util::to_ocv(ie_age), gapi_age, "Test age output" ); + normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output"); +} + +TEST(InferROI, MediaInputNV12) +{ + initDLDTDataPath(); + + cv::gapi::ie::detail::ParamDesc params; + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + params.device_id = "CPU"; + + cv::Size sz{320, 240}; + auto in_y_mat = cv::Mat{sz, CV_8UC1}; + cv::randu(in_y_mat, 0, 255); + auto in_uv_mat = cv::Mat{sz / 2, CV_8UC2}; + cv::randu(in_uv_mat, 0, 255); + + cv::Mat gapi_age, gapi_gender; + cv::Rect rect(cv::Point{64, 60}, cv::Size{96, 96}); + + // Load & run IE network + IE::Blob::Ptr ie_age, ie_gender; + { + auto plugin = cv::gimpl::ie::wrap::getPlugin(params); + auto net = cv::gimpl::ie::wrap::readNetwork(params); + setNetParameters(net, true); + auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params); + auto infer_request = this_network.CreateInferRequest(); + const auto ie_rc = IE::ROI { + 0u + , static_cast(rect.x) + , static_cast(rect.y) + , static_cast(rect.width) + , static_cast(rect.height) + }; + IE::Blob::Ptr roi_blob = IE::make_shared_blob(cv::gapi::ie::util::to_ie(in_y_mat, in_uv_mat), ie_rc); + infer_request.SetBlob("data", roi_blob); + infer_request.Infer(); + ie_age = infer_request.GetBlob("age_conv3"); + ie_gender = infer_request.GetBlob("prob"); + } + + // Configure & run G-API + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + cv::GFrame in; + cv::GOpaque roi; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(roi, in); + cv::GComputation comp(cv::GIn(in, roi), cv::GOut(age, gender)); + + auto frame = MediaFrame::Create(in_y_mat, in_uv_mat); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }); + comp.apply(cv::gin(frame, rect), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp))); + + + // Validate with IE itself (avoid DNN module dependency here) + normAssert(cv::gapi::ie::util::to_ocv(ie_age), gapi_age, "Test age output" ); + normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output"); +} + +TEST_F(ROIList, Infer2MediaInputBGR) +{ + cv::GArray rr; + cv::GFrame in; + cv::GArray age, gender; + std::tie(age, gender) = cv::gapi::infer2(in, rr); + cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender)); + + auto frame = MediaFrame::Create(m_in_mat); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }); + comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp))); + validate(); +} + +TEST_F(ROIListNV12, Infer2MediaInputNV12) +{ + cv::GArray rr; + cv::GFrame in; + cv::GArray age, gender; + std::tie(age, gender) = cv::gapi::infer2(in, rr); + cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender)); + + auto frame = MediaFrame::Create(m_in_y, m_in_uv); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }); + comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp))); + validate(); +} + } // namespace opencv_test #endif // HAVE_INF_ENGINE