diff --git a/modules/gapi/include/opencv2/gapi/infer/ov.hpp b/modules/gapi/include/opencv2/gapi/infer/ov.hpp index 99d701f937..02d6225c7a 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ov.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ov.hpp @@ -679,7 +679,32 @@ protected: }; } // namespace ov + +namespace wip { namespace ov { +/** + * @brief Ask G-API OpenVINO backend to run only inference of model provided. + * + * G-API OpenVINO backend will perform only the inference of the model provided + * without populating input and copying back output data. + * This mode is used to evaluate the pure inference performance of the model without + * taking into account the i/o data transfer. + */ +struct benchmark_mode { }; +/** @} */ + +} // namespace ov +} // namespace wip + } // namespace gapi + +namespace detail +{ + template<> struct CompileArgTag + { + static const char* tag() { return "gapi.wip.ov.benchmark_mode"; } + }; +} + } // namespace cv #endif // OPENCV_GAPI_INFER_OV_HPP diff --git a/modules/gapi/src/backends/ov/govbackend.cpp b/modules/gapi/src/backends/ov/govbackend.cpp index 8384a9d188..a139982e48 100644 --- a/modules/gapi/src/backends/ov/govbackend.cpp +++ b/modules/gapi/src/backends/ov/govbackend.cpp @@ -252,7 +252,8 @@ public: const std::vector & outs, cv::GRunArg::Meta && meta, std::vector && input_objs, - std::vector && output_objs); + std::vector && output_objs, + const cv::gimpl::ov::Options & options); const cv::GArgs& inArgs() const; @@ -281,6 +282,9 @@ public: std::exception_ptr eptr; const cv::GRunArg::Meta& getMeta() { return m_meta; }; + + const cv::gimpl::ov::Options& getOptions() const { return m_options; }; + private: cv::detail::VectorRef& outVecRef(std::size_t idx); @@ -301,6 +305,8 @@ private: // Input parameters passed to an inference operation. cv::GArgs m_args; cv::GShapes m_in_shapes; + + cv::gimpl::ov::Options m_options; }; OVCallContext::OVCallContext(const OVUnit & unit, @@ -309,9 +315,11 @@ OVCallContext::OVCallContext(const OVUnit & const std::vector & outs, cv::GRunArg::Meta && meta, std::vector && input_objs, - std::vector && output_objs) + std::vector && output_objs, + const cv::gimpl::ov::Options & options) : uu(unit), out(output), m_meta(std::move(meta)), - m_input_objs(std::move(input_objs)), m_output_objs(std::move(output_objs)) + m_input_objs(std::move(input_objs)), m_output_objs(std::move(output_objs)), + m_options(options) { for (auto& it : m_input_objs) cv::gimpl::magazine::bindInArg (m_res, it.first, it.second); for (auto& it : m_output_objs) cv::gimpl::magazine::bindOutArg(m_res, it.first, it.second); @@ -577,9 +585,10 @@ static void PostOutputs(::ov::InferRequest &infer_request, ctx->eptr = std::move(eptr); for (auto i : ade::util::iota(ctx->uu.params.num_out)) { - // NB: Copy data back only if execution finished sucessfuly. - // Otherwise just post outputs to keep streaming executor contract. - if (!ctx->eptr) { + // NB: Copy data back only if execution finished sucessfuly + // and inference only mode is disabled. + // Otherwise just post outputs to maintain streaming executor contract. + if (!ctx->eptr && !ctx->getOptions().inference_only) { const auto& out_name = ctx->uu.params.output_names[i]; copyFromOV(infer_request.get_tensor(out_name), ctx->outMatR(i)); @@ -990,6 +999,11 @@ struct Infer: public cv::detail::KernelTag { reqPool.getIdleRequest()->execute( IInferExecutor::Task { [ctx](::ov::InferRequest &infer_request) { + // NB: No need to populate model inputs with data + // if it's inference only mode. + if (ctx->getOptions().inference_only) { + return; + } for (auto i : ade::util::iota(ctx->uu.params.num_in)) { const auto& input_name = ctx->uu.params.input_names[i]; auto input_tensor = infer_request.get_tensor(input_name); @@ -1069,6 +1083,10 @@ struct InferROI: public cv::detail::KernelTag { static void run(std::shared_ptr ctx, cv::gimpl::ov::RequestPool &reqPool) { using namespace std::placeholders; + if (ctx->getOptions().inference_only) { + cv::util::throw_error( + std::logic_error("OV Backend: Inference only mode is not supported for InferROI!")); + } reqPool.getIdleRequest()->execute( IInferExecutor::Task { [ctx](::ov::InferRequest &infer_request) { @@ -1141,6 +1159,10 @@ struct InferList: public cv::detail::KernelTag { static void run(std::shared_ptr ctx, cv::gimpl::ov::RequestPool &reqPool) { + if (ctx->getOptions().inference_only) { + cv::util::throw_error( + std::logic_error("OV Backend: Inference only mode is not supported for InferList!")); + } const auto& in_roi_vec = ctx->inArg(0u).rref(); // NB: In case there is no input data need to post output anyway if (in_roi_vec.empty()) { @@ -1257,6 +1279,10 @@ struct InferList2: public cv::detail::KernelTag { static void run(std::shared_ptr ctx, cv::gimpl::ov::RequestPool &reqPool) { + if (ctx->getOptions().inference_only) { + cv::util::throw_error( + std::logic_error("OV Backend: Inference only mode is not supported for InferList2!")); + } GAPI_Assert(ctx->inArgs().size() > 1u && "This operation must have at least two arguments"); // NB: This blob will be used to make roi from its, so @@ -1348,9 +1374,9 @@ class GOVBackendImpl final: public cv::gapi::GBackend::Priv { } virtual EPtr compile(const ade::Graph &graph, - const cv::GCompileArgs &, + const cv::GCompileArgs &compileArgs, const std::vector &nodes) const override { - return EPtr{new cv::gimpl::ov::GOVExecutable(graph, nodes)}; + return EPtr{new cv::gimpl::ov::GOVExecutable(graph, compileArgs, nodes)}; } virtual cv::GKernelPackage auxiliaryKernels() const override { @@ -1391,9 +1417,12 @@ createInferRequests(::ov::CompiledModel &compiled_model, // GOVExecutable implementation ////////////////////////////////////////////// cv::gimpl::ov::GOVExecutable::GOVExecutable(const ade::Graph &g, + const cv::GCompileArgs &compileArgs, const std::vector &nodes) : m_g(g), m_gm(m_g) { + m_options.inference_only = + cv::gapi::getCompileArg(compileArgs).has_value(); // FIXME: Currently this backend is capable to run a single inference node only. // Need to extend our island fusion with merge/not-to-merge decision making parametrization GConstGOVModel ovm(g); @@ -1471,7 +1500,7 @@ void cv::gimpl::ov::GOVExecutable::run(cv::gimpl::GIslandExecutable::IInput &in const auto &op = m_gm.metadata(this_nh).get(); auto ctx = std::make_shared(uu, out, op.args, op.outs, - std::move(stub_meta), std::move(input_objs), std::move(output_objs)); + std::move(stub_meta), std::move(input_objs), std::move(output_objs), m_options); const auto &kk = giem.metadata(this_nh).get(); diff --git a/modules/gapi/src/backends/ov/govbackend.hpp b/modules/gapi/src/backends/ov/govbackend.hpp index 0ac858dc52..ff9793afad 100644 --- a/modules/gapi/src/backends/ov/govbackend.hpp +++ b/modules/gapi/src/backends/ov/govbackend.hpp @@ -26,6 +26,12 @@ struct OVCompiled { class RequestPool; +struct Options { + // Only performs inference of the model + // without i/o data transfer if enabled. + bool inference_only = false; +}; + class GOVExecutable final: public GIslandExecutable { const ade::Graph &m_g; @@ -42,8 +48,12 @@ class GOVExecutable final: public GIslandExecutable // To manage multiple async requests std::unique_ptr m_reqPool; + // To manage additional execution options + Options m_options; + public: GOVExecutable(const ade::Graph &graph, + const cv::GCompileArgs &compileArgs, const std::vector &nodes); virtual inline bool canReshape() const override { return false; }