From a31fb88fd08d9d55ab8deff8257633d094c6e538 Mon Sep 17 00:00:00 2001 From: Dmitry Matveev Date: Tue, 30 Aug 2022 22:48:29 +0300 Subject: [PATCH 1/2] G-API: Introduce GAbstractExecutor GExecutor is now a subclass of GAbstractExecutor. Other to come --- modules/gapi/CMakeLists.txt | 1 + modules/gapi/src/compiler/gcompiled.cpp | 3 +- modules/gapi/src/compiler/gcompiled_priv.hpp | 6 +- .../gapi/src/executor/gabstractexecutor.cpp | 25 ++++++ .../gapi/src/executor/gabstractexecutor.hpp | 80 +++++++++++++++++++ modules/gapi/src/executor/gexecutor.cpp | 11 +-- modules/gapi/src/executor/gexecutor.hpp | 54 ++----------- 7 files changed, 118 insertions(+), 62 deletions(-) create mode 100644 modules/gapi/src/executor/gabstractexecutor.cpp create mode 100644 modules/gapi/src/executor/gabstractexecutor.hpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 63373ad9c4..962256cc88 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -113,6 +113,7 @@ set(gapi_srcs src/compiler/passes/intrin.cpp # Executor + src/executor/gabstractexecutor.cpp src/executor/gexecutor.cpp src/executor/gtbbexecutor.cpp src/executor/gstreamingexecutor.cpp diff --git a/modules/gapi/src/compiler/gcompiled.cpp b/modules/gapi/src/compiler/gcompiled.cpp index 263878ce0d..df4411e832 100644 --- a/modules/gapi/src/compiler/gcompiled.cpp +++ b/modules/gapi/src/compiler/gcompiled.cpp @@ -14,11 +14,12 @@ #include "compiler/gcompiled_priv.hpp" #include "backends/common/gbackend.hpp" +#include "executor/gexecutor.hpp" // GCompiled private implementation //////////////////////////////////////////// void cv::GCompiled::Priv::setup(const GMetaArgs &_metaArgs, const GMetaArgs &_outMetas, - std::unique_ptr &&_pE) + std::unique_ptr &&_pE) { m_metas = _metaArgs; m_outMetas = _outMetas; diff --git a/modules/gapi/src/compiler/gcompiled_priv.hpp b/modules/gapi/src/compiler/gcompiled_priv.hpp index f21bfc80bc..3f873aba23 100644 --- a/modules/gapi/src/compiler/gcompiled_priv.hpp +++ b/modules/gapi/src/compiler/gcompiled_priv.hpp @@ -12,7 +12,7 @@ #include "opencv2/gapi/util/optional.hpp" #include "compiler/gmodel.hpp" -#include "executor/gexecutor.hpp" +#include "executor/gabstractexecutor.hpp" // NB: BTW, GCompiled is the only "public API" class which // private part (implementation) is hosted in the "compiler/" module. @@ -36,14 +36,14 @@ class GAPI_EXPORTS GCompiled::Priv // If we want to go autonomous, we might to do something with this. GMetaArgs m_metas; // passed by user GMetaArgs m_outMetas; // inferred by compiler - std::unique_ptr m_exec; + std::unique_ptr m_exec; void checkArgs(const cv::gimpl::GRuntimeArgs &args) const; public: void setup(const GMetaArgs &metaArgs, const GMetaArgs &outMetas, - std::unique_ptr &&pE); + std::unique_ptr &&pE); bool isEmpty() const; bool canReshape() const; diff --git a/modules/gapi/src/executor/gabstractexecutor.cpp b/modules/gapi/src/executor/gabstractexecutor.cpp new file mode 100644 index 0000000000..e22b2eeb7c --- /dev/null +++ b/modules/gapi/src/executor/gabstractexecutor.cpp @@ -0,0 +1,25 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2022 Intel Corporation + +#include "precomp.hpp" + +#include + +#include "executor/gabstractexecutor.hpp" + +cv::gimpl::GAbstractExecutor::GAbstractExecutor(std::unique_ptr &&g_model) + : m_orig_graph(std::move(g_model)) + , m_island_graph(GModel::Graph(*m_orig_graph).metadata() + .get().model) + , m_gm(*m_orig_graph) + , m_gim(*m_island_graph) +{ +} + +const cv::gimpl::GModel::Graph& cv::gimpl::GAbstractExecutor::model() const +{ + return m_gm; +} diff --git a/modules/gapi/src/executor/gabstractexecutor.hpp b/modules/gapi/src/executor/gabstractexecutor.hpp new file mode 100644 index 0000000000..1657142021 --- /dev/null +++ b/modules/gapi/src/executor/gabstractexecutor.hpp @@ -0,0 +1,80 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2022 Intel Corporation + + +#ifndef OPENCV_GAPI_GABSTRACT_EXECUTOR_HPP +#define OPENCV_GAPI_GABSTRACT_EXECUTOR_HPP + +#include // unique_ptr, shared_ptr + +#include // tuple, required by magazine +#include // required by magazine + +#include + +#include "backends/common/gbackend.hpp" + +namespace cv { +namespace gimpl { + +// Graph-level executor interface. +// +// This class specifies API for a "super-executor" which orchestrates +// the overall Island graph execution. +// +// Every Island (subgraph) execution is delegated to a particular +// backend and is done opaquely to the GExecutor. +// +// Inputs to a GExecutor instance are: +// - GIslandModel - a high-level graph model which may be seen as a +// "procedure" to execute. +// - GModel - a low-level graph of operations (from which a GIslandModel +// is projected) +// - GComputation runtime arguments - vectors of input/output objects +// +// Every GExecutor is responsible for +// a. Maintaining non-island (intermediate) data objects within graph +// b. Providing GIslandExecutables with input/output data according to +// their protocols +// c. Triggering execution of GIslandExecutables when task/data dependencies +// are met. +// +// By default G-API stores all data on host, and cross-Island +// exchange happens via host buffers (and CV data objects). +// +// Today's exchange data objects are: +// - cv::Mat, cv::RMat - for image buffers +// - cv::Scalar - for single values (with up to four components inside) +// - cv::detail::VectorRef - an untyped wrapper over std::vector +// - cv::detail::OpaqueRef - an untyped wrapper over T +// - cv::MediaFrame - for image textures and surfaces (e.g. in planar format) + +class GAbstractExecutor +{ +protected: + std::unique_ptr m_orig_graph; + std::shared_ptr m_island_graph; + + cv::gimpl::GModel::Graph m_gm; // FIXME: make const? + cv::gimpl::GIslandModel::Graph m_gim; // FIXME: make const? + +public: + explicit GAbstractExecutor(std::unique_ptr &&g_model); + virtual ~GAbstractExecutor() = default; + virtual void run(cv::gimpl::GRuntimeArgs &&args) = 0; + + virtual bool canReshape() const = 0; + virtual void reshape(const GMetaArgs& inMetas, const GCompileArgs& args) = 0; + + virtual void prepareForNewStream() = 0; + + const GModel::Graph& model() const; // FIXME: make it ConstGraph? +}; + +} // namespace gimpl +} // namespace cv + +#endif // OPENCV_GAPI_GABSTRACT_EXECUTOR_HPP diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index 472abaaa14..6853c30d3e 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -16,11 +16,7 @@ #include "compiler/passes/passes.hpp" cv::gimpl::GExecutor::GExecutor(std::unique_ptr &&g_model) - : m_orig_graph(std::move(g_model)) - , m_island_graph(GModel::Graph(*m_orig_graph).metadata() - .get().model) - , m_gm(*m_orig_graph) - , m_gim(*m_island_graph) + : GAbstractExecutor(std::move(g_model)) { // NB: Right now GIslandModel is acyclic, so for a naive execution, // simple unrolling to a list of triggers is enough @@ -424,11 +420,6 @@ void cv::gimpl::GExecutor::run(cv::gimpl::GRuntimeArgs &&args) } } -const cv::gimpl::GModel::Graph& cv::gimpl::GExecutor::model() const -{ - return m_gm; -} - bool cv::gimpl::GExecutor::canReshape() const { // FIXME: Introduce proper reshaping support on GExecutor level diff --git a/modules/gapi/src/executor/gexecutor.hpp b/modules/gapi/src/executor/gexecutor.hpp index 5d797ce604..67182fc8cc 100644 --- a/modules/gapi/src/executor/gexecutor.hpp +++ b/modules/gapi/src/executor/gexecutor.hpp @@ -8,58 +8,18 @@ #ifndef OPENCV_GAPI_GEXECUTOR_HPP #define OPENCV_GAPI_GEXECUTOR_HPP -#include // unique_ptr, shared_ptr - #include // tuple, required by magazine #include // required by magazine -#include - -#include "backends/common/gbackend.hpp" +#include "executor/gabstractexecutor.hpp" namespace cv { namespace gimpl { -// Graph-level executor interface. -// -// This class specifies API for a "super-executor" which orchestrates -// the overall Island graph execution. -// -// Every Island (subgraph) execution is delegated to a particular -// backend and is done opaquely to the GExecutor. -// -// Inputs to a GExecutor instance are: -// - GIslandModel - a high-level graph model which may be seen as a -// "procedure" to execute. -// - GModel - a low-level graph of operations (from which a GIslandModel -// is projected) -// - GComputation runtime arguments - vectors of input/output objects -// -// Every GExecutor is responsible for -// a. Maintaining non-island (intermediate) data objects within graph -// b. Providing GIslandExecutables with input/output data according to -// their protocols -// c. Triggering execution of GIslandExecutables when task/data dependencies -// are met. -// -// By default G-API stores all data on host, and cross-Island -// exchange happens via host buffers (and CV data objects). -// -// Today's exchange data objects are: -// - cv::Mat - for image buffers -// - cv::Scalar - for single values (with up to four components inside) -// - cv::detail::VectorRef - an untyped wrapper over std::vector -// - -class GExecutor +class GExecutor final: public GAbstractExecutor { protected: Mag m_res; - std::unique_ptr m_orig_graph; - std::shared_ptr m_island_graph; - - cv::gimpl::GModel::Graph m_gm; // FIXME: make const? - cv::gimpl::GIslandModel::Graph m_gim; // FIXME: make const? // FIXME: Naive executor details are here for now // but then it should be moved to another place @@ -85,14 +45,12 @@ protected: public: explicit GExecutor(std::unique_ptr &&g_model); - void run(cv::gimpl::GRuntimeArgs &&args); - - bool canReshape() const; - void reshape(const GMetaArgs& inMetas, const GCompileArgs& args); + void run(cv::gimpl::GRuntimeArgs &&args) override; - void prepareForNewStream(); + bool canReshape() const override; + void reshape(const GMetaArgs& inMetas, const GCompileArgs& args) override; - const GModel::Graph& model() const; // FIXME: make it ConstGraph? + void prepareForNewStream() override; }; } // namespace gimpl From a122f0f24802113a0f250f3b71e2b90ed9763bad Mon Sep 17 00:00:00 2001 From: Dmitry Matveev Date: Tue, 30 Aug 2022 23:50:34 +0300 Subject: [PATCH 2/2] G-API: Introduce GAbstractStreamingExecutor Now GStreamingExecutor is its subclass; others to come --- modules/gapi/CMakeLists.txt | 1 + modules/gapi/src/compiler/gstreaming.cpp | 4 +- modules/gapi/src/compiler/gstreaming_priv.hpp | 8 +-- .../executor/gabstractstreamingexecutor.cpp | 23 +++++++++ .../executor/gabstractstreamingexecutor.hpp | 49 +++++++++++++++++++ .../gapi/src/executor/gstreamingexecutor.cpp | 10 +--- .../gapi/src/executor/gstreamingexecutor.hpp | 29 ++++------- 7 files changed, 91 insertions(+), 33 deletions(-) create mode 100644 modules/gapi/src/executor/gabstractstreamingexecutor.cpp create mode 100644 modules/gapi/src/executor/gabstractstreamingexecutor.hpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 962256cc88..edf4e588e0 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -114,6 +114,7 @@ set(gapi_srcs # Executor src/executor/gabstractexecutor.cpp + src/executor/gabstractstreamingexecutor.cpp src/executor/gexecutor.cpp src/executor/gtbbexecutor.cpp src/executor/gstreamingexecutor.cpp diff --git a/modules/gapi/src/compiler/gstreaming.cpp b/modules/gapi/src/compiler/gstreaming.cpp index e45e770427..c0bc0b1d1b 100644 --- a/modules/gapi/src/compiler/gstreaming.cpp +++ b/modules/gapi/src/compiler/gstreaming.cpp @@ -19,14 +19,14 @@ // GStreamingCompiled private implementation /////////////////////////////////// void cv::GStreamingCompiled::Priv::setup(const GMetaArgs &_metaArgs, const GMetaArgs &_outMetas, - std::unique_ptr &&_pE) + std::unique_ptr &&_pE) { m_metas = _metaArgs; m_outMetas = _outMetas; m_exec = std::move(_pE); } -void cv::GStreamingCompiled::Priv::setup(std::unique_ptr &&_pE) +void cv::GStreamingCompiled::Priv::setup(std::unique_ptr &&_pE) { m_exec = std::move(_pE); } diff --git a/modules/gapi/src/compiler/gstreaming_priv.hpp b/modules/gapi/src/compiler/gstreaming_priv.hpp index 1b559ba310..0fd5fc7b7f 100644 --- a/modules/gapi/src/compiler/gstreaming_priv.hpp +++ b/modules/gapi/src/compiler/gstreaming_priv.hpp @@ -9,7 +9,7 @@ #define OPENCV_GAPI_GSTREAMING_COMPILED_PRIV_HPP #include // unique_ptr -#include "executor/gstreamingexecutor.hpp" +#include "executor/gabstractstreamingexecutor.hpp" namespace cv { @@ -26,7 +26,7 @@ class GAPI_EXPORTS GStreamingCompiled::Priv { GMetaArgs m_metas; // passed by user GMetaArgs m_outMetas; // inferred by compiler - std::unique_ptr m_exec; + std::unique_ptr m_exec; // NB: Used by python wrapper to clarify input/output types GTypesInfo m_out_info; @@ -35,8 +35,8 @@ class GAPI_EXPORTS GStreamingCompiled::Priv public: void setup(const GMetaArgs &metaArgs, const GMetaArgs &outMetas, - std::unique_ptr &&pE); - void setup(std::unique_ptr &&pE); + std::unique_ptr &&pE); + void setup(std::unique_ptr &&pE); bool isEmpty() const; const GMetaArgs& metas() const; diff --git a/modules/gapi/src/executor/gabstractstreamingexecutor.cpp b/modules/gapi/src/executor/gabstractstreamingexecutor.cpp new file mode 100644 index 0000000000..07579a9a6f --- /dev/null +++ b/modules/gapi/src/executor/gabstractstreamingexecutor.cpp @@ -0,0 +1,23 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2022 Intel Corporation + +#include "precomp.hpp" + +#include + +#include "executor/gabstractstreamingexecutor.hpp" + +cv::gimpl::GAbstractStreamingExecutor::GAbstractStreamingExecutor(std::unique_ptr &&g_model, + const GCompileArgs &comp_args) + : m_orig_graph(std::move(g_model)) + , m_island_graph(GModel::Graph(*m_orig_graph).metadata() + .get().model) + , m_comp_args(comp_args) + , m_gim(*m_island_graph) + , m_desync(GModel::Graph(*m_orig_graph).metadata() + .contains()) +{ +} diff --git a/modules/gapi/src/executor/gabstractstreamingexecutor.hpp b/modules/gapi/src/executor/gabstractstreamingexecutor.hpp new file mode 100644 index 0000000000..952bbd01db --- /dev/null +++ b/modules/gapi/src/executor/gabstractstreamingexecutor.hpp @@ -0,0 +1,49 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2022 Intel Corporation + +#ifndef OPENCV_GAPI_GABSTRACT_STREAMING_EXECUTOR_HPP +#define OPENCV_GAPI_GABSTRACT_STREAMING_EXECUTOR_HPP + +#include // unique_ptr, shared_ptr + +#include + +#include "backends/common/gbackend.hpp" + +namespace cv { +namespace gimpl { + +class GAbstractStreamingExecutor +{ +protected: + std::unique_ptr m_orig_graph; + std::shared_ptr m_island_graph; + cv::GCompileArgs m_comp_args; + + cv::gimpl::GIslandModel::Graph m_gim; // FIXME: make const? + const bool m_desync; + +public: + explicit GAbstractStreamingExecutor(std::unique_ptr &&g_model, + const cv::GCompileArgs &comp_args); + virtual ~GAbstractStreamingExecutor() = default; + virtual void setSource(GRunArgs &&args) = 0; + virtual void start() = 0; + virtual bool pull(cv::GRunArgsP &&outs) = 0; + virtual bool pull(cv::GOptRunArgsP &&outs) = 0; + + using PyPullResult = std::tuple>; + virtual PyPullResult pull() = 0; + + virtual bool try_pull(cv::GRunArgsP &&outs) = 0; + virtual void stop() = 0; + virtual bool running() const = 0; +}; + +} // namespace gimpl +} // namespace cv + +#endif // OPENCV_GAPI_GABSTRACT_STREAMING_EXECUTOR_HPP diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 557e5ceee4..66aad90697 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -1288,13 +1288,7 @@ public: // proper graph reshape and islands recompilation cv::gimpl::GStreamingExecutor::GStreamingExecutor(std::unique_ptr &&g_model, const GCompileArgs &comp_args) - : m_orig_graph(std::move(g_model)) - , m_island_graph(GModel::Graph(*m_orig_graph).metadata() - .get().model) - , m_comp_args(comp_args) - , m_gim(*m_island_graph) - , m_desync(GModel::Graph(*m_orig_graph).metadata() - .contains()) + : GAbstractStreamingExecutor(std::move(g_model), comp_args) { GModel::Graph gm(*m_orig_graph); // NB: Right now GIslandModel is acyclic, and all the below code assumes that. @@ -1862,7 +1856,7 @@ bool cv::gimpl::GStreamingExecutor::pull(cv::GOptRunArgsP &&outs) GAPI_Assert(false && "Unreachable code"); } -std::tuple> cv::gimpl::GStreamingExecutor::pull() +cv::gimpl::GAbstractStreamingExecutor::PyPullResult cv::gimpl::GStreamingExecutor::pull() { using RunArgs = cv::util::variant; bool is_over = false; diff --git a/modules/gapi/src/executor/gstreamingexecutor.hpp b/modules/gapi/src/executor/gstreamingexecutor.hpp index da27f6a646..cb63d4b7f0 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.hpp +++ b/modules/gapi/src/executor/gstreamingexecutor.hpp @@ -12,7 +12,6 @@ // on concurrent_bounded_queue #endif -#include // unique_ptr, shared_ptr #include // thread #include #include @@ -26,9 +25,7 @@ template using QueueClass = cv::gapi::own::concurrent_bounded_queue< #endif // TBB #include "executor/last_value.hpp" -#include - -#include "backends/common/gbackend.hpp" +#include "executor/gabstractstreamingexecutor.hpp" namespace cv { namespace gimpl { @@ -104,7 +101,7 @@ public: // FIXME: Currently all GExecutor comments apply also // to this one. Please document it separately in the future. -class GStreamingExecutor final +class GStreamingExecutor final: public GAbstractStreamingExecutor { protected: // GStreamingExecutor is a state machine described as follows @@ -131,15 +128,9 @@ protected: RUNNING, } state = State::STOPPED; - std::unique_ptr m_orig_graph; - std::shared_ptr m_island_graph; - cv::GCompileArgs m_comp_args; cv::GMetaArgs m_last_metas; util::optional m_reshapable; - cv::gimpl::GIslandModel::Graph m_gim; // FIXME: make const? - const bool m_desync; - // FIXME: Naive executor details are here for now // but then it should be moved to another place struct OpDesc @@ -202,14 +193,14 @@ public: explicit GStreamingExecutor(std::unique_ptr &&g_model, const cv::GCompileArgs &comp_args); ~GStreamingExecutor(); - void setSource(GRunArgs &&args); - void start(); - bool pull(cv::GRunArgsP &&outs); - bool pull(cv::GOptRunArgsP &&outs); - std::tuple> pull(); - bool try_pull(cv::GRunArgsP &&outs); - void stop(); - bool running() const; + void setSource(GRunArgs &&args) override; + void start() override; + bool pull(cv::GRunArgsP &&outs) override; + bool pull(cv::GOptRunArgsP &&outs) override; + PyPullResult pull() override; + bool try_pull(cv::GRunArgsP &&outs) override; + void stop() override; + bool running() const override; }; } // namespace gimpl