From a3d6994afaaa546b328bfc0f56e2430f61878348 Mon Sep 17 00:00:00 2001 From: xiong-jie-y Date: Thu, 26 May 2022 01:12:51 +0900 Subject: [PATCH] Add stateful kernel to python G-API --- .../include/opencv2/gapi/python/python.hpp | 14 ++- modules/gapi/misc/python/pyopencv_gapi.hpp | 106 +++++++++++++++++- .../src/backends/python/gpythonbackend.cpp | 41 ++++++- 3 files changed, 148 insertions(+), 13 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/python/python.hpp b/modules/gapi/include/opencv2/gapi/python/python.hpp index 0e20bbbb59..72ed6f5850 100644 --- a/modules/gapi/include/opencv2/gapi/python/python.hpp +++ b/modules/gapi/include/opencv2/gapi/python/python.hpp @@ -31,17 +31,26 @@ struct GPythonContext const cv::GArgs &ins; const cv::GMetaArgs &in_metas; const cv::GTypesInfo &out_info; + + bool m_isStateful; + + GArg m_state; }; using Impl = std::function; +using Setup = std::function; class GAPI_EXPORTS GPythonKernel { public: GPythonKernel() = default; - GPythonKernel(Impl run); + GPythonKernel(Impl run, Setup setup); cv::GRunArgs operator()(const GPythonContext& ctx); + + bool m_isStateful = false; + Setup m_setup = nullptr; + private: Impl m_run; }; @@ -51,7 +60,8 @@ class GAPI_EXPORTS GPythonFunctor : public cv::gapi::GFunctor public: using Meta = cv::GKernel::M; - GPythonFunctor(const char* id, const Meta &meta, const Impl& impl); + GPythonFunctor(const char* id, const Meta& meta, const Impl& impl, + const Setup& setup = nullptr); GKernelImpl impl() const override; gapi::GBackend backend() const override; diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 7b760920e7..0812b283b7 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -660,7 +660,7 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, // NB: Doesn't increase reference counter (false), // because PyObject already have ownership. // In case exception decrement reference counter. - cv::detail::PyObjectHolder args(PyTuple_New(ins.size()), false); + cv::detail::PyObjectHolder args(PyTuple_New(ctx.m_isStateful ? ins.size() + 1: ins.size()), false); for (size_t i = 0; i < ins.size(); ++i) { // NB: If meta is monostate then object isn't associated with G-TYPE. @@ -690,6 +690,12 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, } ++in_idx; } + + if (ctx.m_isStateful) + { + PyTuple_SetItem(args.get(), ins.size(), pyopencv_from(ctx.m_state)); + } + // NB: Doesn't increase reference counter (false). // In case PyObject_CallObject return NULL, do nothing in destructor. cv::detail::PyObjectHolder result( @@ -736,6 +742,80 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, return outs; } +static cv::GArg setup_py(cv::detail::PyObjectHolder out_meta, const cv::GMetaArgs& meta, + const cv::GArgs& gargs) +{ + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + cv::GArg out; + + try + { + // NB: Doesn't increase reference counter (false), + // because PyObject already have ownership. + // In case exception decrement reference counter. + cv::detail::PyObjectHolder args(PyTuple_New(meta.size()), false); + size_t idx = 0; + for (auto&& m : meta) + { + switch (m.index()) + { + case cv::GMetaArg::index_of(): + PyTuple_SetItem(args.get(), idx, pyopencv_from(cv::util::get(m))); + break; + case cv::GMetaArg::index_of(): + PyTuple_SetItem(args.get(), idx, + pyopencv_from(cv::util::get(m))); + break; + case cv::GMetaArg::index_of(): + PyTuple_SetItem(args.get(), idx, + pyopencv_from(cv::util::get(m))); + break; + case cv::GMetaArg::index_of(): + PyTuple_SetItem(args.get(), idx, + pyopencv_from(cv::util::get(m))); + break; + case cv::GMetaArg::index_of(): + PyTuple_SetItem(args.get(), idx, pyopencv_from(gargs[idx])); + break; + case cv::GMetaArg::index_of(): + util::throw_error( + std::logic_error("GFrame isn't supported for custom operation")); + break; + } + ++idx; + } + + // PyTuple_SetItem(args.get(), idx, pyopencv_from(gmarg)); + + // NB: Doesn't increase reference counter (false). + // In case PyObject_CallObject return NULL, do nothing in destructor. + cv::detail::PyObjectHolder result(PyObject_CallObject(out_meta.get(), args.get()), false); + + if (PyErr_Occurred()) + { + PyErr_PrintEx(0); + PyErr_Clear(); + throw std::logic_error("Python kernel failed with error!"); + } + // NB: In fact it's impossible situation, because errors were handled above. + GAPI_Assert(result.get() && "Python kernel returned NULL!"); + + if (!pyopencv_to(result.get(), out, ArgInfo("arg", false))) + { + util::throw_error(std::logic_error("Unsupported output meta type")); + } + } + catch (...) + { + PyGILState_Release(gstate); + throw; + } + PyGILState_Release(gstate); + return out; +} + static GMetaArg get_meta_arg(PyObject* obj) { cv::GMetaArg arg; @@ -860,6 +940,10 @@ static PyObject* pyopencv_cv_gapi_kernels(PyObject* , PyObject* py_args, PyObjec "Python kernel should contain run, please use cv.gapi.kernel to define kernel"); return NULL; } + PyObject* setup = nullptr; + if (PyObject_HasAttrString(user_kernel, "setup")) { + setup = PyObject_GetAttrString(user_kernel, "setup"); + } std::string id; if (!pyopencv_to(id_obj, id, ArgInfo("id", false))) @@ -869,10 +953,22 @@ static PyObject* pyopencv_cv_gapi_kernels(PyObject* , PyObject* py_args, PyObjec } using namespace std::placeholders; - gapi::python::GPythonFunctor f(id.c_str(), - std::bind(run_py_meta , cv::detail::PyObjectHolder{out_meta}, _1, _2), - std::bind(run_py_kernel, cv::detail::PyObjectHolder{run} , _1)); - pkg.include(f); + + if (setup) + { + gapi::python::GPythonFunctor f( + id.c_str(), std::bind(run_py_meta, cv::detail::PyObjectHolder{out_meta}, _1, _2), + std::bind(run_py_kernel, cv::detail::PyObjectHolder{run}, _1), + std::bind(setup_py, cv::detail::PyObjectHolder{setup}, _1, _2)); + pkg.include(f); + } + else + { + gapi::python::GPythonFunctor f( + id.c_str(), std::bind(run_py_meta, cv::detail::PyObjectHolder{out_meta}, _1, _2), + std::bind(run_py_kernel, cv::detail::PyObjectHolder{run}, _1)); + pkg.include(f); + } } return pyopencv_from(pkg); } diff --git a/modules/gapi/src/backends/python/gpythonbackend.cpp b/modules/gapi/src/backends/python/gpythonbackend.cpp index 4361bab75d..4bfdc7d3ec 100644 --- a/modules/gapi/src/backends/python/gpythonbackend.cpp +++ b/modules/gapi/src/backends/python/gpythonbackend.cpp @@ -6,15 +6,19 @@ #include // zip_range, indexed +#include "compiler/gmodel.hpp" +#include #include // throw_error #include #include "api/gbackend_priv.hpp" #include "backends/common/gbackend.hpp" -cv::gapi::python::GPythonKernel::GPythonKernel(cv::gapi::python::Impl run) - : m_run(run) +cv::gapi::python::GPythonKernel::GPythonKernel(cv::gapi::python::Impl run, + cv::gapi::python::Setup setup) + : m_run(run), m_setup(setup) { + m_isStateful = (setup != nullptr); } cv::GRunArgs cv::gapi::python::GPythonKernel::operator()(const cv::gapi::python::GPythonContext& ctx) @@ -23,9 +27,10 @@ cv::GRunArgs cv::gapi::python::GPythonKernel::operator()(const cv::gapi::python: } cv::gapi::python::GPythonFunctor::GPythonFunctor(const char* id, - const cv::gapi::python::GPythonFunctor::Meta &meta, - const cv::gapi::python::Impl& impl) - : gapi::GFunctor(id), impl_{GPythonKernel{impl}, meta} + const cv::gapi::python::GPythonFunctor::Meta& meta, + const cv::gapi::python::Impl& impl, + const cv::gapi::python::Setup& setup) + : gapi::GFunctor(id), impl_{GPythonKernel{impl, setup}, meta} { } @@ -68,6 +73,7 @@ class GPythonExecutable final: public cv::gimpl::GIslandExecutable virtual cv::RMat allocate(const cv::GMatDesc&) const override { return {}; } virtual bool canReshape() const override { return true; } + virtual void handleNewStream() override; virtual void reshape(ade::Graph&, const cv::GCompileArgs&) override { // Do nothing here } @@ -80,6 +86,7 @@ public: cv::gimpl::GModel::ConstGraph m_gm; cv::gapi::python::GPythonKernel m_kernel; ade::NodeHandle m_op; + cv::GArg m_node_state; cv::GTypesInfo m_out_info; cv::GMetaArgs m_in_metas; @@ -153,6 +160,15 @@ static void writeBack(cv::GRunArg& arg, cv::GRunArgP& out) } } +void GPythonExecutable::handleNewStream() +{ + if (!m_kernel.m_isStateful) + return; + + m_node_state = m_kernel.m_setup(cv::gimpl::GModel::collectInputMeta(m_gm, m_op), + m_gm.metadata(m_op).get().args); +} + void GPythonExecutable::run(std::vector &&input_objs, std::vector &&output_objs) { @@ -165,8 +181,15 @@ void GPythonExecutable::run(std::vector &&input_objs, std::back_inserter(inputs), std::bind(&packArg, std::ref(m_res), _1)); + cv::gapi::python::GPythonContext ctx{inputs, m_in_metas, m_out_info, false}; + + // For stateful kernel add state to its execution context + if (m_kernel.m_isStateful) + { + ctx.m_state = m_node_state; + ctx.m_isStateful = true; + } - cv::gapi::python::GPythonContext ctx{inputs, m_in_metas, m_out_info}; auto outs = m_kernel(ctx); for (auto&& it : ade::util::zip(outs, output_objs)) @@ -225,6 +248,12 @@ GPythonExecutable::GPythonExecutable(const ade::Graph& g, m_op = *it; m_kernel = cag.metadata(m_op).get().kernel; + // If kernel is stateful then prepare storage for its state. + if (m_kernel.m_isStateful) + { + m_node_state = cv::GArg{ }; + } + // Ensure this the only op in the graph if (std::any_of(it+1, nodes.end(), is_op)) {