Merge pull request #25055 from dmatveev:dm/value_initialized_gmat

G-API: A quick value-initialization support GMat #25055

This PR enables `GMat` objects to be value-initialized in the same way as it was done for `GScalar`s (and, possibly, other types).

- Added some helper methods in backends to distinguish if a certain G-type value initialization is supported or not;
- Added tests, including negative.

Where it is needed:

- Further extension of the OVCV backend (#24379 - will be refreshed soon);
- Further experiments with DNN module;
- Further experiments with "G-API behind UMat" sort of aggregation.

In the current form, PR can be reviewed & merged (@TolyaTalamanov please have a look)

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake

modules/gapi/include/opencv2/gapi/gmat.hpp

@@ -77,6 +77,18 @@ public:
      */
     GAPI_WRAP GMat();                       // Empty constructor
 
+    /**
+     * @brief Constructs a value-initialized GMat
+     *
+     * GMat may be associated with a buffer at graph construction time.
+     * It is useful when some operation has a Mat input which doesn't
+     * change during the program execution, and is set only once.
+     * In this case, there's no need to declare such GMat as graph input.
+     *
+     * @param m a cv::Mat buffer to associate with this GMat object.
+     */
+    GAPI_WRAP explicit GMat(cv::Mat m);     // Value-initialization constructor
+
     /// @private
     GMat(const GNode &n, std::size_t out);  // Operation result constructor
     /// @private

modules/gapi/include/opencv2/gapi/gscalar.hpp

@@ -54,12 +54,11 @@ public:
     /**
      * @brief Constructs a value-initialized GScalar
      *
-     * In contrast with GMat (which can be either an explicit graph input
+     * GScalars may have their values be associated at graph
-     * or a result of some operation), GScalars may have their values
+     * construction time. It is useful when some operation has a
-     * be associated at graph construction time. It is useful when
+     * GScalar input which doesn't change during the program
-     * some operation has a GScalar input which doesn't change during
+     * execution, and is set only once. In this case, there is no need
-     * the program execution, and is set only once. In this case,
+     * to declare such GScalar as a graph input.
-     * there is no need to declare such GScalar as a graph input.
      *
      * @note The value of GScalar may be overwritten by assigning some
      * other GScalar to the object using `operator=` -- on the

modules/gapi/src/api/gbackend.cpp

@@ -80,6 +80,9 @@ bool cv::gapi::GBackend::Priv::allowsMerge(const cv::gimpl::GIslandModel::Graph
     return true;
 }
 
+bool cv::gapi::GBackend::Priv::supportsConst(cv::GShape) const {
+    return false;
+}
 
 // GBackend public implementation //////////////////////////////////////////////
 cv::gapi::GBackend::GBackend()

modules/gapi/src/api/gbackend_priv.hpp

@@ -84,6 +84,14 @@ public:
                              const ade::NodeHandle &slot_nh,
                              const ade::NodeHandle &b_nh) const;
 
+    // Ask backend if it supports CONST_VAL data of the given shape or not.
+    // If the backend does support this data type, a Data node with such
+    // value can be fused into the backend's Island body.
+    // If the backend doesn't support this data type, a Data node won't
+    // be fused into the Islands's body -- will be marked as an in-graph
+    // input connection for this Island.
+    virtual bool supportsConst(cv::GShape shape) const;
+
     virtual ~Priv() = default;
 };
 

modules/gapi/src/api/gcomputation.cpp

@@ -191,8 +191,8 @@ void cv::GComputation::recompile(GMetaArgs&& in_metas, GCompileArgs &&args)
     if (m_priv->m_lastMetas != in_metas)
     {
         if (m_priv->m_lastCompiled &&
-                m_priv->m_lastCompiled.canReshape() &&
+            m_priv->m_lastCompiled.canReshape() &&
-                formats_are_same(m_priv->m_lastMetas, in_metas))
+            formats_are_same(m_priv->m_lastMetas, in_metas))
         {
             m_priv->m_lastCompiled.reshape(in_metas, args);
         }
@@ -203,6 +203,11 @@ void cv::GComputation::recompile(GMetaArgs&& in_metas, GCompileArgs &&args)
         }
         m_priv->m_lastMetas = in_metas;
     }
+    else if (in_metas.size() == 0) {
+        // Happens when the graph is head-less (e.g. starts with const-vals only)
+        // always compile ad-hoc
+        m_priv->m_lastCompiled = compile(GMetaArgs(in_metas), std::move(args));
+    }
 }
 
 void cv::GComputation::apply(GRunArgs &&ins, GRunArgsP &&outs, GCompileArgs &&args)

modules/gapi/src/api/gmat.cpp

@@ -26,6 +26,10 @@ cv::GMat::GMat(const GNode &n, std::size_t out)
 {
 }
 
+cv::GMat::GMat(cv::Mat m)
+    : m_priv(new GOrigin(GShape::GMAT, cv::gimpl::ConstVal(m))) {
+}
+
 cv::GOrigin& cv::GMat::priv()
 {
     return *m_priv;

modules/gapi/src/api/gproto.cpp

@@ -80,6 +80,7 @@ cv::GRunArg cv::value_of(const cv::GOrigin &origin)
     {
     case GShape::GSCALAR: return GRunArg(util::get<cv::Scalar>(origin.value));
     case GShape::GARRAY:  return GRunArg(util::get<cv::detail::VectorRef>(origin.value));
+    case GShape::GMAT:    return GRunArg(util::get<cv::Mat>(origin.value));
     default: util::throw_error(std::logic_error("Unsupported shape for constant"));
     }
 }

modules/gapi/src/backends/cpu/gcpubackend.cpp

@@ -65,6 +65,17 @@ namespace
         {
             return EPtr{new cv::gimpl::GCPUExecutable(graph, compileArgs, nodes)};
         }
+
+        virtual bool supportsConst(cv::GShape shape) const override
+        {
+            // Supports all types of const values
+            return shape == cv::GShape::GOPAQUE
+                || shape == cv::GShape::GSCALAR
+                || shape == cv::GShape::GARRAY;
+            // yes, value-initialized GMats are not supported currently
+            // as in-island data -- compiler will lift these values to the
+            // GIslandModel's SLOT level (will be handled uniformly)
+        }
    };
 }
 

modules/gapi/src/backends/ocl/goclbackend.cpp

@@ -62,6 +62,17 @@ namespace
         {
             return EPtr{new cv::gimpl::GOCLExecutable(graph, nodes)};
         }
+
+        virtual bool supportsConst(cv::GShape shape) const override
+        {
+            // Supports all types of const values
+            return shape == cv::GShape::GOPAQUE
+                || shape == cv::GShape::GSCALAR
+                || shape == cv::GShape::GARRAY;
+            // yes, value-initialized GMats are not supported currently
+            // as in-island data -- compiler will lift these values to the
+            // GIslandModel's SLOT level (will be handled uniformly)
+        }
    };
 }
 

modules/gapi/src/compiler/gobjref.hpp

@@ -24,6 +24,7 @@ namespace gimpl
     < util::monostate
     , cv::Scalar
     , cv::detail::VectorRef
+    , cv::Mat
     >;
 
     struct RcDesc

modules/gapi/src/compiler/passes/exec.cpp

@@ -9,7 +9,7 @@
 
 #include <string>
 #include <list> // list
-#include <iomanip>  // setw, etc
+#include <iomanip> // setw, etc
 #include <fstream> // ofstream
 #include <memory>
 #include <functional>
@@ -85,7 +85,7 @@ namespace
 
         const auto& backend = *src_g.metadata().get<ActiveBackends>().backends.cbegin();
         const auto& proto = src_g.metadata().get<Protocol>();
-        GIsland::node_set all, in_ops, out_ops;
+        GIsland::node_set all, in_ops, out_ops, in_cvals;
 
         all.insert(src_g.nodes().begin(), src_g.nodes().end());
 
@@ -99,7 +99,22 @@ namespace
             all.erase(nh);
             out_ops.insert(nh->inNodes().begin(), nh->inNodes().end());
         }
-
+        for (const auto& nh : src_g.nodes())
+        {
+            if (src_g.metadata(nh).get<NodeType>().t == NodeType::DATA)
+            {
+                const auto &d = src_g.metadata(nh).get<Data>();
+                if (d.storage == Data::Storage::CONST_VAL
+                    && !backend.priv().supportsConst(d.shape)) {
+                    // don't put this node into the island's graph - so the island
+                    // executable don't need to handle value-initialized G-type manually.
+                    // Still mark its readers as inputs
+                    all.erase(nh);
+                    in_cvals.insert(nh);
+                    in_ops.insert(nh->outNodes().begin(), nh->outNodes().end());
+                }
+            }
+        }
         auto isl = std::make_shared<GIsland>(backend,
                                              std::move(all),
                                              std::move(in_ops),
@@ -108,7 +123,8 @@ namespace
 
         auto ih = GIslandModel::mkIslandNode(g, std::move(isl));
 
-        for (const auto& nh : proto.in_nhs)
+        for (const auto& nh : ade::util::chain(ade::util::toRange(proto.in_nhs),
+                                               ade::util::toRange(in_cvals)))
         {
             auto slot = GIslandModel::mkSlotNode(g, nh);
             g.link(slot, ih);

modules/gapi/src/executor/gexecutor.cpp

@@ -208,6 +208,12 @@ void cv::gimpl::GExecutor::initResource(const ade::NodeHandle & nh, const ade::N
     switch (d.shape)
     {
     case GShape::GMAT:
+        if (d.storage == Data::Storage::CONST_VAL)
+        {
+            auto rc = RcDesc{d.rc, d.shape, d.ctor};
+            magazine::bindInArgExec(m_res, rc, m_gm.metadata(orig_nh).get<ConstValue>().arg);
+        }
+        else
         {
             // Let island allocate it's outputs if it can,
             // allocate cv::Mat and wrap it with RMat otherwise

modules/gapi/src/executor/gthreadedexecutor.cpp

@@ -175,7 +175,11 @@ void cv::gimpl::GThreadedExecutor::initResource(const ade::NodeHandle &nh, const
     // to as it is bound externally (e.g. already in the m_state.mag)
 
     switch (d.shape) {
-    case GShape::GMAT: {
+    case GShape::GMAT:
+        if (d.storage == Data::Storage::CONST_VAL) {
+            auto rc = RcDesc{d.rc, d.shape, d.ctor};
+            magazine::bindInArgExec(m_state.mag, rc, m_gm.metadata(orig_nh).get<ConstValue>().arg);
+        } else {
             // Let island allocate it's outputs if it can,
             // allocate cv::Mat and wrap it with RMat otherwise
             GAPI_Assert(!nh->inNodes().empty());

modules/gapi/test/gapi_mat_tests.cpp

@@ -0,0 +1,114 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2024 Intel Corporation
+
+
+#include "test_precomp.hpp"
+
+#include <opencv2/gapi/cpu/core.hpp>
+#include <opencv2/gapi/ocl/core.hpp>
+#include <opencv2/gapi/fluid/core.hpp>
+
+namespace opencv_test
+{
+namespace
+{
+enum class KernelPackage: int
+{
+    OCV,
+    OCL,
+    FLUID,
+};
+std::ostream& operator<< (std::ostream &os, const KernelPackage &e)
+{
+    switch (e)
+    {
+#define _C(X) case KernelPackage::X: os << #X; break
+        _C(OCV);
+        _C(OCL);
+        _C(FLUID);
+#undef _C
+    default: GAPI_Error("Unknown package");
+    }
+    return os;
+}
+} // namespace
+
+struct GMatWithValue : public TestWithParam <KernelPackage> {
+    cv::GKernelPackage getKernelPackage() {
+        switch (GetParam()) {
+        case KernelPackage::OCV: return cv::gapi::core::cpu::kernels();
+        case KernelPackage::OCL: return cv::gapi::core::ocl::kernels();
+        case KernelPackage::FLUID: return cv::gapi::core::fluid::kernels();
+        default: GAPI_Error("Unknown package");
+        }
+    }
+};
+
+TEST_P(GMatWithValue, SingleIsland)
+{
+    cv::Size sz(2, 2);
+    cv::Mat in_mat = cv::Mat::eye(sz, CV_8U);
+
+    cv::GComputationT<cv::GMat(cv::GMat)> addEye([&](cv::GMat in) {
+        return in + cv::GMat(cv::Mat::eye(sz, CV_8U));
+    });
+
+    cv::Mat out_mat;
+    addEye.apply(in_mat, out_mat, cv::compile_args(cv::gapi::use_only{getKernelPackage()}));
+
+    cv::Mat out_mat_ref = in_mat*2;
+    EXPECT_EQ(0, cvtest::norm(out_mat, out_mat_ref, NORM_INF));
+}
+
+TEST_P(GMatWithValue, GraphWithNoInput)
+{
+    cv::Mat cval = cv::Mat::eye(cv::Size(2, 2), CV_8U);
+    cv::GMat gval = cv::GMat(cval);
+    cv::GMat out = cv::gapi::bitwise_not(gval);
+
+    cv::Mat out_mat;
+    cv::GComputation f(cv::GIn(), cv::GOut(out));
+
+    // Compiling this isn't supported for now
+    EXPECT_ANY_THROW(f.compile(cv::descr_of(cval),
+                               cv::compile_args(cv::gapi::use_only{getKernelPackage()})));
+}
+
+INSTANTIATE_TEST_CASE_P(GAPI_GMat, GMatWithValue,
+                        Values(KernelPackage::OCV,
+                               KernelPackage::OCL,
+                               KernelPackage::FLUID));
+
+TEST(GAPI_MatWithValue, MultipleIslands)
+{
+    // This test employs a non-trivial island fusion process
+    // as there's multiple backends in the graph
+
+    cv::Size sz(2, 2);
+    cv::Mat cval2 = cv::Mat::eye(sz, CV_8U) * 2;
+    cv::Mat cval1 = cv::Mat::eye(sz, CV_8U);
+
+    cv::GMat in;
+    cv::GMat tmp = in  + cv::GMat(cval2); // Will be a Fluid operation
+    cv::GMat out = tmp - cv::GMat(cval1); // Will be an OCV operation
+
+    cv::GKernelPackage fluid_kernels = cv::gapi::core::fluid::kernels();
+    cv::GKernelPackage opencv_kernels = cv::gapi::core::cpu::kernels();
+    fluid_kernels.remove<cv::gapi::core::GSub>();
+    opencv_kernels.remove<cv::gapi::core::GAdd>();
+    auto kernels = cv::gapi::combine(fluid_kernels, opencv_kernels);
+
+    cv::Mat in_mat = cv::Mat::zeros(sz, CV_8U);
+    cv::Mat out_mat;
+    auto cc = cv::GComputation(in, out)
+        .compile(cv::descr_of(in_mat),
+                 cv::compile_args(cv::gapi::use_only{kernels}));
+    cc(cv::gin(in_mat), cv::gout(out_mat));
+
+    EXPECT_EQ(0, cvtest::norm(out_mat, cv::Mat::eye(sz, CV_8U), NORM_INF));
+}
+
+} // namespace opencv_test