Merge pull request #20329 from smirnov-alexey:as/mediaframe_serialization

[G-API]: Add serialization mechanism for cv::MediaFrame

* Stub initial interface

* Fix templates for deserialization

* Fix tests

* Disable a warning on windows

* Address review comments

* Change enable_ifs to other template helpers

* Resolve ambiguous template

* Fix warnings in docs

modules/gapi/include/opencv2/gapi/media.hpp

@@ -15,6 +15,16 @@
 #include <opencv2/gapi/gframe.hpp>
 #include <opencv2/gapi/util/any.hpp>
 
+// Forward declaration
+namespace cv {
+namespace gapi {
+namespace s11n {
+struct IOStream;
+struct IIStream;
+} // namespace s11n
+} // namespace gapi
+} // namespace cv
+
 namespace cv {
 
 /** \addtogroup gapi_data_structures
@@ -125,6 +135,16 @@ public:
         return dynamic_cast<T*>(adapter);
     }
 
+    /**
+     * @brief Serialize MediaFrame's data to a byte array.
+     *
+     * @note The actual logic is implemented by frame's adapter class.
+     * Does nothing by default.
+     *
+     * @param os Bytestream to store serialized MediaFrame data in.
+     */
+    void serialize(cv::gapi::s11n::IOStream& os) const;
+
 private:
     struct Priv;
     std::shared_ptr<Priv> m;
@@ -221,6 +241,14 @@ public:
     // FIXME: design a better solution
     // The default implementation does nothing
     virtual cv::util::any blobParams() const;
+    virtual void serialize(cv::gapi::s11n::IOStream&) {
+        GAPI_Assert(false && "Generic serialize method of MediaFrame::IAdapter does nothing by default. "
+                             "Please, implement it in derived class to properly serialize the object.");
+    }
+    virtual void deserialize(cv::gapi::s11n::IIStream&) {
+        GAPI_Assert(false && "Generic deserialize method of MediaFrame::IAdapter does nothing by default. "
+                             "Please, implement it in derived class to properly deserialize the object.");
+    }
 };
 /** @} */
 

modules/gapi/include/opencv2/gapi/rmat.hpp

@@ -14,8 +14,8 @@
 namespace cv {
 namespace gapi {
 namespace s11n {
-    struct IOStream;
-    struct IIStream;
+struct IOStream;
+struct IIStream;
 } // namespace s11n
 } // namespace gapi
 } // namespace cv
@@ -111,10 +111,12 @@ public:
         // is transferred to the device when the view is destroyed
         virtual View access(Access) = 0;
         virtual void serialize(cv::gapi::s11n::IOStream&) {
-            GAPI_Assert(false && "Generic serialize method should never be called for RMat adapter");
+            GAPI_Assert(false && "Generic serialize method of RMat::Adapter does nothing by default. "
+                                 "Please, implement it in derived class to properly serialize the object.");
         }
         virtual void deserialize(cv::gapi::s11n::IIStream&) {
-            GAPI_Assert(false && "Generic deserialize method should never be called for RMat adapter");
+            GAPI_Assert(false && "Generic deserialize method of RMat::Adapter does nothing by default. "
+                                 "Please, implement it in derived class to properly deserialize the object.");
         }
     };
     using AdapterP = std::shared_ptr<Adapter>;

modules/gapi/include/opencv2/gapi/s11n.hpp

@@ -13,6 +13,13 @@
 #include <opencv2/gapi/s11n/base.hpp>
 #include <opencv2/gapi/gcomputation.hpp>
 #include <opencv2/gapi/rmat.hpp>
+#include <opencv2/gapi/media.hpp>
+#include <opencv2/gapi/util/util.hpp>
+
+// FIXME: caused by deserialize_runarg
+#if (defined _WIN32 || defined _WIN64) && defined _MSC_VER
+#pragma warning(disable: 4702)
+#endif
 
 namespace cv {
 namespace gapi {
@@ -34,8 +41,8 @@ namespace detail {
     template<typename... Types>
     cv::GCompileArgs getCompileArgs(const std::vector<char> &bytes);
 
-    template<typename RMatAdapterType>
-    cv::GRunArgs getRunArgsWithRMats(const std::vector<char> &bytes);
+    template<typename... AdapterType>
+    cv::GRunArgs getRunArgsWithAdapters(const std::vector<char> &bytes);
 } // namespace detail
 
 /** @brief Serialize a graph represented by GComputation into an array of bytes.
@@ -133,19 +140,18 @@ type deserialize(const std::vector<char> &bytes) {
 }
 
 /**
- * @brief Deserialize GRunArgs including RMat objects if any from a byte array.
+ * @brief Deserialize GRunArgs including RMat and MediaFrame objects if any from a byte array.
  *
- * RMat adapter type is specified in the template.
- * @note To be used properly specified adapter type must overload its serialize() and
- * deserialize() methods.
+ * Adapter types are specified in the template.
+ * @note To be used properly specified adapter types must overload their deserialize() method.
  * @param bytes vector of bytes to deserialize GRunArgs object from.
- * @return GRunArgs including RMat objects if any.
- * @see RMat
+ * @return GRunArgs including RMat and MediaFrame objects if any.
+ * @see RMat MediaFrame
  */
-template<typename T, typename RMatAdapterType> inline
+template<typename T, typename AtLeastOneAdapterT, typename... AdapterTypes> inline
 typename std::enable_if<std::is_same<T, GRunArgs>::value, GRunArgs>::
 type deserialize(const std::vector<char> &bytes) {
-    return detail::getRunArgsWithRMats<RMatAdapterType>(bytes);
+    return detail::getRunArgsWithAdapters<AtLeastOneAdapterT, AdapterTypes...>(bytes);
 }
 } // namespace gapi
 } // namespace cv
@@ -399,16 +405,39 @@ static cv::util::optional<GCompileArg> exec(const std::string& tag, cv::gapi::s1
 }
 };
 
-template<typename T> struct deserialize_runarg;
+template<typename ...T>
+struct deserialize_arg_with_adapter;
+
+template<typename RA, typename TA>
+struct deserialize_arg_with_adapter<RA, TA> {
+static GRunArg exec(cv::gapi::s11n::IIStream& is) {
+    std::unique_ptr<TA> ptr(new TA);
+    ptr->deserialize(is);
+    return GRunArg { RA(std::move(ptr)) };
+}
+};
+
+template<typename RA>
+struct deserialize_arg_with_adapter<RA, void> {
+static GRunArg exec(cv::gapi::s11n::IIStream&) {
+    GAPI_Assert(false && "No suitable adapter class found during RMat/MediaFrame deserialization. "
+                         "Please, make sure you've passed them in cv::gapi::deserialize() template");
+    return GRunArg{};
+}
+};
 
-template<typename RMatAdapterType>
+template<typename... Types>
 struct deserialize_runarg {
 static GRunArg exec(cv::gapi::s11n::IIStream& is, uint32_t idx) {
     if (idx == GRunArg::index_of<RMat>()) {
-        auto ptr = std::make_shared<RMatAdapterType>();
-        ptr->deserialize(is);
-        return GRunArg { RMat(std::move(ptr)) };
-    } else { // non-RMat arg - use default deserialization
+        // Type or void (if not found)
+        using TA = typename cv::util::find_adapter_impl<RMat::Adapter, Types...>::type;
+        return deserialize_arg_with_adapter<RMat, TA>::exec(is);
+    } else if (idx == GRunArg::index_of<MediaFrame>()) {
+        // Type or void (if not found)
+        using TA = typename cv::util::find_adapter_impl<MediaFrame::IAdapter, Types...>::type;
+        return deserialize_arg_with_adapter<MediaFrame, TA>::exec(is);
+    } else { // not an adapter holding type runarg - use default deserialization
         GRunArg arg;
         getRunArgByIdx(is, arg, idx);
         return arg;
@@ -451,8 +480,8 @@ cv::GCompileArgs getCompileArgs(const std::vector<char> &sArgs) {
     return args;
 }
 
-template<typename RMatAdapterType>
-cv::GRunArgs getRunArgsWithRMats(const std::vector<char> &bytes) {
+template<typename... AdapterTypes>
+cv::GRunArgs getRunArgsWithAdapters(const std::vector<char> &bytes) {
     std::unique_ptr<cv::gapi::s11n::IIStream> pIs = cv::gapi::s11n::detail::getInStream(bytes);
     cv::gapi::s11n::IIStream& is = *pIs;
     cv::GRunArgs args;
@@ -462,7 +491,7 @@ cv::GRunArgs getRunArgsWithRMats(const std::vector<char> &bytes) {
     for (uint32_t i = 0; i < sz; ++i) {
         uint32_t idx = 0;
         is >> idx;
-        args.push_back(cv::gapi::detail::deserialize_runarg<RMatAdapterType>::exec(is, idx));
+        args.push_back(cv::gapi::detail::deserialize_runarg<AdapterTypes...>::exec(is, idx));
     }
 
     return args;

modules/gapi/include/opencv2/gapi/util/util.hpp

@@ -153,7 +153,29 @@ overload_lamba_set<L...> overload_lambdas(L&& ...lambdas)
 {
     return overload_lamba_set<L...>(std::forward<L>(lambdas)...);
 }
-}
+
+template<typename ...T>
+struct find_adapter_impl;
+
+template<typename AdapterT, typename T>
+struct find_adapter_impl<AdapterT, T>
+{
+    using type = typename std::conditional<std::is_base_of<AdapterT, T>::value,
+                                           T,
+                                           void>::type;
+    static constexpr bool found = std::is_base_of<AdapterT, T>::value;
+};
+
+template<typename AdapterT, typename T, typename... Types>
+struct find_adapter_impl<AdapterT, T, Types...>
+{
+    using type = typename std::conditional<std::is_base_of<AdapterT, T>::value,
+                                           T,
+                                           typename find_adapter_impl<AdapterT, Types...>::type>::type;
+    static constexpr bool found = std::is_base_of<AdapterT, T>::value ||
+                                  find_adapter_impl<AdapterT, Types...>::found;
+};
+} // namespace util
 } // namespace cv
 
 // \endcond

modules/gapi/src/api/media.cpp

@@ -35,6 +35,10 @@ cv::MediaFrame::IAdapter* cv::MediaFrame::getAdapter() const {
     return m->adapter.get();
 }
 
+void cv::MediaFrame::serialize(cv::gapi::s11n::IOStream& os) const {
+    return m->adapter->serialize(os);
+}
+
 cv::MediaFrame::View::View(Ptrs&& ptrs, Strides&& strs, Callback &&cb)
     : ptr   (std::move(ptrs))
     , stride(std::move(strs))

modules/gapi/src/api/s11n.cpp

@@ -76,14 +76,14 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args)
         {
 #if !defined(GAPI_STANDALONE)
         case T::index_of<cv::UMat>() :
-            outputs.emplace_back((cv::UMat*)(&(cv::util::get<cv::UMat>(res_obj))));
+            outputs.emplace_back(&(cv::util::get<cv::UMat>(res_obj)));
             break;
 #endif
         case cv::GRunArg::index_of<cv::Mat>() :
-            outputs.emplace_back((cv::Mat*)(&(cv::util::get<cv::Mat>(res_obj))));
+            outputs.emplace_back(&(cv::util::get<cv::Mat>(res_obj)));
             break;
         case cv::GRunArg::index_of<cv::Scalar>() :
-            outputs.emplace_back((cv::Scalar*)(&(cv::util::get<cv::Scalar>(res_obj))));
+            outputs.emplace_back(&(cv::util::get<cv::Scalar>(res_obj)));
             break;
         case T::index_of<cv::detail::VectorRef>() :
             outputs.emplace_back(cv::util::get<cv::detail::VectorRef>(res_obj));
@@ -92,7 +92,10 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args)
             outputs.emplace_back(cv::util::get<cv::detail::OpaqueRef>(res_obj));
             break;
         case cv::GRunArg::index_of<cv::RMat>() :
-            outputs.emplace_back((cv::RMat*)(&(cv::util::get<cv::RMat>(res_obj))));
+            outputs.emplace_back(&(cv::util::get<cv::RMat>(res_obj)));
+            break;
+        case cv::GRunArg::index_of<cv::MediaFrame>() :
+            outputs.emplace_back(&(cv::util::get<cv::MediaFrame>(res_obj)));
             break;
         default:
             GAPI_Assert(false && "This value type is not supported!"); // ...maybe because of STANDALONE mode.
@@ -130,6 +133,9 @@ cv::GRunArg cv::gapi::bind(cv::GRunArgP &out)
     case T::index_of<cv::RMat*>() :
         return cv::GRunArg(*cv::util::get<cv::RMat*>(out));
 
+    case T::index_of<cv::MediaFrame*>() :
+        return cv::GRunArg(*cv::util::get<cv::MediaFrame*>(out));
+
     default:
         // ...maybe our types were extended
         GAPI_Assert(false && "This value type is UNKNOWN!");

modules/gapi/src/backends/common/serialization.cpp

@@ -201,18 +201,20 @@ IOStream& operator<< (IOStream& os, const cv::RMat& mat) {
     return os;
 }
 IIStream& operator>> (IIStream& is, cv::RMat&) {
-    util::throw_error(std::logic_error("operator>> for RMat should never be called"));
+    util::throw_error(std::logic_error("operator>> for RMat should never be called. "
+                                        "Instead, cv::gapi::deserialize<cv::GRunArgs, AdapterTypes...>() "
+                                        "should be used"));
     return is;
 }
 
-IOStream& operator<< (IOStream& os, const cv::MediaFrame &) {
-    // Stub
-    GAPI_Assert(false && "cv::MediaFrame serialization is not supported!");
+IOStream& operator<< (IOStream& os, const cv::MediaFrame &frame) {
+    frame.serialize(os);
     return os;
 }
 IIStream& operator>> (IIStream& is, cv::MediaFrame &) {
-    // Stub
-    GAPI_Assert(false && "cv::MediaFrame serialization is not supported!");
+    util::throw_error(std::logic_error("operator>> for MediaFrame should never be called. "
+                                        "Instead, cv::gapi::deserialize<cv::GRunArgs, AdapterTypes...>() "
+                                        "should be used"));
     return is;
 }
 

modules/gapi/test/s11n/gapi_s11n_tests.cpp

@@ -2,6 +2,7 @@
 
 #include "backends/common/serialization.hpp"
 #include <opencv2/gapi/rmat.hpp>
+#include <opencv2/gapi/media.hpp>
 #include <../src/backends/common/gbackend.hpp> // asView
 
 namespace {
@@ -148,6 +149,29 @@ public:
     int getVal() { return m_value; }
     std::string getStr() { return m_str; }
 };
+
+class MyMediaFrameAdapter : public cv::MediaFrame::IAdapter {
+    cv::Mat m_mat;
+    int m_value;
+    std::string m_str;
+public:
+    MyMediaFrameAdapter() = default;
+    MyMediaFrameAdapter(cv::Mat m, int value, const std::string& str)
+        : m_mat(m), m_value(value), m_str(str)
+    {}
+    virtual cv::MediaFrame::View access(cv::MediaFrame::Access) override {
+        return cv::MediaFrame::View({m_mat.data}, {m_mat.step});
+    }
+    virtual cv::GFrameDesc meta() const override { return {cv::MediaFormat::BGR, m_mat.size()}; }
+    virtual void serialize(cv::gapi::s11n::IOStream& os) override {
+        os << m_value << m_str;
+    }
+    virtual void deserialize(cv::gapi::s11n::IIStream& is) override {
+        is >> m_value >> m_str;
+    }
+    int getVal() { return m_value; }
+    std::string getStr() { return m_str; }
+};
 }
 
 namespace opencv_test {
@@ -581,6 +605,17 @@ TEST_F(S11N_Basic, Test_Vector_Of_Strings) {
     EXPECT_EQ("42", des[2]);
 }
 
+TEST_F(S11N_Basic, Test_RunArg) {
+    cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
+    auto v = cv::GRunArgs{ cv::GRunArg{ mat } };
+
+    const std::vector<char> sargsin = cv::gapi::serialize(v);
+    cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs>(sargsin);
+    cv::Mat out_mat = cv::util::get<cv::Mat>(out[0]);
+
+    EXPECT_EQ(0, cv::norm(mat, out_mat));
+}
+
 TEST_F(S11N_Basic, Test_RunArg_RMat) {
     cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
     cv::RMat rmat = cv::make_rmat<MyRMatAdapter>(mat, 42, "It actually works");
@@ -614,6 +649,87 @@ TEST_F(S11N_Basic, Test_RunArg_RMat_Scalar_Mat) {
     EXPECT_EQ(0, cv::norm(mat, out_mat));
 }
 
+TEST_F(S11N_Basic, Test_RunArg_MediaFrame) {
+    cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
+    auto frame = cv::MediaFrame::Create<MyMediaFrameAdapter>(mat, 42, "It actually works");
+    auto v = cv::GRunArgs{ cv::GRunArg{ frame } };
+
+    const std::vector<char> sargsin = cv::gapi::serialize(v);
+    cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs, MyMediaFrameAdapter>(sargsin);
+    cv::MediaFrame out_mat = cv::util::get<cv::MediaFrame>(out[0]);
+    auto adapter = out_mat.get<MyMediaFrameAdapter>();
+    EXPECT_EQ(42, adapter->getVal());
+    EXPECT_EQ("It actually works", adapter->getStr());
+}
+
+TEST_F(S11N_Basic, Test_RunArg_MediaFrame_Scalar_Mat) {
+    cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
+    auto frame = cv::MediaFrame::Create<MyMediaFrameAdapter>(mat, 42, "It actually works");
+    cv::Scalar sc(111);
+    auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ sc }, cv::GRunArg{ mat } };
+
+    const std::vector<char> sargsin = cv::gapi::serialize(v);
+    cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs, MyMediaFrameAdapter>(sargsin);
+    cv::MediaFrame out_frame = cv::util::get<cv::MediaFrame>(out[0]);
+    auto adapter = out_frame.get<MyMediaFrameAdapter>();
+    EXPECT_EQ(42, adapter->getVal());
+    EXPECT_EQ("It actually works", adapter->getStr());
+
+    cv::Scalar out_sc = cv::util::get<cv::Scalar>(out[1]);
+    EXPECT_EQ(sc, out_sc);
+
+    cv::Mat out_mat = cv::util::get<cv::Mat>(out[2]);
+    EXPECT_EQ(0, cv::norm(mat, out_mat));
+}
+
+TEST_F(S11N_Basic, Test_RunArg_MediaFrame_RMat) {
+    cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
+    cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
+
+    auto frame = cv::MediaFrame::Create<MyMediaFrameAdapter>(mat, 42, "It actually works");
+    auto rmat = cv::make_rmat<MyRMatAdapter>(mat2, 24, "Hello there");
+
+    auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ rmat } };
+
+    const std::vector<char> sargsin = cv::gapi::serialize(v);
+    cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs, MyMediaFrameAdapter, MyRMatAdapter>(sargsin);
+
+    cv::MediaFrame out_frame = cv::util::get<cv::MediaFrame>(out[0]);
+    cv::RMat out_rmat = cv::util::get<cv::RMat>(out[1]);
+
+    auto adapter = out_frame.get<MyMediaFrameAdapter>();
+    EXPECT_EQ(42, adapter->getVal());
+    EXPECT_EQ("It actually works", adapter->getStr());
+
+    auto adapter2 = out_rmat.get<MyRMatAdapter>();
+    EXPECT_EQ(24, adapter2->getVal());
+    EXPECT_EQ("Hello there", adapter2->getStr());
+}
+
+TEST_F(S11N_Basic, Test_RunArg_RMat_MediaFrame) {
+    cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
+    cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
+
+    auto frame = cv::MediaFrame::Create<MyMediaFrameAdapter>(mat, 42, "It actually works");
+    auto rmat = cv::make_rmat<MyRMatAdapter>(mat2, 24, "Hello there");
+
+    auto v = cv::GRunArgs{ cv::GRunArg{ rmat }, cv::GRunArg{ frame } };
+
+    const std::vector<char> sargsin = cv::gapi::serialize(v);
+    cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs, MyMediaFrameAdapter, MyRMatAdapter>(sargsin);
+
+    cv::RMat out_rmat = cv::util::get<cv::RMat>(out[0]);
+    cv::MediaFrame out_frame = cv::util::get<cv::MediaFrame>(out[1]);
+
+    auto adapter = out_frame.get<MyMediaFrameAdapter>();
+    EXPECT_EQ(42, adapter->getVal());
+    EXPECT_EQ("It actually works", adapter->getStr());
+
+    auto adapter2 = out_rmat.get<MyRMatAdapter>();
+    EXPECT_EQ(24, adapter2->getVal());
+    EXPECT_EQ("Hello there", adapter2->getStr());
+}
+
 namespace {
     template <cv::detail::OpaqueKind K, typename T>
     bool verifyOpaqueKind(T&& in) {