diff --git a/modules/gapi/include/opencv2/gapi/media.hpp b/modules/gapi/include/opencv2/gapi/media.hpp index aa7d6d6a1f..19aaef3fd1 100644 --- a/modules/gapi/include/opencv2/gapi/media.hpp +++ b/modules/gapi/include/opencv2/gapi/media.hpp @@ -15,6 +15,16 @@ #include #include +// Forward declaration +namespace cv { +namespace gapi { +namespace s11n { +struct IOStream; +struct IIStream; +} // namespace s11n +} // namespace gapi +} // namespace cv + namespace cv { /** \addtogroup gapi_data_structures @@ -125,6 +135,16 @@ public: return dynamic_cast(adapter); } + /** + * @brief Serialize MediaFrame's data to a byte array. + * + * @note The actual logic is implemented by frame's adapter class. + * Does nothing by default. + * + * @param os Bytestream to store serialized MediaFrame data in. + */ + void serialize(cv::gapi::s11n::IOStream& os) const; + private: struct Priv; std::shared_ptr m; @@ -221,6 +241,14 @@ public: // FIXME: design a better solution // The default implementation does nothing virtual cv::util::any blobParams() const; + virtual void serialize(cv::gapi::s11n::IOStream&) { + GAPI_Assert(false && "Generic serialize method of MediaFrame::IAdapter does nothing by default. " + "Please, implement it in derived class to properly serialize the object."); + } + virtual void deserialize(cv::gapi::s11n::IIStream&) { + GAPI_Assert(false && "Generic deserialize method of MediaFrame::IAdapter does nothing by default. " + "Please, implement it in derived class to properly deserialize the object."); + } }; /** @} */ diff --git a/modules/gapi/include/opencv2/gapi/rmat.hpp b/modules/gapi/include/opencv2/gapi/rmat.hpp index cc27f48664..6b289001e7 100644 --- a/modules/gapi/include/opencv2/gapi/rmat.hpp +++ b/modules/gapi/include/opencv2/gapi/rmat.hpp @@ -14,8 +14,8 @@ namespace cv { namespace gapi { namespace s11n { - struct IOStream; - struct IIStream; +struct IOStream; +struct IIStream; } // namespace s11n } // namespace gapi } // namespace cv @@ -111,10 +111,12 @@ public: // is transferred to the device when the view is destroyed virtual View access(Access) = 0; virtual void serialize(cv::gapi::s11n::IOStream&) { - GAPI_Assert(false && "Generic serialize method should never be called for RMat adapter"); + GAPI_Assert(false && "Generic serialize method of RMat::Adapter does nothing by default. " + "Please, implement it in derived class to properly serialize the object."); } virtual void deserialize(cv::gapi::s11n::IIStream&) { - GAPI_Assert(false && "Generic deserialize method should never be called for RMat adapter"); + GAPI_Assert(false && "Generic deserialize method of RMat::Adapter does nothing by default. " + "Please, implement it in derived class to properly deserialize the object."); } }; using AdapterP = std::shared_ptr; diff --git a/modules/gapi/include/opencv2/gapi/s11n.hpp b/modules/gapi/include/opencv2/gapi/s11n.hpp index ca8e32c98b..53800970d1 100644 --- a/modules/gapi/include/opencv2/gapi/s11n.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n.hpp @@ -13,6 +13,13 @@ #include #include #include +#include +#include + +// FIXME: caused by deserialize_runarg +#if (defined _WIN32 || defined _WIN64) && defined _MSC_VER +#pragma warning(disable: 4702) +#endif namespace cv { namespace gapi { @@ -34,8 +41,8 @@ namespace detail { template cv::GCompileArgs getCompileArgs(const std::vector &bytes); - template - cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes); + template + cv::GRunArgs getRunArgsWithAdapters(const std::vector &bytes); } // namespace detail /** @brief Serialize a graph represented by GComputation into an array of bytes. @@ -133,19 +140,18 @@ type deserialize(const std::vector &bytes) { } /** - * @brief Deserialize GRunArgs including RMat objects if any from a byte array. + * @brief Deserialize GRunArgs including RMat and MediaFrame objects if any from a byte array. * - * RMat adapter type is specified in the template. - * @note To be used properly specified adapter type must overload its serialize() and - * deserialize() methods. + * Adapter types are specified in the template. + * @note To be used properly specified adapter types must overload their deserialize() method. * @param bytes vector of bytes to deserialize GRunArgs object from. - * @return GRunArgs including RMat objects if any. - * @see RMat + * @return GRunArgs including RMat and MediaFrame objects if any. + * @see RMat MediaFrame */ -template inline +template inline typename std::enable_if::value, GRunArgs>:: type deserialize(const std::vector &bytes) { - return detail::getRunArgsWithRMats(bytes); + return detail::getRunArgsWithAdapters(bytes); } } // namespace gapi } // namespace cv @@ -399,16 +405,39 @@ static cv::util::optional exec(const std::string& tag, cv::gapi::s1 } }; -template struct deserialize_runarg; +template +struct deserialize_arg_with_adapter; + +template +struct deserialize_arg_with_adapter { +static GRunArg exec(cv::gapi::s11n::IIStream& is) { + std::unique_ptr ptr(new TA); + ptr->deserialize(is); + return GRunArg { RA(std::move(ptr)) }; +} +}; + +template +struct deserialize_arg_with_adapter { +static GRunArg exec(cv::gapi::s11n::IIStream&) { + GAPI_Assert(false && "No suitable adapter class found during RMat/MediaFrame deserialization. " + "Please, make sure you've passed them in cv::gapi::deserialize() template"); + return GRunArg{}; +} +}; -template +template struct deserialize_runarg { static GRunArg exec(cv::gapi::s11n::IIStream& is, uint32_t idx) { if (idx == GRunArg::index_of()) { - auto ptr = std::make_shared(); - ptr->deserialize(is); - return GRunArg { RMat(std::move(ptr)) }; - } else { // non-RMat arg - use default deserialization + // Type or void (if not found) + using TA = typename cv::util::find_adapter_impl::type; + return deserialize_arg_with_adapter::exec(is); + } else if (idx == GRunArg::index_of()) { + // Type or void (if not found) + using TA = typename cv::util::find_adapter_impl::type; + return deserialize_arg_with_adapter::exec(is); + } else { // not an adapter holding type runarg - use default deserialization GRunArg arg; getRunArgByIdx(is, arg, idx); return arg; @@ -451,8 +480,8 @@ cv::GCompileArgs getCompileArgs(const std::vector &sArgs) { return args; } -template -cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes) { +template +cv::GRunArgs getRunArgsWithAdapters(const std::vector &bytes) { std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(bytes); cv::gapi::s11n::IIStream& is = *pIs; cv::GRunArgs args; @@ -462,7 +491,7 @@ cv::GRunArgs getRunArgsWithRMats(const std::vector &bytes) { for (uint32_t i = 0; i < sz; ++i) { uint32_t idx = 0; is >> idx; - args.push_back(cv::gapi::detail::deserialize_runarg::exec(is, idx)); + args.push_back(cv::gapi::detail::deserialize_runarg::exec(is, idx)); } return args; diff --git a/modules/gapi/include/opencv2/gapi/util/util.hpp b/modules/gapi/include/opencv2/gapi/util/util.hpp index c6ad0632e2..eb435a3eef 100644 --- a/modules/gapi/include/opencv2/gapi/util/util.hpp +++ b/modules/gapi/include/opencv2/gapi/util/util.hpp @@ -153,7 +153,29 @@ overload_lamba_set overload_lambdas(L&& ...lambdas) { return overload_lamba_set(std::forward(lambdas)...); } -} + +template +struct find_adapter_impl; + +template +struct find_adapter_impl +{ + using type = typename std::conditional::value, + T, + void>::type; + static constexpr bool found = std::is_base_of::value; +}; + +template +struct find_adapter_impl +{ + using type = typename std::conditional::value, + T, + typename find_adapter_impl::type>::type; + static constexpr bool found = std::is_base_of::value || + find_adapter_impl::found; +}; +} // namespace util } // namespace cv // \endcond diff --git a/modules/gapi/src/api/media.cpp b/modules/gapi/src/api/media.cpp index 884fc9e83d..b1c455d40a 100644 --- a/modules/gapi/src/api/media.cpp +++ b/modules/gapi/src/api/media.cpp @@ -35,6 +35,10 @@ cv::MediaFrame::IAdapter* cv::MediaFrame::getAdapter() const { return m->adapter.get(); } +void cv::MediaFrame::serialize(cv::gapi::s11n::IOStream& os) const { + return m->adapter->serialize(os); +} + cv::MediaFrame::View::View(Ptrs&& ptrs, Strides&& strs, Callback &&cb) : ptr (std::move(ptrs)) , stride(std::move(strs)) diff --git a/modules/gapi/src/api/s11n.cpp b/modules/gapi/src/api/s11n.cpp index 97f5a95c42..bd7f46c88a 100644 --- a/modules/gapi/src/api/s11n.cpp +++ b/modules/gapi/src/api/s11n.cpp @@ -76,14 +76,14 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args) { #if !defined(GAPI_STANDALONE) case T::index_of() : - outputs.emplace_back((cv::UMat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; #endif case cv::GRunArg::index_of() : - outputs.emplace_back((cv::Mat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; case cv::GRunArg::index_of() : - outputs.emplace_back((cv::Scalar*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; case T::index_of() : outputs.emplace_back(cv::util::get(res_obj)); @@ -92,7 +92,10 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args) outputs.emplace_back(cv::util::get(res_obj)); break; case cv::GRunArg::index_of() : - outputs.emplace_back((cv::RMat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); + break; + case cv::GRunArg::index_of() : + outputs.emplace_back(&(cv::util::get(res_obj))); break; default: GAPI_Assert(false && "This value type is not supported!"); // ...maybe because of STANDALONE mode. @@ -130,6 +133,9 @@ cv::GRunArg cv::gapi::bind(cv::GRunArgP &out) case T::index_of() : return cv::GRunArg(*cv::util::get(out)); + case T::index_of() : + return cv::GRunArg(*cv::util::get(out)); + default: // ...maybe our types were extended GAPI_Assert(false && "This value type is UNKNOWN!"); diff --git a/modules/gapi/src/backends/common/serialization.cpp b/modules/gapi/src/backends/common/serialization.cpp index f2c956874c..619b2feb74 100644 --- a/modules/gapi/src/backends/common/serialization.cpp +++ b/modules/gapi/src/backends/common/serialization.cpp @@ -201,18 +201,20 @@ IOStream& operator<< (IOStream& os, const cv::RMat& mat) { return os; } IIStream& operator>> (IIStream& is, cv::RMat&) { - util::throw_error(std::logic_error("operator>> for RMat should never be called")); + util::throw_error(std::logic_error("operator>> for RMat should never be called. " + "Instead, cv::gapi::deserialize() " + "should be used")); return is; } -IOStream& operator<< (IOStream& os, const cv::MediaFrame &) { - // Stub - GAPI_Assert(false && "cv::MediaFrame serialization is not supported!"); +IOStream& operator<< (IOStream& os, const cv::MediaFrame &frame) { + frame.serialize(os); return os; } IIStream& operator>> (IIStream& is, cv::MediaFrame &) { - // Stub - GAPI_Assert(false && "cv::MediaFrame serialization is not supported!"); + util::throw_error(std::logic_error("operator>> for MediaFrame should never be called. " + "Instead, cv::gapi::deserialize() " + "should be used")); return is; } diff --git a/modules/gapi/test/s11n/gapi_s11n_tests.cpp b/modules/gapi/test/s11n/gapi_s11n_tests.cpp index c2b17521d9..4c6e63b552 100644 --- a/modules/gapi/test/s11n/gapi_s11n_tests.cpp +++ b/modules/gapi/test/s11n/gapi_s11n_tests.cpp @@ -2,6 +2,7 @@ #include "backends/common/serialization.hpp" #include +#include #include <../src/backends/common/gbackend.hpp> // asView namespace { @@ -148,6 +149,29 @@ public: int getVal() { return m_value; } std::string getStr() { return m_str; } }; + +class MyMediaFrameAdapter : public cv::MediaFrame::IAdapter { + cv::Mat m_mat; + int m_value; + std::string m_str; +public: + MyMediaFrameAdapter() = default; + MyMediaFrameAdapter(cv::Mat m, int value, const std::string& str) + : m_mat(m), m_value(value), m_str(str) + {} + virtual cv::MediaFrame::View access(cv::MediaFrame::Access) override { + return cv::MediaFrame::View({m_mat.data}, {m_mat.step}); + } + virtual cv::GFrameDesc meta() const override { return {cv::MediaFormat::BGR, m_mat.size()}; } + virtual void serialize(cv::gapi::s11n::IOStream& os) override { + os << m_value << m_str; + } + virtual void deserialize(cv::gapi::s11n::IIStream& is) override { + is >> m_value >> m_str; + } + int getVal() { return m_value; } + std::string getStr() { return m_str; } +}; } namespace opencv_test { @@ -581,6 +605,17 @@ TEST_F(S11N_Basic, Test_Vector_Of_Strings) { EXPECT_EQ("42", des[2]); } +TEST_F(S11N_Basic, Test_RunArg) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto v = cv::GRunArgs{ cv::GRunArg{ mat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::Mat out_mat = cv::util::get(out[0]); + + EXPECT_EQ(0, cv::norm(mat, out_mat)); +} + TEST_F(S11N_Basic, Test_RunArg_RMat) { cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); cv::RMat rmat = cv::make_rmat(mat, 42, "It actually works"); @@ -614,6 +649,87 @@ TEST_F(S11N_Basic, Test_RunArg_RMat_Scalar_Mat) { EXPECT_EQ(0, cv::norm(mat, out_mat)); } +TEST_F(S11N_Basic, Test_RunArg_MediaFrame) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto v = cv::GRunArgs{ cv::GRunArg{ frame } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::MediaFrame out_mat = cv::util::get(out[0]); + auto adapter = out_mat.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); +} + +TEST_F(S11N_Basic, Test_RunArg_MediaFrame_Scalar_Mat) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + cv::Scalar sc(111); + auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ sc }, cv::GRunArg{ mat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::MediaFrame out_frame = cv::util::get(out[0]); + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + cv::Scalar out_sc = cv::util::get(out[1]); + EXPECT_EQ(sc, out_sc); + + cv::Mat out_mat = cv::util::get(out[2]); + EXPECT_EQ(0, cv::norm(mat, out_mat)); +} + +TEST_F(S11N_Basic, Test_RunArg_MediaFrame_RMat) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto rmat = cv::make_rmat(mat2, 24, "Hello there"); + + auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ rmat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + + cv::MediaFrame out_frame = cv::util::get(out[0]); + cv::RMat out_rmat = cv::util::get(out[1]); + + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + auto adapter2 = out_rmat.get(); + EXPECT_EQ(24, adapter2->getVal()); + EXPECT_EQ("Hello there", adapter2->getStr()); +} + +TEST_F(S11N_Basic, Test_RunArg_RMat_MediaFrame) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto rmat = cv::make_rmat(mat2, 24, "Hello there"); + + auto v = cv::GRunArgs{ cv::GRunArg{ rmat }, cv::GRunArg{ frame } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + + cv::RMat out_rmat = cv::util::get(out[0]); + cv::MediaFrame out_frame = cv::util::get(out[1]); + + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + auto adapter2 = out_rmat.get(); + EXPECT_EQ(24, adapter2->getVal()); + EXPECT_EQ("Hello there", adapter2->getStr()); +} + namespace { template bool verifyOpaqueKind(T&& in) {