mirror of https://github.com/opencv/opencv.git
Merge pull request #18793 from dmatveev:dm/in_graph_metadata
G-API: Introduce runtime in-graph metadata * G-API: In-graph metadata -- initial implementation * G-API: Finish the in-graph metadata implementation for Streaming * G-API: Fix standalone build & warnings for in-graph metadata * G-API: In-graph meta -- fixed review comments * G-API: Fix issues with desync causing failing testspull/18868/head
Dmitry Matveev
4 years ago
committed by
GitHub
parent
b5c162175b
commit
b866d0dc38
16 changed files with 681 additions and 55 deletions
@ -0,0 +1,79 @@ |
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
//
|
||||
// Copyright (C) 2020 Intel Corporation
|
||||
|
||||
|
||||
#ifndef OPENCV_GAPI_GSTREAMING_META_HPP |
||||
#define OPENCV_GAPI_GSTREAMING_META_HPP |
||||
|
||||
#include <opencv2/gapi/gopaque.hpp> |
||||
#include <opencv2/gapi/gcall.hpp> |
||||
#include <opencv2/gapi/gkernel.hpp> |
||||
#include <opencv2/gapi/gtype_traits.hpp> |
||||
|
||||
namespace cv { |
||||
namespace gapi { |
||||
namespace streaming { |
||||
|
||||
// FIXME: the name is debatable
|
||||
namespace meta_tag { |
||||
static constexpr const char * timestamp = "org.opencv.gapi.meta.timestamp"; |
||||
static constexpr const char * seq_id = "org.opencv.gapi.meta.seq_id"; |
||||
} // namespace meta_tag
|
||||
|
||||
namespace detail { |
||||
struct GMeta { |
||||
static const char *id() { |
||||
return "org.opencv.streaming.meta"; |
||||
} |
||||
// A universal yield for meta(), same as in GDesync
|
||||
template<typename... R, int... IIs> |
||||
static std::tuple<R...> yield(cv::GCall &call, cv::detail::Seq<IIs...>) { |
||||
return std::make_tuple(cv::detail::Yield<R>::yield(call, IIs)...); |
||||
} |
||||
// Also a universal outMeta stub here
|
||||
static GMetaArgs getOutMeta(const GMetaArgs &args, const GArgs &) { |
||||
return args; |
||||
} |
||||
}; |
||||
} // namespace detail
|
||||
|
||||
template<typename T, typename G> |
||||
cv::GOpaque<T> meta(G g, const std::string &tag) { |
||||
using O = cv::GOpaque<T>; |
||||
cv::GKernel k{ |
||||
detail::GMeta::id() // kernel id
|
||||
, tag // kernel tag. Use meta tag here
|
||||
, &detail::GMeta::getOutMeta // outMeta callback
|
||||
, {cv::detail::GTypeTraits<O>::shape} // output Shape
|
||||
, {cv::detail::GTypeTraits<G>::op_kind} // input data kinds
|
||||
, {cv::detail::GObtainCtor<O>::get()} // output template ctors
|
||||
}; |
||||
cv::GCall call(std::move(k)); |
||||
call.pass(g); |
||||
return std::get<0>(detail::GMeta::yield<O>(call, cv::detail::MkSeq<1>::type())); |
||||
} |
||||
|
||||
template<typename G> |
||||
cv::GOpaque<int64_t> timestamp(G g) { |
||||
return meta<int64_t>(g, meta_tag::timestamp); |
||||
} |
||||
|
||||
template<typename G> |
||||
cv::GOpaque<int64_t> seq_id(G g) { |
||||
return meta<int64_t>(g, meta_tag::seq_id); |
||||
} |
||||
|
||||
template<typename G> |
||||
cv::GOpaque<int64_t> seqNo(G g) { |
||||
// Old name, compatibility only
|
||||
return seq_id(g); |
||||
} |
||||
|
||||
} // namespace streaming
|
||||
} // namespace gapi
|
||||
} // namespace cv
|
||||
|
||||
#endif // OPENCV_GAPI_GSTREAMING_META_HPP
|
||||
@ -0,0 +1,33 @@ |
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
//
|
||||
// Copyright (C) 2020 Intel Corporation
|
||||
|
||||
#include "precomp.hpp" |
||||
#include <opencv2/gapi/garg.hpp> |
||||
|
||||
cv::GRunArg::GRunArg() { |
||||
} |
||||
|
||||
cv::GRunArg::GRunArg(const cv::GRunArg &arg) |
||||
: cv::GRunArgBase(static_cast<const cv::GRunArgBase&>(arg)) |
||||
, meta(arg.meta) { |
||||
} |
||||
|
||||
cv::GRunArg::GRunArg(cv::GRunArg &&arg) |
||||
: cv::GRunArgBase(std::move(static_cast<const cv::GRunArgBase&>(arg))) |
||||
, meta(std::move(arg.meta)) { |
||||
} |
||||
|
||||
cv::GRunArg& cv::GRunArg::operator= (const cv::GRunArg &arg) { |
||||
cv::GRunArgBase::operator=(static_cast<const cv::GRunArgBase&>(arg)); |
||||
meta = arg.meta; |
||||
return *this; |
||||
} |
||||
|
||||
cv::GRunArg& cv::GRunArg::operator= (cv::GRunArg &&arg) { |
||||
cv::GRunArgBase::operator=(std::move(static_cast<const cv::GRunArgBase&>(arg))); |
||||
meta = std::move(arg.meta); |
||||
return *this; |
||||
} |
||||
@ -0,0 +1,105 @@ |
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
//
|
||||
// Copyright (C) 2020 Intel Corporation
|
||||
|
||||
#include "precomp.hpp" |
||||
|
||||
#include <opencv2/gapi/gcommon.hpp> // compile args |
||||
#include <opencv2/gapi/util/any.hpp> // any |
||||
#include <opencv2/gapi/streaming/meta.hpp> // GMeta |
||||
|
||||
#include "compiler/gobjref.hpp" // RcDesc |
||||
#include "compiler/gmodel.hpp" // GModel, Op |
||||
#include "backends/common/gbackend.hpp" |
||||
#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK! |
||||
|
||||
#include "backends/common/gmetabackend.hpp" |
||||
|
||||
namespace { |
||||
|
||||
class GraphMetaExecutable final: public cv::gimpl::GIslandExecutable { |
||||
std::string m_meta_tag; |
||||
|
||||
public: |
||||
GraphMetaExecutable(const ade::Graph& g, |
||||
const std::vector<ade::NodeHandle>& nodes); |
||||
bool canReshape() const override; |
||||
void reshape(ade::Graph&, const cv::GCompileArgs&) override; |
||||
|
||||
void run(std::vector<InObj> &&input_objs, |
||||
std::vector<OutObj> &&output_objs) override; |
||||
}; |
||||
|
||||
bool GraphMetaExecutable::canReshape() const { |
||||
return true; |
||||
} |
||||
void GraphMetaExecutable::reshape(ade::Graph&, const cv::GCompileArgs&) { |
||||
// do nothing here
|
||||
} |
||||
|
||||
GraphMetaExecutable::GraphMetaExecutable(const ade::Graph& g, |
||||
const std::vector<ade::NodeHandle>& nodes) { |
||||
// There may be only one node in the graph
|
||||
GAPI_Assert(nodes.size() == 1u); |
||||
|
||||
cv::gimpl::GModel::ConstGraph cg(g); |
||||
const auto &op = cg.metadata(nodes[0]).get<cv::gimpl::Op>(); |
||||
GAPI_Assert(op.k.name == cv::gapi::streaming::detail::GMeta::id()); |
||||
m_meta_tag = op.k.tag; |
||||
} |
||||
|
||||
void GraphMetaExecutable::run(std::vector<InObj> &&input_objs, |
||||
std::vector<OutObj> &&output_objs) { |
||||
GAPI_Assert(input_objs.size() == 1u); |
||||
GAPI_Assert(output_objs.size() == 1u); |
||||
|
||||
const cv::GRunArg in_arg = input_objs[0].second; |
||||
cv::GRunArgP out_arg = output_objs[0].second; |
||||
|
||||
auto it = in_arg.meta.find(m_meta_tag); |
||||
if (it == in_arg.meta.end()) { |
||||
cv::util::throw_error |
||||
(std::logic_error("Run-time meta " |
||||
+ m_meta_tag |
||||
+ " is not found in object " |
||||
+ std::to_string(static_cast<int>(input_objs[0].first.shape)) |
||||
+ "/" |
||||
+ std::to_string(input_objs[0].first.id))); |
||||
} |
||||
cv::util::get<cv::detail::OpaqueRef>(out_arg) = it->second; |
||||
} |
||||
|
||||
class GraphMetaBackendImpl final: public cv::gapi::GBackend::Priv { |
||||
virtual void unpackKernel(ade::Graph &, |
||||
const ade::NodeHandle &, |
||||
const cv::GKernelImpl &) override { |
||||
// Do nothing here
|
||||
} |
||||
|
||||
virtual EPtr compile(const ade::Graph& graph, |
||||
const cv::GCompileArgs&, |
||||
const std::vector<ade::NodeHandle>& nodes, |
||||
const std::vector<cv::gimpl::Data>&, |
||||
const std::vector<cv::gimpl::Data>&) const override { |
||||
return EPtr{new GraphMetaExecutable(graph, nodes)}; |
||||
} |
||||
}; |
||||
|
||||
cv::gapi::GBackend graph_meta_backend() { |
||||
static cv::gapi::GBackend this_backend(std::make_shared<GraphMetaBackendImpl>()); |
||||
return this_backend; |
||||
} |
||||
|
||||
struct InGraphMetaKernel final: public cv::detail::KernelTag { |
||||
using API = cv::gapi::streaming::detail::GMeta; |
||||
static cv::gapi::GBackend backend() { return graph_meta_backend(); } |
||||
static int kernel() { return 42; } |
||||
}; |
||||
|
||||
} // anonymous namespace
|
||||
|
||||
cv::gapi::GKernelPackage cv::gimpl::meta::kernels() { |
||||
return cv::gapi::kernels<InGraphMetaKernel>(); |
||||
} |
||||
@ -0,0 +1,16 @@ |
||||
#ifndef OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP |
||||
#define OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP |
||||
|
||||
#include <opencv2/gapi/gkernel.hpp> |
||||
|
||||
namespace cv { |
||||
namespace gimpl { |
||||
namespace meta { |
||||
|
||||
cv::gapi::GKernelPackage kernels(); |
||||
|
||||
} // namespace meta
|
||||
} // namespace gimpl
|
||||
} // namespace cv
|
||||
|
||||
#endif // OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP
|
||||
@ -0,0 +1,195 @@ |
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
//
|
||||
// Copyright (C) 2020 Intel Corporation
|
||||
|
||||
#include <tuple> |
||||
#include <unordered_set> |
||||
|
||||
#include "test_precomp.hpp" |
||||
#include "opencv2/gapi/streaming/meta.hpp" |
||||
#include "opencv2/gapi/streaming/cap.hpp" |
||||
|
||||
namespace opencv_test { |
||||
|
||||
namespace { |
||||
void initTestDataPath() { |
||||
#ifndef WINRT |
||||
static bool initialized = false; |
||||
if (!initialized) |
||||
{ |
||||
// Since G-API has no own test data (yet), it is taken from the common space
|
||||
const char* testDataPath = getenv("OPENCV_TEST_DATA_PATH"); |
||||
if (testDataPath != nullptr) { |
||||
cvtest::addDataSearchPath(testDataPath); |
||||
initialized = true; |
||||
} |
||||
} |
||||
#endif // WINRT
|
||||
} |
||||
} // anonymous namespace
|
||||
|
||||
TEST(GraphMeta, Trad_AccessInput) { |
||||
cv::GMat in; |
||||
cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3)); |
||||
cv::GOpaque<int> out2 = cv::gapi::streaming::meta<int>(in, "foo"); |
||||
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2)); |
||||
|
||||
cv::Mat in_mat = cv::Mat::eye(cv::Size(64, 64), CV_8UC1); |
||||
cv::Mat out_mat; |
||||
int out_meta = 0; |
||||
|
||||
// manually set metadata in the input fields
|
||||
auto inputs = cv::gin(in_mat); |
||||
inputs[0].meta["foo"] = 42; |
||||
|
||||
graph.apply(std::move(inputs), cv::gout(out_mat, out_meta)); |
||||
EXPECT_EQ(42, out_meta); |
||||
} |
||||
|
||||
TEST(GraphMeta, Trad_AccessTmp) { |
||||
cv::GMat in; |
||||
cv::GMat tmp = cv::gapi::blur(in, cv::Size(3,3)); |
||||
cv::GMat out1 = tmp+1; |
||||
cv::GOpaque<float> out2 = cv::gapi::streaming::meta<float>(tmp, "bar"); |
||||
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2)); |
||||
|
||||
cv::Mat in_mat = cv::Mat::eye(cv::Size(64, 64), CV_8UC1); |
||||
cv::Mat out_mat; |
||||
float out_meta = 0.f; |
||||
|
||||
// manually set metadata in the input fields
|
||||
auto inputs = cv::gin(in_mat); |
||||
inputs[0].meta["bar"] = 1.f; |
||||
|
||||
graph.apply(std::move(inputs), cv::gout(out_mat, out_meta)); |
||||
EXPECT_EQ(1.f, out_meta); |
||||
} |
||||
|
||||
TEST(GraphMeta, Trad_AccessOutput) { |
||||
cv::GMat in; |
||||
cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3)); |
||||
cv::GOpaque<std::string> out2 = cv::gapi::streaming::meta<std::string>(out1, "baz"); |
||||
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2)); |
||||
|
||||
cv::Mat in_mat = cv::Mat::eye(cv::Size(64, 64), CV_8UC1); |
||||
cv::Mat out_mat; |
||||
std::string out_meta; |
||||
|
||||
// manually set metadata in the input fields
|
||||
auto inputs = cv::gin(in_mat); |
||||
|
||||
// NOTE: Assigning explicitly an std::string is important,
|
||||
// otherwise a "const char*" will be stored and won't be
|
||||
// translated properly by util::any since std::string is
|
||||
// used within the graph.
|
||||
inputs[0].meta["baz"] = std::string("opencv"); |
||||
|
||||
graph.apply(std::move(inputs), cv::gout(out_mat, out_meta)); |
||||
EXPECT_EQ("opencv", out_meta); |
||||
} |
||||
|
||||
TEST(GraphMeta, Streaming_AccessInput) { |
||||
initTestDataPath(); |
||||
|
||||
cv::GMat in; |
||||
cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3)); |
||||
cv::GOpaque<int64_t> out2 = cv::gapi::streaming::seq_id(in); |
||||
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2)); |
||||
|
||||
auto ccomp = graph.compileStreaming(); |
||||
ccomp.setSource<cv::gapi::wip::GCaptureSource>(findDataFile("cv/video/768x576.avi", false)); |
||||
ccomp.start(); |
||||
|
||||
cv::Mat out_mat; |
||||
int64_t out_meta = 0; |
||||
int64_t expected_counter = 0; |
||||
|
||||
while (ccomp.pull(cv::gout(out_mat, out_meta))) { |
||||
EXPECT_EQ(expected_counter, out_meta); |
||||
++expected_counter; |
||||
} |
||||
} |
||||
|
||||
TEST(GraphMeta, Streaming_AccessOutput) { |
||||
initTestDataPath(); |
||||
|
||||
cv::GMat in; |
||||
cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3)); |
||||
cv::GOpaque<int64_t> out2 = cv::gapi::streaming::seq_id(out1); |
||||
cv::GOpaque<int64_t> out3 = cv::gapi::streaming::timestamp(out1); |
||||
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2, out3)); |
||||
|
||||
auto ccomp = graph.compileStreaming(); |
||||
ccomp.setSource<cv::gapi::wip::GCaptureSource>(findDataFile("cv/video/768x576.avi", false)); |
||||
ccomp.start(); |
||||
|
||||
cv::Mat out_mat; |
||||
int64_t out_meta = 0; |
||||
int64_t out_timestamp = 0; |
||||
int64_t expected_counter = 0; |
||||
int64_t prev_timestamp = -1; |
||||
|
||||
while (ccomp.pull(cv::gout(out_mat, out_meta, out_timestamp))) { |
||||
EXPECT_EQ(expected_counter, out_meta); |
||||
++expected_counter; |
||||
|
||||
EXPECT_NE(prev_timestamp, out_timestamp); |
||||
prev_timestamp = out_timestamp; |
||||
} |
||||
} |
||||
|
||||
TEST(GraphMeta, Streaming_AccessDesync) { |
||||
initTestDataPath(); |
||||
|
||||
cv::GMat in; |
||||
cv::GOpaque<int64_t> out1 = cv::gapi::streaming::seq_id(in); |
||||
cv::GOpaque<int64_t> out2 = cv::gapi::streaming::timestamp(in); |
||||
cv::GMat out3 = cv::gapi::blur(in, cv::Size(3,3)); |
||||
|
||||
cv::GMat tmp = cv::gapi::streaming::desync(in); |
||||
cv::GScalar mean = cv::gapi::mean(tmp); |
||||
cv::GOpaque<int64_t> out4 = cv::gapi::streaming::seq_id(mean); |
||||
cv::GOpaque<int64_t> out5 = cv::gapi::streaming::timestamp(mean); |
||||
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2, out3, out4, out5)); |
||||
|
||||
auto ccomp = graph.compileStreaming(); |
||||
ccomp.setSource<cv::gapi::wip::GCaptureSource>(findDataFile("cv/video/768x576.avi", false)); |
||||
ccomp.start(); |
||||
|
||||
cv::optional<int64_t> out_sync_id; |
||||
cv::optional<int64_t> out_sync_ts; |
||||
cv::optional<cv::Mat> out_sync_mat; |
||||
|
||||
cv::optional<int64_t> out_desync_id; |
||||
cv::optional<int64_t> out_desync_ts; |
||||
|
||||
std::unordered_set<int64_t> sync_ids; |
||||
std::unordered_set<int64_t> desync_ids; |
||||
|
||||
while (ccomp.pull(cv::gout(out_sync_id, out_sync_ts, out_sync_mat, |
||||
out_desync_id, out_desync_ts))) { |
||||
if (out_sync_id.has_value()) { |
||||
CV_Assert(out_sync_ts.has_value()); |
||||
CV_Assert(out_sync_mat.has_value()); |
||||
sync_ids.insert(out_sync_id.value()); |
||||
} |
||||
if (out_desync_id.has_value()) { |
||||
CV_Assert(out_desync_ts.has_value()); |
||||
desync_ids.insert(out_desync_id.value()); |
||||
} |
||||
} |
||||
// Visually report that everything is really ok
|
||||
std::cout << sync_ids.size() << " vs " << desync_ids.size() << std::endl; |
||||
|
||||
// Desync path should generate less objects than the synchronized one
|
||||
EXPECT_GE(sync_ids.size(), desync_ids.size()); |
||||
|
||||
// ..but all desynchronized IDs must be present in the synchronized set
|
||||
for (auto &&d_id : desync_ids) { |
||||
EXPECT_TRUE(sync_ids.count(d_id) > 0); |
||||
} |
||||
} |
||||
|
||||
} // namespace opencv_test
|
||||
Loading…
Reference in new issue