Merge pull request #19009 from TolyaTalamanov:at/media-frame-copy
[G-API] GStreamingBackend * Snapshot * Implement StreamingBackend * Refactoring * Refactoring 2 * Clean up * Add missing functionality to support MediaFrame as output * Partially address review comments * Fix build * Implement reshape for gstreamingbackend and add a test on it * Address more comments * Add format.hpp to gapi.hpp * Fix debug build * Address review comments Co-authored-by: Smirnov Alexey <alexey.smirnov@intel.com>pull/19085/head
Anatoliy Talamanov
4 years ago
committed by
GitHub
parent
9f01b97e14
commit
8ed0fc6f0c
23 changed files with 730 additions and 8 deletions
@ -0,0 +1,52 @@ |
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
//
|
||||
// Copyright (C) 2020 Intel Corporation
|
||||
|
||||
#ifndef OPENCV_GAPI_GSTREAMING_FORMAT_HPP |
||||
#define OPENCV_GAPI_GSTREAMING_FORMAT_HPP |
||||
|
||||
#include <opencv2/gapi/gkernel.hpp> // GKernelPackage |
||||
|
||||
namespace cv { |
||||
namespace gapi { |
||||
namespace streaming { |
||||
|
||||
cv::gapi::GKernelPackage kernels(); |
||||
cv::gapi::GBackend backend(); |
||||
|
||||
// FIXME: Make a generic kernel
|
||||
G_API_OP(GCopy, <GFrame(GFrame)>, "org.opencv.streaming.copy") |
||||
{ |
||||
static GFrameDesc outMeta(const GFrameDesc& in) { return in; } |
||||
}; |
||||
|
||||
G_API_OP(GBGR, <GMat(GFrame)>, "org.opencv.streaming.BGR") |
||||
{ |
||||
static GMatDesc outMeta(const GFrameDesc& in) { return GMatDesc{CV_8U, 3, in.size}; } |
||||
}; |
||||
|
||||
/** @brief Gets copy from the input frame
|
||||
|
||||
@note Function textual ID is "org.opencv.streaming.copy" |
||||
|
||||
@param in Input frame |
||||
@return Copy of the input frame |
||||
*/ |
||||
GAPI_EXPORTS cv::GFrame copy(const cv::GFrame& in); |
||||
|
||||
/** @brief Gets bgr plane from input frame
|
||||
|
||||
@note Function textual ID is "org.opencv.streaming.BGR" |
||||
|
||||
@param in Input frame |
||||
@return Image in BGR format |
||||
*/ |
||||
GAPI_EXPORTS cv::GMat BGR (const cv::GFrame& in); |
||||
|
||||
} // namespace streaming
|
||||
} // namespace gapi
|
||||
} // namespace cv
|
||||
|
||||
#endif // OPENCV_GAPI_GSTREAMING_FORMAT_HPP
|
||||
@ -0,0 +1,203 @@ |
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
//
|
||||
// Copyright (C) 2020 Intel Corporation
|
||||
|
||||
#include <opencv2/imgproc.hpp> |
||||
#include <opencv2/gapi/util/throw.hpp> // throw_error |
||||
#include <opencv2/gapi/streaming/format.hpp> // kernels |
||||
|
||||
#include "api/gbackend_priv.hpp" |
||||
#include "backends/common/gbackend.hpp" |
||||
|
||||
#include "gstreamingbackend.hpp" |
||||
#include "gstreamingkernel.hpp" |
||||
|
||||
namespace { |
||||
|
||||
struct StreamingCreateFunction |
||||
{ |
||||
static const char *name() { return "StreamingCreateFunction"; } |
||||
cv::gapi::streaming::CreateActorFunction createActorFunction; |
||||
}; |
||||
|
||||
using StreamingGraph = ade::TypedGraph |
||||
< cv::gimpl::Op |
||||
, StreamingCreateFunction |
||||
>; |
||||
|
||||
using ConstStreamingGraph = ade::ConstTypedGraph |
||||
< cv::gimpl::Op |
||||
, StreamingCreateFunction |
||||
>; |
||||
|
||||
|
||||
class GStreamingIntrinExecutable final: public cv::gimpl::GIslandExecutable |
||||
{ |
||||
virtual void run(std::vector<InObj> &&, |
||||
std::vector<OutObj> &&) override { |
||||
GAPI_Assert(false && "Not implemented"); |
||||
} |
||||
|
||||
virtual void run(GIslandExecutable::IInput &in, |
||||
GIslandExecutable::IOutput &out) override; |
||||
|
||||
virtual bool allocatesOutputs() const override { return true; } |
||||
// Return an empty RMat since we will reuse the input.
|
||||
// There is no need to allocate and copy 4k image here.
|
||||
virtual cv::RMat allocate(const cv::GMatDesc&) const override { return {}; } |
||||
|
||||
virtual bool canReshape() const override { return true; } |
||||
virtual void reshape(ade::Graph&, const cv::GCompileArgs&) override { |
||||
// Do nothing here
|
||||
} |
||||
|
||||
public: |
||||
GStreamingIntrinExecutable(const ade::Graph &, |
||||
const std::vector<ade::NodeHandle> &); |
||||
|
||||
const ade::Graph& m_g; |
||||
cv::gimpl::GModel::ConstGraph m_gm; |
||||
cv::gapi::streaming::IActor::Ptr m_actor; |
||||
}; |
||||
|
||||
void GStreamingIntrinExecutable::run(GIslandExecutable::IInput &in, |
||||
GIslandExecutable::IOutput &out) |
||||
{ |
||||
m_actor->run(in, out); |
||||
} |
||||
|
||||
class GStreamingBackendImpl final: public cv::gapi::GBackend::Priv |
||||
{ |
||||
virtual void unpackKernel(ade::Graph &graph, |
||||
const ade::NodeHandle &op_node, |
||||
const cv::GKernelImpl &impl) override |
||||
{ |
||||
StreamingGraph gm(graph); |
||||
const auto &kimpl = cv::util::any_cast<cv::gapi::streaming::GStreamingKernel>(impl.opaque); |
||||
gm.metadata(op_node).set(StreamingCreateFunction{kimpl.createActorFunction}); |
||||
} |
||||
|
||||
virtual EPtr compile(const ade::Graph &graph, |
||||
const cv::GCompileArgs &, |
||||
const std::vector<ade::NodeHandle> &nodes) const override |
||||
{ |
||||
return EPtr{new GStreamingIntrinExecutable(graph, nodes)}; |
||||
} |
||||
|
||||
virtual bool controlsMerge() const override |
||||
{ |
||||
return true; |
||||
} |
||||
|
||||
virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &, |
||||
const ade::NodeHandle &, |
||||
const ade::NodeHandle &, |
||||
const ade::NodeHandle &) const override |
||||
{ |
||||
return false; |
||||
} |
||||
}; |
||||
|
||||
GStreamingIntrinExecutable::GStreamingIntrinExecutable(const ade::Graph& g, |
||||
const std::vector<ade::NodeHandle>& nodes) |
||||
: m_g(g), m_gm(m_g) |
||||
{ |
||||
using namespace cv::gimpl; |
||||
const auto is_op = [this](const ade::NodeHandle &nh) |
||||
{ |
||||
return m_gm.metadata(nh).get<NodeType>().t == NodeType::OP; |
||||
}; |
||||
|
||||
auto it = std::find_if(nodes.begin(), nodes.end(), is_op); |
||||
GAPI_Assert(it != nodes.end() && "No operators found for this island?!"); |
||||
|
||||
ConstStreamingGraph cag(m_g); |
||||
m_actor = cag.metadata(*it).get<StreamingCreateFunction>().createActorFunction(); |
||||
|
||||
// Ensure this the only op in the graph
|
||||
if (std::any_of(it+1, nodes.end(), is_op)) |
||||
{ |
||||
cv::util::throw_error |
||||
(std::logic_error |
||||
("Internal error: Streaming subgraph has multiple operations")); |
||||
} |
||||
} |
||||
|
||||
} // anonymous namespace
|
||||
|
||||
cv::gapi::GBackend cv::gapi::streaming::backend() |
||||
{ |
||||
static cv::gapi::GBackend this_backend(std::make_shared<GStreamingBackendImpl>()); |
||||
return this_backend; |
||||
} |
||||
|
||||
cv::gapi::GKernelPackage cv::gapi::streaming::kernels() |
||||
{ |
||||
return cv::gapi::kernels<cv::gimpl::Copy, cv::gimpl::BGR>(); |
||||
} |
||||
|
||||
void cv::gimpl::Copy::Actor::run(cv::gimpl::GIslandExecutable::IInput &in, |
||||
cv::gimpl::GIslandExecutable::IOutput &out) |
||||
{ |
||||
while (true) |
||||
{ |
||||
const auto in_msg = in.get(); |
||||
if (cv::util::holds_alternative<cv::gimpl::EndOfStream>(in_msg)) |
||||
{ |
||||
out.post(cv::gimpl::EndOfStream{}); |
||||
return; |
||||
} |
||||
|
||||
const cv::GRunArgs &in_args = cv::util::get<cv::GRunArgs>(in_msg); |
||||
GAPI_Assert(in_args.size() == 1u); |
||||
|
||||
cv::GRunArgP out_arg = out.get(0); |
||||
*cv::util::get<cv::MediaFrame*>(out_arg) = cv::util::get<cv::MediaFrame>(in_args[0]); |
||||
out.post(std::move(out_arg)); |
||||
} |
||||
} |
||||
|
||||
void cv::gimpl::BGR::Actor::run(cv::gimpl::GIslandExecutable::IInput &in, |
||||
cv::gimpl::GIslandExecutable::IOutput &out) |
||||
{ |
||||
while (true) |
||||
{ |
||||
const auto in_msg = in.get(); |
||||
if (cv::util::holds_alternative<cv::gimpl::EndOfStream>(in_msg)) |
||||
{ |
||||
out.post(cv::gimpl::EndOfStream{}); |
||||
return; |
||||
} |
||||
|
||||
const cv::GRunArgs &in_args = cv::util::get<cv::GRunArgs>(in_msg); |
||||
GAPI_Assert(in_args.size() == 1u); |
||||
|
||||
cv::GRunArgP out_arg = out.get(0); |
||||
auto frame = cv::util::get<cv::MediaFrame>(in_args[0]); |
||||
const auto& desc = frame.desc(); |
||||
|
||||
auto& rmat = *cv::util::get<cv::RMat*>(out_arg); |
||||
switch (desc.fmt) |
||||
{ |
||||
case cv::MediaFormat::BGR: |
||||
rmat = cv::make_rmat<cv::gimpl::RMatMediaBGRAdapter>(frame); |
||||
break; |
||||
case cv::MediaFormat::NV12: |
||||
{ |
||||
cv::Mat bgr; |
||||
auto view = frame.access(cv::MediaFrame::Access::R); |
||||
cv::Mat y_plane (desc.size, CV_8UC1, view.ptr[0]); |
||||
cv::Mat uv_plane(desc.size / 2, CV_8UC2, view.ptr[1]); |
||||
cv::cvtColorTwoPlane(y_plane, uv_plane, bgr, cv::COLOR_YUV2BGR_NV12); |
||||
rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(bgr); |
||||
break; |
||||
} |
||||
default: |
||||
cv::util::throw_error( |
||||
std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::BGR")); |
||||
} |
||||
out.post(std::move(out_arg)); |
||||
} |
||||
} |
||||
@ -0,0 +1,89 @@ |
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
//
|
||||
// Copyright (C) 2020 Intel Corporation
|
||||
|
||||
#ifndef OPENCV_GAPI_GSTREAMINGBACKEND_HPP |
||||
#define OPENCV_GAPI_GSTREAMINGBACKEND_HPP |
||||
|
||||
#include <opencv2/gapi/gkernel.hpp> |
||||
#include <opencv2/gapi/streaming/format.hpp> |
||||
#include "gstreamingkernel.hpp" |
||||
|
||||
namespace cv { |
||||
namespace gimpl { |
||||
|
||||
struct RMatMediaBGRAdapter final: public cv::RMat::Adapter |
||||
{ |
||||
RMatMediaBGRAdapter(cv::MediaFrame frame) : m_frame(frame) { }; |
||||
|
||||
virtual cv::RMat::View access(cv::RMat::Access a) override |
||||
{ |
||||
auto view = m_frame.access(a == cv::RMat::Access::W ? cv::MediaFrame::Access::W |
||||
: cv::MediaFrame::Access::R); |
||||
auto ptr = reinterpret_cast<uchar*>(view.ptr[0]); |
||||
auto stride = view.stride[0]; |
||||
|
||||
std::shared_ptr<cv::MediaFrame::View> view_ptr = |
||||
std::make_shared<cv::MediaFrame::View>(std::move(view)); |
||||
auto callback = [view_ptr]() mutable { view_ptr.reset(); }; |
||||
|
||||
return cv::RMat::View(desc(), ptr, stride, callback); |
||||
} |
||||
|
||||
virtual cv::GMatDesc desc() const override |
||||
{ |
||||
const auto& desc = m_frame.desc(); |
||||
GAPI_Assert(desc.fmt == cv::MediaFormat::BGR); |
||||
return cv::GMatDesc{CV_8U, 3, desc.size}; |
||||
} |
||||
|
||||
cv::MediaFrame m_frame; |
||||
}; |
||||
|
||||
struct Copy: public cv::detail::KernelTag |
||||
{ |
||||
using API = cv::gapi::streaming::GCopy; |
||||
|
||||
static gapi::GBackend backend() { return cv::gapi::streaming::backend(); } |
||||
|
||||
class Actor final: public cv::gapi::streaming::IActor |
||||
{ |
||||
public: |
||||
explicit Actor() {} |
||||
virtual void run(cv::gimpl::GIslandExecutable::IInput &in, |
||||
cv::gimpl::GIslandExecutable::IOutput &out) override; |
||||
}; |
||||
|
||||
static cv::gapi::streaming::IActor::Ptr create() |
||||
{ |
||||
return cv::gapi::streaming::IActor::Ptr(new Actor()); |
||||
} |
||||
|
||||
static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; }; |
||||
}; |
||||
|
||||
struct BGR: public cv::detail::KernelTag |
||||
{ |
||||
using API = cv::gapi::streaming::GBGR; |
||||
static gapi::GBackend backend() { return cv::gapi::streaming::backend(); } |
||||
|
||||
class Actor final: public cv::gapi::streaming::IActor { |
||||
public: |
||||
explicit Actor() {} |
||||
virtual void run(cv::gimpl::GIslandExecutable::IInput &in, |
||||
cv::gimpl::GIslandExecutable::IOutput&out) override; |
||||
}; |
||||
|
||||
static cv::gapi::streaming::IActor::Ptr create() |
||||
{ |
||||
return cv::gapi::streaming::IActor::Ptr(new Actor()); |
||||
} |
||||
static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; }; |
||||
}; |
||||
|
||||
} // namespace gimpl
|
||||
} // namespace cv
|
||||
|
||||
#endif // OPENCV_GAPI_GSTREAMINGBACKEND_HPP
|
||||
@ -0,0 +1,37 @@ |
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
//
|
||||
// Copyright (C) 2020 Intel Corporation
|
||||
|
||||
|
||||
#ifndef OPENCV_GAPI_GSTREAMINGKERNEL_HPP |
||||
#define OPENCV_GAPI_GSTREAMINGKERNEL_HPP |
||||
|
||||
#include "compiler/gislandmodel.hpp" |
||||
|
||||
namespace cv { |
||||
namespace gapi { |
||||
namespace streaming { |
||||
|
||||
class IActor { |
||||
public: |
||||
using Ptr = std::shared_ptr<IActor>; |
||||
|
||||
virtual void run(cv::gimpl::GIslandExecutable::IInput &in, |
||||
cv::gimpl::GIslandExecutable::IOutput &out) = 0; |
||||
|
||||
virtual ~IActor() = default; |
||||
}; |
||||
|
||||
using CreateActorFunction = std::function<IActor::Ptr()>; |
||||
struct GStreamingKernel |
||||
{ |
||||
CreateActorFunction createActorFunction; |
||||
}; |
||||
|
||||
} // namespace streaming
|
||||
} // namespace gapi
|
||||
} // namespace cv
|
||||
|
||||
#endif // OPENCV_GAPI_GSTREAMINGKERNEL_HPP
|
||||
Loading…
Reference in new issue