Chiebot-Mirror
/
opencv
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #21504 from smirnov-alexey:as/oak_infer

Browse Source 
[GAPI] Support basic inference in OAK backend

* Combined commit which enables basic inference and other extra capabilities of OAK backend

* Remove unnecessary target options from the cmakelist
pull/21827/head
Alexey Smirnov 3 years ago committed by GitHub
parent b2e20a82ba
commit 7ed557497d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 870 additions and 159 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 12
      modules/gapi/CMakeLists.txt
  2. 66
      modules/gapi/include/opencv2/gapi/oak/infer.hpp
  3. 29
      modules/gapi/include/opencv2/gapi/oak/oak.hpp
  4. 122
      modules/gapi/samples/oak_basic_infer.cpp
  5. 48
      modules/gapi/samples/oak_copy.cpp
  6. 10
      modules/gapi/samples/oak_rgb_camera_encoding.cpp
  7. 11
      modules/gapi/samples/oak_small_hetero_pipeline.cpp
  8. 15
      modules/gapi/src/backends/oak/goak.cpp
  9. 32
      modules/gapi/src/backends/oak/goak_memory_adapters.cpp
  10. 660
      modules/gapi/src/backends/oak/goakbackend.cpp
  11. 22
      modules/gapi/src/backends/oak/oak_memory_adapters.hpp
  12. 2
      modules/gapi/src/compiler/gmodel.hpp

12
modules/gapi/CMakeLists.txt
Unescape View File

@ -138,7 +138,7 @@ set(gapi_srcs
# OAK Backend (optional)
src/backends/oak/goak.cpp
src/backends/oak/goakbackend.cpp
src/backends/oak/goak_media_adapter.cpp
src/backends/oak/goak_memory_adapters.cpp
# OCL Backend (currently built-in)
src/backends/ocl/goclbackend.cpp
@ -375,13 +375,3 @@ if(HAVE_GAPI_ONEVPL)
endif()
endif()
endif()
if(HAVE_OAK)
# FIXME: consider better solution
if(TARGET example_gapi_oak_rgb_camera_encoding)
ocv_target_compile_definitions(example_gapi_oak_rgb_camera_encoding PRIVATE -DHAVE_OAK)
endif()
if(TARGET example_gapi_oak_small_hetero_pipeline)
ocv_target_compile_definitions(example_gapi_oak_small_hetero_pipeline PRIVATE -DHAVE_OAK)
endif()
endif()

66
modules/gapi/include/opencv2/gapi/oak/infer.hpp
Unescape View File

@ -0,0 +1,66 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2022 Intel Corporation
#ifndef OPENCV_GAPI_OAK_INFER_HPP
#define OPENCV_GAPI_OAK_INFER_HPP
#include <unordered_map>
#include <string>
#include <array>
#include <tuple>
#include <opencv2/gapi/opencv_includes.hpp>
#include <opencv2/gapi/util/any.hpp>
#include <opencv2/core/cvdef.h> // GAPI_EXPORTS
#include <opencv2/gapi/gkernel.hpp> // GKernelPackage
namespace cv {
namespace gapi {
namespace oak {
namespace detail {
/**
* @brief This structure contains description of inference parameters
* which is specific to OAK models.
*/
struct ParamDesc {
std::string blob_file;
};
} // namespace detail
/**
* Contains description of inference parameters and kit of functions that
* fill this parameters.
*/
template<typename Net> class Params {
public:
/** @brief Class constructor.
Constructs Params based on model information and sets default values for other
inference description parameters.
@param model Path to model (.blob file)
*/
explicit Params(const std::string &model) {
desc.blob_file = model;
};
// BEGIN(G-API's network parametrization API)
GBackend backend() const { return cv::gapi::oak::backend(); }
std::string tag() const { return Net::tag(); }
cv::util::any params() const { return { desc }; }
// END(G-API's network parametrization API)
protected:
detail::ParamDesc desc;
};
} // namespace oak
} // namespace gapi
} // namespace cv
#endif // OPENCV_GAPI_OAK_INFER_HPP

29
modules/gapi/include/opencv2/gapi/oak/oak.hpp
Unescape View File

@ -89,28 +89,55 @@ G_API_OP(GSobelXY, <GFrame(GFrame, const cv::Mat&, const cv::Mat&)>, "org.opencv
}
};
G_API_OP(GCopy, <GFrame(GFrame)>, "org.opencv.oak.copy") {
static GFrameDesc outMeta(const GFrameDesc& in) {
return in;
}
};
// FIXME: add documentation on operations below
GAPI_EXPORTS GArray<uint8_t> encode(const GFrame& in, const EncoderConfig&);
GAPI_EXPORTS GFrame sobelXY(const GFrame& in,
const cv::Mat& hk,
const cv::Mat& vk);
GAPI_EXPORTS GFrame copy(const GFrame& in);
// OAK backend & kernels ////////////////////////////////////////////////////////
GAPI_EXPORTS cv::gapi::GBackend backend();
GAPI_EXPORTS cv::gapi::GKernelPackage kernels();
// Camera object ///////////////////////////////////////////////////////////////
struct GAPI_EXPORTS ColorCameraParams {};
struct GAPI_EXPORTS ColorCameraParams {
/**
* Format of the frame one gets from the camera
*/
bool interleaved = false;
// FIXME: extend
enum class BoardSocket: int { RGB, BGR };
BoardSocket board_socket = BoardSocket::RGB;
// FIXME: extend
enum class Resolution: int { THE_1080_P };
Resolution resolution = Resolution::THE_1080_P;
};
class GAPI_EXPORTS ColorCamera: public cv::gapi::wip::IStreamSource {
cv::MediaFrame m_dummy;
ColorCameraParams m_params;
virtual bool pull(cv::gapi::wip::Data &data) override;
virtual GMetaArg descr_of() const override;
public:
ColorCamera();
explicit ColorCamera(const ColorCameraParams& params);
};
} // namespace oak

122
modules/gapi/samples/oak_basic_infer.cpp
Unescape View File

@ -0,0 +1,122 @@
#include <algorithm>
#include <iostream>
#include <sstream>
#include <opencv2/imgproc.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/gapi.hpp>
#include <opencv2/gapi/core.hpp>
#include <opencv2/gapi/imgproc.hpp>
#include <opencv2/gapi/infer.hpp>
#include <opencv2/gapi/infer/parsers.hpp>
#include <opencv2/gapi/render.hpp>
#include <opencv2/gapi/cpu/gcpukernel.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/gapi/oak/oak.hpp>
#include <opencv2/gapi/oak/infer.hpp>
const std::string keys =
"{ h help | | Print this help message }"
"{ detector | | Path to compiled .blob face detector model }"
"{ duration | 100 | Number of frames to pull from camera and run inference on }";
namespace custom {
G_API_NET(FaceDetector, <cv::GMat(cv::GFrame)>, "sample.custom.face-detector");
using GDetections = cv::GArray<cv::Rect>;
using GSize = cv::GOpaque<cv::Size>;
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
G_API_OP(BBoxes, <GPrims(GDetections)>, "sample.custom.b-boxes") {
static cv::GArrayDesc outMeta(const cv::GArrayDesc &) {
return cv::empty_array_desc();
}
};
GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
// This kernel converts the rectangles into G-API's
// rendering primitives
static void run(const std::vector<cv::Rect> &in_face_rcs,
std::vector<cv::gapi::wip::draw::Prim> &out_prims) {
out_prims.clear();
const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) {
return cv::gapi::wip::draw::Rect(rc, clr, 2);
};
for (auto &&rc : in_face_rcs) {
out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green
}
}
};
} // namespace custom
int main(int argc, char *argv[]) {
cv::CommandLineParser cmd(argc, argv, keys);
if (cmd.has("help")) {
cmd.printMessage();
return 0;
}
const auto det_name = cmd.get<std::string>("detector");
const auto duration = cmd.get<int>("duration");
if (det_name.empty()) {
std::cerr << "FATAL: path to detection model is not provided for the sample."
<< "Please specify it with --detector options."
<< std::endl;
return 1;
}
// Prepare G-API kernels and networks packages:
auto detector = cv::gapi::oak::Params<custom::FaceDetector>(det_name);
auto networks = cv::gapi::networks(detector);
auto kernels = cv::gapi::combine(
cv::gapi::kernels<custom::OCVBBoxes>(),
cv::gapi::oak::kernels());
auto args = cv::compile_args(kernels, networks);
// Initialize graph structure
cv::GFrame in;
cv::GFrame copy = cv::gapi::oak::copy(in); // NV12 transfered to host + passthrough copy for infer
cv::GOpaque<cv::Size> sz = cv::gapi::streaming::size(copy);
// infer is not affected by the actual copy here
cv::GMat blob = cv::gapi::infer<custom::FaceDetector>(copy);
// FIXME: OAK infer detects faces slightly out of frame bounds
cv::GArray<cv::Rect> rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, false);
auto rendered = cv::gapi::wip::draw::renderFrame(copy, custom::BBoxes::on(rcs));
// on-the-fly conversion NV12->BGR
cv::GMat out = cv::gapi::streaming::BGR(rendered);
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out, rcs))
.compileStreaming(std::move(args));
// Graph execution
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
pipeline.start();
cv::Mat out_mat;
std::vector<cv::Rect> out_dets;
int frames = 0;
while (pipeline.pull(cv::gout(out_mat, out_dets))) {
std::string name = "oak_infer_frame_" + std::to_string(frames) + ".png";
cv::imwrite(name, out_mat);
if (!out_dets.empty()) {
std::cout << "Got " << out_dets.size() << " detections on frame #" << frames << std::endl;
}
++frames;
if (frames == duration) {
pipeline.stop();
break;
}
}
std::cout << "Pipeline finished. Processed " << frames << " frames" << std::endl;
return 0;
}

48
modules/gapi/samples/oak_copy.cpp
Unescape View File

@ -0,0 +1,48 @@
#include <opencv2/gapi.hpp>
#include <opencv2/gapi/core.hpp>
#include <opencv2/gapi/cpu/core.hpp>
#include <opencv2/gapi/gframe.hpp>
#include <opencv2/gapi/media.hpp>
#include <opencv2/gapi/oak/oak.hpp>
#include <opencv2/gapi/streaming/format.hpp> // BGR accessor
#include <opencv2/highgui.hpp> // CommandLineParser
const std::string keys =
"{ h help | | Print this help message }"
"{ output | output.png | Path to the output file }";
int main(int argc, char *argv[]) {
cv::CommandLineParser cmd(argc, argv, keys);
if (cmd.has("help")) {
cmd.printMessage();
return 0;
}
const std::string output_name = cmd.get<std::string>("output");
cv::GFrame in;
// Actually transfers data to host
cv::GFrame copy = cv::gapi::oak::copy(in);
// Default camera works only with nv12 format
cv::GMat out = cv::gapi::streaming::Y(copy);
auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{},
cv::gapi::oak::kernels());
auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)).compileStreaming(std::move(args));
// Graph execution /////////////////////////////////////////////////////////
cv::Mat out_mat(1920, 1080, CV_8UC1);
pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
pipeline.start();
// pull 1 frame
pipeline.pull(cv::gout(out_mat));
cv::imwrite(output_name, out_mat);
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
}

10
modules/gapi/samples/oak_rgb_camera_encoding.cpp
Unescape View File

@ -13,8 +13,6 @@ const std::string keys =
"{ h help | | Print this help message }"
"{ output | output.h265 | Path to the output .h265 video file }";
#ifdef HAVE_OAK
int main(int argc, char *argv[]) {
cv::CommandLineParser cmd(argc, argv, keys);
if (cmd.has("help")) {
@ -60,11 +58,3 @@ int main(int argc, char *argv[]) {
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
}
#else // HAVE_OAK
int main() {
GAPI_Assert(false && "Built without OAK support");
return -1;
}
#endif // HAVE_OAK

11
modules/gapi/samples/oak_small_hetero_pipeline.cpp
Unescape View File

@ -13,8 +13,6 @@ const std::string keys =
"{ h help | | Print this help message }"
"{ output | output.png | Path to the output file }";
#ifdef HAVE_OAK
int main(int argc, char *argv[]) {
cv::CommandLineParser cmd(argc, argv, keys);
if (cmd.has("help")) {
@ -58,12 +56,3 @@ int main(int argc, char *argv[]) {
std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
}
#else // HAVE_OAK
int main() {
GAPI_Assert(false && "Built without OAK support");
return -1;
}
#endif // HAVE_OAK

15
modules/gapi/src/backends/oak/goak.cpp
Unescape View File

@ -7,7 +7,7 @@
#include <opencv2/gapi/oak/oak.hpp>
#include <opencv2/gapi/cpu/gcpukernel.hpp>
#include "oak_media_adapter.hpp"
#include "oak_memory_adapters.hpp"
#include <thread>
#include <chrono>
@ -24,6 +24,10 @@ GFrame sobelXY(const GFrame& in, const cv::Mat& hk, const cv::Mat& vk) {
return GSobelXY::on(in, hk, vk);
}
GFrame copy(const GFrame& in) {
return GCopy::on(in);
}
// This is a dummy oak::ColorCamera class that just makes our pipelining
// machinery work. The real data comes from the physical camera which
// is handled by DepthAI library.
@ -31,6 +35,11 @@ ColorCamera::ColorCamera()
: m_dummy(cv::MediaFrame::Create<cv::gapi::oak::OAKMediaAdapter>()) {
}
ColorCamera::ColorCamera(const ColorCameraParams& params)
: m_dummy(cv::MediaFrame::Create<cv::gapi::oak::OAKMediaAdapter>()),
m_params(params) {
}
bool ColorCamera::pull(cv::gapi::wip::Data &data) {
// FIXME: Avoid passing this formal frame to the pipeline
std::this_thread::sleep_for(std::chrono::milliseconds(10));
@ -39,7 +48,9 @@ bool ColorCamera::pull(cv::gapi::wip::Data &data) {
}
cv::GMetaArg ColorCamera::descr_of() const {
return cv::GMetaArg{cv::descr_of(m_dummy)};
// FIXME: support other resolutions
GAPI_Assert(m_params.resolution == ColorCameraParams::Resolution::THE_1080_P);
return cv::GMetaArg{cv::GFrameDesc{cv::MediaFormat::NV12, cv::Size{1920, 1080}}};
}
} // namespace oak

32
modules/gapi/src/backends/oak/goak_media_adapter.cpp → modules/gapi/src/backends/oak/goak_memory_adapters.cpp
Unescape View File

@ -4,17 +4,15 @@
//
// Copyright (C) 2021 Intel Corporation
#include "oak_media_adapter.hpp"
#include "oak_memory_adapters.hpp"
namespace cv {
namespace gapi {
namespace oak {
OAKMediaAdapter::OAKMediaAdapter(cv::Size sz, cv::MediaFormat fmt, std::vector<uint8_t>&& buffer) {
OAKMediaAdapter::OAKMediaAdapter(cv::Size sz, cv::MediaFormat fmt, std::vector<uint8_t>&& buffer)
: m_sz(sz), m_fmt(fmt), m_buffer(buffer) {
GAPI_Assert(fmt == cv::MediaFormat::NV12 && "OAKMediaAdapter only supports NV12 format for now");
m_sz = sz;
m_fmt = fmt;
m_buffer = buffer;
}
MediaFrame::View OAKMediaAdapter::OAKMediaAdapter::access(MediaFrame::Access) {
@ -27,6 +25,30 @@ MediaFrame::View OAKMediaAdapter::OAKMediaAdapter::access(MediaFrame::Access) {
cv::GFrameDesc OAKMediaAdapter::OAKMediaAdapter::meta() const { return {m_fmt, m_sz}; }
OAKRMatAdapter::OAKRMatAdapter(const cv::Size& size,
int precision,
std::vector<float>&& buffer)
: m_size(size), m_precision(precision), m_buffer(buffer) {
GAPI_Assert(m_precision == CV_16F);
std::vector<int> wrapped_dims{1, 1, m_size.width, m_size.height};
// FIXME: check layout and add strides
m_desc = cv::GMatDesc(m_precision, wrapped_dims);
m_mat = cv::Mat(static_cast<int>(wrapped_dims.size()),
wrapped_dims.data(),
CV_16FC1, // FIXME: cover other precisions
m_buffer.data());
}
cv::GMatDesc OAKRMatAdapter::desc() const {
return m_desc;
}
cv::RMat::View OAKRMatAdapter::access(cv::RMat::Access) {
return cv::RMat::View{m_desc, m_mat.data};
}
} // namespace oak
} // namespace gapi
} // namespace cv

660
modules/gapi/src/backends/oak/goakbackend.cpp
Unescape View File

@ -2,9 +2,10 @@
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2021 Intel Corporation
// Copyright (C) 2021-2022 Intel Corporation
#include <opencv2/gapi/gkernel.hpp> // GKernelPackage
#include <opencv2/gapi/oak/oak.hpp> // kernels()
#ifdef HAVE_OAK
@ -18,21 +19,24 @@
#include <api/gbackend_priv.hpp>
#include <backends/common/gbackend.hpp>
#include <opencv2/gapi/infer.hpp> // GInferBase
#include <opencv2/gapi/streaming/meta.hpp> // streaming::meta_tag
#include "depthai/depthai.hpp"
#include <opencv2/gapi/oak/oak.hpp>
#include "oak_media_adapter.hpp"
#include "oak_memory_adapters.hpp"
#include <opencv2/gapi/oak/infer.hpp> // infer params
namespace cv { namespace gimpl {
// Forward declaration
class GOAKContext;
struct OAKNodeInfo;
class OAKKernelParams;
class GOAKExecutable final: public GIslandExecutable {
friend class GOAKContext;
friend class OAKKernelParams;
virtual void run(std::vector<InObj>&&,
std::vector<OutObj>&&) override {
GAPI_Assert(false && "Not implemented");
@ -41,7 +45,8 @@ class GOAKExecutable final: public GIslandExecutable {
virtual void run(GIslandExecutable::IInput &in,
GIslandExecutable::IOutput &out) override;
void LinkToParents(ade::NodeHandle handle);
void linkToParent(ade::NodeHandle handle);
void linkCopy(ade::NodeHandle handle);
class ExtractTypeHelper : protected dai::Node {
public:
@ -61,6 +66,7 @@ class GOAKExecutable final: public GIslandExecutable {
std::shared_ptr<dai::node::XLinkOut> xlink_output;
std::shared_ptr<dai::DataOutputQueue> out_queue;
std::string out_queue_name;
size_t gapi_out_data_index;
};
cv::GArg packInArg(const GArg &arg, std::vector<ExtractTypeHelper::InputPtr>& oak_ins);
@ -79,11 +85,16 @@ class GOAKExecutable final: public GIslandExecutable {
cv::Size m_camera_size;
// Backend outputs
std::vector<OAKOutQueueInfo> m_out_queues;
std::unordered_map<ade::NodeHandle,
OAKOutQueueInfo,
ade::HandleHasher<ade::Node>> m_out_queues;
// Backend inputs
std::vector<std::pair<std::string, dai::Buffer>> m_in_queues;
std::unordered_set<ade::NodeHandle,
ade::HandleHasher<ade::Node>> m_passthrough_copy_nodes;
// Note: dai::Pipeline should be the only one for the whole pipeline,
// so there is no way to insert any non-OAK node in graph between other OAK nodes.
// The only heterogeneous case possible is if we insert other backends after or before
@ -91,6 +102,14 @@ class GOAKExecutable final: public GIslandExecutable {
std::unique_ptr<dai::Device> m_device;
std::unique_ptr<dai::Pipeline> m_pipeline;
// Camera config
cv::gapi::oak::ColorCameraParams m_ccp;
// Infer info
std::unordered_map<ade::NodeHandle,
cv::gapi::oak::detail::ParamDesc,
ade::HandleHasher<ade::Node>> m_oak_infer_info;
public:
GOAKExecutable(const ade::Graph& g,
const cv::GCompileArgs& args,
@ -122,6 +141,12 @@ public:
std::vector<cv::GArg>& args,
std::vector<OutputPtr>& results);
GOAKContext(const std::unique_ptr<dai::Pipeline>& pipeline,
const cv::Size& camera_size,
const cv::gapi::oak::detail::ParamDesc& infer_info,
std::vector<cv::GArg>& args,
std::vector<OutputPtr>& results);
// Generic accessor API
template<typename T>
T& inArg(int input) { return m_args.at(input).get<T>(); }
@ -130,12 +155,14 @@ public:
InputPtr& in(int input);
OutputPtr& out(int output);
const std::unique_ptr<dai::Pipeline>& pipeline();
const std::unique_ptr<dai::Pipeline>& pipeline() const;
const cv::Size& camera_size() const;
const cv::gapi::oak::detail::ParamDesc& ii() const;
private:
const std::unique_ptr<dai::Pipeline>& m_pipeline;
const cv::Size& m_camera_size;
const cv::Size m_camera_size;
const cv::gapi::oak::detail::ParamDesc m_infer_info;
std::vector<cv::GArg>& m_args;
std::vector<OutputPtr>& m_outputs;
};
@ -144,9 +171,18 @@ GOAKContext::GOAKContext(const std::unique_ptr<dai::Pipeline>& pipeline,
const cv::Size& camera_size,
std::vector<cv::GArg>& args,
std::vector<OutputPtr>& results)
: m_pipeline(pipeline), m_camera_size(camera_size), m_args(args), m_outputs(results) {}
: m_pipeline(pipeline), m_camera_size(camera_size),
m_args(args), m_outputs(results) {}
GOAKContext::GOAKContext(const std::unique_ptr<dai::Pipeline>& pipeline,
const cv::Size& camera_size,
const cv::gapi::oak::detail::ParamDesc& infer_info,
std::vector<cv::GArg>& args,
std::vector<OutputPtr>& results)
: m_pipeline(pipeline), m_camera_size(camera_size),
m_infer_info(infer_info), m_args(args), m_outputs(results) {}
const std::unique_ptr<dai::Pipeline>& GOAKContext::pipeline() {
const std::unique_ptr<dai::Pipeline>& GOAKContext::pipeline() const {
return m_pipeline;
}
@ -154,6 +190,10 @@ const cv::Size& GOAKContext::camera_size() const {
return m_camera_size;
}
const cv::gapi::oak::detail::ParamDesc& GOAKContext::ii() const {
return m_infer_info;
}
GOAKContext::InputPtr& GOAKContext::in(int input) {
return inArg<std::reference_wrapper<GOAKContext::InputPtr>>(input).get();
}
@ -162,6 +202,14 @@ GOAKContext::OutputPtr& GOAKContext::out(int output) {
return m_outputs.at(output);
}
class OAKKernelParams {
public:
const std::unique_ptr<dai::Pipeline>& pipeline;
const cv::Size& camera_size;
const cv::gapi::oak::detail::ParamDesc& infer_info;
std::vector<std::pair<std::string, dai::Buffer>>& in_queues;
};
namespace detail {
template<class T> struct get_in;
template<> struct get_in<cv::GFrame> {
@ -179,13 +227,10 @@ template<> struct get_out<cv::GFrame> {
template<typename U> struct get_out<cv::GArray<U>> {
static GOAKContext::OutputPtr& get(GOAKContext &ctx, int idx) { return ctx.out(idx); }
};
// FIXME: add support of other types
struct OAKKernelParams {
const std::unique_ptr<dai::Pipeline>& pipeline;
const cv::Size& camera_size;
std::vector<std::pair<std::string, dai::Buffer>>& m_in_queues;
template<> struct get_out<cv::GMat> {
static GOAKContext::OutputPtr& get(GOAKContext &ctx, int idx) { return ctx.out(idx); }
};
// FIXME: add support of other types
template<typename, typename, typename>
struct OAKCallHelper;
@ -200,6 +245,7 @@ struct OAKCallHelper<Impl, std::tuple<Ins...>, std::tuple<Outs...> > {
, cv::detail::Seq<OIs...>) {
return Impl::put(OAKKernelParams{ctx.pipeline(),
ctx.camera_size(),
ctx.ii(),
in_queues_params},
get_in<Ins>::get(ctx, IIs)...,
get_out<Outs>::get(ctx, OIs)...);
@ -229,23 +275,84 @@ struct OAKComponent
static const char *name() { return "OAK Component"; }
GOAKKernel k;
};
}} // namespace gimpl // namespace cv
} // namespace gimpl
} // namespace cv
using OAKGraph = ade::TypedGraph
< cv::gimpl::OAKComponent
// FIXME: extend
< cv::gimpl::Protocol
, cv::gimpl::Op
, cv::gimpl::NetworkParams
, cv::gimpl::CustomMetaFunction
// OAK specific
, cv::gimpl::OAKComponent
>;
using ConstOAKGraph = ade::ConstTypedGraph
< cv::gimpl::OAKComponent
// FIXME: extend
< cv::gimpl::Protocol
, cv::gimpl::Op
, cv::gimpl::NetworkParams
, cv::gimpl::CustomMetaFunction
// OAK specific
, cv::gimpl::OAKComponent
>;
// This function links dai operation nodes - parent's output to child's input.
namespace
{
std::pair<dai::TensorInfo, dai::TensorInfo>
parseDaiInferMeta(const cv::gapi::oak::detail::ParamDesc& pd) {
dai::OpenVINO::Blob blob(pd.blob_file);
GAPI_Assert(blob.networkInputs.size() == 1);
GAPI_Assert(blob.networkOutputs.size() == 1);
return {blob.networkInputs.begin()->second,
blob.networkOutputs.begin()->second};
}
std::string
getDaiInferOutLayerName(const cv::gapi::oak::detail::ParamDesc& pd) {
dai::OpenVINO::Blob blob(pd.blob_file);
GAPI_Assert(blob.networkInputs.size() == 1);
GAPI_Assert(blob.networkOutputs.size() == 1);
return blob.networkOutputs.begin()->first;
}
} // anonymous namespace
// Custom meta function for OAK backend for infer
static cv::GMetaArgs customOutMeta(const ade::Graph &gr,
const ade::NodeHandle &nh,
const cv::GMetaArgs &/*in_metas*/,
const cv::GArgs &/*in_args*/) {
cv::GMetaArgs result;
const auto &np = ConstOAKGraph(gr).metadata(nh).get<cv::gimpl::NetworkParams>();
const auto &pd = cv::util::any_cast<cv::gapi::oak::detail::ParamDesc>(np.opaque);
// FIXME: Infer kernel and backend does rather the same
auto in_out_tensor_info = parseDaiInferMeta(pd);
GAPI_Assert(in_out_tensor_info.second.dataType ==
dai::TensorInfo::DataType::FP16);
// FIXME: add proper layout converter here
GAPI_Assert(in_out_tensor_info.second.order ==
dai::TensorInfo::StorageOrder::NCHW);
// FIXME: DAI returns vector<unsigned>, remove workaround
std::vector<int> wrapped_dims;
for (const auto& d : in_out_tensor_info.second.dims) {
wrapped_dims.push_back(d);
}
result = {cv::GMetaArg{cv::GMatDesc(CV_16F, 1, cv::Size(wrapped_dims[1], wrapped_dims[0]), false)}};
return result;
}
// This function links DAI operation nodes - parent's output to child's input.
// It utilizes G-API graph to search for operation's node it's previous operation in graph
// when links them in dai graph.
void cv::gimpl::GOAKExecutable::LinkToParents(ade::NodeHandle handle)
// when links them in DAI graph.
void cv::gimpl::GOAKExecutable::linkToParent(ade::NodeHandle handle)
{
ade::NodeHandle parent;
for (const auto& data_nh : handle.get()->inNodes()) {
@ -253,6 +360,13 @@ void cv::gimpl::GOAKExecutable::LinkToParents(ade::NodeHandle handle)
GAPI_Assert(data_nh.get()->inNodes().size() == 1);
parent = data_nh.get()->inNodes().front();
// Don't link if parent is copy - the case is handled differently
// in linkCopy
const auto& op = m_gm.metadata(parent).get<Op>();
if (op.k.name == "org.opencv.oak.copy") {
continue;
}
// Assuming that OAK nodes are aligned for linking.
// FIXME: potential rework might be needed then
// counterexample is found.
@ -269,6 +383,89 @@ void cv::gimpl::GOAKExecutable::LinkToParents(ade::NodeHandle handle)
}
}
// This function links DAI operations for Copy OP in G-API graph
void cv::gimpl::GOAKExecutable::linkCopy(ade::NodeHandle handle) {
// 1. Check that there are no back-to-back Copy OPs in graph
auto copy_out = handle.get()->outNodes();
GAPI_Assert(copy_out.size() == 1);
for (const auto& copy_next_op : copy_out.front().get()->outNodes()) {
const auto& op = m_gm.metadata(copy_next_op).get<Op>();
if (op.k.name == "org.opencv.oak.copy") {
GAPI_Assert(false && "Back-to-back Copy operations are not supported in graph");
}
}
// 2. Link passthrough case
if (m_passthrough_copy_nodes.find(handle) != m_passthrough_copy_nodes.end()) {
ExtractTypeHelper::OutputPtr parent;
bool parent_is_camera = false;
// Copy has only 1 input data
GAPI_Assert(handle.get()->inNodes().size() == 1);
auto in_ops = handle.get()->inNodes().front().get()->inNodes();
if (in_ops.size() == 0) {
// No parent nodes - parent = camera
parent = &m_camera_input->video;
parent_is_camera = true;
} else {
// Data has only 1 input
GAPI_Assert(in_ops.size() == 1);
auto node = m_oak_nodes.at(in_ops.front());
// Should only have 1 output
GAPI_Assert(node.outputs.size() == 1);
parent = node.outputs[0];
}
// Now link DAI parent output to Copy's child's inputs ignoring the Copy operation
// FIXME: simplify this loop
auto copy_out_data = handle.get()->outNodes();
// Copy has only 1 output
GAPI_Assert(copy_out_data.size() == 1);
for (const auto& copy_next_op : copy_out_data.front().get()->outNodes()) {
if (m_oak_nodes.find(copy_next_op) != m_oak_nodes.end()) {
// FIXME: consider a better approach
if (parent_is_camera) {
if (m_oak_infer_info.find(copy_next_op) != m_oak_infer_info.end()) {
parent = &m_camera_input->preview;
} else {
parent = &m_camera_input->video;
}
}
// Found next Copy OP which needs to be linked to Copy's parent
GAPI_Assert(m_oak_nodes.at(copy_next_op).inputs.size() == 1 &&
"Internal OAK nodes are not aligned for linking (Copy operation)");
parent->link(*(m_oak_nodes.at(copy_next_op).inputs.front()));
}
}
}
// 3. Link output Copy case
if (m_out_queues.find(handle) != m_out_queues.end()) {
// DAI XLinkOutput node
auto xout = m_out_queues[handle].xlink_output->input;
// Find parent node
// FIXME: copypasted from case 2 above
ExtractTypeHelper::OutputPtr parent;
// Copy has only 1 input data
GAPI_Assert(handle.get()->inNodes().size() == 1);
auto in_ops = handle.get()->inNodes().front().get()->inNodes();
if (in_ops.size() == 0) {
// No parent nodes - parent = camera
parent = &m_camera_input->video;
} else {
// Data has only 1 input
GAPI_Assert(in_ops.size() == 1);
auto node = m_oak_nodes.at(in_ops.front());
// Should only have 1 output
GAPI_Assert(node.outputs.size() == 1);
parent = node.outputs[0];
}
// Link parent to xout
parent->link(xout);
}
}
cv::GArg
cv::gimpl::GOAKExecutable::packInArg(const GArg &arg,
std::vector<ExtractTypeHelper::InputPtr>& oak_ins) {
@ -298,9 +495,8 @@ void cv::gimpl::GOAKExecutable::packOutArg(const RcDesc &rc,
std::vector<ExtractTypeHelper::OutputPtr>& oak_outs) {
switch (rc.shape) {
case GShape::GFRAME:
oak_outs.push_back(nullptr);
break;
case GShape::GARRAY:
case GShape::GMAT:
oak_outs.push_back(nullptr);
break;
default:
@ -309,6 +505,33 @@ void cv::gimpl::GOAKExecutable::packOutArg(const RcDesc &rc,
}
}
namespace {
static dai::CameraBoardSocket extractCameraBoardSocket(cv::gapi::oak::ColorCameraParams ccp) {
switch (ccp.board_socket) {
case cv::gapi::oak::ColorCameraParams::BoardSocket::RGB:
return dai::CameraBoardSocket::RGB;
// FIXME: extend
default:
// basically unreachable
GAPI_Assert("Unsupported camera board socket");
return {};
}
}
static dai::ColorCameraProperties::SensorResolution
extractCameraResolution(cv::gapi::oak::ColorCameraParams ccp) {
switch (ccp.resolution) {
case cv::gapi::oak::ColorCameraParams::Resolution::THE_1080_P:
return dai::ColorCameraProperties::SensorResolution::THE_1080_P;
// FIXME: extend
default:
// basically unreachable
GAPI_Assert("Unsupported camera board socket");
return {};
}
}
} // anonymous namespace
cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
const cv::GCompileArgs &args,
const std::vector<ade::NodeHandle>& nodes,
@ -344,16 +567,52 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
}
}
m_ccp = cv::gimpl::getCompileArg<cv::gapi::oak::ColorCameraParams>(args)
.value_or(cv::gapi::oak::ColorCameraParams{});
// FIXME: change the hard-coded behavior (XLinkIn path)
auto camRgb = m_pipeline->create<dai::node::ColorCamera>();
// FIXME: extract camera compile arguments here and properly convert them for dai
camRgb->setBoardSocket(dai::CameraBoardSocket::RGB);
camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
camRgb->setBoardSocket(extractCameraBoardSocket(m_ccp));
camRgb->setResolution(extractCameraResolution(m_ccp));
camRgb->setInterleaved(m_ccp.interleaved);
// Extract infer params
for (const auto& nh : nodes) {
if (m_gm.metadata(nh).get<NodeType>().t == NodeType::OP) {
if (ConstOAKGraph(m_g).metadata(nh).contains<cv::gimpl::NetworkParams>()) {
const auto &np = ConstOAKGraph(m_g).metadata(nh).get<cv::gimpl::NetworkParams>();
const auto &pp = cv::util::any_cast<cv::gapi::oak::detail::ParamDesc>(np.opaque);
m_oak_infer_info[nh] = pp;
break;
}
}
}
// FIXME: handle multiple infers
if (!m_oak_infer_info.empty()) {
GAPI_Assert(m_oak_infer_info.size() == 1);
// FIXME: move to infer node?
auto in_out_tensor_info = parseDaiInferMeta(m_oak_infer_info.begin()->second);
if (in_out_tensor_info.first.dataType ==
dai::TensorInfo::DataType::FP16 ||
in_out_tensor_info.first.dataType ==
dai::TensorInfo::DataType::FP32) {
camRgb->setFp16(true);
} else {
camRgb->setFp16(false);
}
// FIXME: add proper layout converter here
GAPI_Assert(in_out_tensor_info.first.order ==
dai::TensorInfo::StorageOrder::NCHW);
camRgb->setPreviewSize(in_out_tensor_info.first.dims[0], in_out_tensor_info.first.dims[1]);
}
// Set camera output. Fixme: consider working with other camera outputs
m_camera_input = camRgb;
// FIXME: change when other camera censors are introduced
std::tuple<int, int> video_size = camRgb->getVideoSize();
std::tuple<int, int> video_size = m_camera_input->getVideoSize();
m_camera_size = cv::Size{std::get<0>(video_size), std::get<1>(video_size)};
// Prepare XLinkOut nodes for each output object in graph
@ -361,7 +620,23 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
auto xout = m_pipeline->create<dai::node::XLinkOut>();
std::string xout_name = "xout" + std::to_string(i);
xout->setStreamName(xout_name);
m_out_queues.push_back({xout, nullptr, xout_name});
// Find parent OP's nh
ade::NodeHandle parent_op_nh;
for (const auto& nh : nodes) {
for (const auto& outdata : nh.get()->outNodes()) {
if (m_gm.metadata(outdata).get<NodeType>().t == NodeType::DATA) {
auto rc = m_gm.metadata(outdata).get<cv::gimpl::Data>().rc;
auto shape = m_gm.metadata(outdata).get<cv::gimpl::Data>().shape;
// Match outs_data with the actual operation
if (rc == outs_data[i].rc && shape == outs_data[i].shape) {
parent_op_nh = nh;
}
}
}
}
m_out_queues[parent_op_nh] = {xout, nullptr, xout_name, i};
}
// Create OAK node for each node in this backend
@ -375,33 +650,66 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
m_oak_nodes.at(nh).inputs.reserve(op.args.size());
m_oak_nodes.at(nh).outputs.reserve(op.outs.size());
// Copy operation in graph can fall into 3 cases:
// 1) Copy is an output of the island -
// in that case we link it to XLinkOut node from m_out_queues
// 2) Copy is between other two operations in the same OAK island -
// in that case we link its parent operation (could be camera) to
// the child one (those copy operations are placed in m_passthrough_copy_nodes)
// 3) Copy can fall into cases 1) and 2) at the same time
// Prepare passthrough Copy operations
if (op.k.name == "org.opencv.oak.copy") {
// Copy has only 1 output
auto copy_out = nh.get()->outNodes();
GAPI_Assert(copy_out.size() == 1);
for (const auto& copy_next_op : copy_out.front().get()->outNodes()) {
// Check that copy is a passthrough OP
if (std::find(nodes.begin(), nodes.end(), copy_next_op) != nodes.end()) {
m_passthrough_copy_nodes.insert(nh);
break;
}
}
}
std::vector<cv::GArg> in_ctx_args;
in_ctx_args.reserve(op.args.size());
for (auto &op_arg : op.args) in_ctx_args.push_back(packInArg(op_arg,
m_oak_nodes.at(nh).inputs));
m_oak_nodes.at(nh).inputs));
for (auto &&op_out : op.outs) packOutArg(op_out, m_oak_nodes.at(nh).outputs);
GAPI_Assert(!m_oak_nodes.at(nh).inputs.empty());
GAPI_Assert(!m_oak_nodes.at(nh).outputs.empty());
GOAKContext ctx(m_pipeline, m_camera_size, in_ctx_args, m_oak_nodes.at(nh).outputs);
m_oak_nodes.at(nh).node = u.k.m_put_f(ctx, m_in_queues);
GAPI_Assert(m_oak_nodes.at(nh).node != nullptr);
if (ConstOAKGraph(m_g).metadata(nh).contains<cv::gimpl::NetworkParams>()) {
GOAKContext ctx(m_pipeline, m_camera_size, m_oak_infer_info[nh],
in_ctx_args, m_oak_nodes.at(nh).outputs);
m_oak_nodes.at(nh).node = u.k.m_put_f(ctx, m_in_queues);
} else {
GOAKContext ctx(m_pipeline, m_camera_size,
in_ctx_args, m_oak_nodes.at(nh).outputs);
m_oak_nodes.at(nh).node = u.k.m_put_f(ctx, m_in_queues);
}
// Check that all inputs and outputs are properly filled after constructing kernels
// to then link it together
// FIXME: add more logging
const auto& node = m_oak_nodes.at(nh);
if (std::any_of(node.inputs.cbegin(), node.inputs.cend(),
[](ExtractTypeHelper::InputPtr ptr) {
return ptr == nullptr;
})) {
GAPI_Assert(false && "DAI input are not set");
}
if (std::any_of(node.outputs.cbegin(), node.outputs.cend(),
[](ExtractTypeHelper::OutputPtr ptr) {
return ptr == nullptr;
})) {
GAPI_Assert(false && "DAI outputs are not set");
const auto& node_info = m_oak_nodes.at(nh);
// Copy operations don't set their inputs/outputs properly
if (op.k.name != "org.opencv.oak.copy") {
GAPI_Assert(node_info.node != nullptr);
if (std::any_of(node_info.inputs.cbegin(), node_info.inputs.cend(),
[](ExtractTypeHelper::InputPtr ptr) {
return ptr == nullptr;
})) {
GAPI_Assert(false && "DAI input are not set");
}
if (std::any_of(node_info.outputs.cbegin(), node_info.outputs.cend(),
[](ExtractTypeHelper::OutputPtr ptr) {
return ptr == nullptr;
})) {
GAPI_Assert(false && "DAI outputs are not set");
}
}
}
}
@ -413,15 +721,26 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
ade::HandleHasher<ade::Node>> out_nodes;
std::unordered_set<ade::NodeHandle,
ade::HandleHasher<ade::Node>> inter_nodes;
std::unordered_set<ade::NodeHandle,
ade::HandleHasher<ade::Node>> copy_nodes;
// TODO: optimize this loop
for (const auto& node : m_oak_nodes) {
auto nh = node.first;
// Check if it's a Copy OP - will be handled differently when linking
GAPI_Assert(m_gm.metadata(nh).get<NodeType>().t == NodeType::OP);
const auto& op = m_gm.metadata(nh).get<Op>();
if (op.k.name == "org.opencv.oak.copy") {
copy_nodes.insert(nh);
continue;
}
// Fill input op nodes
for (const auto& d : ins_data) {
for (const auto& indata : nh.get()->inNodes()) {
auto rc = m_gm.metadata(indata).get<cv::gimpl::Data>().rc;
if (rc == d.rc) {
auto shape = m_gm.metadata(indata).get<cv::gimpl::Data>().shape;
if (rc == d.rc && shape == d.shape) {
in_nodes.insert(nh);
}
}
@ -430,7 +749,8 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
for (const auto& d : outs_data) {
for (const auto& outdata : nh.get()->outNodes()) {
auto rc = m_gm.metadata(outdata).get<cv::gimpl::Data>().rc;
if (rc == d.rc) {
auto shape = m_gm.metadata(outdata).get<cv::gimpl::Data>().shape;
if (rc == d.rc && shape == d.shape) {
out_nodes.insert(nh);
}
}
@ -446,42 +766,47 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g,
// 1. Link input nodes to camera
for (const auto& nh : in_nodes) {
GAPI_Assert(m_oak_nodes.at(nh).inputs.size() == 1);
// FIXME: covert other camera outputs
m_camera_input->video.link(*(m_oak_nodes.at(nh).inputs[0]));
// FIXME: cover other camera outputs
// Link preview to infer, video to all other nodes
if (m_oak_infer_info.find(nh) == m_oak_infer_info.end()) {
m_camera_input->video.link(*(m_oak_nodes.at(nh).inputs[0]));
} else {
m_camera_input->preview.link(*(m_oak_nodes.at(nh).inputs[0]));
}
}
// 2. Link output nodes to XLinkOut nodes
size_t out_counter = 0;
for (const auto& nh : out_nodes) {
GAPI_Assert(out_counter + m_oak_nodes.at(nh).outputs.size() <= m_out_queues.size());
for (const auto& out : m_oak_nodes.at(nh).outputs) {
out->link(m_out_queues[out_counter++].xlink_output->input);
out->link(m_out_queues[nh].xlink_output->input);
}
// Input nodes in OAK doesn't have parent operation - just camera (for now)
if (in_nodes.find(nh) == in_nodes.end()) {
LinkToParents(nh);
linkToParent(nh);
}
}
// 3. Link internal nodes to their parents
for (const auto& nh : inter_nodes) {
// Input nodes in OAK doesn't have parent operation - just camera (for now)
if (in_nodes.find(nh) == in_nodes.end()) {
LinkToParents(nh);
}
linkToParent(nh);
}
// 4. Link copy nodes
for (const auto& nh : copy_nodes) {
linkCopy(nh);
}
m_device = std::unique_ptr<dai::Device>(new dai::Device(*m_pipeline));
// Prepare OAK output queues
GAPI_Assert(m_out_queues.size() == outs_data.size());
for (const auto out_it : ade::util::indexed(outs_data))
for (const auto out_it : ade::util::indexed(m_out_queues))
{
auto& q = m_out_queues[ade::util::index(out_it)];
auto& q = ade::util::value(out_it).second;
GAPI_Assert(q.out_queue == nullptr); // shouldn't be not filled till this point
// FIXME: add queue parameters
// Currently: 30 - max DAI queue capacity, true - blocking queue
q.out_queue = m_device->getOutputQueue(q.out_queue_name, 30, true);
// Currently: 4 - max DAI queue capacity, true - blocking queue
q.out_queue = m_device->getOutputQueue(q.out_queue_name, 4, true);
}
}
@ -507,17 +832,25 @@ void cv::gimpl::GOAKExecutable::run(GIslandExecutable::IInput &in,
q->send(in_q.second);
}
for (size_t i = 0; i < m_out_queues.size(); ++i) {
auto q = m_out_queues[i].out_queue;
// TODO: support other DAI types if needed
// Note: we utilize getData() method that returns std::vector of data
// on which we gain ownership
auto oak_frame = q->get<dai::ImgFrame>();
for (size_t i = 0; i < m_in_queues.size(); ++i) {
auto q = m_device->getInputQueue(m_in_queues[i].first);
q->send(m_in_queues[i].second);
}
for (const auto el : m_out_queues) {
const auto out_q = el.second;
auto& q = out_q.out_queue;
auto out_arg = out.get(out_q.gapi_out_data_index);
auto out_arg = out.get(i);
// FIXME: misc info to be utilized in switch below
cv::GRunArg::Meta meta;
std::shared_ptr<dai::ImgFrame> oak_frame;
switch(out_arg.index()) {
case cv::GRunArgP::index_of<cv::MediaFrame*>():
{
oak_frame = q->get<dai::ImgFrame>();
// FIXME: hard-coded NV12
*cv::util::get<cv::MediaFrame*>(out_arg) =
cv::MediaFrame::Create<cv::gapi::oak::OAKMediaAdapter>(
@ -525,56 +858,61 @@ void cv::gimpl::GOAKExecutable::run(GIslandExecutable::IInput &in,
static_cast<int>(oak_frame->getHeight())),
cv::MediaFormat::NV12,
std::move(oak_frame->getData()));
using namespace cv::gapi::streaming::meta_tag;
meta[timestamp] = oak_frame->getTimestamp();
meta[seq_id] = oak_frame->getSequenceNum();
break;
}
case cv::GRunArgP::index_of<cv::detail::VectorRef>():
{
oak_frame = q->get<dai::ImgFrame>();
cv::util::get<cv::detail::VectorRef>(out_arg).wref<uint8_t>() = std::move(oak_frame->getData());
using namespace cv::gapi::streaming::meta_tag;
meta[timestamp] = oak_frame->getTimestamp();
meta[seq_id] = oak_frame->getSequenceNum();
break;
// FIXME: Add support for remaining types
default:
GAPI_Assert(false && "Unsupported type in OAK backend");
}
case cv::GRunArgP::index_of<cv::RMat*>(): // only supported for infer
{
auto nn_data = q->get<dai::NNData>();
using namespace cv::gapi::streaming::meta_tag;
cv::GRunArg::Meta meta;
meta[timestamp] = oak_frame->getTimestamp();
meta[seq_id] = oak_frame->getSequenceNum();
auto out_layer_name = getDaiInferOutLayerName(m_oak_infer_info.begin()->second);
auto in_out_tensor_info = parseDaiInferMeta(m_oak_infer_info.begin()->second);
out.meta(out_arg, meta);
out.post(std::move(out_arg));
}
}
auto layer = std::move(nn_data->getLayerFp16(out_layer_name));
// Built-in kernels for OAK /////////////////////////////////////////////////////
// FIXME: add proper layout converter here
GAPI_Assert(in_out_tensor_info.second.order ==
dai::TensorInfo::StorageOrder::NCHW);
// FIMXE: only 1-channel data is supported for now
GAPI_Assert(in_out_tensor_info.second.dims[2] == 1);
class GOAKBackendImpl final : public cv::gapi::GBackend::Priv {
virtual void unpackKernel(ade::Graph &graph,
const ade::NodeHandle &op_node,
const cv::GKernelImpl &impl) override {
OAKGraph gm(graph);
*cv::util::get<cv::RMat*>(out_arg) =
cv::make_rmat<cv::gapi::oak::OAKRMatAdapter>(
cv::Size(in_out_tensor_info.second.dims[1],
in_out_tensor_info.second.dims[0]),
CV_16F, // FIXME: cover other precisions
std::move(layer)
);
const auto &kimpl = cv::util::any_cast<cv::gimpl::GOAKKernel>(impl.opaque);
gm.metadata(op_node).set(cv::gimpl::OAKComponent{kimpl});
}
using namespace cv::gapi::streaming::meta_tag;
meta[timestamp] = nn_data->getTimestamp();
meta[seq_id] = nn_data->getSequenceNum();
virtual EPtr compile(const ade::Graph &graph,
const cv::GCompileArgs &args,
const std::vector<ade::NodeHandle> &nodes,
const std::vector<cv::gimpl::Data>& ins_data,
const std::vector<cv::gimpl::Data>& outs_data) const override {
cv::gimpl::GModel::ConstGraph gm(graph);
// FIXME: pass streaming/non-streaming option to support non-camera case
// NB: how could we have non-OAK source in streaming mode, then OAK backend in
// streaming mode but without camera input?
if (!gm.metadata().contains<cv::gimpl::Streaming>()) {
GAPI_Assert(false && "OAK backend only supports Streaming mode for now");
break;
}
// FIXME: Add support for remaining types
default:
GAPI_Assert(false && "Unsupported type in OAK backend");
}
return EPtr{new cv::gimpl::GOAKExecutable(graph, args, nodes, ins_data, outs_data)};
}
};
cv::gapi::GBackend cv::gapi::oak::backend() {
static cv::gapi::GBackend this_backend(std::make_shared<GOAKBackendImpl>());
return this_backend;
out.meta(out_arg, meta);
out.post(std::move(out_arg));
}
}
namespace cv {
@ -604,10 +942,13 @@ static dai::VideoEncoderProperties::Profile convertEncProfile(cv::gapi::oak::Enc
// Kernels ///////////////////////////////////////////////////////////////
template<class Impl, class K>
class GOAKKernelImpl: public detail::OAKCallHelper<Impl, typename K::InArgs, typename K::OutArgs>
// FIXME: consider a better solution - hard-coded API
// Is there a way to extract API from somewhereelse/utilize structs
// like in streaming/infer backends (mainly infer and copy operations)
template<class Impl, class K, class InArgs = typename K::InArgs, class OutArgs = typename K::OutArgs>
class GOAKKernelImpl: public detail::OAKCallHelper<Impl, InArgs, OutArgs>
, public cv::detail::KernelTag {
using P = detail::OAKCallHelper<Impl, typename K::InArgs, typename K::OutArgs>;
using P = detail::OAKCallHelper<Impl, InArgs, OutArgs>;
public:
using API = K;
static cv::gapi::GBackend backend() { return cv::gapi::oak::backend(); }
@ -617,17 +958,51 @@ public:
#define GAPI_OAK_KERNEL(Name, API) \
struct Name: public cv::gimpl::oak::GOAKKernelImpl<Name, API>
#define GAPI_OAK_FIXED_API_KERNEL(Name, API, InArgs, OutArgs) \
struct Name: public cv::gimpl::oak::GOAKKernelImpl<Name, API, InArgs, OutArgs>
namespace {
GAPI_OAK_FIXED_API_KERNEL(GOAKInfer, cv::GInferBase, std::tuple<cv::GFrame>, std::tuple<cv::GMat>) {
static std::shared_ptr<dai::Node> put(const cv::gimpl::OAKKernelParams& params,
GOAKContext::InputPtr& in,
GOAKContext::OutputPtr& out) {
auto nn = params.pipeline->create<dai::node::NeuralNetwork>();
nn->input.setBlocking(true);
nn->input.setQueueSize(1);
// FIXME: add G-API built-in preproc here (currently it's only setPreviewSize() on the camera node)
// Note: for some reason currently it leads to:
// "Fatal error. Please report to developers. Log: 'ImageManipHelper' '61'"
nn->setBlobPath(params.infer_info.blob_file);
in = &(nn->input);
out = &(nn->out);
return nn;
}
};
GAPI_OAK_KERNEL(GOAKCopy, cv::gapi::oak::GCopy) {
static std::shared_ptr<dai::Node> put(const cv::gimpl::OAKKernelParams&,
GOAKContext::InputPtr&,
GOAKContext::OutputPtr&) {
// Do nothing in Copy OP since it's either already represented
// by XLinkOut node (bonded to output queues) or it's a passthrough OP
return nullptr;
}
};
GAPI_OAK_KERNEL(GOAKEncFrame, cv::gapi::oak::GEncFrame) {
static std::shared_ptr<dai::Node> put(const cv::gimpl::detail::OAKKernelParams& params,
static std::shared_ptr<dai::Node> put(const cv::gimpl::OAKKernelParams& params,
GOAKContext::InputPtr& in,
const cv::gapi::oak::EncoderConfig& cfg,
GOAKContext::OutputPtr& out) {
auto videoEnc = params.pipeline->create<dai::node::VideoEncoder>();
// FIXME: convert all the parameters to dai
videoEnc->setDefaultProfilePreset(cfg.width, cfg.height,
cfg.frameRate,
videoEnc->setDefaultProfilePreset(cfg.frameRate,
convertEncProfile(cfg.profile));
in = &(videoEnc->input);
@ -638,7 +1013,7 @@ GAPI_OAK_KERNEL(GOAKEncFrame, cv::gapi::oak::GEncFrame) {
};
GAPI_OAK_KERNEL(GOAKSobelXY, cv::gapi::oak::GSobelXY) {
static std::shared_ptr<dai::Node> put(const cv::gimpl::detail::OAKKernelParams& params,
static std::shared_ptr<dai::Node> put(const cv::gimpl::OAKKernelParams& params,
GOAKContext::InputPtr& in,
const cv::Mat& hk,
const cv::Mat& vk,
@ -664,7 +1039,7 @@ GAPI_OAK_KERNEL(GOAKSobelXY, cv::gapi::oak::GSobelXY) {
xinEdgeCfg->out.link(edgeDetector->inputConfig);
params.m_in_queues.push_back({"sobel_cfg", cfg});
params.in_queues.push_back({"sobel_cfg", cfg});
in = &(edgeDetector->inputImage);
out = &(edgeDetector->outputImage);
@ -672,11 +1047,55 @@ GAPI_OAK_KERNEL(GOAKSobelXY, cv::gapi::oak::GSobelXY) {
return edgeDetector;
}
};
} // anonymous namespace
} // namespace oak
} // namespace gimpl
} // namespace cv
class GOAKBackendImpl final : public cv::gapi::GBackend::Priv {
virtual void unpackKernel(ade::Graph &graph,
const ade::NodeHandle &op_node,
const cv::GKernelImpl &impl) override {
using namespace cv::gimpl;
OAKGraph gm(graph);
const auto &kimpl = cv::util::any_cast<GOAKKernel>(impl.opaque);
gm.metadata(op_node).set(OAKComponent{kimpl});
// Set custom meta for infer
if (gm.metadata(op_node).contains<cv::gimpl::NetworkParams>()) {
gm.metadata(op_node).set(CustomMetaFunction{customOutMeta});
}
}
virtual EPtr compile(const ade::Graph &graph,
const cv::GCompileArgs &args,
const std::vector<ade::NodeHandle> &nodes,
const std::vector<cv::gimpl::Data>& ins_data,
const std::vector<cv::gimpl::Data>& outs_data) const override {
cv::gimpl::GModel::ConstGraph gm(graph);
// FIXME: pass streaming/non-streaming option to support non-camera case
// NB: how could we have non-OAK source in streaming mode, then OAK backend in
// streaming mode but without camera input?
if (!gm.metadata().contains<cv::gimpl::Streaming>()) {
GAPI_Assert(false && "OAK backend only supports Streaming mode for now");
}
return EPtr{new cv::gimpl::GOAKExecutable(graph, args, nodes, ins_data, outs_data)};
}
virtual cv::GKernelPackage auxiliaryKernels() const override {
return cv::gapi::kernels< cv::gimpl::oak::GOAKInfer
>();
}
};
cv::gapi::GBackend cv::gapi::oak::backend() {
static cv::gapi::GBackend this_backend(std::make_shared<GOAKBackendImpl>());
return this_backend;
}
namespace cv {
namespace gapi {
namespace oak {
@ -684,6 +1103,7 @@ namespace oak {
cv::gapi::GKernelPackage kernels() {
return cv::gapi::kernels< cv::gimpl::oak::GOAKEncFrame
, cv::gimpl::oak::GOAKSobelXY
, cv::gimpl::oak::GOAKCopy
>();
}
@ -697,13 +1117,17 @@ namespace cv {
namespace gapi {
namespace oak {
cv::gapi::GKernelPackage kernels();
cv::gapi::GKernelPackage kernels() {
GAPI_Assert(false && "Built without OAK support");
return {};
}
cv::gapi::GBackend backend() {
GAPI_Assert(false && "Built without OAK support");
static cv::gapi::GBackend this_backend(nullptr);
return this_backend;
}
} // namespace oak
} // namespace gapi
} // namespace cv

22
modules/gapi/src/backends/oak/oak_media_adapter.hpp → modules/gapi/src/backends/oak/oak_memory_adapters.hpp
Unescape View File

@ -10,11 +10,15 @@
#include <memory>
#include <opencv2/gapi/media.hpp>
#include <opencv2/gapi/rmat.hpp>
namespace cv {
namespace gapi {
namespace oak {
// Used for OAK backends outputs only.
// Filled from DepthAI's ImgFrame type and owns the memory.
// Used mainly for CV operations.
class GAPI_EXPORTS OAKMediaAdapter final : public cv::MediaFrame::IAdapter {
public:
OAKMediaAdapter() = default;
@ -28,6 +32,24 @@ private:
std::vector<uint8_t> m_buffer;
};
// Used for OAK backends outputs only.
// Filled from DepthAI's NNData type and owns the memory.
// Used only for infer operations.
class GAPI_EXPORTS OAKRMatAdapter final : public cv::RMat::Adapter {
public:
OAKRMatAdapter() = default;
OAKRMatAdapter(const cv::Size& size, int precision, std::vector<float>&& buffer);
cv::GMatDesc desc() const override;
cv::RMat::View access(cv::RMat::Access) override;
~OAKRMatAdapter() = default;
private:
cv::Size m_size;
int m_precision;
std::vector<float> m_buffer;
cv::GMatDesc m_desc;
cv::Mat m_mat;
};
} // namespace oak
} // namespace gapi
} // namespace cv

2
modules/gapi/src/compiler/gmodel.hpp
Unescape View File

@ -70,7 +70,7 @@ struct Data
// FIXME: This is a _pure_ duplication of RcDesc now! (except storage)
GShape shape; // FIXME: Probably to be replaced by GMetaArg?
int rc;
int rc; // rc is unique but local to shape
GMetaArg meta;
HostCtor ctor; // T-specific helper to deal with unknown types in our code
cv::detail::OpaqueKind kind; // FIXME: is needed to store GArray/GOpaque type

Write Preview
Loading…
Cancel
Save