From 7ed557497dfacaab494959ac686891ce9e463dfa Mon Sep 17 00:00:00 2001 From: Alexey Smirnov Date: Tue, 5 Apr 2022 21:00:32 +0300 Subject: [PATCH] Merge pull request #21504 from smirnov-alexey:as/oak_infer [GAPI] Support basic inference in OAK backend * Combined commit which enables basic inference and other extra capabilities of OAK backend * Remove unnecessary target options from the cmakelist --- modules/gapi/CMakeLists.txt | 12 +- .../gapi/include/opencv2/gapi/oak/infer.hpp | 66 ++ modules/gapi/include/opencv2/gapi/oak/oak.hpp | 29 +- modules/gapi/samples/oak_basic_infer.cpp | 122 ++++ modules/gapi/samples/oak_copy.cpp | 48 ++ .../gapi/samples/oak_rgb_camera_encoding.cpp | 10 - .../samples/oak_small_hetero_pipeline.cpp | 11 - modules/gapi/src/backends/oak/goak.cpp | 15 +- ...a_adapter.cpp => goak_memory_adapters.cpp} | 32 +- modules/gapi/src/backends/oak/goakbackend.cpp | 660 ++++++++++++++---- ...ia_adapter.hpp => oak_memory_adapters.hpp} | 22 + modules/gapi/src/compiler/gmodel.hpp | 2 +- 12 files changed, 870 insertions(+), 159 deletions(-) create mode 100644 modules/gapi/include/opencv2/gapi/oak/infer.hpp create mode 100644 modules/gapi/samples/oak_basic_infer.cpp create mode 100644 modules/gapi/samples/oak_copy.cpp rename modules/gapi/src/backends/oak/{goak_media_adapter.cpp => goak_memory_adapters.cpp} (54%) rename modules/gapi/src/backends/oak/{oak_media_adapter.hpp => oak_memory_adapters.hpp} (56%) diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 29036c4e26..579bcfc1be 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -138,7 +138,7 @@ set(gapi_srcs # OAK Backend (optional) src/backends/oak/goak.cpp src/backends/oak/goakbackend.cpp - src/backends/oak/goak_media_adapter.cpp + src/backends/oak/goak_memory_adapters.cpp # OCL Backend (currently built-in) src/backends/ocl/goclbackend.cpp @@ -375,13 +375,3 @@ if(HAVE_GAPI_ONEVPL) endif() endif() endif() - -if(HAVE_OAK) - # FIXME: consider better solution - if(TARGET example_gapi_oak_rgb_camera_encoding) - ocv_target_compile_definitions(example_gapi_oak_rgb_camera_encoding PRIVATE -DHAVE_OAK) - endif() - if(TARGET example_gapi_oak_small_hetero_pipeline) - ocv_target_compile_definitions(example_gapi_oak_small_hetero_pipeline PRIVATE -DHAVE_OAK) - endif() -endif() diff --git a/modules/gapi/include/opencv2/gapi/oak/infer.hpp b/modules/gapi/include/opencv2/gapi/oak/infer.hpp new file mode 100644 index 0000000000..4a1b9f6db6 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/oak/infer.hpp @@ -0,0 +1,66 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2022 Intel Corporation + +#ifndef OPENCV_GAPI_OAK_INFER_HPP +#define OPENCV_GAPI_OAK_INFER_HPP + +#include +#include +#include +#include + +#include +#include + +#include // GAPI_EXPORTS +#include // GKernelPackage + +namespace cv { +namespace gapi { +namespace oak { + +namespace detail { +/** +* @brief This structure contains description of inference parameters +* which is specific to OAK models. +*/ +struct ParamDesc { + std::string blob_file; +}; +} // namespace detail + +/** + * Contains description of inference parameters and kit of functions that + * fill this parameters. + */ +template class Params { +public: + /** @brief Class constructor. + + Constructs Params based on model information and sets default values for other + inference description parameters. + + @param model Path to model (.blob file) + */ + explicit Params(const std::string &model) { + desc.blob_file = model; + }; + + // BEGIN(G-API's network parametrization API) + GBackend backend() const { return cv::gapi::oak::backend(); } + std::string tag() const { return Net::tag(); } + cv::util::any params() const { return { desc }; } + // END(G-API's network parametrization API) + +protected: + detail::ParamDesc desc; +}; + +} // namespace oak +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_OAK_INFER_HPP diff --git a/modules/gapi/include/opencv2/gapi/oak/oak.hpp b/modules/gapi/include/opencv2/gapi/oak/oak.hpp index 05fb09946f..ba1ea46cc7 100644 --- a/modules/gapi/include/opencv2/gapi/oak/oak.hpp +++ b/modules/gapi/include/opencv2/gapi/oak/oak.hpp @@ -89,28 +89,55 @@ G_API_OP(GSobelXY, , "org.opencv } }; +G_API_OP(GCopy, , "org.opencv.oak.copy") { + static GFrameDesc outMeta(const GFrameDesc& in) { + return in; + } +}; + +// FIXME: add documentation on operations below + GAPI_EXPORTS GArray encode(const GFrame& in, const EncoderConfig&); GAPI_EXPORTS GFrame sobelXY(const GFrame& in, const cv::Mat& hk, const cv::Mat& vk); +GAPI_EXPORTS GFrame copy(const GFrame& in); + // OAK backend & kernels //////////////////////////////////////////////////////// GAPI_EXPORTS cv::gapi::GBackend backend(); GAPI_EXPORTS cv::gapi::GKernelPackage kernels(); // Camera object /////////////////////////////////////////////////////////////// -struct GAPI_EXPORTS ColorCameraParams {}; +struct GAPI_EXPORTS ColorCameraParams { + /** + * Format of the frame one gets from the camera + */ + bool interleaved = false; + + // FIXME: extend + enum class BoardSocket: int { RGB, BGR }; + + BoardSocket board_socket = BoardSocket::RGB; + + // FIXME: extend + enum class Resolution: int { THE_1080_P }; + + Resolution resolution = Resolution::THE_1080_P; +}; class GAPI_EXPORTS ColorCamera: public cv::gapi::wip::IStreamSource { cv::MediaFrame m_dummy; + ColorCameraParams m_params; virtual bool pull(cv::gapi::wip::Data &data) override; virtual GMetaArg descr_of() const override; public: ColorCamera(); + explicit ColorCamera(const ColorCameraParams& params); }; } // namespace oak diff --git a/modules/gapi/samples/oak_basic_infer.cpp b/modules/gapi/samples/oak_basic_infer.cpp new file mode 100644 index 0000000000..ee234e3806 --- /dev/null +++ b/modules/gapi/samples/oak_basic_infer.cpp @@ -0,0 +1,122 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +const std::string keys = + "{ h help | | Print this help message }" + "{ detector | | Path to compiled .blob face detector model }" + "{ duration | 100 | Number of frames to pull from camera and run inference on }"; + +namespace custom { + +G_API_NET(FaceDetector, , "sample.custom.face-detector"); + +using GDetections = cv::GArray; +using GSize = cv::GOpaque; +using GPrims = cv::GArray; + +G_API_OP(BBoxes, , "sample.custom.b-boxes") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc &) { + return cv::empty_array_desc(); + } +}; + +GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { + // This kernel converts the rectangles into G-API's + // rendering primitives + static void run(const std::vector &in_face_rcs, + std::vector &out_prims) { + out_prims.clear(); + const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) { + return cv::gapi::wip::draw::Rect(rc, clr, 2); + }; + for (auto &&rc : in_face_rcs) { + out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green + } + } +}; + +} // namespace custom + +int main(int argc, char *argv[]) { + cv::CommandLineParser cmd(argc, argv, keys); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + + const auto det_name = cmd.get("detector"); + const auto duration = cmd.get("duration"); + + if (det_name.empty()) { + std::cerr << "FATAL: path to detection model is not provided for the sample." + << "Please specify it with --detector options." + << std::endl; + return 1; + } + + // Prepare G-API kernels and networks packages: + auto detector = cv::gapi::oak::Params(det_name); + auto networks = cv::gapi::networks(detector); + + auto kernels = cv::gapi::combine( + cv::gapi::kernels(), + cv::gapi::oak::kernels()); + + auto args = cv::compile_args(kernels, networks); + + // Initialize graph structure + cv::GFrame in; + cv::GFrame copy = cv::gapi::oak::copy(in); // NV12 transfered to host + passthrough copy for infer + cv::GOpaque sz = cv::gapi::streaming::size(copy); + + // infer is not affected by the actual copy here + cv::GMat blob = cv::gapi::infer(copy); + // FIXME: OAK infer detects faces slightly out of frame bounds + cv::GArray rcs = cv::gapi::parseSSD(blob, sz, 0.5f, true, false); + auto rendered = cv::gapi::wip::draw::renderFrame(copy, custom::BBoxes::on(rcs)); + // on-the-fly conversion NV12->BGR + cv::GMat out = cv::gapi::streaming::BGR(rendered); + + auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out, rcs)) + .compileStreaming(std::move(args)); + + // Graph execution + pipeline.setSource(cv::gapi::wip::make_src()); + pipeline.start(); + + cv::Mat out_mat; + std::vector out_dets; + int frames = 0; + while (pipeline.pull(cv::gout(out_mat, out_dets))) { + std::string name = "oak_infer_frame_" + std::to_string(frames) + ".png"; + + cv::imwrite(name, out_mat); + + if (!out_dets.empty()) { + std::cout << "Got " << out_dets.size() << " detections on frame #" << frames << std::endl; + } + + ++frames; + if (frames == duration) { + pipeline.stop(); + break; + } + } + std::cout << "Pipeline finished. Processed " << frames << " frames" << std::endl; + return 0; +} diff --git a/modules/gapi/samples/oak_copy.cpp b/modules/gapi/samples/oak_copy.cpp new file mode 100644 index 0000000000..3c4d455f52 --- /dev/null +++ b/modules/gapi/samples/oak_copy.cpp @@ -0,0 +1,48 @@ +#include +#include +#include +#include +#include + +#include +#include // BGR accessor + +#include // CommandLineParser + +const std::string keys = + "{ h help | | Print this help message }" + "{ output | output.png | Path to the output file }"; + +int main(int argc, char *argv[]) { + cv::CommandLineParser cmd(argc, argv, keys); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + + const std::string output_name = cmd.get("output"); + + cv::GFrame in; + // Actually transfers data to host + cv::GFrame copy = cv::gapi::oak::copy(in); + // Default camera works only with nv12 format + cv::GMat out = cv::gapi::streaming::Y(copy); + + auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{}, + cv::gapi::oak::kernels()); + + auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)).compileStreaming(std::move(args)); + + // Graph execution ///////////////////////////////////////////////////////// + cv::Mat out_mat(1920, 1080, CV_8UC1); + + pipeline.setSource(cv::gapi::wip::make_src()); + pipeline.start(); + + // pull 1 frame + pipeline.pull(cv::gout(out_mat)); + + cv::imwrite(output_name, out_mat); + + std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl; +} diff --git a/modules/gapi/samples/oak_rgb_camera_encoding.cpp b/modules/gapi/samples/oak_rgb_camera_encoding.cpp index ac6b5cc5f0..ee07ef09e4 100644 --- a/modules/gapi/samples/oak_rgb_camera_encoding.cpp +++ b/modules/gapi/samples/oak_rgb_camera_encoding.cpp @@ -13,8 +13,6 @@ const std::string keys = "{ h help | | Print this help message }" "{ output | output.h265 | Path to the output .h265 video file }"; -#ifdef HAVE_OAK - int main(int argc, char *argv[]) { cv::CommandLineParser cmd(argc, argv, keys); if (cmd.has("help")) { @@ -60,11 +58,3 @@ int main(int argc, char *argv[]) { std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl; } -#else // HAVE_OAK - -int main() { - GAPI_Assert(false && "Built without OAK support"); - return -1; -} - -#endif // HAVE_OAK diff --git a/modules/gapi/samples/oak_small_hetero_pipeline.cpp b/modules/gapi/samples/oak_small_hetero_pipeline.cpp index dadb9d0f3b..5df3ba5da1 100644 --- a/modules/gapi/samples/oak_small_hetero_pipeline.cpp +++ b/modules/gapi/samples/oak_small_hetero_pipeline.cpp @@ -13,8 +13,6 @@ const std::string keys = "{ h help | | Print this help message }" "{ output | output.png | Path to the output file }"; -#ifdef HAVE_OAK - int main(int argc, char *argv[]) { cv::CommandLineParser cmd(argc, argv, keys); if (cmd.has("help")) { @@ -58,12 +56,3 @@ int main(int argc, char *argv[]) { std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl; } - -#else // HAVE_OAK - -int main() { - GAPI_Assert(false && "Built without OAK support"); - return -1; -} - -#endif // HAVE_OAK diff --git a/modules/gapi/src/backends/oak/goak.cpp b/modules/gapi/src/backends/oak/goak.cpp index 6d9044aefa..022f8f596f 100644 --- a/modules/gapi/src/backends/oak/goak.cpp +++ b/modules/gapi/src/backends/oak/goak.cpp @@ -7,7 +7,7 @@ #include #include -#include "oak_media_adapter.hpp" +#include "oak_memory_adapters.hpp" #include #include @@ -24,6 +24,10 @@ GFrame sobelXY(const GFrame& in, const cv::Mat& hk, const cv::Mat& vk) { return GSobelXY::on(in, hk, vk); } +GFrame copy(const GFrame& in) { + return GCopy::on(in); +} + // This is a dummy oak::ColorCamera class that just makes our pipelining // machinery work. The real data comes from the physical camera which // is handled by DepthAI library. @@ -31,6 +35,11 @@ ColorCamera::ColorCamera() : m_dummy(cv::MediaFrame::Create()) { } +ColorCamera::ColorCamera(const ColorCameraParams& params) + : m_dummy(cv::MediaFrame::Create()), + m_params(params) { +} + bool ColorCamera::pull(cv::gapi::wip::Data &data) { // FIXME: Avoid passing this formal frame to the pipeline std::this_thread::sleep_for(std::chrono::milliseconds(10)); @@ -39,7 +48,9 @@ bool ColorCamera::pull(cv::gapi::wip::Data &data) { } cv::GMetaArg ColorCamera::descr_of() const { - return cv::GMetaArg{cv::descr_of(m_dummy)}; + // FIXME: support other resolutions + GAPI_Assert(m_params.resolution == ColorCameraParams::Resolution::THE_1080_P); + return cv::GMetaArg{cv::GFrameDesc{cv::MediaFormat::NV12, cv::Size{1920, 1080}}}; } } // namespace oak diff --git a/modules/gapi/src/backends/oak/goak_media_adapter.cpp b/modules/gapi/src/backends/oak/goak_memory_adapters.cpp similarity index 54% rename from modules/gapi/src/backends/oak/goak_media_adapter.cpp rename to modules/gapi/src/backends/oak/goak_memory_adapters.cpp index c8e6bbb59b..5805f7933f 100644 --- a/modules/gapi/src/backends/oak/goak_media_adapter.cpp +++ b/modules/gapi/src/backends/oak/goak_memory_adapters.cpp @@ -4,17 +4,15 @@ // // Copyright (C) 2021 Intel Corporation -#include "oak_media_adapter.hpp" +#include "oak_memory_adapters.hpp" namespace cv { namespace gapi { namespace oak { -OAKMediaAdapter::OAKMediaAdapter(cv::Size sz, cv::MediaFormat fmt, std::vector&& buffer) { +OAKMediaAdapter::OAKMediaAdapter(cv::Size sz, cv::MediaFormat fmt, std::vector&& buffer) +: m_sz(sz), m_fmt(fmt), m_buffer(buffer) { GAPI_Assert(fmt == cv::MediaFormat::NV12 && "OAKMediaAdapter only supports NV12 format for now"); - m_sz = sz; - m_fmt = fmt; - m_buffer = buffer; } MediaFrame::View OAKMediaAdapter::OAKMediaAdapter::access(MediaFrame::Access) { @@ -27,6 +25,30 @@ MediaFrame::View OAKMediaAdapter::OAKMediaAdapter::access(MediaFrame::Access) { cv::GFrameDesc OAKMediaAdapter::OAKMediaAdapter::meta() const { return {m_fmt, m_sz}; } +OAKRMatAdapter::OAKRMatAdapter(const cv::Size& size, + int precision, + std::vector&& buffer) + : m_size(size), m_precision(precision), m_buffer(buffer) { + GAPI_Assert(m_precision == CV_16F); + + std::vector wrapped_dims{1, 1, m_size.width, m_size.height}; + + // FIXME: check layout and add strides + m_desc = cv::GMatDesc(m_precision, wrapped_dims); + m_mat = cv::Mat(static_cast(wrapped_dims.size()), + wrapped_dims.data(), + CV_16FC1, // FIXME: cover other precisions + m_buffer.data()); +} + +cv::GMatDesc OAKRMatAdapter::desc() const { + return m_desc; +} + +cv::RMat::View OAKRMatAdapter::access(cv::RMat::Access) { + return cv::RMat::View{m_desc, m_mat.data}; +} + } // namespace oak } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/oak/goakbackend.cpp b/modules/gapi/src/backends/oak/goakbackend.cpp index dc0daaead3..83a2ca88d2 100644 --- a/modules/gapi/src/backends/oak/goakbackend.cpp +++ b/modules/gapi/src/backends/oak/goakbackend.cpp @@ -2,9 +2,10 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2021 Intel Corporation +// Copyright (C) 2021-2022 Intel Corporation #include // GKernelPackage +#include // kernels() #ifdef HAVE_OAK @@ -18,21 +19,24 @@ #include #include +#include // GInferBase #include // streaming::meta_tag #include "depthai/depthai.hpp" -#include -#include "oak_media_adapter.hpp" +#include "oak_memory_adapters.hpp" + +#include // infer params namespace cv { namespace gimpl { // Forward declaration class GOAKContext; -struct OAKNodeInfo; +class OAKKernelParams; class GOAKExecutable final: public GIslandExecutable { friend class GOAKContext; + friend class OAKKernelParams; virtual void run(std::vector&&, std::vector&&) override { GAPI_Assert(false && "Not implemented"); @@ -41,7 +45,8 @@ class GOAKExecutable final: public GIslandExecutable { virtual void run(GIslandExecutable::IInput &in, GIslandExecutable::IOutput &out) override; - void LinkToParents(ade::NodeHandle handle); + void linkToParent(ade::NodeHandle handle); + void linkCopy(ade::NodeHandle handle); class ExtractTypeHelper : protected dai::Node { public: @@ -61,6 +66,7 @@ class GOAKExecutable final: public GIslandExecutable { std::shared_ptr xlink_output; std::shared_ptr out_queue; std::string out_queue_name; + size_t gapi_out_data_index; }; cv::GArg packInArg(const GArg &arg, std::vector& oak_ins); @@ -79,11 +85,16 @@ class GOAKExecutable final: public GIslandExecutable { cv::Size m_camera_size; // Backend outputs - std::vector m_out_queues; + std::unordered_map> m_out_queues; // Backend inputs std::vector> m_in_queues; + std::unordered_set> m_passthrough_copy_nodes; + // Note: dai::Pipeline should be the only one for the whole pipeline, // so there is no way to insert any non-OAK node in graph between other OAK nodes. // The only heterogeneous case possible is if we insert other backends after or before @@ -91,6 +102,14 @@ class GOAKExecutable final: public GIslandExecutable { std::unique_ptr m_device; std::unique_ptr m_pipeline; + // Camera config + cv::gapi::oak::ColorCameraParams m_ccp; + + // Infer info + std::unordered_map> m_oak_infer_info; + public: GOAKExecutable(const ade::Graph& g, const cv::GCompileArgs& args, @@ -122,6 +141,12 @@ public: std::vector& args, std::vector& results); + GOAKContext(const std::unique_ptr& pipeline, + const cv::Size& camera_size, + const cv::gapi::oak::detail::ParamDesc& infer_info, + std::vector& args, + std::vector& results); + // Generic accessor API template T& inArg(int input) { return m_args.at(input).get(); } @@ -130,12 +155,14 @@ public: InputPtr& in(int input); OutputPtr& out(int output); - const std::unique_ptr& pipeline(); + const std::unique_ptr& pipeline() const; const cv::Size& camera_size() const; + const cv::gapi::oak::detail::ParamDesc& ii() const; private: const std::unique_ptr& m_pipeline; - const cv::Size& m_camera_size; + const cv::Size m_camera_size; + const cv::gapi::oak::detail::ParamDesc m_infer_info; std::vector& m_args; std::vector& m_outputs; }; @@ -144,9 +171,18 @@ GOAKContext::GOAKContext(const std::unique_ptr& pipeline, const cv::Size& camera_size, std::vector& args, std::vector& results) - : m_pipeline(pipeline), m_camera_size(camera_size), m_args(args), m_outputs(results) {} + : m_pipeline(pipeline), m_camera_size(camera_size), + m_args(args), m_outputs(results) {} + +GOAKContext::GOAKContext(const std::unique_ptr& pipeline, + const cv::Size& camera_size, + const cv::gapi::oak::detail::ParamDesc& infer_info, + std::vector& args, + std::vector& results) + : m_pipeline(pipeline), m_camera_size(camera_size), + m_infer_info(infer_info), m_args(args), m_outputs(results) {} -const std::unique_ptr& GOAKContext::pipeline() { +const std::unique_ptr& GOAKContext::pipeline() const { return m_pipeline; } @@ -154,6 +190,10 @@ const cv::Size& GOAKContext::camera_size() const { return m_camera_size; } +const cv::gapi::oak::detail::ParamDesc& GOAKContext::ii() const { + return m_infer_info; +} + GOAKContext::InputPtr& GOAKContext::in(int input) { return inArg>(input).get(); } @@ -162,6 +202,14 @@ GOAKContext::OutputPtr& GOAKContext::out(int output) { return m_outputs.at(output); } +class OAKKernelParams { +public: + const std::unique_ptr& pipeline; + const cv::Size& camera_size; + const cv::gapi::oak::detail::ParamDesc& infer_info; + std::vector>& in_queues; +}; + namespace detail { template struct get_in; template<> struct get_in { @@ -179,13 +227,10 @@ template<> struct get_out { template struct get_out> { static GOAKContext::OutputPtr& get(GOAKContext &ctx, int idx) { return ctx.out(idx); } }; -// FIXME: add support of other types - -struct OAKKernelParams { - const std::unique_ptr& pipeline; - const cv::Size& camera_size; - std::vector>& m_in_queues; +template<> struct get_out { + static GOAKContext::OutputPtr& get(GOAKContext &ctx, int idx) { return ctx.out(idx); } }; +// FIXME: add support of other types template struct OAKCallHelper; @@ -200,6 +245,7 @@ struct OAKCallHelper, std::tuple > { , cv::detail::Seq) { return Impl::put(OAKKernelParams{ctx.pipeline(), ctx.camera_size(), + ctx.ii(), in_queues_params}, get_in::get(ctx, IIs)..., get_out::get(ctx, OIs)...); @@ -229,23 +275,84 @@ struct OAKComponent static const char *name() { return "OAK Component"; } GOAKKernel k; }; - -}} // namespace gimpl // namespace cv +} // namespace gimpl +} // namespace cv using OAKGraph = ade::TypedGraph - < cv::gimpl::OAKComponent - // FIXME: extend + < cv::gimpl::Protocol + , cv::gimpl::Op + , cv::gimpl::NetworkParams + , cv::gimpl::CustomMetaFunction + // OAK specific + , cv::gimpl::OAKComponent >; using ConstOAKGraph = ade::ConstTypedGraph - < cv::gimpl::OAKComponent - // FIXME: extend + < cv::gimpl::Protocol + , cv::gimpl::Op + , cv::gimpl::NetworkParams + , cv::gimpl::CustomMetaFunction + // OAK specific + , cv::gimpl::OAKComponent >; -// This function links dai operation nodes - parent's output to child's input. +namespace +{ +std::pair +parseDaiInferMeta(const cv::gapi::oak::detail::ParamDesc& pd) { + dai::OpenVINO::Blob blob(pd.blob_file); + + GAPI_Assert(blob.networkInputs.size() == 1); + GAPI_Assert(blob.networkOutputs.size() == 1); + + return {blob.networkInputs.begin()->second, + blob.networkOutputs.begin()->second}; +} + +std::string +getDaiInferOutLayerName(const cv::gapi::oak::detail::ParamDesc& pd) { + dai::OpenVINO::Blob blob(pd.blob_file); + + GAPI_Assert(blob.networkInputs.size() == 1); + GAPI_Assert(blob.networkOutputs.size() == 1); + + return blob.networkOutputs.begin()->first; +} +} // anonymous namespace + +// Custom meta function for OAK backend for infer +static cv::GMetaArgs customOutMeta(const ade::Graph &gr, + const ade::NodeHandle &nh, + const cv::GMetaArgs &/*in_metas*/, + const cv::GArgs &/*in_args*/) { + cv::GMetaArgs result; + const auto &np = ConstOAKGraph(gr).metadata(nh).get(); + const auto &pd = cv::util::any_cast(np.opaque); + + // FIXME: Infer kernel and backend does rather the same + auto in_out_tensor_info = parseDaiInferMeta(pd); + + GAPI_Assert(in_out_tensor_info.second.dataType == + dai::TensorInfo::DataType::FP16); + + // FIXME: add proper layout converter here + GAPI_Assert(in_out_tensor_info.second.order == + dai::TensorInfo::StorageOrder::NCHW); + + // FIXME: DAI returns vector, remove workaround + std::vector wrapped_dims; + for (const auto& d : in_out_tensor_info.second.dims) { + wrapped_dims.push_back(d); + } + result = {cv::GMetaArg{cv::GMatDesc(CV_16F, 1, cv::Size(wrapped_dims[1], wrapped_dims[0]), false)}}; + + return result; +} + +// This function links DAI operation nodes - parent's output to child's input. // It utilizes G-API graph to search for operation's node it's previous operation in graph -// when links them in dai graph. -void cv::gimpl::GOAKExecutable::LinkToParents(ade::NodeHandle handle) +// when links them in DAI graph. +void cv::gimpl::GOAKExecutable::linkToParent(ade::NodeHandle handle) { ade::NodeHandle parent; for (const auto& data_nh : handle.get()->inNodes()) { @@ -253,6 +360,13 @@ void cv::gimpl::GOAKExecutable::LinkToParents(ade::NodeHandle handle) GAPI_Assert(data_nh.get()->inNodes().size() == 1); parent = data_nh.get()->inNodes().front(); + // Don't link if parent is copy - the case is handled differently + // in linkCopy + const auto& op = m_gm.metadata(parent).get(); + if (op.k.name == "org.opencv.oak.copy") { + continue; + } + // Assuming that OAK nodes are aligned for linking. // FIXME: potential rework might be needed then // counterexample is found. @@ -269,6 +383,89 @@ void cv::gimpl::GOAKExecutable::LinkToParents(ade::NodeHandle handle) } } +// This function links DAI operations for Copy OP in G-API graph +void cv::gimpl::GOAKExecutable::linkCopy(ade::NodeHandle handle) { + // 1. Check that there are no back-to-back Copy OPs in graph + auto copy_out = handle.get()->outNodes(); + GAPI_Assert(copy_out.size() == 1); + for (const auto& copy_next_op : copy_out.front().get()->outNodes()) { + const auto& op = m_gm.metadata(copy_next_op).get(); + if (op.k.name == "org.opencv.oak.copy") { + GAPI_Assert(false && "Back-to-back Copy operations are not supported in graph"); + } + } + + // 2. Link passthrough case + if (m_passthrough_copy_nodes.find(handle) != m_passthrough_copy_nodes.end()) { + ExtractTypeHelper::OutputPtr parent; + bool parent_is_camera = false; + // Copy has only 1 input data + GAPI_Assert(handle.get()->inNodes().size() == 1); + auto in_ops = handle.get()->inNodes().front().get()->inNodes(); + if (in_ops.size() == 0) { + // No parent nodes - parent = camera + parent = &m_camera_input->video; + parent_is_camera = true; + } else { + // Data has only 1 input + GAPI_Assert(in_ops.size() == 1); + auto node = m_oak_nodes.at(in_ops.front()); + // Should only have 1 output + GAPI_Assert(node.outputs.size() == 1); + parent = node.outputs[0]; + } + + // Now link DAI parent output to Copy's child's inputs ignoring the Copy operation + // FIXME: simplify this loop + auto copy_out_data = handle.get()->outNodes(); + // Copy has only 1 output + GAPI_Assert(copy_out_data.size() == 1); + for (const auto& copy_next_op : copy_out_data.front().get()->outNodes()) { + if (m_oak_nodes.find(copy_next_op) != m_oak_nodes.end()) { + // FIXME: consider a better approach + if (parent_is_camera) { + if (m_oak_infer_info.find(copy_next_op) != m_oak_infer_info.end()) { + parent = &m_camera_input->preview; + } else { + parent = &m_camera_input->video; + } + } + // Found next Copy OP which needs to be linked to Copy's parent + GAPI_Assert(m_oak_nodes.at(copy_next_op).inputs.size() == 1 && + "Internal OAK nodes are not aligned for linking (Copy operation)"); + parent->link(*(m_oak_nodes.at(copy_next_op).inputs.front())); + } + } + } + + // 3. Link output Copy case + if (m_out_queues.find(handle) != m_out_queues.end()) { + // DAI XLinkOutput node + auto xout = m_out_queues[handle].xlink_output->input; + + // Find parent node + // FIXME: copypasted from case 2 above + ExtractTypeHelper::OutputPtr parent; + // Copy has only 1 input data + GAPI_Assert(handle.get()->inNodes().size() == 1); + auto in_ops = handle.get()->inNodes().front().get()->inNodes(); + if (in_ops.size() == 0) { + // No parent nodes - parent = camera + parent = &m_camera_input->video; + } else { + // Data has only 1 input + GAPI_Assert(in_ops.size() == 1); + auto node = m_oak_nodes.at(in_ops.front()); + // Should only have 1 output + GAPI_Assert(node.outputs.size() == 1); + parent = node.outputs[0]; + } + + // Link parent to xout + parent->link(xout); + } +} + cv::GArg cv::gimpl::GOAKExecutable::packInArg(const GArg &arg, std::vector& oak_ins) { @@ -298,9 +495,8 @@ void cv::gimpl::GOAKExecutable::packOutArg(const RcDesc &rc, std::vector& oak_outs) { switch (rc.shape) { case GShape::GFRAME: - oak_outs.push_back(nullptr); - break; case GShape::GARRAY: + case GShape::GMAT: oak_outs.push_back(nullptr); break; default: @@ -309,6 +505,33 @@ void cv::gimpl::GOAKExecutable::packOutArg(const RcDesc &rc, } } +namespace { +static dai::CameraBoardSocket extractCameraBoardSocket(cv::gapi::oak::ColorCameraParams ccp) { + switch (ccp.board_socket) { + case cv::gapi::oak::ColorCameraParams::BoardSocket::RGB: + return dai::CameraBoardSocket::RGB; + // FIXME: extend + default: + // basically unreachable + GAPI_Assert("Unsupported camera board socket"); + return {}; + } +} + +static dai::ColorCameraProperties::SensorResolution +extractCameraResolution(cv::gapi::oak::ColorCameraParams ccp) { + switch (ccp.resolution) { + case cv::gapi::oak::ColorCameraParams::Resolution::THE_1080_P: + return dai::ColorCameraProperties::SensorResolution::THE_1080_P; + // FIXME: extend + default: + // basically unreachable + GAPI_Assert("Unsupported camera board socket"); + return {}; + } +} +} // anonymous namespace + cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g, const cv::GCompileArgs &args, const std::vector& nodes, @@ -344,16 +567,52 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g, } } + m_ccp = cv::gimpl::getCompileArg(args) + .value_or(cv::gapi::oak::ColorCameraParams{}); + // FIXME: change the hard-coded behavior (XLinkIn path) auto camRgb = m_pipeline->create(); // FIXME: extract camera compile arguments here and properly convert them for dai - camRgb->setBoardSocket(dai::CameraBoardSocket::RGB); - camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P); + camRgb->setBoardSocket(extractCameraBoardSocket(m_ccp)); + camRgb->setResolution(extractCameraResolution(m_ccp)); + camRgb->setInterleaved(m_ccp.interleaved); + + // Extract infer params + for (const auto& nh : nodes) { + if (m_gm.metadata(nh).get().t == NodeType::OP) { + if (ConstOAKGraph(m_g).metadata(nh).contains()) { + const auto &np = ConstOAKGraph(m_g).metadata(nh).get(); + const auto &pp = cv::util::any_cast(np.opaque); + m_oak_infer_info[nh] = pp; + break; + } + } + } + + // FIXME: handle multiple infers + if (!m_oak_infer_info.empty()) { + GAPI_Assert(m_oak_infer_info.size() == 1); + // FIXME: move to infer node? + auto in_out_tensor_info = parseDaiInferMeta(m_oak_infer_info.begin()->second); + + if (in_out_tensor_info.first.dataType == + dai::TensorInfo::DataType::FP16 || + in_out_tensor_info.first.dataType == + dai::TensorInfo::DataType::FP32) { + camRgb->setFp16(true); + } else { + camRgb->setFp16(false); + } + + // FIXME: add proper layout converter here + GAPI_Assert(in_out_tensor_info.first.order == + dai::TensorInfo::StorageOrder::NCHW); + camRgb->setPreviewSize(in_out_tensor_info.first.dims[0], in_out_tensor_info.first.dims[1]); + } - // Set camera output. Fixme: consider working with other camera outputs m_camera_input = camRgb; // FIXME: change when other camera censors are introduced - std::tuple video_size = camRgb->getVideoSize(); + std::tuple video_size = m_camera_input->getVideoSize(); m_camera_size = cv::Size{std::get<0>(video_size), std::get<1>(video_size)}; // Prepare XLinkOut nodes for each output object in graph @@ -361,7 +620,23 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g, auto xout = m_pipeline->create(); std::string xout_name = "xout" + std::to_string(i); xout->setStreamName(xout_name); - m_out_queues.push_back({xout, nullptr, xout_name}); + + // Find parent OP's nh + ade::NodeHandle parent_op_nh; + for (const auto& nh : nodes) { + for (const auto& outdata : nh.get()->outNodes()) { + if (m_gm.metadata(outdata).get().t == NodeType::DATA) { + auto rc = m_gm.metadata(outdata).get().rc; + auto shape = m_gm.metadata(outdata).get().shape; + // Match outs_data with the actual operation + if (rc == outs_data[i].rc && shape == outs_data[i].shape) { + parent_op_nh = nh; + } + } + } + } + + m_out_queues[parent_op_nh] = {xout, nullptr, xout_name, i}; } // Create OAK node for each node in this backend @@ -375,33 +650,66 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g, m_oak_nodes.at(nh).inputs.reserve(op.args.size()); m_oak_nodes.at(nh).outputs.reserve(op.outs.size()); + // Copy operation in graph can fall into 3 cases: + // 1) Copy is an output of the island - + // in that case we link it to XLinkOut node from m_out_queues + // 2) Copy is between other two operations in the same OAK island - + // in that case we link its parent operation (could be camera) to + // the child one (those copy operations are placed in m_passthrough_copy_nodes) + // 3) Copy can fall into cases 1) and 2) at the same time + + // Prepare passthrough Copy operations + if (op.k.name == "org.opencv.oak.copy") { + // Copy has only 1 output + auto copy_out = nh.get()->outNodes(); + GAPI_Assert(copy_out.size() == 1); + for (const auto& copy_next_op : copy_out.front().get()->outNodes()) { + // Check that copy is a passthrough OP + if (std::find(nodes.begin(), nodes.end(), copy_next_op) != nodes.end()) { + m_passthrough_copy_nodes.insert(nh); + break; + } + } + } + std::vector in_ctx_args; in_ctx_args.reserve(op.args.size()); for (auto &op_arg : op.args) in_ctx_args.push_back(packInArg(op_arg, - m_oak_nodes.at(nh).inputs)); + m_oak_nodes.at(nh).inputs)); for (auto &&op_out : op.outs) packOutArg(op_out, m_oak_nodes.at(nh).outputs); GAPI_Assert(!m_oak_nodes.at(nh).inputs.empty()); GAPI_Assert(!m_oak_nodes.at(nh).outputs.empty()); - GOAKContext ctx(m_pipeline, m_camera_size, in_ctx_args, m_oak_nodes.at(nh).outputs); - m_oak_nodes.at(nh).node = u.k.m_put_f(ctx, m_in_queues); - GAPI_Assert(m_oak_nodes.at(nh).node != nullptr); + if (ConstOAKGraph(m_g).metadata(nh).contains()) { + GOAKContext ctx(m_pipeline, m_camera_size, m_oak_infer_info[nh], + in_ctx_args, m_oak_nodes.at(nh).outputs); + m_oak_nodes.at(nh).node = u.k.m_put_f(ctx, m_in_queues); + } else { + GOAKContext ctx(m_pipeline, m_camera_size, + in_ctx_args, m_oak_nodes.at(nh).outputs); + m_oak_nodes.at(nh).node = u.k.m_put_f(ctx, m_in_queues); + } // Check that all inputs and outputs are properly filled after constructing kernels // to then link it together // FIXME: add more logging - const auto& node = m_oak_nodes.at(nh); - if (std::any_of(node.inputs.cbegin(), node.inputs.cend(), - [](ExtractTypeHelper::InputPtr ptr) { - return ptr == nullptr; - })) { - GAPI_Assert(false && "DAI input are not set"); - } - if (std::any_of(node.outputs.cbegin(), node.outputs.cend(), - [](ExtractTypeHelper::OutputPtr ptr) { - return ptr == nullptr; - })) { - GAPI_Assert(false && "DAI outputs are not set"); + const auto& node_info = m_oak_nodes.at(nh); + // Copy operations don't set their inputs/outputs properly + if (op.k.name != "org.opencv.oak.copy") { + GAPI_Assert(node_info.node != nullptr); + if (std::any_of(node_info.inputs.cbegin(), node_info.inputs.cend(), + [](ExtractTypeHelper::InputPtr ptr) { + return ptr == nullptr; + })) { + GAPI_Assert(false && "DAI input are not set"); + } + + if (std::any_of(node_info.outputs.cbegin(), node_info.outputs.cend(), + [](ExtractTypeHelper::OutputPtr ptr) { + return ptr == nullptr; + })) { + GAPI_Assert(false && "DAI outputs are not set"); + } } } } @@ -413,15 +721,26 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g, ade::HandleHasher> out_nodes; std::unordered_set> inter_nodes; + std::unordered_set> copy_nodes; // TODO: optimize this loop for (const auto& node : m_oak_nodes) { auto nh = node.first; + // Check if it's a Copy OP - will be handled differently when linking + GAPI_Assert(m_gm.metadata(nh).get().t == NodeType::OP); + const auto& op = m_gm.metadata(nh).get(); + if (op.k.name == "org.opencv.oak.copy") { + copy_nodes.insert(nh); + continue; + } + // Fill input op nodes for (const auto& d : ins_data) { for (const auto& indata : nh.get()->inNodes()) { auto rc = m_gm.metadata(indata).get().rc; - if (rc == d.rc) { + auto shape = m_gm.metadata(indata).get().shape; + if (rc == d.rc && shape == d.shape) { in_nodes.insert(nh); } } @@ -430,7 +749,8 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g, for (const auto& d : outs_data) { for (const auto& outdata : nh.get()->outNodes()) { auto rc = m_gm.metadata(outdata).get().rc; - if (rc == d.rc) { + auto shape = m_gm.metadata(outdata).get().shape; + if (rc == d.rc && shape == d.shape) { out_nodes.insert(nh); } } @@ -446,42 +766,47 @@ cv::gimpl::GOAKExecutable::GOAKExecutable(const ade::Graph& g, // 1. Link input nodes to camera for (const auto& nh : in_nodes) { GAPI_Assert(m_oak_nodes.at(nh).inputs.size() == 1); - // FIXME: covert other camera outputs - m_camera_input->video.link(*(m_oak_nodes.at(nh).inputs[0])); + // FIXME: cover other camera outputs + // Link preview to infer, video to all other nodes + if (m_oak_infer_info.find(nh) == m_oak_infer_info.end()) { + m_camera_input->video.link(*(m_oak_nodes.at(nh).inputs[0])); + } else { + m_camera_input->preview.link(*(m_oak_nodes.at(nh).inputs[0])); + } } // 2. Link output nodes to XLinkOut nodes - size_t out_counter = 0; for (const auto& nh : out_nodes) { - GAPI_Assert(out_counter + m_oak_nodes.at(nh).outputs.size() <= m_out_queues.size()); for (const auto& out : m_oak_nodes.at(nh).outputs) { - out->link(m_out_queues[out_counter++].xlink_output->input); + out->link(m_out_queues[nh].xlink_output->input); } // Input nodes in OAK doesn't have parent operation - just camera (for now) if (in_nodes.find(nh) == in_nodes.end()) { - LinkToParents(nh); + linkToParent(nh); } } // 3. Link internal nodes to their parents for (const auto& nh : inter_nodes) { - // Input nodes in OAK doesn't have parent operation - just camera (for now) - if (in_nodes.find(nh) == in_nodes.end()) { - LinkToParents(nh); - } + linkToParent(nh); + } + + // 4. Link copy nodes + for (const auto& nh : copy_nodes) { + linkCopy(nh); } m_device = std::unique_ptr(new dai::Device(*m_pipeline)); // Prepare OAK output queues GAPI_Assert(m_out_queues.size() == outs_data.size()); - for (const auto out_it : ade::util::indexed(outs_data)) + for (const auto out_it : ade::util::indexed(m_out_queues)) { - auto& q = m_out_queues[ade::util::index(out_it)]; + auto& q = ade::util::value(out_it).second; GAPI_Assert(q.out_queue == nullptr); // shouldn't be not filled till this point // FIXME: add queue parameters - // Currently: 30 - max DAI queue capacity, true - blocking queue - q.out_queue = m_device->getOutputQueue(q.out_queue_name, 30, true); + // Currently: 4 - max DAI queue capacity, true - blocking queue + q.out_queue = m_device->getOutputQueue(q.out_queue_name, 4, true); } } @@ -507,17 +832,25 @@ void cv::gimpl::GOAKExecutable::run(GIslandExecutable::IInput &in, q->send(in_q.second); } - for (size_t i = 0; i < m_out_queues.size(); ++i) { - auto q = m_out_queues[i].out_queue; - // TODO: support other DAI types if needed - // Note: we utilize getData() method that returns std::vector of data - // on which we gain ownership - auto oak_frame = q->get(); + for (size_t i = 0; i < m_in_queues.size(); ++i) { + auto q = m_device->getInputQueue(m_in_queues[i].first); + q->send(m_in_queues[i].second); + } + + for (const auto el : m_out_queues) { + const auto out_q = el.second; + auto& q = out_q.out_queue; + + auto out_arg = out.get(out_q.gapi_out_data_index); - auto out_arg = out.get(i); + // FIXME: misc info to be utilized in switch below + cv::GRunArg::Meta meta; + std::shared_ptr oak_frame; switch(out_arg.index()) { case cv::GRunArgP::index_of(): + { + oak_frame = q->get(); // FIXME: hard-coded NV12 *cv::util::get(out_arg) = cv::MediaFrame::Create( @@ -525,56 +858,61 @@ void cv::gimpl::GOAKExecutable::run(GIslandExecutable::IInput &in, static_cast(oak_frame->getHeight())), cv::MediaFormat::NV12, std::move(oak_frame->getData())); + + using namespace cv::gapi::streaming::meta_tag; + meta[timestamp] = oak_frame->getTimestamp(); + meta[seq_id] = oak_frame->getSequenceNum(); + break; + } case cv::GRunArgP::index_of(): + { + oak_frame = q->get(); cv::util::get(out_arg).wref() = std::move(oak_frame->getData()); + + using namespace cv::gapi::streaming::meta_tag; + meta[timestamp] = oak_frame->getTimestamp(); + meta[seq_id] = oak_frame->getSequenceNum(); + break; - // FIXME: Add support for remaining types - default: - GAPI_Assert(false && "Unsupported type in OAK backend"); } + case cv::GRunArgP::index_of(): // only supported for infer + { + auto nn_data = q->get(); - using namespace cv::gapi::streaming::meta_tag; - cv::GRunArg::Meta meta; - meta[timestamp] = oak_frame->getTimestamp(); - meta[seq_id] = oak_frame->getSequenceNum(); + auto out_layer_name = getDaiInferOutLayerName(m_oak_infer_info.begin()->second); + auto in_out_tensor_info = parseDaiInferMeta(m_oak_infer_info.begin()->second); - out.meta(out_arg, meta); - out.post(std::move(out_arg)); - } -} + auto layer = std::move(nn_data->getLayerFp16(out_layer_name)); -// Built-in kernels for OAK ///////////////////////////////////////////////////// + // FIXME: add proper layout converter here + GAPI_Assert(in_out_tensor_info.second.order == + dai::TensorInfo::StorageOrder::NCHW); + // FIMXE: only 1-channel data is supported for now + GAPI_Assert(in_out_tensor_info.second.dims[2] == 1); -class GOAKBackendImpl final : public cv::gapi::GBackend::Priv { - virtual void unpackKernel(ade::Graph &graph, - const ade::NodeHandle &op_node, - const cv::GKernelImpl &impl) override { - OAKGraph gm(graph); + *cv::util::get(out_arg) = + cv::make_rmat( + cv::Size(in_out_tensor_info.second.dims[1], + in_out_tensor_info.second.dims[0]), + CV_16F, // FIXME: cover other precisions + std::move(layer) + ); - const auto &kimpl = cv::util::any_cast(impl.opaque); - gm.metadata(op_node).set(cv::gimpl::OAKComponent{kimpl}); - } + using namespace cv::gapi::streaming::meta_tag; + meta[timestamp] = nn_data->getTimestamp(); + meta[seq_id] = nn_data->getSequenceNum(); - virtual EPtr compile(const ade::Graph &graph, - const cv::GCompileArgs &args, - const std::vector &nodes, - const std::vector& ins_data, - const std::vector& outs_data) const override { - cv::gimpl::GModel::ConstGraph gm(graph); - // FIXME: pass streaming/non-streaming option to support non-camera case - // NB: how could we have non-OAK source in streaming mode, then OAK backend in - // streaming mode but without camera input? - if (!gm.metadata().contains()) { - GAPI_Assert(false && "OAK backend only supports Streaming mode for now"); + break; + } + // FIXME: Add support for remaining types + default: + GAPI_Assert(false && "Unsupported type in OAK backend"); } - return EPtr{new cv::gimpl::GOAKExecutable(graph, args, nodes, ins_data, outs_data)}; - } -}; -cv::gapi::GBackend cv::gapi::oak::backend() { - static cv::gapi::GBackend this_backend(std::make_shared()); - return this_backend; + out.meta(out_arg, meta); + out.post(std::move(out_arg)); + } } namespace cv { @@ -604,10 +942,13 @@ static dai::VideoEncoderProperties::Profile convertEncProfile(cv::gapi::oak::Enc // Kernels /////////////////////////////////////////////////////////////// -template -class GOAKKernelImpl: public detail::OAKCallHelper +// FIXME: consider a better solution - hard-coded API +// Is there a way to extract API from somewhereelse/utilize structs +// like in streaming/infer backends (mainly infer and copy operations) +template +class GOAKKernelImpl: public detail::OAKCallHelper , public cv::detail::KernelTag { - using P = detail::OAKCallHelper; + using P = detail::OAKCallHelper; public: using API = K; static cv::gapi::GBackend backend() { return cv::gapi::oak::backend(); } @@ -617,17 +958,51 @@ public: #define GAPI_OAK_KERNEL(Name, API) \ struct Name: public cv::gimpl::oak::GOAKKernelImpl +#define GAPI_OAK_FIXED_API_KERNEL(Name, API, InArgs, OutArgs) \ + struct Name: public cv::gimpl::oak::GOAKKernelImpl + namespace { +GAPI_OAK_FIXED_API_KERNEL(GOAKInfer, cv::GInferBase, std::tuple, std::tuple) { + static std::shared_ptr put(const cv::gimpl::OAKKernelParams& params, + GOAKContext::InputPtr& in, + GOAKContext::OutputPtr& out) { + auto nn = params.pipeline->create(); + + nn->input.setBlocking(true); + nn->input.setQueueSize(1); + + // FIXME: add G-API built-in preproc here (currently it's only setPreviewSize() on the camera node) + // Note: for some reason currently it leads to: + // "Fatal error. Please report to developers. Log: 'ImageManipHelper' '61'" + + nn->setBlobPath(params.infer_info.blob_file); + + in = &(nn->input); + out = &(nn->out); + + return nn; + } +}; + +GAPI_OAK_KERNEL(GOAKCopy, cv::gapi::oak::GCopy) { + static std::shared_ptr put(const cv::gimpl::OAKKernelParams&, + GOAKContext::InputPtr&, + GOAKContext::OutputPtr&) { + // Do nothing in Copy OP since it's either already represented + // by XLinkOut node (bonded to output queues) or it's a passthrough OP + return nullptr; + } +}; + GAPI_OAK_KERNEL(GOAKEncFrame, cv::gapi::oak::GEncFrame) { - static std::shared_ptr put(const cv::gimpl::detail::OAKKernelParams& params, + static std::shared_ptr put(const cv::gimpl::OAKKernelParams& params, GOAKContext::InputPtr& in, const cv::gapi::oak::EncoderConfig& cfg, GOAKContext::OutputPtr& out) { auto videoEnc = params.pipeline->create(); // FIXME: convert all the parameters to dai - videoEnc->setDefaultProfilePreset(cfg.width, cfg.height, - cfg.frameRate, + videoEnc->setDefaultProfilePreset(cfg.frameRate, convertEncProfile(cfg.profile)); in = &(videoEnc->input); @@ -638,7 +1013,7 @@ GAPI_OAK_KERNEL(GOAKEncFrame, cv::gapi::oak::GEncFrame) { }; GAPI_OAK_KERNEL(GOAKSobelXY, cv::gapi::oak::GSobelXY) { - static std::shared_ptr put(const cv::gimpl::detail::OAKKernelParams& params, + static std::shared_ptr put(const cv::gimpl::OAKKernelParams& params, GOAKContext::InputPtr& in, const cv::Mat& hk, const cv::Mat& vk, @@ -664,7 +1039,7 @@ GAPI_OAK_KERNEL(GOAKSobelXY, cv::gapi::oak::GSobelXY) { xinEdgeCfg->out.link(edgeDetector->inputConfig); - params.m_in_queues.push_back({"sobel_cfg", cfg}); + params.in_queues.push_back({"sobel_cfg", cfg}); in = &(edgeDetector->inputImage); out = &(edgeDetector->outputImage); @@ -672,11 +1047,55 @@ GAPI_OAK_KERNEL(GOAKSobelXY, cv::gapi::oak::GSobelXY) { return edgeDetector; } }; + } // anonymous namespace } // namespace oak } // namespace gimpl } // namespace cv +class GOAKBackendImpl final : public cv::gapi::GBackend::Priv { + virtual void unpackKernel(ade::Graph &graph, + const ade::NodeHandle &op_node, + const cv::GKernelImpl &impl) override { + using namespace cv::gimpl; + + OAKGraph gm(graph); + + const auto &kimpl = cv::util::any_cast(impl.opaque); + gm.metadata(op_node).set(OAKComponent{kimpl}); + + // Set custom meta for infer + if (gm.metadata(op_node).contains()) { + gm.metadata(op_node).set(CustomMetaFunction{customOutMeta}); + } + } + + virtual EPtr compile(const ade::Graph &graph, + const cv::GCompileArgs &args, + const std::vector &nodes, + const std::vector& ins_data, + const std::vector& outs_data) const override { + cv::gimpl::GModel::ConstGraph gm(graph); + // FIXME: pass streaming/non-streaming option to support non-camera case + // NB: how could we have non-OAK source in streaming mode, then OAK backend in + // streaming mode but without camera input? + if (!gm.metadata().contains()) { + GAPI_Assert(false && "OAK backend only supports Streaming mode for now"); + } + return EPtr{new cv::gimpl::GOAKExecutable(graph, args, nodes, ins_data, outs_data)}; + } + + virtual cv::GKernelPackage auxiliaryKernels() const override { + return cv::gapi::kernels< cv::gimpl::oak::GOAKInfer + >(); + } +}; + +cv::gapi::GBackend cv::gapi::oak::backend() { + static cv::gapi::GBackend this_backend(std::make_shared()); + return this_backend; +} + namespace cv { namespace gapi { namespace oak { @@ -684,6 +1103,7 @@ namespace oak { cv::gapi::GKernelPackage kernels() { return cv::gapi::kernels< cv::gimpl::oak::GOAKEncFrame , cv::gimpl::oak::GOAKSobelXY + , cv::gimpl::oak::GOAKCopy >(); } @@ -697,13 +1117,17 @@ namespace cv { namespace gapi { namespace oak { -cv::gapi::GKernelPackage kernels(); - cv::gapi::GKernelPackage kernels() { GAPI_Assert(false && "Built without OAK support"); return {}; } +cv::gapi::GBackend backend() { + GAPI_Assert(false && "Built without OAK support"); + static cv::gapi::GBackend this_backend(nullptr); + return this_backend; +} + } // namespace oak } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/oak/oak_media_adapter.hpp b/modules/gapi/src/backends/oak/oak_memory_adapters.hpp similarity index 56% rename from modules/gapi/src/backends/oak/oak_media_adapter.hpp rename to modules/gapi/src/backends/oak/oak_memory_adapters.hpp index 9c81f5a953..69677978d4 100644 --- a/modules/gapi/src/backends/oak/oak_media_adapter.hpp +++ b/modules/gapi/src/backends/oak/oak_memory_adapters.hpp @@ -10,11 +10,15 @@ #include #include +#include namespace cv { namespace gapi { namespace oak { +// Used for OAK backends outputs only. +// Filled from DepthAI's ImgFrame type and owns the memory. +// Used mainly for CV operations. class GAPI_EXPORTS OAKMediaAdapter final : public cv::MediaFrame::IAdapter { public: OAKMediaAdapter() = default; @@ -28,6 +32,24 @@ private: std::vector m_buffer; }; +// Used for OAK backends outputs only. +// Filled from DepthAI's NNData type and owns the memory. +// Used only for infer operations. +class GAPI_EXPORTS OAKRMatAdapter final : public cv::RMat::Adapter { +public: + OAKRMatAdapter() = default; + OAKRMatAdapter(const cv::Size& size, int precision, std::vector&& buffer); + cv::GMatDesc desc() const override; + cv::RMat::View access(cv::RMat::Access) override; + ~OAKRMatAdapter() = default; +private: + cv::Size m_size; + int m_precision; + std::vector m_buffer; + cv::GMatDesc m_desc; + cv::Mat m_mat; +}; + } // namespace oak } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/compiler/gmodel.hpp b/modules/gapi/src/compiler/gmodel.hpp index d016766fb5..592678dae7 100644 --- a/modules/gapi/src/compiler/gmodel.hpp +++ b/modules/gapi/src/compiler/gmodel.hpp @@ -70,7 +70,7 @@ struct Data // FIXME: This is a _pure_ duplication of RcDesc now! (except storage) GShape shape; // FIXME: Probably to be replaced by GMetaArg? - int rc; + int rc; // rc is unique but local to shape GMetaArg meta; HostCtor ctor; // T-specific helper to deal with unknown types in our code cv::detail::OpaqueKind kind; // FIXME: is needed to store GArray/GOpaque type