diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index f5aca38d1b..b49ec38b3c 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -62,7 +62,9 @@ set(gapi_srcs src/compiler/passes/meta.cpp src/compiler/passes/kernels.cpp src/compiler/passes/exec.cpp + src/compiler/passes/transformations.cpp src/compiler/passes/pattern_matching.cpp + src/compiler/passes/perform_substitution.cpp # Executor src/executor/gexecutor.cpp diff --git a/modules/gapi/src/compiler/gcompiler.cpp b/modules/gapi/src/compiler/gcompiler.cpp index a3f0fa4064..861574307e 100644 --- a/modules/gapi/src/compiler/gcompiler.cpp +++ b/modules/gapi/src/compiler/gcompiler.cpp @@ -29,6 +29,7 @@ #include "compiler/gcompiler.hpp" #include "compiler/gcompiled_priv.hpp" #include "compiler/passes/passes.hpp" +#include "compiler/passes/pattern_matching.hpp" #include "executor/gexecutor.hpp" #include "backends/common/gbackend.hpp" @@ -103,6 +104,84 @@ namespace return result; } + // Creates ADE graph from input/output proto args + std::unique_ptr makeGraph(const cv::GProtoArgs &ins, const cv::GProtoArgs &outs) { + std::unique_ptr pG(new ade::Graph); + ade::Graph& g = *pG; + + cv::gimpl::GModel::Graph gm(g); + cv::gimpl::GModel::init(gm); + cv::gimpl::GModelBuilder builder(g); + auto proto_slots = builder.put(ins, outs); + + // Store Computation's protocol in metadata + cv::gimpl::Protocol p; + std::tie(p.inputs, p.outputs, p.in_nhs, p.out_nhs) = proto_slots; + gm.metadata().set(p); + + return pG; + } + + using adeGraphs = std::vector>; + + // Creates ADE graphs (patterns and substitutes) from pkg's transformations + void makeTransformationGraphs(const cv::gapi::GKernelPackage& pkg, + adeGraphs& patterns, + adeGraphs& substitutes) { + const auto& transforms = pkg.get_transformations(); + const auto size = transforms.size(); + if (0u == size) return; + + // pre-generate all required graphs + patterns.resize(size); + substitutes.resize(size); + for (auto it : ade::util::zip(ade::util::toRange(transforms), + ade::util::toRange(patterns), + ade::util::toRange(substitutes))) { + const auto& t = std::get<0>(it); + auto& p = std::get<1>(it); + auto& s = std::get<2>(it); + + auto pattern_comp = t.pattern(); + p = makeGraph(pattern_comp.priv().m_ins, pattern_comp.priv().m_outs); + + auto substitute_comp = t.substitute(); + s = makeGraph(substitute_comp.priv().m_ins, substitute_comp.priv().m_outs); + } + } + + void checkTransformations(const cv::gapi::GKernelPackage& pkg, + const adeGraphs& patterns, + const adeGraphs& substitutes) { + const auto& transforms = pkg.get_transformations(); + const auto size = transforms.size(); + if (0u == size) return; + + GAPI_Assert(size == patterns.size()); + GAPI_Assert(size == substitutes.size()); + + const auto empty = [] (const cv::gimpl::SubgraphMatch& m) { + return m.inputDataNodes.empty() && m.startOpNodes.empty() + && m.finishOpNodes.empty() && m.outputDataNodes.empty() + && m.inputTestDataNodes.empty() && m.outputTestDataNodes.empty(); + }; + + // **FIXME**: verify other types of endless loops. now, only checking if pattern exists in + // substitute within __the same__ transformation + for (size_t i = 0; i < size; ++i) { + const auto& p = patterns[i]; + const auto& s = substitutes[i]; + + auto matchInSubstitute = cv::gimpl::findMatches(*p, *s); + if (!empty(matchInSubstitute)) { + std::stringstream ss; + ss << "Error: (in transformation with description: '" + << transforms[i].description + << "') pattern is detected inside substitute"; + throw std::runtime_error(ss.str()); + } + } + } } // anonymous namespace @@ -129,10 +208,26 @@ cv::gimpl::GCompiler::GCompiler(const cv::GComputation &c, // NN backends (present here via network package) always add their // inference kernels via auxiliary...() + // sanity check transformations + { + adeGraphs patterns, substitutes; + // FIXME: returning vectors of unique_ptrs from makeTransformationGraphs results in + // compile error (at least) on GCC 9 with usage of copy-ctor of std::unique_ptr, so + // using initialization by lvalue reference instead + makeTransformationGraphs(m_all_kernels, patterns, substitutes); + checkTransformations(m_all_kernels, patterns, substitutes); + + // NB: saving generated patterns to m_all_patterns to be used later in passes + m_all_patterns = std::move(patterns); + } + auto dump_path = getGraphDumpDirectory(m_args); m_e.addPassStage("init"); m_e.addPass("init", "check_cycles", ade::passes::CheckCycles()); + m_e.addPass("init", "apply_transformations", + std::bind(passes::applyTransformations, _1, std::cref(m_all_kernels), + std::cref(m_all_patterns))); // Note: and re-using patterns here m_e.addPass("init", "expand_kernels", std::bind(passes::expandKernels, _1, m_all_kernels)); // NB: package is copied @@ -255,23 +350,7 @@ cv::gimpl::GCompiler::GPtr cv::gimpl::GCompiler::generateGraph() { validateInputMeta(); validateOutProtoArgs(); - - // Generate ADE graph from expression-based computation - std::unique_ptr pG(new ade::Graph); - ade::Graph& g = *pG; - - GModel::Graph gm(g); - cv::gimpl::GModel::init(gm); - cv::gimpl::GModelBuilder builder(g); - auto proto_slots = builder.put(m_c.priv().m_ins, m_c.priv().m_outs); - GAPI_LOG_INFO(NULL, "Generated graph: " << g.nodes().size() << " nodes" << std::endl); - - // Store Computation's protocol in metadata - Protocol p; - std::tie(p.inputs, p.outputs, p.in_nhs, p.out_nhs) = proto_slots; - gm.metadata().set(p); - - return pG; + return makeGraph(m_c.priv().m_ins, m_c.priv().m_outs); } void cv::gimpl::GCompiler::runPasses(ade::Graph &g) diff --git a/modules/gapi/src/compiler/gcompiler.hpp b/modules/gapi/src/compiler/gcompiler.hpp index b4b8157dbf..d55f84d56e 100644 --- a/modules/gapi/src/compiler/gcompiler.hpp +++ b/modules/gapi/src/compiler/gcompiler.hpp @@ -29,6 +29,8 @@ class GAPI_EXPORTS GCompiler cv::gapi::GKernelPackage m_all_kernels; cv::gapi::GNetPackage m_all_networks; + std::vector> m_all_patterns; // built patterns from transformations + void validateInputMeta(); void validateOutProtoArgs(); diff --git a/modules/gapi/src/compiler/passes/passes.hpp b/modules/gapi/src/compiler/passes/passes.hpp index 91436dcd6f..e102a2f0e9 100644 --- a/modules/gapi/src/compiler/passes/passes.hpp +++ b/modules/gapi/src/compiler/passes/passes.hpp @@ -59,6 +59,10 @@ void resolveKernels(ade::passes::PassContext &ctx, void fuseIslands(ade::passes::PassContext &ctx); void syncIslandTags(ade::passes::PassContext &ctx); +void applyTransformations(ade::passes::PassContext &ctx, + const gapi::GKernelPackage &pkg, + const std::vector> &preGeneratedPatterns); + }} // namespace gimpl::passes } // namespace cv diff --git a/modules/gapi/src/compiler/passes/pattern_matching.cpp b/modules/gapi/src/compiler/passes/pattern_matching.cpp index b4dc2ee1c2..7be2da73ae 100644 --- a/modules/gapi/src/compiler/passes/pattern_matching.cpp +++ b/modules/gapi/src/compiler/passes/pattern_matching.cpp @@ -54,8 +54,7 @@ bool compareDataNodes(const ade::NodeHandle& first, const std::vector().shape != - secondMeta.get().shape) { + if (firstMeta.get().shape != secondMeta.get().shape) { return false; } @@ -180,7 +179,7 @@ inline bool IS_ENDPOINT(const ade::NodeHandle& nh){ // Try to rely on the nh Data::Storage::OUTPUT return nh->outEdges().empty(); } -} +} // anonymous namespace // Routine relies on the logic that 1 DATA node may have only 1 input edge. cv::gimpl::SubgraphMatch @@ -509,7 +508,7 @@ cv::gimpl::findMatches(const cv::gimpl::GModel::Graph& patternGraph, // Create vector with the correctly ordered IN data nodes in the test subgraph std::vector inputTestDataNodes; for (const auto& patternInNode : patternInputDataNodes) { - inputTestDataNodes.push_back(inputApiMatch[patternInNode]); + inputTestDataNodes.push_back(inputApiMatch.at(patternInNode)); } // Traversing current result for ending OPs @@ -560,7 +559,7 @@ cv::gimpl::findMatches(const cv::gimpl::GModel::Graph& patternGraph, // Create vector with the correctly ordered OUT data nodes in the test subgraph std::vector outputTestDataNodes; for (const auto& patternOutNode : patternOutputDataNodes) { - outputTestDataNodes.push_back(outputApiMatch[patternOutNode]); + outputTestDataNodes.push_back(outputApiMatch.at(patternOutNode)); } SubgraphMatch subgraph; diff --git a/modules/gapi/src/compiler/passes/pattern_matching.hpp b/modules/gapi/src/compiler/passes/pattern_matching.hpp index 0d1537ee74..2955e9c42a 100644 --- a/modules/gapi/src/compiler/passes/pattern_matching.hpp +++ b/modules/gapi/src/compiler/passes/pattern_matching.hpp @@ -41,7 +41,6 @@ namespace gimpl { return !inputDataNodes.empty() && !startOpNodes.empty() && !finishOpNodes.empty() && !outputDataNodes.empty() && !inputTestDataNodes.empty() && !outputTestDataNodes.empty(); - } S nodes() const { @@ -93,6 +92,11 @@ namespace gimpl { GAPI_EXPORTS SubgraphMatch findMatches(const cv::gimpl::GModel::Graph& patternGraph, const cv::gimpl::GModel::Graph& compGraph); + GAPI_EXPORTS void performSubstitution(cv::gimpl::GModel::Graph& graph, + const cv::gimpl::Protocol& patternP, + const cv::gimpl::Protocol& substituteP, + const cv::gimpl::SubgraphMatch& patternToGraphMatch); + } //namespace gimpl } //namespace cv #endif // OPENCV_GAPI_PATTERN_MATCHING_HPP diff --git a/modules/gapi/src/compiler/passes/perform_substitution.cpp b/modules/gapi/src/compiler/passes/perform_substitution.cpp new file mode 100644 index 0000000000..a1d9098102 --- /dev/null +++ b/modules/gapi/src/compiler/passes/perform_substitution.cpp @@ -0,0 +1,85 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2019 Intel Corporation + +#include "pattern_matching.hpp" + +#include "ade/util/zip_range.hpp" + +namespace cv { namespace gimpl { +namespace { +using Graph = GModel::Graph; + +template +ade::NodeHandle getNh(Iterator it) { return *it; } + +template<> +ade::NodeHandle getNh(SubgraphMatch::M::const_iterator it) { return it->second; } + +template +void erase(Graph& g, const Container& c) +{ + for (auto first = c.begin(); first != c.end(); ++first) { + ade::NodeHandle node = getNh(first); + if (node == nullptr) continue; // some nodes might already be erased + g.erase(node); + } +} +} // anonymous namespace + +void performSubstitution(GModel::Graph& graph, + const Protocol& patternP, + const Protocol& substituteP, + const SubgraphMatch& patternToGraphMatch) +{ + // 1. substitute input nodes + const auto& patternIns = patternP.in_nhs; + const auto& substituteIns = substituteP.in_nhs; + + for (auto it : ade::util::zip(ade::util::toRange(patternIns), + ade::util::toRange(substituteIns))) { + // Note: we don't replace input DATA nodes here, only redirect their output edges + const auto& patternDataNode = std::get<0>(it); + const auto& substituteDataNode = std::get<1>(it); + const auto& graphDataNode = patternToGraphMatch.inputDataNodes.at(patternDataNode); + GModel::redirectReaders(graph, substituteDataNode, graphDataNode); + } + + // 2. substitute output nodes + const auto& patternOuts = patternP.out_nhs; + const auto& substituteOuts = substituteP.out_nhs; + + for (auto it : ade::util::zip(ade::util::toRange(patternOuts), + ade::util::toRange(substituteOuts))) { + // Note: we don't replace output DATA nodes here, only redirect their input edges + const auto& patternDataNode = std::get<0>(it); + const auto& substituteDataNode = std::get<1>(it); + const auto& graphDataNode = patternToGraphMatch.outputDataNodes.at(patternDataNode); + // delete existing edges (otherwise we cannot redirect) + for (auto e : graphDataNode->inEdges()) { + graph.erase(e); + } + GModel::redirectWriter(graph, substituteDataNode, graphDataNode); + } + + // 3. erase redundant nodes: + // erase input data nodes of __substitute__ + erase(graph, substituteIns); + + // erase old start OP nodes of __main graph__ + erase(graph, patternToGraphMatch.startOpNodes); + + // erase old internal nodes of __main graph__ + erase(graph, patternToGraphMatch.internalLayers); + + // erase old finish OP nodes of __main graph__ + erase(graph, patternToGraphMatch.finishOpNodes); + + // erase output data nodes of __substitute__ + erase(graph, substituteOuts); +} + +} // namespace gimpl +} // namespace cv diff --git a/modules/gapi/src/compiler/passes/transformations.cpp b/modules/gapi/src/compiler/passes/transformations.cpp new file mode 100644 index 0000000000..f3b963858e --- /dev/null +++ b/modules/gapi/src/compiler/passes/transformations.cpp @@ -0,0 +1,139 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2019 Intel Corporation + +#include "precomp.hpp" + +#include +#include + +#include "api/gcomputation_priv.hpp" + +#include "compiler/gmodel.hpp" +#include "compiler/gmodelbuilder.hpp" +#include "compiler/passes/passes.hpp" +#include "compiler/passes/pattern_matching.hpp" + +#include + +namespace cv { namespace gimpl { namespace passes { +namespace +{ +using Graph = GModel::Graph; +using Metadata = typename Graph::CMetadataT; + +// Checks pairs of {pattern node, substitute node} and asserts if there are any incompatibilities +void checkDataNodes(const Graph& pattern, + const Graph& substitute, + const std::vector& patternNodes, + const std::vector& substituteNodes) +{ + for (auto it : ade::util::zip(patternNodes, substituteNodes)) { + auto pNodeMeta = pattern.metadata(std::get<0>(it)); + auto sNodeMeta = substitute.metadata(std::get<1>(it)); + GAPI_Assert(pNodeMeta.get().t == NodeType::DATA); + GAPI_Assert(pNodeMeta.get().t == sNodeMeta.get().t); + GAPI_Assert(pNodeMeta.get().shape == sNodeMeta.get().shape); + } +} + +// Checks compatibility of pattern and substitute graphs based on in/out nodes +void checkCompatibility(const Graph& pattern, + const Graph& substitute, + const Protocol& patternP, + const Protocol& substituteP) +{ + const auto& patternDataInputs = patternP.in_nhs; + const auto& patternDataOutputs = patternP.out_nhs; + + const auto& substituteDataInputs = substituteP.in_nhs; + const auto& substituteDataOutputs = substituteP.out_nhs; + + // number of data nodes must be the same + GAPI_Assert(patternDataInputs.size() == substituteDataInputs.size()); + GAPI_Assert(patternDataOutputs.size() == substituteDataOutputs.size()); + + // for each pattern input node, verify a corresponding substitute input node + checkDataNodes(pattern, substitute, patternDataInputs, substituteDataInputs); + + // for each pattern output node, verify a corresponding substitute output node + checkDataNodes(pattern, substitute, patternDataOutputs, substituteDataOutputs); +} + +// Tries to substitute __single__ pattern with substitute in the given graph +bool tryToSubstitute(ade::Graph& main, + const std::unique_ptr& patternG, + const cv::GComputation& substitute) +{ + GModel::Graph gm(main); + + // 1. find a pattern in main graph + auto match1 = findMatches(*patternG, gm); + if (!match1.ok()) { + return false; + } + + // 2. build substitute graph inside the main graph + cv::gimpl::GModelBuilder builder(main); + const auto& proto_slots = builder.put(substitute.priv().m_ins, substitute.priv().m_outs); + Protocol substituteP; + std::tie(substituteP.inputs, substituteP.outputs, substituteP.in_nhs, substituteP.out_nhs) = + proto_slots; + + const Protocol& patternP = GModel::Graph(*patternG).metadata().get(); + + // 3. check that pattern and substitute are compatible + // FIXME: in theory, we should always have compatible pattern/substitute. if not, we're in + // half-completed state where some transformations are already applied - what can we do + // to handle the situation better? -- use transactional API as in fuse_islands pass? + checkCompatibility(*patternG, gm, patternP, substituteP); + + // 4. make substitution + performSubstitution(gm, patternP, substituteP, match1); + + return true; +} +} // anonymous namespace + +void applyTransformations(ade::passes::PassContext& ctx, + const gapi::GKernelPackage& pkg, + const std::vector>& patterns) +{ + const auto& transforms = pkg.get_transformations(); + const auto size = transforms.size(); + if (0u == size) return; + // Note: patterns are already generated at this point + GAPI_Assert(patterns.size() == transforms.size()); + + // transform as long as it is possible + bool canTransform = true; + while (canTransform) + { + canTransform = false; + + // iterate through every transformation and try to transform graph parts + for (auto it : ade::util::zip(ade::util::toRange(transforms), ade::util::toRange(patterns))) + { + const auto& t = std::get<0>(it); + auto& pattern = std::get<1>(it); // Note: using pre-created graphs + GAPI_Assert(nullptr != pattern); + + // if transformation is successful (pattern found and substituted), it is possible that + // other transformations will also be successful, so set canTransform to the returned + // value from tryToSubstitute + canTransform = tryToSubstitute(ctx.graph, pattern, t.substitute()); + + // Note: apply the __same__ substitution as many times as possible and only after go to + // the next one. BUT it can happen that after applying some substitution, some + // _previous_ patterns will also be found and these will be applied first + if (canTransform) { + break; + } + } + } +} +} // namespace passes +} // namespace gimpl +} // namespace cv diff --git a/modules/gapi/test/internal/gapi_int_perform_substitution_test.cpp b/modules/gapi/test/internal/gapi_int_perform_substitution_test.cpp new file mode 100644 index 0000000000..f75503b101 --- /dev/null +++ b/modules/gapi/test/internal/gapi_int_perform_substitution_test.cpp @@ -0,0 +1,649 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2019 Intel Corporation + + +#include "../test_precomp.hpp" + +#include + +#include +#include +#include + +#include "compiler/gmodel.hpp" +#include "compiler/gmodel_priv.hpp" + +#include "api/gcomputation_priv.hpp" +#include "compiler/gcompiler.hpp" +#include "compiler/gmodelbuilder.hpp" +#include "compiler/passes/passes.hpp" + +#include "compiler/passes/pattern_matching.hpp" + +#include "../common/gapi_tests_common.hpp" + +#include "logger.hpp" + +namespace opencv_test +{ +// -------------------------------------------------------------------------------------- +// Accuracy integration tests (GComputation-level) + +namespace { +// FIXME: replace listener with something better (e.g. check graph via GModel?) +// custom "listener" to check what kernels are called within the test +struct KernelListener { std::map counts; }; +KernelListener& getListener() { + static KernelListener l; + return l; +} + +using CompCreator = std::function; +using CompileArgsCreator = std::function; +using Verifier = std::function; +} // anonymous namespace + +// Custom kernels && transformations below: + +G_TYPED_KERNEL(MyNV12toBGR, , "test.my_nv12_to_bgr") { + static GMatDesc outMeta(GMatDesc in_y, GMatDesc in_uv) { + return cv::gapi::imgproc::GNV12toBGR::outMeta(in_y, in_uv); + } +}; +GAPI_OCV_KERNEL(MyNV12toBGRImpl, MyNV12toBGR) +{ + static void run(const cv::Mat& in_y, const cv::Mat& in_uv, cv::Mat &out) + { + getListener().counts[MyNV12toBGR::id()]++; + cv::cvtColorTwoPlane(in_y, in_uv, out, cv::COLOR_YUV2BGR_NV12); + } +}; +G_TYPED_KERNEL(MyPlanarResize, , "test.my_planar_resize") { + static GMatDesc outMeta(GMatDesc in, Size sz, int interp) { + return cv::gapi::core::GResizeP::outMeta(in, sz, interp); + } +}; +GAPI_OCV_KERNEL(MyPlanarResizeImpl, MyPlanarResize) { + static void run(const cv::Mat& in, cv::Size out_sz, int interp, cv::Mat &out) + { + getListener().counts[MyPlanarResize::id()]++; + int inH = in.rows / 3; + int inW = in.cols; + int outH = out.rows / 3; + int outW = out.cols; + for (int i = 0; i < 3; i++) { + auto in_plane = in(cv::Rect(0, i*inH, inW, inH)); + auto out_plane = out(cv::Rect(0, i*outH, outW, outH)); + cv::resize(in_plane, out_plane, out_sz, 0, 0, interp); + } + } +}; +G_TYPED_KERNEL(MyInterleavedResize, , "test.my_interleaved_resize") { + static GMatDesc outMeta(GMatDesc in, Size sz, int interp) { + return cv::gapi::core::GResize::outMeta(in, sz, 0.0, 0.0, interp); + } +}; +GAPI_OCV_KERNEL(MyInterleavedResizeImpl, MyInterleavedResize) { + static void run(const cv::Mat& in, cv::Size out_sz, int interp, cv::Mat &out) + { + getListener().counts[MyInterleavedResize::id()]++; + cv::resize(in, out, out_sz, 0.0, 0.0, interp); + } +}; +G_TYPED_KERNEL(MyToNCHW, , "test.my_to_nchw") { + static GMatDesc outMeta(GMatDesc in) { + GAPI_Assert(in.depth == CV_8U); + GAPI_Assert(in.chan == 3); + GAPI_Assert(in.planar == false); + return in.asPlanar(); + } +}; +GAPI_OCV_KERNEL(MyToNCHWImpl, MyToNCHW) { + static void run(const cv::Mat& in, cv::Mat& out) + { + getListener().counts[MyToNCHW::id()]++; + auto sz = in.size(); + auto w = sz.width; + auto h = sz.height; + cv::Mat ins[3] = {}; + cv::split(in, ins); + for (int i = 0; i < 3; i++) { + auto in_plane = ins[i]; + auto out_plane = out(cv::Rect(0, i*h, w, h)); + in_plane.copyTo(out_plane); + } + } +}; +using GMat4 = std::tuple; +G_TYPED_KERNEL_M(MySplit4, , "test.my_split4") { + static std::tuple outMeta(GMatDesc in) { + const auto out_depth = in.depth; + const auto out_desc = in.withType(out_depth, 1); + return std::make_tuple(out_desc, out_desc, out_desc, out_desc); + } +}; +GAPI_OCV_KERNEL(MySplit4Impl, MySplit4) { + static void run(const cv::Mat& in, cv::Mat& out1, cv::Mat& out2, cv::Mat& out3, cv::Mat& out4) + { + getListener().counts[MySplit4::id()]++; + cv::Mat outs[] = { out1, out2, out3, out4 }; + cv::split(in, outs); + } +}; + +GAPI_TRANSFORM(NV12Transform, , "test.nv12_transform") +{ + static cv::GMat pattern(const cv::GMat& y, const cv::GMat& uv) + { + GMat out = cv::gapi::NV12toBGR(y, uv); + return out; + } + + static cv::GMat substitute(const cv::GMat& y, const cv::GMat& uv) + { + GMat out = MyNV12toBGR::on(y, uv); + return out; + } +}; +GAPI_TRANSFORM(ResizeTransform, , "3 x Resize -> Interleaved Resize") +{ + static cv::GMat pattern(const cv::GMat& in) + { + GMat b, g, r; + std::tie(b, g, r) = cv::gapi::split3(in); + const auto resize = std::bind(&cv::gapi::resize, std::placeholders::_1, + cv::Size(100, 100), 0, 0, cv::INTER_AREA); + return cv::gapi::merge3(resize(b), resize(g), resize(r)); + } + + static cv::GMat substitute(const cv::GMat& in) + { + return MyInterleavedResize::on(in, cv::Size(100, 100), cv::INTER_AREA); + } +}; +GAPI_TRANSFORM(ResizeTransformToCustom, , "Resize -> Custom Resize") +{ + static cv::GMat pattern(const cv::GMat& in) + { + return cv::gapi::resize(in, cv::Size(100, 100), 0, 0, cv::INTER_AREA); + } + + static cv::GMat substitute(const cv::GMat& in) + { + return MyInterleavedResize::on(in, cv::Size(100, 100), cv::INTER_AREA); + } +}; +GAPI_TRANSFORM(ChainTransform1, , "Resize + toNCHW -> toNCHW + PlanarResize") +{ + static GMatP pattern(const cv::GMat& in) + { + return MyToNCHW::on(cv::gapi::resize(in, cv::Size(60, 60))); + } + + static GMatP substitute(const cv::GMat& in) + { + return MyPlanarResize::on(MyToNCHW::on(in), cv::Size(60, 60), cv::INTER_LINEAR); + } +}; +GAPI_TRANSFORM(ChainTransform2, , "NV12toBGR + toNCHW -> NV12toBGRp") +{ + static GMatP pattern(const GMat& y, const GMat& uv) + { + return MyToNCHW::on(MyNV12toBGR::on(y, uv)); + } + + static GMatP substitute(const GMat& y, const GMat& uv) + { + return cv::gapi::NV12toBGRp(y, uv); + } +}; +GAPI_TRANSFORM(Split4Transform, , "Split4 -> Custom Split4") +{ + static GMat4 pattern(const GMat& in) + { + return cv::gapi::split4(in); + } + + static GMat4 substitute(const GMat& in) + { + return MySplit4::on(in); + } +}; +GAPI_TRANSFORM(Split4Merge3Transform, , "Split4 + Merge3 -> Custom Split4 + Merge3") +{ + static GMat pattern(const GMat& in) + { + GMat tmp1, tmp2, tmp3, unused; + std::tie(tmp1, tmp2, tmp3, unused) = cv::gapi::split4(in); + return cv::gapi::merge3(tmp1, tmp2, tmp3); + } + + static GMat substitute(const GMat& in) + { + GMat tmp1, tmp2, tmp3, unused; + std::tie(tmp1, tmp2, tmp3, unused) = MySplit4::on(in); + return cv::gapi::merge3(tmp1, tmp2, tmp3); + } +}; +GAPI_TRANSFORM(Merge4Split4Transform, , + "Merge4 + Split4 -> Merge4 + Custom Split4") +{ + static GMat4 pattern(const GMat& in1, const GMat& in2, const GMat& in3, + const GMat& in4) + { + return cv::gapi::split4(cv::gapi::merge4(in1, in2, in3, in4)); + } + + static GMat4 substitute(const GMat& in1, const GMat& in2, const GMat& in3, + const GMat& in4) + { + return MySplit4::on(cv::gapi::merge4(in1, in2, in3, in4)); + } +}; + +// -------------------------------------------------------------------------------------- +// Integration tests + +TEST(PatternMatchingIntegrationBasic, OneTransformationApplied) +{ + cv::Size in_sz(640, 480); + cv::Mat input(in_sz, CV_8UC3); + cv::randu(input, cv::Scalar::all(0), cv::Scalar::all(100)); + cv::Mat orig_graph_output, transformed_graph_output; + + auto orig_args = cv::compile_args(); + auto transform_args = cv::compile_args( + cv::gapi::kernels()); + + auto& listener = getListener(); + listener.counts.clear(); // clear counters before testing + + const auto make_computation = [] () { + GMat in; + GMat b, g, r; + std::tie(b, g, r) = cv::gapi::split3(in); + const auto resize = std::bind(&cv::gapi::resize, std::placeholders::_1, + cv::Size(100, 100), 0, 0, cv::INTER_AREA); + GMat out = cv::gapi::merge3(resize(b), resize(g), resize(r)); + return cv::GComputation(cv::GIn(in), cv::GOut(out)); + }; + + { + // Run original graph + auto mainC = make_computation(); + mainC.apply(cv::gin(input), cv::gout(orig_graph_output), std::move(orig_args)); + } + + // Generate transformed graph (passing transformations via compile args) + auto mainC = make_computation(); // get new copy with new Priv + mainC.apply(cv::gin(input), cv::gout(transformed_graph_output), std::move(transform_args)); + + // Compare + ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output)); + + // Custom verification via listener + ASSERT_EQ(1u, listener.counts.size()); + // called in transformed graph: + ASSERT_NE(listener.counts.cend(), listener.counts.find(MyInterleavedResize::id())); + ASSERT_EQ(1u, listener.counts.at(MyInterleavedResize::id())); +} + +TEST(PatternMatchingIntegrationBasic, SameTransformationAppliedSeveralTimes) +{ + cv::Size in_sz(640, 480); + cv::Mat input(in_sz, CV_8UC3); + cv::randu(input, cv::Scalar::all(0), cv::Scalar::all(100)); + cv::Mat orig_graph_output, transformed_graph_output; + + auto orig_args = cv::compile_args(); + auto transform_args = cv::compile_args( + cv::gapi::kernels()); + + auto& listener = getListener(); + listener.counts.clear(); // clear counters before testing + + const auto make_computation = [] () { + GMat in; + GMat b, g, r; + std::tie(b, g, r) = cv::gapi::split3(in); + const auto resize = std::bind(&cv::gapi::resize, std::placeholders::_1, + cv::Size(100, 100), 0, 0, cv::INTER_AREA); + GMat out = cv::gapi::merge3(resize(b), resize(g), resize(r)); + return cv::GComputation(cv::GIn(in), cv::GOut(out)); + }; + + { + // Run original graph + auto mainC = make_computation(); + mainC.apply(cv::gin(input), cv::gout(orig_graph_output), std::move(orig_args)); + } + + // Generate transformed graph (passing transformations via compile args) + auto mainC = make_computation(); // get new copy with new Priv + mainC.apply(cv::gin(input), cv::gout(transformed_graph_output), std::move(transform_args)); + + // Compare + ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output)); + + // Custom verification via listener + ASSERT_EQ(1u, listener.counts.size()); + // called in transformed graph: + ASSERT_NE(listener.counts.cend(), listener.counts.find(MyInterleavedResize::id())); + ASSERT_EQ(3u, listener.counts.at(MyInterleavedResize::id())); +} + +TEST(PatternMatchingIntegrationBasic, OneNV12toBGRTransformationApplied) +{ + cv::Size in_sz(640, 480); + cv::Mat y(in_sz, CV_8UC1), uv(cv::Size(in_sz.width / 2, in_sz.height / 2), CV_8UC2); + cv::randu(y, cv::Scalar::all(0), cv::Scalar::all(100)); + cv::randu(uv, cv::Scalar::all(100), cv::Scalar::all(200)); + cv::Mat orig_graph_output, transformed_graph_output; + + auto orig_args = cv::compile_args(); + auto transform_args = cv::compile_args(cv::gapi::kernels()); + + auto& listener = getListener(); + listener.counts.clear(); // clear counters before testing + + const auto make_computation = [] () { + GMat in1, in2; + GMat bgr = cv::gapi::NV12toBGR(in1, in2); + GMat out = cv::gapi::resize(bgr, cv::Size(100, 100)); + return cv::GComputation(cv::GIn(in1, in2), cv::GOut(out)); + }; + + { + // Run original graph + auto mainC = make_computation(); + mainC.apply(cv::gin(y, uv), cv::gout(orig_graph_output), std::move(orig_args)); + } + + // Generate transformed graph (passing transformations via compile args) + auto mainC = make_computation(); // get new copy with new Priv + mainC.apply(cv::gin(y, uv), cv::gout(transformed_graph_output), std::move(transform_args)); + + // Compare + ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output)); + + // Custom verification via listener + ASSERT_EQ(1u, listener.counts.size()); + // called in transformed graph: + ASSERT_NE(listener.counts.cend(), listener.counts.find(MyNV12toBGR::id())); + ASSERT_EQ(1u, listener.counts.at(MyNV12toBGR::id())); +} + +TEST(PatternMatchingIntegrationBasic, TwoTransformationsApplied) +{ + cv::Size in_sz(640, 480); + cv::Mat y(in_sz, CV_8UC1), uv(cv::Size(in_sz.width / 2, in_sz.height / 2), CV_8UC2); + cv::randu(y, cv::Scalar::all(0), cv::Scalar::all(100)); + cv::randu(uv, cv::Scalar::all(100), cv::Scalar::all(200)); + cv::Mat orig_graph_output, transformed_graph_output; + + auto orig_args = cv::compile_args(); + auto transform_args = cv::compile_args( + cv::gapi::kernels()); // compile args with transformations + + auto& listener = getListener(); + listener.counts.clear(); // clear counters before testing + + const auto make_computation = [] () { + GMat in1, in2; + GMat bgr = cv::gapi::NV12toBGR(in1, in2); + GMat b, g, r; + std::tie(b, g, r) = cv::gapi::split3(bgr); + const auto resize = std::bind(&cv::gapi::resize, std::placeholders::_1, + cv::Size(100, 100), 0, 0, cv::INTER_AREA); + GMat tmp1 = cv::gapi::resize(bgr, cv::Size(90, 90)); + GMat tmp2 = cv::gapi::bitwise_not(cv::gapi::merge3(resize(b), resize(g), resize(r))); + GMat out = cv::gapi::resize(tmp1 + GScalar(10.0), cv::Size(100, 100)) + tmp2; + return cv::GComputation(cv::GIn(in1, in2), cv::GOut(out)); + }; + + { + // Run original graph + auto mainC = make_computation(); + mainC.apply(cv::gin(y, uv), cv::gout(orig_graph_output), std::move(orig_args)); + } + + // Generate transformed graph (passing transformations via compile args) + auto mainC = make_computation(); // get new copy with new Priv + mainC.apply(cv::gin(y, uv), cv::gout(transformed_graph_output), std::move(transform_args)); + + // Compare + ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output)); + + // Custom verification via listener + ASSERT_EQ(2u, listener.counts.size()); + // called in transformed graph: + ASSERT_NE(listener.counts.cend(), listener.counts.find(MyNV12toBGR::id())); + ASSERT_EQ(1u, listener.counts.at(MyNV12toBGR::id())); + ASSERT_NE(listener.counts.cend(), listener.counts.find(MyInterleavedResize::id())); + ASSERT_EQ(1u, listener.counts.at(MyInterleavedResize::id())); +} + +struct PatternMatchingIntegrationE2E : testing::Test +{ + cv::GComputation makeComputation() { + GMat in1, in2; + GMat bgr = MyNV12toBGR::on(in1, in2); + GMat resized = cv::gapi::resize(bgr, cv::Size(60, 60)); + GMatP out = MyToNCHW::on(resized); + return cv::GComputation(cv::GIn(in1, in2), cv::GOut(out)); + } + + void runTest(cv::GCompileArgs&& transform_args) { + cv::Size in_sz(640, 480); + cv::Mat y(in_sz, CV_8UC1), uv(cv::Size(in_sz.width / 2, in_sz.height / 2), CV_8UC2); + cv::randu(y, cv::Scalar::all(0), cv::Scalar::all(100)); + cv::randu(uv, cv::Scalar::all(100), cv::Scalar::all(200)); + cv::Mat orig_graph_output, transformed_graph_output; + + auto& listener = getListener(); + listener.counts.clear(); // clear counters before testing + { + // Run original graph + auto mainC = makeComputation(); + mainC.apply(cv::gin(y, uv), cv::gout(orig_graph_output), + cv::compile_args(cv::gapi::kernels())); + } + + // Generate transformed graph (passing transformations via compile args) + auto mainC = makeComputation(); // get new copy with new Priv + mainC.apply(cv::gin(y, uv), cv::gout(transformed_graph_output), std::move(transform_args)); + + // Compare + ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output)); + + // Custom verification via listener + ASSERT_EQ(3u, listener.counts.size()); + // called in original graph: + ASSERT_NE(listener.counts.cend(), listener.counts.find(MyNV12toBGR::id())); + ASSERT_NE(listener.counts.cend(), listener.counts.find(MyToNCHW::id())); + ASSERT_EQ(1u, listener.counts.at(MyNV12toBGR::id())); + ASSERT_EQ(1u, listener.counts.at(MyToNCHW::id())); + // called in transformed graph: + ASSERT_NE(listener.counts.cend(), listener.counts.find(MyPlanarResize::id())); + ASSERT_EQ(1u, listener.counts.at(MyPlanarResize::id())); + } +}; + +TEST_F(PatternMatchingIntegrationE2E, ChainTransformationsApplied) +{ + runTest(cv::compile_args( + cv::gapi::kernels())); +} + +TEST_F(PatternMatchingIntegrationE2E, ReversedChainTransformationsApplied) +{ + runTest(cv::compile_args( + cv::gapi::kernels())); +} + +struct PatternMatchingIntegrationUnusedNodes : testing::Test +{ + cv::GComputation makeComputation() { + GMat in1, in2; + GMat bgr = cv::gapi::NV12toBGR(in1, in2); + GMat b1, g1, r1; + std::tie(b1, g1, r1) = cv::gapi::split3(bgr); + // FIXME: easier way to call split4?? + GMat merged4 = cv::gapi::merge4(b1, g1, r1, b1); + GMat b2, g2, r2, unused; + std::tie(b2, g2, r2, unused) = cv::gapi::split4(merged4); + GMat out = cv::gapi::merge3(b2, g2, r2); + return cv::GComputation(cv::GIn(in1, in2), cv::GOut(out)); + } + + void runTest(cv::GCompileArgs&& transform_args) { + cv::Size in_sz(640, 480); + cv::Mat y(in_sz, CV_8UC1), uv(cv::Size(in_sz.width / 2, in_sz.height / 2), CV_8UC2); + cv::randu(y, cv::Scalar::all(0), cv::Scalar::all(100)); + cv::randu(uv, cv::Scalar::all(100), cv::Scalar::all(200)); + + cv::Mat orig_graph_output, transformed_graph_output; + + auto& listener = getListener(); + listener.counts.clear(); // clear counters before testing + { + // Run original graph + auto mainC = makeComputation(); + mainC.apply(cv::gin(y, uv), cv::gout(orig_graph_output), + cv::compile_args(cv::gapi::kernels())); + } + + // Generate transformed graph (passing transformations via compile args) + auto mainC = makeComputation(); // get new copy with new Priv + mainC.apply(cv::gin(y, uv), cv::gout(transformed_graph_output), std::move(transform_args)); + + // Compare + ASSERT_TRUE(AbsExact()(orig_graph_output, transformed_graph_output)); + + // Custom verification via listener + ASSERT_EQ(1u, listener.counts.size()); + // called in transformed graph: + ASSERT_NE(listener.counts.cend(), listener.counts.find(MySplit4::id())); + ASSERT_EQ(1u, listener.counts.at(MySplit4::id())); + } +}; + +TEST_F(PatternMatchingIntegrationUnusedNodes, SingleOpTransformApplied) +{ + runTest(cv::compile_args(cv::gapi::kernels())); +} + +// FIXME: enable once unused nodes are properly handled by Transformation API +TEST_F(PatternMatchingIntegrationUnusedNodes, DISABLED_TransformWithInternalUnusedNodeApplied) +{ + runTest(cv::compile_args(cv::gapi::kernels())); +} + +TEST_F(PatternMatchingIntegrationUnusedNodes, TransformWithOutputUnusedNodeApplied) +{ + runTest(cv::compile_args(cv::gapi::kernels())); +} + +// -------------------------------------------------------------------------------------- +// Bad arg integration tests (GCompiler-level) - General + +struct PatternMatchingIntegrationBadArgTests : testing::Test +{ + cv::GComputation makeComputation() { + GMat in; + GMat a, b, c, d; + std::tie(a, b, c, d) = MySplit4::on(in); // using custom Split4 to check if it's called + GMat out = cv::gapi::merge3(a + b, cv::gapi::bitwise_not(c), d * cv::GScalar(2.0)); + return cv::GComputation(cv::GIn(in), cv::GOut(out)); + } + + void runTest(cv::GCompileArgs&& transform_args) { + cv::Size in_sz(640, 480); + cv::Mat input(in_sz, CV_8UC4); + cv::randu(input, cv::Scalar::all(70), cv::Scalar::all(140)); + + cv::Mat output; + + // Generate transformed graph (passing transformations via compile args) + auto mainC = makeComputation(); // get new copy with new Priv + ASSERT_NO_THROW(mainC.apply(cv::gin(input), cv::gout(output), std::move(transform_args))); + } +}; + +TEST_F(PatternMatchingIntegrationBadArgTests, NoTransformations) +{ + auto transform_args = cv::compile_args(cv::gapi::kernels()); + + auto& listener = getListener(); + listener.counts.clear(); // clear counters before testing + + runTest(std::move(transform_args)); + + // Custom verification via listener + ASSERT_EQ(1u, listener.counts.size()); + ASSERT_NE(listener.counts.cend(), listener.counts.find(MySplit4::id())); + ASSERT_EQ(1u, listener.counts.at(MySplit4::id())); +} + +TEST_F(PatternMatchingIntegrationBadArgTests, WrongTransformation) +{ + // Here Split4Transform::pattern is "looking for" cv::gapi::split4 but it's not used + auto transform_args = cv::compile_args(cv::gapi::kernels()); + + auto& listener = getListener(); + listener.counts.clear(); // clear counters before testing + + runTest(std::move(transform_args)); + + // Custom verification via listener + ASSERT_EQ(1u, listener.counts.size()); + ASSERT_NE(listener.counts.cend(), listener.counts.find(MySplit4::id())); + ASSERT_EQ(1u, listener.counts.at(MySplit4::id())); +} + +// -------------------------------------------------------------------------------------- +// Bad arg integration tests (GCompiler-level) - Endless Loops + +GAPI_TRANSFORM(EndlessLoopTransform, , "pattern in substitute") +{ + static cv::GMat pattern(const cv::GMat& in) + { + return cv::gapi::resize(in, cv::Size(100, 100), 0, 0, cv::INTER_LINEAR); + } + + static cv::GMat substitute(const cv::GMat& in) + { + cv::GMat b, g, r; + std::tie(b, g, r) = cv::gapi::split3(in); + auto resize = std::bind(&cv::gapi::resize, + std::placeholders::_1, cv::Size(100, 100), 0, 0, cv::INTER_LINEAR); + cv::GMat out = cv::gapi::merge3(resize(b), resize(g), resize(r)); + return out; + } +}; + +TEST(PatternMatchingIntegrationEndlessLoops, PatternInSubstituteInOneTransform) +{ + cv::Size in_sz(640, 480); + cv::Mat input(in_sz, CV_8UC3); + cv::randu(input, cv::Scalar::all(0), cv::Scalar::all(100)); + + auto c = [] () { + GMat in; + GMat tmp = cv::gapi::resize(in, cv::Size(100, 100), 0, 0, cv::INTER_LINEAR); + GMat out = cv::gapi::bitwise_not(tmp); + return cv::GComputation(cv::GIn(in), cv::GOut(out)); + }(); + + EXPECT_THROW( + cv::gimpl::GCompiler(c, cv::descr_of(cv::gin(input)), + cv::compile_args(cv::gapi::kernels())), + std::exception); +} + + +} // namespace opencv_test