From e06efd532989b541d63126b39533683b64fea906 Mon Sep 17 00:00:00 2001 From: AsyaPronina <155jj@mail.ru> Date: Fri, 9 Aug 2019 18:57:56 +0300 Subject: [PATCH] G-API: Added graph pattern matching routine and basic tests --- modules/gapi/CMakeLists.txt | 1 + modules/gapi/src/compiler/gmodel_priv.hpp | 8 + .../src/compiler/passes/pattern_matching.cpp | 582 +++++++++++ .../src/compiler/passes/pattern_matching.hpp | 98 ++ .../gapi/test/common/gapi_core_tests_inl.hpp | 2 +- .../gapi_int_pattern_matching_test.cpp | 942 ++++++++++++++++++ 6 files changed, 1632 insertions(+), 1 deletion(-) create mode 100644 modules/gapi/src/compiler/passes/pattern_matching.cpp create mode 100644 modules/gapi/src/compiler/passes/pattern_matching.hpp create mode 100644 modules/gapi/test/internal/gapi_int_pattern_matching_test.cpp diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 2c5a4cf1ab..3198580d19 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -60,6 +60,7 @@ set(gapi_srcs src/compiler/passes/meta.cpp src/compiler/passes/kernels.cpp src/compiler/passes/exec.cpp + src/compiler/passes/pattern_matching.cpp # Executor src/executor/gexecutor.cpp diff --git a/modules/gapi/src/compiler/gmodel_priv.hpp b/modules/gapi/src/compiler/gmodel_priv.hpp index 3a3b444b8a..b40fff30c4 100644 --- a/modules/gapi/src/compiler/gmodel_priv.hpp +++ b/modules/gapi/src/compiler/gmodel_priv.hpp @@ -46,6 +46,14 @@ template inline ade::NodeHandle dataNodeOf(const ConstLayoutGraph& g return detail::dataNodeOf(g, cv::gimpl::proto::origin_of(GProtoArg{t})); } +inline ade::NodeHandle producerOf(const cv::gimpl::GModel::Graph& gm, ade::NodeHandle dh) +{ + GAPI_Assert(gm.metadata(dh).get().t == NodeType::DATA); + auto ins = dh->inNodes(); + return ins.empty() ? ade::NodeHandle{ } : *ins.begin(); +} + + }}} #endif // OPENCV_GAPI_GMODEL_PRIV_HPP diff --git a/modules/gapi/src/compiler/passes/pattern_matching.cpp b/modules/gapi/src/compiler/passes/pattern_matching.cpp new file mode 100644 index 0000000000..b4dc2ee1c2 --- /dev/null +++ b/modules/gapi/src/compiler/passes/pattern_matching.cpp @@ -0,0 +1,582 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2019 Intel Corporation + +#include + +#include "pattern_matching.hpp" + +namespace { +using Graph = cv::gimpl::GModel::Graph; +using Metadata = typename Graph::CMetadataT; +using VisitedMatchings = std::list>; + +using LabeledNodes = std::unordered_map + < // reader node + ade::NodeHandle + // if the reader node above is: + // - DATA node: then vector is 1-element vector containing port number of + // the input edge + // - OP node: then vector is ports' vector of current connections between + // this node and an parent active DATA node + , std::vector + , ade::HandleHasher + >; + +using MultipleMatchings = std::unordered_map + // pattern OP node + < ade::NodeHandle + // nodes in the test graph which match to the pattern OP node above + , std::vector + , ade::HandleHasher + >; + +// Returns true if two DATA nodes are semantically and structurally identical: +// - both nodes have the same GShape +// - both nodes are produced by the same port numbers +// - both nodes have the same number of output edges +// (output edges' ports are not checked here) +// +// @param first - first node to compare +// @param firstPorts - a single element vector with first DATA node's producer output port +// @param firstMeta - metadata of first +// @param second - second node to compare +// @param secondPorts - a single element vector with second DATA node's producer output port +// @param secondMeta - metadata of second +bool compareDataNodes(const ade::NodeHandle& first, const std::vector& firstPorts, + const Metadata& firstMeta, + const ade::NodeHandle& second, const std::vector& secondPorts, + const Metadata& secondMeta) { + if (secondMeta.get().t != cv::gimpl::NodeType::DATA) { + throw std::logic_error("NodeType of passed node as second argument" + "shall be NodeType::DATA!"); + } + + if (firstMeta.get().shape != + secondMeta.get().shape) { + return false; + } + + if (*firstPorts.begin() != *secondPorts.begin()) { + return false; + } + + const auto& firstOutputEdges = first->outEdges(); + const auto& secondOutputEdges = second->outEdges(); + + if (firstOutputEdges.size() != secondOutputEdges.size()) { + return false; + } + + // FIXME: Because of new changes which introduce existence of unused DATA nodes + // check that first and second nodes have the same type of DATA::Storage. + + return true; +}; + +// Returns true if two OP nodes semantically and structurally identical: +// - both nodes have the same kernel name +// - both nodes are produced by the same port numbers +// - if any of the nodes are in the array with visited matchings, then: +// first node is equal to found matching first argument and +// second node is equal to found matching second argument +// +// @param first - first node to compare +// @param firstPorts - ports' vector of current connections between first node and an parent active +// DATA node +// @param firstMeta - metadata of first +// @param second - second node to compare +// @param secondPorts - ports' vector of current connections between second node and an parent +// active DATA node +// @param secondMeta - metadata of second +// @param [out] isAlreadyVisited - set to true if first and second nodes have been already visited +bool compareOpNodes(const VisitedMatchings& matchedVisitedNodes, + const ade::NodeHandle& first, std::vector firstPorts, + const Metadata& firstMeta, + const ade::NodeHandle& second, std::vector secondPorts, + const Metadata& secondMeta, + bool& isAlreadyVisited) { + if (secondMeta.get().t != cv::gimpl::NodeType::OP) { + throw std::logic_error("NodeType of passed node as second argument shall be NodeType::OP!"); + } + + // Assuming that if kernels names are the same then + // output DATA nodes counts from kernels are the same. + // Assuming that if kernels names are the same then + // input DATA nodes counts to kernels are the same. + if (firstMeta.get().k.name != secondMeta.get().k.name) { + return false; + } + + std::sort(firstPorts.begin(), firstPorts.end()); + std::sort(secondPorts.begin(), secondPorts.end()); + if (firstPorts != secondPorts) { + return false; + } + + // Shall work, but it is good to test on the cases where multiple start pattern OP nodes + // maps to the test's one. + auto foundIt = std::find_if(matchedVisitedNodes.begin(), matchedVisitedNodes.end(), + [&first, &second](const std::pair& match) + {return first == match.first || second == match.second; }); + if (foundIt != matchedVisitedNodes.end()) { + if (first != foundIt->first || second != foundIt->second) { + return false; + } + + isAlreadyVisited = true; + } + + return true; +}; + +// Retrieves and return sample from the cartesian product of candidates sets +VisitedMatchings sampleFromProduct(std::size_t sampleIdx, // index of the sample in the product + const MultipleMatchings& candidatesSets) // map of nodes to sets + // of candidates + { + VisitedMatchings matchingsSample; + + std::size_t quo = sampleIdx; + for (const auto& setForNode : candidatesSets) { + // TODO: order is not determined: for ex., for last node. + // May be use ordered set and map to ensure order? + auto size = setForNode.second.size(); + + // The below code block decodes sampleIdx into a particular sample from cartesian product + // of candidates sets. + std::size_t index = quo % size; + quo = quo / size; + const auto& candidate = setForNode.second[index]; + matchingsSample.push_back({ setForNode.first, candidate }); + } + + return matchingsSample; +} + +// Depending on type of the node retrieve port number (IN/OUT) of the edge entering this node. +std::size_t labelOf (const ade::NodeHandle& node, // reader node + const ade::EdgeHandle& edge, // edge entering the reader node + const Graph& graph) // graph containing node and edge + { + + if (graph.metadata(node).get().t == cv::gimpl::NodeType::OP) { + return graph.metadata(edge).get().port; + } + else { + return graph.metadata(edge).get().port; + } +}; + +inline bool IS_STARTPOINT(const ade::NodeHandle& nh){ + return nh->inEdges().empty(); +} + +inline bool IS_ENDPOINT(const ade::NodeHandle& nh){ + // FIXME: Because of new changes which introduce existence of unused DATA nodes + // Try to rely on the nh Data::Storage::OUTPUT + return nh->outEdges().empty(); +} +} + +// Routine relies on the logic that 1 DATA node may have only 1 input edge. +cv::gimpl::SubgraphMatch +cv::gimpl::findMatches(const cv::gimpl::GModel::Graph& patternGraph, + const cv::gimpl::GModel::Graph& testGraph) { + + //TODO: Possibly, we may add N^2 check whether this graph may match or not at all. + // Check that all pattern OP nodes exist in computational graph. + + //--------------------------------------------------------------- + // Identify operations which start and end our pattern + SubgraphMatch::S patternStartOpNodes, patternEndOpNodes; + + const auto& patternInputDataNodes = patternGraph.metadata().get().in_nhs; + const auto& patternOutputDataNodes = patternGraph.metadata().get().out_nhs; + + for (const auto& node : patternInputDataNodes) { + auto opNodes = node->outNodes(); + patternStartOpNodes.insert(opNodes.begin(), opNodes.end()); + } + + for (const auto& node : patternOutputDataNodes) { + auto opNodes = node->inNodes(); + // May be switched to patternEndOpNodes.insert(*opNodes.begin()); + patternEndOpNodes.insert(opNodes.begin(), opNodes.end()); + } + + std::unordered_map, // nodes in the test graph which match + // to the pattern OP node + ade::HandleHasher> allMatchingsForStartOpNodes; + + //Filling of allMatchingsForStartOpNodes + std::size_t possibleStartPointsCount = 1; + + // For every starting OP node of pattern identify matching candidates(there may be many) + // in test graph. + auto testOpNodes = ade::util::filter(testGraph.nodes(), + [&](const ade::NodeHandle& node) { + return testGraph.metadata(node). + get().t + == cv::gimpl::NodeType::OP; + }); + for (const auto& patternStartOpNode : patternStartOpNodes) { + const auto& patternOpMeta = patternGraph.metadata(patternStartOpNode); + + auto& possibleMatchings = allMatchingsForStartOpNodes[patternStartOpNode]; + std::copy_if(testOpNodes.begin(), testOpNodes.end(), std::back_inserter(possibleMatchings), + [&](const ade::NodeHandle& testOpNode) { + const auto& testOpMeta = testGraph.metadata(testOpNode); + + bool stub = false; + return compareOpNodes({ }, + patternStartOpNode, { }, patternOpMeta, + testOpNode, { }, testOpMeta, + stub); + }); + + if (possibleMatchings.size() == 0) { + // Pattern graph is not matched + return SubgraphMatch { }; + } + + possibleStartPointsCount *= possibleMatchings.size(); + } + + SubgraphMatch::M subgraphStartOps; + SubgraphMatch::M subgraphEndOps; + // FIXME: consider moving to S + std::list subgraphInternals; + + + // Structural matching first, semantic matching second. + + // 'patternFound' means pattern is matched. + bool patternFound = false; + std::size_t i = 0; + while (!patternFound && (i < possibleStartPointsCount)) { + subgraphStartOps.clear(); + subgraphEndOps.clear(); + subgraphInternals.clear(); + + // List of the pairs representing matchings of pattern node to the test node. + VisitedMatchings matchedVisitedNodes; + + // Cartesian product of candidate sets for start OP nodes gives set of samples + // as possible matchings for start OP nodes. + // Let allMatchingsForStartOpNodes looks like: x1 : [ y1 ] + // x2 : [ y2, y3 ] + // Cartesian product of two these candidates sets (for x1 and x2 pattern nodes + // correspondingly) produces two samples of matchings for x1, x2: + // [ (x1, y1), (x2, y2) ] + // [ (x1, y1), (x2, y3) ] + // + + // Here we fill matchedVisitedNodes list with the next sample from the cartesian product + // of candidates sets. + // i is traversing full cartesian product of candidates sets. + matchedVisitedNodes = sampleFromProduct(i, allMatchingsForStartOpNodes); + + bool stop = false; + + // matchIt is an iterator to a pair of pattern ade::NodeHandle to test's ade::nodeHandle. + auto matchIt = matchedVisitedNodes.begin(); + std::size_t size = matchedVisitedNodes.size(); + + while (!stop) { + // The following loop traverses through the current level of matchings. + // Every iteration we consider only one certain pair of matched nodes. + for (std::size_t index = 0u; index < size && !stop; ++index, ++matchIt) { + + // Check if a given matchIt->first node is an pattern-ending OP node. + // If it is just remember it in a special map. + bool cond1 = std::find(patternEndOpNodes.begin(), + patternEndOpNodes.end(), + matchIt->first) + != patternEndOpNodes.end(); + if (cond1) { + subgraphEndOps[matchIt->first] = matchIt->second; + } + + // Check if a given matchIt->first node is an pattern-starting OP node. + // If it is just remember it in a special map. + bool cond2 = std::find(patternStartOpNodes.begin(), + patternStartOpNodes.end(), + matchIt->first) + != patternStartOpNodes.end(); + if (cond2) { + subgraphStartOps[matchIt->first] = matchIt->second; + } + + // If neither of conditions are true mark the test node as an internal one. + if (!cond1 && !cond2) { + subgraphInternals.push_back(matchIt->second); + } + + //------------------------------------------------------------------------------- + // Given the current pattern/test matching of nodes, traverse their descendants. + // For every descendant store the port of the edge connecting to it. + // NOTE: the nature of port number may vary: it may be either IN for OP nodes + // or OUT for DATA ones + LabeledNodes patternOutputNodesLabeled; + LabeledNodes testOutputNodesLabeled; + + auto patternOutputEdges = matchIt->first->outEdges(); + auto testOutputEdges = matchIt->second->outEdges(); + + for (const auto& patternOutputEdge : patternOutputEdges) { + const auto& dstNh = patternOutputEdge->dstNode(); + if (!IS_ENDPOINT(dstNh)) { + //Assuming that there is no case for the op node without output data nodes. + patternOutputNodesLabeled[dstNh]. + push_back(labelOf(dstNh, patternOutputEdge, patternGraph)); + } + } + + for (const auto& testOutputEdge : testOutputEdges) { + const auto& dstNh = testOutputEdge->dstNode(); + testOutputNodesLabeled[dstNh]. + push_back(labelOf(dstNh, testOutputEdge, testGraph)); + } + + //--------------------------------------------------------------------------------- + // Traverse through labeled descendants of pattern node and for every descedant + // find a matching in labeled descendants of corresponding test node + for (const auto& patternNode : patternOutputNodesLabeled) { + bool isAlreadyVisited = false; + const auto& patternNodeMeta = patternGraph.metadata(patternNode.first); + + auto testIt = std::find_if(testOutputNodesLabeled.begin(), + testOutputNodesLabeled.end(), + [&](const std::pair>& testNode) { + const auto& testNodeMeta = testGraph.metadata(testNode.first); + + auto patternNodeType = patternNodeMeta.get().t; + + switch(patternNodeType) { + case cv::gimpl::NodeType::DATA: + return compareDataNodes(patternNode.first, patternNode.second, + patternNodeMeta, + testNode.first, testNode.second, + testNodeMeta); + case cv::gimpl::NodeType::OP: + return compareOpNodes(matchedVisitedNodes, + patternNode.first, patternNode.second, + patternNodeMeta, + testNode.first, testNode.second, + testNodeMeta, + isAlreadyVisited); + default: + GAPI_Assert(false && "Unsupported Node type!"); + } + + return false; + }); + + if (testIt == testOutputNodesLabeled.end()) { + stop = true; + break; + } + + // Update matchedVisitedNodes list with found pair of nodes if the pair + // has not been visited before. + if (!isAlreadyVisited) { + matchedVisitedNodes.push_back({ patternNode.first, testIt->first }); + } + } // Loop traversed patternOutputNodesLabeled + } // Loop traversed matchedVisitedNodes + + // Suppose, pattern and test graphs' structures without input DATA nodes look like: + // Pattern graph Test graph + // op1 op2 t_op1 t_op2 + // +-----+ +-----+ +-----+ +-----+ + // v v v v v v v v + // d1 d2 d3 d4 t_d1 t_d2 t_d3 t_d4 + // v v v v v v v v + // ... ... ... ... ... ... ... ... + + // matchedVisitedNodes content before previous loop execution: + // op1 <--> t_op1, op2 <--> t_op2 + // matchedVisitedNodes content after previous loop execution (extended with the next + // level of matchings): + // op1 <--> t_op1, op2 <--> t_op2 | d1 <--> t_d1, d2 <--> t_d2, d3 <--> t_d3, d4 <--> t_d4 + // ^ + // | + // matchIt + // + // matchIt iterator points to the first matching in next level if the next level exists. + // If there is no next level, matchIt == matchedVisitedNodes.end() and all pattern + // levels (except ones for IN/OUT data nodes) have been already processed, so, + // pattern subgraph is found. + + if (!stop) { + // Check if pattetn subgraph is found + if (matchIt == matchedVisitedNodes.end()) { + // Found + stop = true; + patternFound = true; + } + + // Update 'size' with the size of the new level of matchings + size = static_cast(std::distance(matchIt, matchedVisitedNodes.end())); + } + } + + if (!patternFound){ + // Pattern subgraph is not matched. + // Switch to the next combination of starting points + ++i; + continue; + } + + SubgraphMatch::M inputApiMatch; + SubgraphMatch::M outputApiMatch; + + // Traversing current result for starting OPs + for (auto it = subgraphStartOps.begin(); + it != subgraphStartOps.end() && patternFound; ++it) { + const auto& match = *it; + auto patternInputEdges = match.first->inEdges(); + auto testInputEdges = match.second->inEdges(); + + SubgraphMatch::S patternUniqInNodes(match.first->inNodes().begin(), + match.first->inNodes().end()); + SubgraphMatch::S testUniqInNodes(match.second->inNodes().begin(), + match.second->inNodes().end()); + + if (patternUniqInNodes.size() < testUniqInNodes.size()) { + inputApiMatch.clear(); + patternFound = false; + break; + } + // Else, patternInNodes.size() > testInNodes.size() is considered as valid case. + + // Match pattern input DATA nodes with boundary matched test DATA nodes. + for (const auto& patternInEdge : patternInputEdges) { + + // Not all start OP nodes are located in the beginning of the pattern graph + // Start OP may have one input DATA node as an Protocol IN node and other + // input DATA nodes produced from another operations + if (!IS_STARTPOINT(patternInEdge->srcNode())) { + continue; + } + + auto patternInputPort = + patternGraph.metadata(patternInEdge).get().port; + + auto matchedIt = std::find_if(testInputEdges.begin(), testInputEdges.end(), + [&](const ade::EdgeHandle& testInEdge) -> bool { + auto testInputPort = + testGraph.metadata(testInEdge).get().port; + + if (patternInputPort != testInputPort) { + return false; + } + + auto foundIt = inputApiMatch.find(patternInEdge->srcNode()); + if (foundIt != inputApiMatch.end()) { + if (testInEdge->srcNode() != foundIt->second) { + return false; + } + return true; + } + + // Update inputApiMatch map only if the pair of nodes isn't in the map already + inputApiMatch[patternInEdge->srcNode()] = testInEdge->srcNode(); + return true; + }); + + if (matchedIt == testInputEdges.end()) { + inputApiMatch.clear(); + patternFound = false; + break; + } + } // Loop traversed patternInputEdges + } // Loop traversed sugraphStartOps + + if (!patternFound) { + // Pattern IN data nodes can not be matched. + // Switch to the next combination of starting points + ++i; + continue; + } + + // Create vector with the correctly ordered IN data nodes in the test subgraph + std::vector inputTestDataNodes; + for (const auto& patternInNode : patternInputDataNodes) { + inputTestDataNodes.push_back(inputApiMatch[patternInNode]); + } + + // Traversing current result for ending OPs + // There is an assumption that if the pattern subgraph is matched, then + // OUT data nodes shall be definitely matched + for (const auto& match : subgraphEndOps) { + auto patternOutputEdges = match.first->outEdges(); + auto testOutputEdges = match.second->outEdges(); + + GAPI_Assert(patternOutputEdges.size() == testOutputEdges.size() + && + "Ending OP nodes are matched, so OPs' outputs count shall be the same!"); + + // Match pattern output DATA nodes with boundary matched test DATA nodes. + for (const auto& patternOutEdge : patternOutputEdges) { + + // Not all end OP nodes are located in the ending of the pattern graph + // End OP node may have one output DATA node as an Protocol OUT node and other + // output DATA nodes as input for another operations + if (!IS_ENDPOINT(patternOutEdge->dstNode())) { + continue; + } + + auto patternOutputPort = + patternGraph.metadata(patternOutEdge).get().port; + + auto matchedIt = std::find_if(testOutputEdges.begin(), testOutputEdges.end(), + [&](const ade::EdgeHandle& testOutEdge) -> bool { + auto testOutputPort = + testGraph.metadata(testOutEdge).get().port; + + if (patternOutputPort != testOutputPort) { + return false; + } + + outputApiMatch[patternOutEdge->dstNode()] = testOutEdge->dstNode(); + return true; + }); + + GAPI_Assert(matchedIt != testOutputEdges.end() + && + "There shall be a match for every OUT data node from ending OP node," + "if ending OP node matches"); + } + + } + + // Create vector with the correctly ordered OUT data nodes in the test subgraph + std::vector outputTestDataNodes; + for (const auto& patternOutNode : patternOutputDataNodes) { + outputTestDataNodes.push_back(outputApiMatch[patternOutNode]); + } + + SubgraphMatch subgraph; + + subgraph.inputDataNodes = std::move(inputApiMatch); + subgraph.startOpNodes = std::move(subgraphStartOps); + subgraph.internalLayers = std::move(subgraphInternals); + subgraph.finishOpNodes = std::move(subgraphEndOps); + subgraph.outputDataNodes = std::move(outputApiMatch); + + subgraph.inputTestDataNodes = std::move(inputTestDataNodes); + subgraph.outputTestDataNodes = std::move(outputTestDataNodes); + + return subgraph; + + } + + return SubgraphMatch { }; +} diff --git a/modules/gapi/src/compiler/passes/pattern_matching.hpp b/modules/gapi/src/compiler/passes/pattern_matching.hpp new file mode 100644 index 0000000000..0d1537ee74 --- /dev/null +++ b/modules/gapi/src/compiler/passes/pattern_matching.hpp @@ -0,0 +1,98 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2019 Intel Corporation + + +#ifndef OPENCV_GAPI_PATTERN_MATCHING_HPP +#define OPENCV_GAPI_PATTERN_MATCHING_HPP + +#include +#include +#include +#include + +#include "compiler/gmodel.hpp" + +namespace cv { +namespace gimpl { + + struct SubgraphMatch { + using M = std::unordered_map< ade::NodeHandle // Pattern graph node + , ade::NodeHandle // Test graph node + , ade::HandleHasher + >; + using S = std::unordered_set< ade::NodeHandle + , ade::HandleHasher + >; + M inputDataNodes; + M startOpNodes; + M finishOpNodes; + M outputDataNodes; + + std::vector inputTestDataNodes; + std::vector outputTestDataNodes; + + std::list internalLayers; + + // FIXME: switch to operator bool() instead + bool ok() const { + return !inputDataNodes.empty() && !startOpNodes.empty() + && !finishOpNodes.empty() && !outputDataNodes.empty() + && !inputTestDataNodes.empty() && !outputTestDataNodes.empty(); + + } + + S nodes() const { + S allNodes {}; + + allNodes.insert(inputTestDataNodes.begin(), inputTestDataNodes.end()); + + for (const auto& startOpMatch : startOpNodes) { + allNodes.insert(startOpMatch.second); + } + + for (const auto& finishOpMatch : finishOpNodes) { + allNodes.insert(finishOpMatch.second); + } + + allNodes.insert(outputTestDataNodes.begin(), outputTestDataNodes.end()); + + allNodes.insert(internalLayers.begin(), internalLayers.end()); + + return allNodes; + } + + S startOps() { + S sOps; + for (const auto& opMatch : startOpNodes) { + sOps.insert(opMatch.second); + } + return sOps; + } + + S finishOps() { + S fOps; + for (const auto& opMatch : finishOpNodes) { + fOps.insert(opMatch.second); + } + return fOps; + } + + std::vector protoIns() { + return inputTestDataNodes; + } + + + std::vector protoOuts() { + return outputTestDataNodes; + } + }; + + GAPI_EXPORTS SubgraphMatch findMatches(const cv::gimpl::GModel::Graph& patternGraph, + const cv::gimpl::GModel::Graph& compGraph); + +} //namespace gimpl +} //namespace cv +#endif // OPENCV_GAPI_PATTERN_MATCHING_HPP diff --git a/modules/gapi/test/common/gapi_core_tests_inl.hpp b/modules/gapi/test/common/gapi_core_tests_inl.hpp index 5b22ef98b5..9ce0e59ef8 100644 --- a/modules/gapi/test/common/gapi_core_tests_inl.hpp +++ b/modules/gapi/test/common/gapi_core_tests_inl.hpp @@ -14,7 +14,7 @@ namespace opencv_test { -TEST_P(MathOpTest, MatricesAccuracyTest ) +TEST_P(MathOpTest, MatricesAccuracyTest) { // G-API code & corresponding OpenCV code //////////////////////////////// cv::GMat in1, in2, out; diff --git a/modules/gapi/test/internal/gapi_int_pattern_matching_test.cpp b/modules/gapi/test/internal/gapi_int_pattern_matching_test.cpp new file mode 100644 index 0000000000..841caed9df --- /dev/null +++ b/modules/gapi/test/internal/gapi_int_pattern_matching_test.cpp @@ -0,0 +1,942 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2019 Intel Corporation + + +#include "../test_precomp.hpp" + +#include + +#include "compiler/gmodel.hpp" +#include "compiler/gmodel_priv.hpp" + +#include "api/gcomputation_priv.hpp" +#include "compiler/gcompiler.hpp" +#include "compiler/gmodelbuilder.hpp" +#include "compiler/passes/passes.hpp" + +#include "compiler/passes/pattern_matching.hpp" + +#include "../common/gapi_tests_common.hpp" + +#include "logger.hpp" + +namespace opencv_test +{ + +namespace matching_test { +namespace { +using V = std::vector; +using S = std::unordered_set< ade::NodeHandle + , ade::HandleHasher + >; + +void initGModel(ade::Graph& gr, + cv::GProtoInputArgs&& in, + cv::GProtoOutputArgs&& out) { + + cv::gimpl::GModel::Graph gm(gr); + cv::gimpl::GModel::init(gm); + auto proto_slots = cv::gimpl::GModelBuilder(gr) + .put(in.m_args, out.m_args); + + cv::gimpl::Protocol p; + std::tie(p.inputs, p.outputs, p.in_nhs, p.out_nhs) = proto_slots; + gm.metadata().set(p); +} + +bool isConsumedBy(cv::gimpl::GModel::Graph gm, ade::NodeHandle data_nh, ade::NodeHandle op_nh) { + auto oi = cv::gimpl::GModel::orderedInputs(gm, op_nh); + return std::find(oi.begin(), oi.end(), data_nh) != oi.end(); +} + +std::string opName(cv::gimpl::GModel::Graph gm, ade::NodeHandle op_nh) { + return gm.metadata(op_nh).get().k.name; +} + +} +} // matching_test + +TEST(PatternMatching, TestFuncDoesNotChangeTestGraph) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat out = cv::gapi::bitwise_not(in); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(out)); + } + + // Test + ade::Graph tg; + GMat in; + GMat out = cv::gapi::bitwise_not(in); + matching_test::initGModel(tg, cv::GIn(in), cv::GOut(out)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + matching_test::S nodes{ tgm.nodes().begin(), tgm.nodes().end() }; + + const auto in_nh = cv::gimpl::GModel::dataNodeOf(tgm, in); + const auto out_nh = cv::gimpl::GModel::dataNodeOf(tgm, out); + + auto input_data_nhs = tgm.metadata().get().in_nhs; + auto output_data_nhs = tgm.metadata().get().out_nhs; + + EXPECT_EQ(1u, input_data_nhs.size()); + EXPECT_EQ(1u, output_data_nhs.size()); + EXPECT_EQ(in_nh, *input_data_nhs.begin()); + EXPECT_EQ(out_nh, *output_data_nhs.begin()); + EXPECT_EQ(0u, in_nh->inEdges().size()); + EXPECT_EQ(0u, out_nh->outEdges().size()); + EXPECT_EQ(1u, in_nh->outEdges().size()); + EXPECT_EQ(1u, out_nh->inEdges().size()); + + const auto op_nh = cv::gimpl::GModel::producerOf(tgm, out_nh); //bitwise_not + EXPECT_EQ(cv::gapi::core::GNot::id(), matching_test::opName(tgm, op_nh)); + EXPECT_EQ(1u, op_nh->inEdges().size()); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, in_nh, op_nh)); + EXPECT_EQ(1u, op_nh->outEdges().size()); +} + +TEST(PatternMatching, TestSimple1) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat out = cv::gapi::bitwise_not(in); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(out)); + } + + // Test + ade::Graph tg; + GMat in; + GMat out = cv::gapi::bitwise_not(in); + matching_test::initGModel(tg, cv::GIn(in), cv::GOut(out)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(3u, nodes.size()); + + const auto in_nh = cv::gimpl::GModel::dataNodeOf(tgm, in); + const auto out_nh = cv::gimpl::GModel::dataNodeOf(tgm, out); + const auto op_nh = cv::gimpl::GModel::producerOf(tgm, out_nh); + + EXPECT_EQ(matching_test::S({in_nh, out_nh, op_nh}), nodes); + EXPECT_EQ(cv::gapi::core::GNot::id(), matching_test::opName(tgm, op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, in_nh, op_nh)); + EXPECT_EQ(matching_test::S{op_nh}, match.startOps()); + EXPECT_EQ(matching_test::S{op_nh}, match.finishOps()); + EXPECT_EQ(matching_test::V{in_nh}, match.protoIns()); + EXPECT_EQ(matching_test::V{out_nh}, match.protoOuts()); +} + +TEST(PatternMatching, TestSimple2) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat out = cv::gapi::bitwise_not(in); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(out)); + } + + // Test + ade::Graph tg; + GMat in; + GMat tmp = cv::gapi::bitwise_not(in); + GMat out = cv::gapi::blur(tmp, cv::Size(3, 3)); + matching_test::initGModel(tg, cv::GIn(in), cv::GOut(out)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(3u, nodes.size()); + + const auto in_nh = cv::gimpl::GModel::dataNodeOf(tgm, in); + const auto tmp_nh = cv::gimpl::GModel::dataNodeOf(tgm, tmp); + const auto op_nh = cv::gimpl::GModel::producerOf(tgm, tmp_nh); + + EXPECT_EQ(matching_test::S({in_nh, tmp_nh, op_nh}), nodes); + EXPECT_EQ(cv::gapi::core::GNot::id(), matching_test::opName(tgm, op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, in_nh, op_nh)); + EXPECT_EQ(matching_test::S{op_nh}, match.startOps()); + EXPECT_EQ(matching_test::S{op_nh}, match.finishOps()); + EXPECT_EQ(matching_test::V{in_nh}, match.protoIns()); + EXPECT_EQ(matching_test::V{tmp_nh}, match.protoOuts()); +} + +TEST(PatternMatching, TestSimple3) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat out = cv::gapi::bitwise_not(in); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(out)); + } + + // Test + ade::Graph tg; + GMat in; + GMat tmp = cv::gapi::blur(in, cv::Size(3, 3)); + GMat out = cv::gapi::bitwise_not(tmp); + matching_test::initGModel(tg, cv::GIn(in), cv::GOut(out)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(3u, nodes.size()); + + const auto tmp_nh = cv::gimpl::GModel::dataNodeOf(tgm, tmp); + const auto out_nh = cv::gimpl::GModel::dataNodeOf(tgm, out); + const auto op_nh = cv::gimpl::GModel::producerOf(tgm, out_nh); + + EXPECT_EQ(matching_test::S({tmp_nh, out_nh, op_nh}), nodes); + EXPECT_EQ(cv::gapi::core::GNot::id(), matching_test::opName(tgm, op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, tmp_nh, op_nh)); + EXPECT_EQ(matching_test::S{op_nh}, match.startOps()); + EXPECT_EQ(matching_test::S{op_nh}, match.finishOps()); + EXPECT_EQ(matching_test::V{tmp_nh}, match.protoIns()); + EXPECT_EQ(matching_test::V{out_nh}, match.protoOuts()); +} + +TEST(PatternMatching, TestMultiplePatternOuts) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat dx, dy; + std::tie(dx, dy) = cv::gapi::SobelXY(in, -1, 1); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(dx, dy)); + } + + // Test + ade::Graph tg; + GMat in; + GMat dx, dy; + std::tie(dx, dy) = cv::gapi::SobelXY(in, -1, 1); + matching_test::initGModel(tg, cv::GIn(in), cv::GOut(dx, dy)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(4u, nodes.size()); + + const auto in_nh = cv::gimpl::GModel::dataNodeOf(tgm, in); + const auto dx_nh = cv::gimpl::GModel::dataNodeOf(tgm, dx); + const auto dy_nh = cv::gimpl::GModel::dataNodeOf(tgm, dy); + const auto op_nh = cv::gimpl::GModel::producerOf(tgm, dx_nh); + EXPECT_EQ(op_nh, cv::gimpl::GModel::producerOf(tgm, dy_nh)); + + EXPECT_EQ(matching_test::S({in_nh, dx_nh, dy_nh, op_nh}), nodes); + EXPECT_EQ(cv::gapi::imgproc::GSobelXY::id(), matching_test::opName(tgm, op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, in_nh, op_nh)); + EXPECT_EQ(matching_test::S{op_nh}, match.startOps()); + EXPECT_EQ(matching_test::S{op_nh}, match.finishOps()); + EXPECT_EQ(matching_test::V{in_nh}, match.protoIns()); + EXPECT_EQ(matching_test::V({dx_nh, dy_nh}), match.protoOuts()); +} + +TEST(PatternMatching, TestPreprocSplit3) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat tmp = cv::gapi::resize(in, cv::Size{224, 224}); + GMat b, g, r; + std::tie(b, g, r) = cv::gapi::split3(tmp); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(b, g, r)); + } + + // Test + ade::Graph tg; + GMat y, uv; + GMat bgr = cv::gapi::NV12toBGR(y, uv); + GMat tmp = cv::gapi::resize(bgr, cv::Size{224, 224}); + GMat b, g, r; + std::tie(b, g, r) = cv::gapi::split3(tmp); + matching_test::initGModel(tg, cv::GIn(y, uv), cv::GOut(b, g, r)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(7u, nodes.size()); + + const auto bgr_nh = cv::gimpl::GModel::dataNodeOf(tgm, bgr); + const auto tmp_nh = cv::gimpl::GModel::dataNodeOf(tgm, tmp); + const auto b_nh = cv::gimpl::GModel::dataNodeOf(tgm, b); + const auto g_nh = cv::gimpl::GModel::dataNodeOf(tgm, g); + const auto r_nh = cv::gimpl::GModel::dataNodeOf(tgm, r); + + const auto op1_nh = cv::gimpl::GModel::producerOf(tgm, tmp_nh); // 1st resize + const auto op2_nh = cv::gimpl::GModel::producerOf(tgm, b_nh); // 2nd split3 + EXPECT_EQ(op2_nh, cv::gimpl::GModel::producerOf(tgm, g_nh)); + EXPECT_EQ(op2_nh, cv::gimpl::GModel::producerOf(tgm, r_nh)); + + EXPECT_EQ(matching_test::S({bgr_nh, tmp_nh, b_nh, g_nh, + r_nh, op1_nh, op2_nh}), + nodes); + + EXPECT_EQ(cv::gapi::core::GResize::id(), matching_test::opName(tgm, op1_nh)); + EXPECT_EQ(cv::gapi::core::GSplit3::id(), matching_test::opName(tgm, op2_nh)); + + EXPECT_EQ(1u, tmp_nh->outEdges().size()); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, bgr_nh, op1_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, tmp_nh, op2_nh)); + + EXPECT_EQ(matching_test::S{ op1_nh }, match.startOps()); + EXPECT_EQ(matching_test::S{ op2_nh }, match.finishOps()); + EXPECT_EQ(matching_test::V{ bgr_nh }, match.protoIns()); + EXPECT_EQ(matching_test::V({ b_nh, g_nh, r_nh }), match.protoOuts()); +} + +G_TYPED_KERNEL(GToNCHW, , "test.toNCHW") { + static GMatDesc outMeta(GMatDesc in) { + GAPI_Assert(in.depth == CV_8U); + GAPI_Assert(in.chan == 3); + GAPI_Assert(in.planar == false); + return in.asPlanar(); + } +}; + +static GMat toNCHW(const GMat& src) +{ + return GToNCHW::on(src); +} + +TEST(PatternMatching, TestPreprocToNCHW) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat tmp = cv::gapi::resize(in, cv::Size{224, 224}); + GMat plr = toNCHW(tmp); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(plr)); + } + + // Test + ade::Graph tg; + GMat y, uv; + GMat bgr = cv::gapi::NV12toBGR(y, uv); + GMat tmp = cv::gapi::resize(bgr, cv::Size{224, 224}); + GMat plr = toNCHW(tmp); + matching_test::initGModel(tg, cv::GIn(y, uv), cv::GOut(plr)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(5u, nodes.size()); + + const auto bgr_nh = cv::gimpl::GModel::dataNodeOf(tgm, bgr); + const auto tmp_nh = cv::gimpl::GModel::dataNodeOf(tgm, tmp); + const auto plr_nh = cv::gimpl::GModel::dataNodeOf(tgm, plr); + + const auto op1_nh = cv::gimpl::GModel::producerOf(tgm, tmp_nh); // 1st resize + const auto op2_nh = cv::gimpl::GModel::producerOf(tgm, plr_nh); // 2nd toNCHW + + EXPECT_EQ(matching_test::S({bgr_nh, tmp_nh, plr_nh, op1_nh, op2_nh}), + nodes); + + EXPECT_EQ(cv::gapi::core::GResize::id(), matching_test::opName(tgm, op1_nh)); + EXPECT_EQ(GToNCHW::id(), matching_test::opName(tgm, op2_nh)); + + EXPECT_EQ(1u, tmp_nh->outEdges().size()); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, bgr_nh, op1_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, tmp_nh, op2_nh)); + + EXPECT_EQ(matching_test::S{ op1_nh }, match.startOps()); + EXPECT_EQ(matching_test::S{ op2_nh }, match.finishOps()); + EXPECT_EQ(matching_test::V{ bgr_nh }, match.protoIns()); + EXPECT_EQ(matching_test::V{ plr_nh }, match.protoOuts()); +} + +//FIXME: To switch from filter2d kernel (which shall be matched by params too) to another one +TEST(PatternMatching, MatchChainInTheMiddle) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat tmp = cv::gapi::filter2D(in, -1, {}); + GMat out = cv::gapi::filter2D(tmp, -1, {}); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(out)); + } + + // Test + ade::Graph tg; + GMat in; + GMat tmp1 = cv::gapi::erode3x3(in); + GMat tmp2 = cv::gapi::filter2D(tmp1, -1, {}); + GMat tmp3 = cv::gapi::filter2D(tmp2, -1, {}); + GMat out = cv::gapi::dilate3x3(tmp3); + matching_test::initGModel(tg, cv::GIn(in), cv::GOut(out)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(5u, nodes.size()); + + const auto tmp1_nh = cv::gimpl::GModel::dataNodeOf(tgm, tmp1); + const auto tmp2_nh = cv::gimpl::GModel::dataNodeOf(tgm, tmp2); + const auto tmp3_nh = cv::gimpl::GModel::dataNodeOf(tgm, tmp3); + const auto op1_nh = cv::gimpl::GModel::producerOf(tgm, tmp2_nh); // 1st filter2D + const auto op2_nh = cv::gimpl::GModel::producerOf(tgm, tmp3_nh); // 2nd filter2D + + EXPECT_EQ(matching_test::S({tmp1_nh, tmp2_nh, tmp3_nh, op1_nh, op2_nh}), nodes); + + EXPECT_EQ(cv::gapi::imgproc::GFilter2D::id(), matching_test::opName(tgm, op1_nh)); + EXPECT_EQ(cv::gapi::imgproc::GFilter2D::id(), matching_test::opName(tgm, op2_nh)); + + EXPECT_EQ(1u, tmp2_nh->outEdges().size()); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, tmp1_nh, op1_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, tmp2_nh, op2_nh)); + + EXPECT_EQ(matching_test::S({op1_nh}), match.startOps()); + EXPECT_EQ(matching_test::S({op2_nh}), match.finishOps()); + EXPECT_EQ(matching_test::V{ tmp1_nh }, match.protoIns()); + EXPECT_EQ(matching_test::V{ tmp3_nh }, match.protoOuts()); +} + +TEST(PatternMatching, TestMultipleStartOps1) +{ + // Pattern + ade::Graph pg; + { + GMat in1, in2; + GMat er = cv::gapi::erode3x3(in1); + GMat dil = cv::gapi::dilate3x3(in2); + GMat out = cv::gapi::add(er, dil); + matching_test::initGModel(pg, cv::GIn(in1, in2), cv::GOut(out)); + } + + // Test + ade::Graph tg; + + GMat in1, in2, in3, in4, in5, in6; + GMat er1 = cv::gapi::erode3x3(in1); + GMat er2 = cv::gapi::erode3x3(in2); + GMat er3 = cv::gapi::erode3x3(in3); + GMat er4 = cv::gapi::erode3x3(in4); + GMat dil1 = cv::gapi::dilate3x3(in5); + GMat dil2 = cv::gapi::dilate3x3(in6); + GMat out1 = cv::gapi::add(er1, er2); + GMat out2 = cv::gapi::add(er3, dil2); + matching_test::initGModel(tg, cv::GIn(in1, in2, in3, in4, in5, in6), cv::GOut(out1, out2, er4, dil1)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(8u, nodes.size()); + + const auto in3_nh = cv::gimpl::GModel::dataNodeOf(tgm, in3); + const auto in6_nh = cv::gimpl::GModel::dataNodeOf(tgm, in6); + const auto er3_nh = cv::gimpl::GModel::dataNodeOf(tgm, er3); + const auto dil2_nh = cv::gimpl::GModel::dataNodeOf(tgm, dil2); + const auto out2_nh = cv::gimpl::GModel::dataNodeOf(tgm, out2); + + const auto er_op_nh = cv::gimpl::GModel::producerOf(tgm, er3_nh); + const auto dil_op_nh = cv::gimpl::GModel::producerOf(tgm, dil2_nh); + const auto add_op_nh = cv::gimpl::GModel::producerOf(tgm, out2_nh); + + EXPECT_EQ(matching_test::S({in3_nh, in6_nh, er3_nh, dil2_nh, out2_nh, + er_op_nh, dil_op_nh, add_op_nh}), + nodes); + + EXPECT_EQ(cv::gapi::imgproc::GErode::id(), matching_test::opName(tgm, er_op_nh)); + EXPECT_EQ(cv::gapi::imgproc::GDilate::id(), matching_test::opName(tgm, dil_op_nh)); + EXPECT_EQ(cv::gapi::core::GAdd::id(), matching_test::opName(tgm, add_op_nh)); + + EXPECT_EQ(1u, er3_nh->outEdges().size()); + EXPECT_EQ(1u, dil2_nh->outEdges().size()); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, in3_nh, er_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, in6_nh, dil_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, er3_nh, add_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, dil2_nh, add_op_nh)); + + EXPECT_EQ(matching_test::S({ er_op_nh, dil_op_nh }), match.startOps()); + EXPECT_EQ(matching_test::S{ add_op_nh }, match.finishOps()); + EXPECT_EQ(matching_test::V({ in3_nh, in6_nh }), match.protoIns()); + EXPECT_EQ(matching_test::V{ out2_nh }, match.protoOuts()); +} + +TEST(PatternMatching, TestMultipleStartOps2) +{ + // Pattern + ade::Graph pg; + { + GMat in1, in2; + GMat er = cv::gapi::erode3x3(in1); + GMat dil = cv::gapi::dilate3x3(in2); + GMat out = cv::gapi::add(er, dil); + matching_test::initGModel(pg, cv::GIn(in1, in2), cv::GOut(out)); + } + + // Test + ade::Graph tg; + + GMat in1, in2; + GMat er = cv::gapi::erode3x3(in1); + GMat dil1 = cv::gapi::dilate3x3(in2); + GMat dil2 = cv::gapi::dilate3x3(dil1); + GMat out = cv::gapi::add(er, dil2); + matching_test::initGModel(tg, cv::GIn(in1, in2), cv::GOut(out)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(8u, nodes.size()); + + const auto in1_nh = cv::gimpl::GModel::dataNodeOf(tgm, in1); + const auto dil1_nh = cv::gimpl::GModel::dataNodeOf(tgm, dil1); + const auto er_nh = cv::gimpl::GModel::dataNodeOf(tgm, er); + const auto dil2_nh = cv::gimpl::GModel::dataNodeOf(tgm, dil2); + const auto out_nh = cv::gimpl::GModel::dataNodeOf(tgm, out); + + const auto er_op_nh = cv::gimpl::GModel::producerOf(tgm, er_nh); + const auto dil_op_nh = cv::gimpl::GModel::producerOf(tgm, dil2_nh); + const auto add_op_nh = cv::gimpl::GModel::producerOf(tgm, out_nh); + + EXPECT_EQ(matching_test::S({in1_nh, dil1_nh, er_nh, dil2_nh, out_nh, + er_op_nh, dil_op_nh, add_op_nh}), + nodes); + + EXPECT_EQ(cv::gapi::imgproc::GErode::id(), matching_test::opName(tgm, er_op_nh)); + EXPECT_EQ(cv::gapi::imgproc::GDilate::id(), matching_test::opName(tgm, dil_op_nh)); + EXPECT_EQ(cv::gapi::core::GAdd::id(), matching_test::opName(tgm, add_op_nh)); + + EXPECT_EQ(1u, er_nh->outEdges().size()); + EXPECT_EQ(1u, dil2_nh->outEdges().size()); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, in1_nh, er_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, dil1_nh, dil_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, er_nh, add_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, dil2_nh, add_op_nh)); + + EXPECT_EQ(matching_test::S({ er_op_nh, dil_op_nh }), match.startOps()); + EXPECT_EQ(matching_test::S{ add_op_nh }, match.finishOps()); + EXPECT_EQ(matching_test::V({ in1_nh, dil1_nh }), match.protoIns()); + EXPECT_EQ(matching_test::V{ out_nh }, match.protoOuts()); +} + +TEST(PatternMatching, TestInexactMatchOfInOutData) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat out = cv::gapi::dilate3x3(in); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(out)); + } + + // Test + ade::Graph tg; + GMat in; + GMat out1 = cv::gapi::erode3x3(in); + GMat out2 = cv::gapi::boxFilter(in, -1, cv::Size(3, 3)); + GMat tmp = cv::gapi::dilate3x3(in); + GScalar out3 = cv::gapi::sum(tmp); + GScalar out4 = cv::gapi::mean(tmp); + matching_test::initGModel(tg, cv::GIn(in), cv::GOut(out1, out2, out3, out4)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(3u, nodes.size()); + + const auto in_nh = cv::gimpl::GModel::dataNodeOf(tgm, in); + const auto tmp_nh = cv::gimpl::GModel::dataNodeOf(tgm, tmp); + + const auto op_nh = cv::gimpl::GModel::producerOf(tgm, tmp_nh); // dilate3x3 + + EXPECT_EQ(matching_test::S({in_nh, tmp_nh, op_nh}), + nodes); + + EXPECT_EQ(cv::gapi::imgproc::GDilate::id(), matching_test::opName(tgm, op_nh)); + + EXPECT_TRUE(matching_test::isConsumedBy(tgm, in_nh, op_nh)); + + + EXPECT_EQ(matching_test::S{ op_nh }, match.startOps()); + EXPECT_EQ(matching_test::S{ op_nh }, match.finishOps()); + EXPECT_EQ(matching_test::V{ in_nh }, match.protoIns()); + EXPECT_EQ(matching_test::V{ tmp_nh }, match.protoOuts()); + + EXPECT_GT(in_nh->outEdges().size(), 1u); + EXPECT_GT(tmp_nh->outEdges().size(), 1u); +} + +//FIXME: The start ops matching shall be reworked to more smarter way. +// Start ops matching shall get rid of non valid matchings sample, +// where two identical start ops in the pattern refer to the only one in the test. +TEST(PatternMatching, TestManySameStartOpsAndHinge) +{ + // Pattern + ade::Graph pg; + { + GMat in1, in2, in3; + GMat er1 = cv::gapi::erode3x3(in1); + GMat er2 = cv::gapi::erode3x3(in2); + GMat er3 = cv::gapi::erode3x3(in3); + GMat mrg = cv::gapi::merge3(er1, er2, er3); + matching_test::initGModel(pg, cv::GIn(in1, in2, in3), cv::GOut(mrg)); + } + + // Test + ade::Graph tg; + GMat in1, in2, in3; + GMat er1 = cv::gapi::erode3x3(in1); + GMat er2 = cv::gapi::erode3x3(in2); + GMat er3 = cv::gapi::erode3x3(in3); + GMat mrg = cv::gapi::merge3(er1, er2, er3); + matching_test::initGModel(tg, cv::GIn(in1, in2, in3), cv::GOut(mrg)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(11u, nodes.size()); + EXPECT_EQ(matching_test::S(tgm.nodes().begin(), tgm.nodes().end()), + nodes); +} + +//FIXME: The start ops matching shall be reworked to more smarter way. +// Start ops matching shall get rid of non valid matchings sample, +// where two identical start ops in the pattern refer to the only one in the test. +TEST(PatternMatching, TestManySameStartOpsAndHinge2) +{ + // Pattern + ade::Graph pg; + { + GMat in1, in2, in3; + GMat er1 = cv::gapi::erode3x3(in1); + GMat er2 = cv::gapi::erode3x3(in2); + GMat er3 = cv::gapi::erode3x3(in3); + GMat dil1 = cv::gapi::dilate3x3(er1); + GMat dil2 = cv::gapi::dilate3x3(er2); + GMat dil3 = cv::gapi::dilate3x3(er3); + GMat mrg = cv::gapi::merge3(dil1, dil2, dil3); + matching_test::initGModel(pg, cv::GIn(in1, in2, in3), cv::GOut(mrg)); + } + + // Test + ade::Graph tg; + GMat in1, in2, in3; + GMat er1 = cv::gapi::erode3x3(in1); + GMat er2 = cv::gapi::erode3x3(in2); + GMat er3 = cv::gapi::erode3x3(in3); + GMat dil1 = cv::gapi::dilate3x3(er1); + GMat dil2 = cv::gapi::dilate3x3(er2); + GMat dil3 = cv::gapi::dilate3x3(er3); + GMat mrg = cv::gapi::merge3(dil1, dil2, dil3); + matching_test::initGModel(tg, cv::GIn(in1, in2, in3), cv::GOut(mrg)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(17u, nodes.size()); + EXPECT_EQ(matching_test::S(tgm.nodes().begin(), tgm.nodes().end()), + nodes); +} + +//FIXME: The start ops matching shall be reworked to more smarter way. +// Start ops matching shall get rid of non valid matchings sample, +// where two identical start ops in the pattern refer to the only one in the test. +TEST(PatternMatching, TestTwoChainsOnTheHingeIsomorphism) +{ + // Pattern + ade::Graph pg; + { + GMat in1, in2; + GMat er1 = cv::gapi::erode3x3(in1); + GMat er2 = cv::gapi::erode3x3(in2); + GMat mdb = cv::gapi::medianBlur(er1, 3); + GMat gb = cv::gapi::gaussianBlur(er2, cv::Size(5, 5), 0.12); + GMat conc = cv::gapi::concatVert(mdb, gb); + matching_test::initGModel(pg, cv::GIn(in1, in2), cv::GOut(conc)); + } + + // Test + ade::Graph tg; + GMat in1, in2; + GMat er1 = cv::gapi::erode3x3(in1); + GMat er2 = cv::gapi::erode3x3(in2); + GMat gb = cv::gapi::gaussianBlur(er1, cv::Size(5, 5), 0.12); + GMat mdb = cv::gapi::medianBlur(er2, 3); + GMat conc = cv::gapi::concatVert(mdb, gb); + matching_test::initGModel(tg, cv::GIn(in1, in2), cv::GOut(conc)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(12u, nodes.size()); + EXPECT_EQ(matching_test::S(tgm.nodes().begin(), tgm.nodes().end()), + nodes); + + const auto in1_nh = cv::gimpl::GModel::dataNodeOf(tgm, in1); + const auto in2_nh = cv::gimpl::GModel::dataNodeOf(tgm, in2); + + EXPECT_EQ(matching_test::V({ in2_nh, in1_nh }), match.protoIns()); +} + +TEST(PatternMatching, TestPatternHasMoreInDataNodes) +{ + // Pattern + ade::Graph pg; + { + GMat in1, in2, in3; + GMat out = cv::gapi::merge3(in1, in2, in3); + matching_test::initGModel(pg, cv::GIn(in1, in2, in3), cv::GOut(out)); + } + + // Test + ade::Graph tg; + GMat in; + GMat out = cv::gapi::merge3(in, in, in); + matching_test::initGModel(tg, cv::GIn(in), cv::GOut(out)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(3u, nodes.size()); + EXPECT_EQ(matching_test::S(tgm.nodes().begin(), tgm.nodes().end()), + nodes); + + const auto in_nh = cv::gimpl::GModel::dataNodeOf(tgm, in); + + EXPECT_EQ(matching_test::V({ in_nh, in_nh, in_nh }), match.protoIns()); +} + +TEST(PatternMatching, TestPatternHasFewerInDataNodes) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat out = cv::gapi::merge3(in, in, in); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(out)); + } + + // Test + ade::Graph tg; + GMat in1, in2, in3; + GMat out = cv::gapi::merge3(in1, in2, in3); + matching_test::initGModel(tg, cv::GIn(in1, in2, in3), cv::GOut(out)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_FALSE(match.ok()); +} + +TEST(PatternMatching, TestTwoMatchingsOneCorrect) +{ + // Pattern + ade::Graph pg; + { + GMat in1, in2; + GMat n = cv::gapi::bitwise_not(in1); + GMat e = cv::gapi::erode3x3(in1); + GMat d = cv::gapi::dilate3x3(in2); + GMat out = cv::gapi::merge3(n, e, d); + matching_test::initGModel(pg, cv::GIn(in1, in2), cv::GOut(out)); + } + + // Test + ade::Graph tg; + GMat in1, in2; + GMat n = cv::gapi::bitwise_not(in1); + GMat e = cv::gapi::erode3x3(in2); + GMat d = cv::gapi::dilate3x3(in2); + GMat mrg = cv::gapi::merge3(n, e, d); + GMat i, sqi; + std::tie(i, sqi) = cv::gapi::integral(mrg); + GMat n1 = cv::gapi::bitwise_not(i); + GMat e1 = cv::gapi::erode3x3(i); + GMat d1 = cv::gapi::dilate3x3(sqi); + GMat out = cv::gapi::merge3(n1, e1, d1); + matching_test::initGModel(tg, cv::GIn(in1, in2), cv::GOut(out)); + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_TRUE(match.ok()); + + auto nodes = match.nodes(); + EXPECT_EQ(10u, nodes.size()); + + const auto i_nh = cv::gimpl::GModel::dataNodeOf(tgm, i); + const auto sqi_nh = cv::gimpl::GModel::dataNodeOf(tgm, sqi); + const auto n1_nh = cv::gimpl::GModel::dataNodeOf(tgm, n1); + const auto e1_nh = cv::gimpl::GModel::dataNodeOf(tgm, e1); + const auto d1_nh = cv::gimpl::GModel::dataNodeOf(tgm, d1); + const auto out_nh = cv::gimpl::GModel::dataNodeOf(tgm, out); + + const auto n_op_nh = cv::gimpl::GModel::producerOf(tgm, n1_nh); + const auto e_op_nh = cv::gimpl::GModel::producerOf(tgm, e1_nh); + const auto d_op_nh = cv::gimpl::GModel::producerOf(tgm, d1_nh); + const auto m_op_nh = cv::gimpl::GModel::producerOf(tgm, out_nh); + + EXPECT_EQ(matching_test::S({i_nh, sqi_nh, n1_nh, e1_nh, d1_nh, out_nh, + n_op_nh, e_op_nh, d_op_nh, m_op_nh}), nodes); + + EXPECT_EQ(cv::gapi::core::GNot::id(), matching_test::opName(tgm, n_op_nh)); + EXPECT_EQ(cv::gapi::imgproc::GErode::id(), matching_test::opName(tgm, e_op_nh)); + EXPECT_EQ(cv::gapi::imgproc::GDilate::id(), matching_test::opName(tgm, d_op_nh)); + EXPECT_EQ(cv::gapi::core::GMerge3::id(), matching_test::opName(tgm, m_op_nh)); + + EXPECT_TRUE(matching_test::isConsumedBy(tgm, i_nh, n_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, i_nh, e_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, sqi_nh, d_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, n1_nh, m_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, e1_nh, m_op_nh)); + EXPECT_TRUE(matching_test::isConsumedBy(tgm, d1_nh, m_op_nh)); + EXPECT_EQ(1u, n1_nh->outEdges().size()); + EXPECT_EQ(1u, e1_nh->outEdges().size()); + EXPECT_EQ(1u, d1_nh->outEdges().size()); + + EXPECT_EQ(matching_test::S({n_op_nh, e_op_nh, d_op_nh}), match.startOps()); + EXPECT_EQ(matching_test::S{m_op_nh}, match.finishOps()); + EXPECT_EQ(matching_test::V({i_nh, sqi_nh}), match.protoIns()); + EXPECT_EQ(matching_test::V{out_nh}, match.protoOuts());} + +TEST(PatternMatching, CheckNoMatch) +{ + // Pattern + ade::Graph pg; + { + GMat in; + GMat tmp = cv::gapi::filter2D(in, -1, {}); + GMat out = cv::gapi::filter2D(tmp, -1, {}); + matching_test::initGModel(pg, cv::GIn(in), cv::GOut(out)); + } + + // Test + ade::Graph tg; + { + GMat in; + GMat tmp1 = cv::gapi::erode3x3(in); + GMat out = cv::gapi::dilate3x3(tmp1); + matching_test::initGModel(tg, cv::GIn(in), cv::GOut(out)); + } + + // Pattern Matching + cv::gimpl::GModel::Graph pgm(pg); + cv::gimpl::GModel::Graph tgm(tg); + cv::gimpl::SubgraphMatch match = cv::gimpl::findMatches(pg, tg); + + // Inspecting results: + EXPECT_FALSE(match.ok()); +} + +TEST(PatternMatching, adeSmokeTest) +{ + ade::Graph g; + ade::NodeHandle src = g.createNode(); + ade::NodeHandle dst = g.createNode(); + g.link(src, dst); + g.link(src, dst); + + EXPECT_EQ(2u, dst->inNodes().size()); +} + +} // namespace opencv_test