From 3672a14b42d02073ee72ff16e5441b7fa9a2344c Mon Sep 17 00:00:00 2001 From: alexlyulkov Date: Wed, 27 Nov 2024 09:15:20 +0300 Subject: [PATCH] Merge pull request #26394 from alexlyulkov:al/new-engine-tf-parser Modified tensorflow parser for the new dnn engine #26394 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/include/opencv2/dnn/dnn.hpp | 22 +- .../misc/java/test/DnnListRegressionTest.java | 8 +- .../dnn/misc/java/test/DnnTensorFlowTest.java | 27 +- modules/dnn/misc/objc/gen_dict.json | 4 +- modules/dnn/perf/perf_net.cpp | 2 +- modules/dnn/src/dnn_read.cpp | 8 +- modules/dnn/src/layers/convolution_layer.cpp | 3 +- modules/dnn/src/layers/reshape_layer.cpp | 1 - modules/dnn/src/model.cpp | 19 +- modules/dnn/src/net_impl2.cpp | 1 + .../src/tensorflow/tf_graph_simplifier.cpp | 6 +- .../src/tensorflow/tf_graph_simplifier.hpp | 2 +- modules/dnn/src/tensorflow/tf_importer.cpp | 763 +++++++++--------- modules/dnn/test/test_tf_importer.cpp | 37 +- 14 files changed, 493 insertions(+), 410 deletions(-) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 6331353140..bc7f57d0df 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -1090,17 +1090,28 @@ CV__DNN_INLINE_NS_BEGIN * @param config path to the .pbtxt file that contains text graph definition in protobuf format. * Resulting Net object is built by text graph using weights from a binary one that * let us make it more flexible. + * @param engine select DNN engine to be used. With auto selection the new engine is used. + * @param extraOutputs specify model outputs explicitly, in addition to the outputs the graph analyzer finds. + * Please pay attention that the new DNN does not support non-CPU back-ends for now. * @returns Net object. */ - CV_EXPORTS_W Net readNetFromTensorflow(CV_WRAP_FILE_PATH const String &model, CV_WRAP_FILE_PATH const String &config = String()); + CV_EXPORTS_W Net readNetFromTensorflow(CV_WRAP_FILE_PATH const String &model, + CV_WRAP_FILE_PATH const String &config = String(), + int engine=ENGINE_AUTO, + const std::vector& extraOutputs = std::vector()); /** @brief Reads a network model stored in TensorFlow framework's format. * @param bufferModel buffer containing the content of the pb file * @param bufferConfig buffer containing the content of the pbtxt file + * @param engine select DNN engine to be used. With auto selection the new engine is used. + * @param extraOutputs specify model outputs explicitly, in addition to the outputs the graph analyzer finds. + * Please pay attention that the new DNN does not support non-CPU back-ends for now. * @returns Net object. */ CV_EXPORTS_W Net readNetFromTensorflow(const std::vector& bufferModel, - const std::vector& bufferConfig = std::vector()); + const std::vector& bufferConfig = std::vector(), + int engine=ENGINE_AUTO, + const std::vector& extraOutputs = std::vector()); /** @brief Reads a network model stored in TensorFlow framework's format. * @details This is an overloaded member function, provided for convenience. @@ -1109,9 +1120,14 @@ CV__DNN_INLINE_NS_BEGIN * @param lenModel length of bufferModel * @param bufferConfig buffer containing the content of the pbtxt file * @param lenConfig length of bufferConfig + * @param engine select DNN engine to be used. With auto selection the new engine is used. + * @param extraOutputs specify model outputs explicitly, in addition to the outputs the graph analyzer finds. + * Please pay attention that the new DNN does not support non-CPU back-ends for now. */ CV_EXPORTS Net readNetFromTensorflow(const char *bufferModel, size_t lenModel, - const char *bufferConfig = NULL, size_t lenConfig = 0); + const char *bufferConfig = NULL, size_t lenConfig = 0, + int engine=ENGINE_AUTO, + const std::vector& extraOutputs = std::vector()); /** @brief Reads a network model stored in TFLite framework's format. * @param model path to the .tflite file with binary flatbuffers description of the network architecture diff --git a/modules/dnn/misc/java/test/DnnListRegressionTest.java b/modules/dnn/misc/java/test/DnnListRegressionTest.java index 21c7dfb338..81944d1da9 100644 --- a/modules/dnn/misc/java/test/DnnListRegressionTest.java +++ b/modules/dnn/misc/java/test/DnnListRegressionTest.java @@ -69,10 +69,10 @@ public class DnnListRegressionTest extends OpenCVTestCase { Mat inputBlob = Dnn.blobFromImage(image, 1.0, new Size(224, 224), new Scalar(0), true, true); assertNotNull("Converting image to blob failed!", inputBlob); - net.setInput(inputBlob, "input"); + net.setInput(inputBlob, ""); } - public void testSetInputsNames() { + /*public void testSetInputsNames() { List inputs = new ArrayList(); inputs.add("input"); try { @@ -80,12 +80,12 @@ public class DnnListRegressionTest extends OpenCVTestCase { } catch(Exception e) { fail("Net setInputsNames failed: " + e.getMessage()); } - } + }*/ public void testForward() { List outs = new ArrayList(); List outNames = new ArrayList(); - outNames.add("softmax2"); + //outNames.add(""); try { net.forward(outs,outNames); } catch(Exception e) { diff --git a/modules/dnn/misc/java/test/DnnTensorFlowTest.java b/modules/dnn/misc/java/test/DnnTensorFlowTest.java index 469573642b..4c22b5673d 100644 --- a/modules/dnn/misc/java/test/DnnTensorFlowTest.java +++ b/modules/dnn/misc/java/test/DnnTensorFlowTest.java @@ -74,20 +74,15 @@ public class DnnTensorFlowTest extends OpenCVTestCase { } public void testGetLayer() { - List layernames = net.getLayerNames(); - - assertFalse("Test net returned no layers!", layernames.isEmpty()); - - String testLayerName = layernames.get(0); - - DictValue layerId = new DictValue(testLayerName); - - assertEquals("DictValue did not return the string, which was used in constructor!", testLayerName, layerId.getStringValue()); - - Layer layer = net.getLayer(layerId); - - assertEquals("Layer name does not match the expected value!", testLayerName, layer.get_name()); - + List layerNames = net.getLayerNames(); + assertFalse("Test net returned no layers!", layerNames.isEmpty()); + + int layerId = 0; + for (String layerName: layerNames) { + Layer layer = net.getLayer(layerId); + assertEquals("Layer name does not match the expected value!", layerName, layer.get_name()); + layerId++; + } } public void checkInceptionNet(Net net) @@ -98,12 +93,12 @@ public class DnnTensorFlowTest extends OpenCVTestCase { Mat inputBlob = Dnn.blobFromImage(image, 1.0, new Size(224, 224), new Scalar(0), true, true); assertNotNull("Converting image to blob failed!", inputBlob); - net.setInput(inputBlob, "input"); + net.setInput(inputBlob, ""); Mat result = new Mat(); try { net.setPreferableBackend(Dnn.DNN_BACKEND_OPENCV); - result = net.forward("softmax2"); + result = net.forward(""); } catch (Exception e) { fail("DNN forward failed: " + e.getMessage()); diff --git a/modules/dnn/misc/objc/gen_dict.json b/modules/dnn/misc/objc/gen_dict.json index fc426efab5..48c8c92f26 100644 --- a/modules/dnn/misc/objc/gen_dict.json +++ b/modules/dnn/misc/objc/gen_dict.json @@ -7,8 +7,8 @@ "(Net*)readNetFromDarknet:(ByteVector*)bufferCfg bufferModel:(ByteVector*)bufferModel" : { "readNetFromDarknet" : {"name" : "readNetFromDarknetBuffer"} }, "(Net*)readNetFromONNX:(NSString*)onnxFile engine:(int)engine" : { "readNetFromONNX" : {"name" : "readNetFromONNXFile"} }, "(Net*)readNetFromONNX:(ByteVector*)buffer engine:(int)engine" : { "readNetFromONNX" : {"name" : "readNetFromONNXBuffer"} }, - "(Net*)readNetFromTensorflow:(NSString*)model config:(NSString*)config" : { "readNetFromTensorflow" : {"name" : "readNetFromTensorflowFile"} }, - "(Net*)readNetFromTensorflow:(ByteVector*)bufferModel bufferConfig:(ByteVector*)bufferConfig" : { "readNetFromTensorflow" : {"name" : "readNetFromTensorflowBuffer"} }, + "(Net*)readNetFromTensorflow:(NSString*)model config:(NSString*)config engine:(int)engine extraOutputs:(NSArray*)extraOutputs" : { "readNetFromTensorflow" : {"name" : "readNetFromTensorflowFile"} }, + "(Net*)readNetFromTensorflow:(ByteVector*)bufferModel bufferConfig:(ByteVector*)bufferConfig engine:(int)engine extraOutputs:(NSArray*)extraOutputs" : { "readNetFromTensorflow" : {"name" : "readNetFromTensorflowBuffer"} }, "(Net*)readNetFromTFLite:(NSString*)model engine:(int)engine" : { "readNetFromTFLite" : {"name" : "readNetFromTFLiteFile"} }, "(Net*)readNetFromTFLite:(ByteVector*)buffer engine:(int)engine" : { "readNetFromTFLite" : {"name" : "readNetFromTFLiteBuffer"} } }, diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp index be489e627f..01cbd74c3b 100644 --- a/modules/dnn/perf/perf_net.cpp +++ b/modules/dnn/perf/perf_net.cpp @@ -119,7 +119,7 @@ PERF_TEST_P_(DNNTestNetwork, SqueezeNet_v1_1) PERF_TEST_P_(DNNTestNetwork, Inception_5h) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) throw SkipTestException(""); - processNet("dnn/tensorflow_inception_graph.pb", "", cv::Size(224, 224), "softmax2"); + processNet("dnn/tensorflow_inception_graph.pb", "", cv::Size(224, 224)); } PERF_TEST_P_(DNNTestNetwork, SSD) diff --git a/modules/dnn/src/dnn_read.cpp b/modules/dnn/src/dnn_read.cpp index 8aeb10ced8..f28f7d0188 100644 --- a/modules/dnn/src/dnn_read.cpp +++ b/modules/dnn/src/dnn_read.cpp @@ -27,11 +27,11 @@ Net readNet(const String& _model, const String& _config, const String& _framewor { if (modelExt == "pbtxt" || configExt == "pb") std::swap(model, config); - return readNetFromTensorflow(model, config); + return readNetFromTensorflow(model, config, engine); } if (framework == "tflite" || modelExt == "tflite") { - return readNetFromTFLite(model); + return readNetFromTFLite(model, engine); } if (framework == "darknet" || modelExt == "weights" || configExt == "weights" || modelExt == "cfg" || configExt == "cfg") { @@ -63,13 +63,13 @@ Net readNet(const String& _framework, const std::vector& bufferModel, else if (framework == "caffe") return readNetFromCaffe(bufferConfig, bufferModel, engine); else if (framework == "tensorflow") - return readNetFromTensorflow(bufferModel, bufferConfig); + return readNetFromTensorflow(bufferModel, bufferConfig, engine); else if (framework == "darknet") return readNetFromDarknet(bufferConfig, bufferModel); else if (framework == "dldt" || framework == "openvino") return readNetFromModelOptimizer(bufferConfig, bufferModel); else if (framework == "tflite") - return readNetFromTFLite(bufferModel); + return readNetFromTFLite(bufferModel, engine); CV_Error(Error::StsError, "Cannot determine an origin framework with a name " + framework); } diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index cbee52ac70..153415bc25 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -322,7 +322,8 @@ public: internals.clear(); - CV_Assert(inputs.size() != 0); + CV_Assert(!inputs.empty()); + CV_Assert(inputs[0].size() > 2); std::vector inpShape(inputs[0].begin() + 2, inputs[0].end()); int outCn = weightShape[0]; diff --git a/modules/dnn/src/layers/reshape_layer.cpp b/modules/dnn/src/layers/reshape_layer.cpp index 1c31a36f98..eb30a3947a 100644 --- a/modules/dnn/src/layers/reshape_layer.cpp +++ b/modules/dnn/src/layers/reshape_layer.cpp @@ -234,7 +234,6 @@ public: std::vector &outputs, std::vector &internals) const CV_OVERRIDE { - if (inputs.size() == 1 || inputs.size() == requiredOutputs) { outputs.clear(); diff --git a/modules/dnn/src/model.cpp b/modules/dnn/src/model.cpp index c36279e464..e6a259d0d8 100644 --- a/modules/dnn/src/model.cpp +++ b/modules/dnn/src/model.cpp @@ -46,9 +46,19 @@ public: net = network; outNames = net.getUnconnectedOutLayersNames(); + + Ptr graph = net.getMainGraph(); std::vector inLayerShapes; - std::vector outLayerShapes; - net.getLayerShapes(MatShape(), CV_32F, 0, inLayerShapes, outLayerShapes); + + if (graph) { + const std::vector& inputs = graph->inputs(); + for (auto inp: inputs) { + inLayerShapes.push_back(net.argData(inp).shape); + } + } else { + std::vector outLayerShapes; + net.getLayerShapes(MatShape(), CV_32F, 0, inLayerShapes, outLayerShapes); + } if (!inLayerShapes.empty() && inLayerShapes[0].size() == 4) size = Size(inLayerShapes[0][3], inLayerShapes[0][2]); else @@ -1182,6 +1192,11 @@ struct TextDetectionModel_EAST_Impl : public TextDetectionModel_Impl CV_CheckEQ(geometry.dims, 4, ""); CV_CheckEQ(scoreMap.size[0], 1, ""); CV_CheckEQ(geometry.size[0], 1, ""); + + if (geometry.size[1] == 1 && scoreMap.size[1] == 5) { + std::swap(geometry, scoreMap); + } + CV_CheckEQ(scoreMap.size[1], 1, ""); CV_CheckEQ(geometry.size[1], 5, ""); CV_CheckEQ(scoreMap.size[2], geometry.size[2], ""); diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index 003c9e4056..37f1721339 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -199,6 +199,7 @@ Mat& Net::Impl::argTensor(Arg arg) const if (adata.kind == DNN_ARG_TEMP) { CV_Assert(__tensors__.at(arg.idx).empty()); int bufidx = bufidxs.at(arg.idx); + CV_Assert(bufidx >= 0); return const_cast(buffers.at(bufidx)); } return const_cast(__tensors__.at(arg.idx)); diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp index 45fcacbe34..6382efb629 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp @@ -773,7 +773,7 @@ public: } }; -void simplifySubgraphs(tensorflow::GraphDef& net) +void simplifySubgraphs(tensorflow::GraphDef& net, bool newEngine) { std::vector > subgraphs; subgraphs.push_back(Ptr(new BatchNormSubgraph())); @@ -791,7 +791,6 @@ void simplifySubgraphs(tensorflow::GraphDef& net) subgraphs.push_back(Ptr(new UpsamplingKerasSubgraph("ResizeBilinear"))); subgraphs.push_back(Ptr(new SoftMaxSlimSubgraph())); subgraphs.push_back(Ptr(new SoftMaxSlimV2Subgraph())); - subgraphs.push_back(Ptr(new ReshapeAsShapeSubgraph())); subgraphs.push_back(Ptr(new KerasMVNSubgraph())); subgraphs.push_back(Ptr(new PReLUSubgraph(true))); subgraphs.push_back(Ptr(new PReLUSubgraph(false))); @@ -799,6 +798,9 @@ void simplifySubgraphs(tensorflow::GraphDef& net) subgraphs.push_back(Ptr(new ResizeBilinearSubgraphDown())); subgraphs.push_back(Ptr(new ClipByValueSubgraph())); + if (!newEngine) + subgraphs.push_back(Ptr(new ReshapeAsShapeSubgraph())); + for (int i = 0; i < net.node_size(); ++i) { tensorflow::NodeDef* layer = net.mutable_node(i); diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp index ad20968f6d..b22a8f2f00 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp @@ -17,7 +17,7 @@ CV__DNN_INLINE_NS_BEGIN void RemoveIdentityOps(tensorflow::GraphDef& net); -void simplifySubgraphs(tensorflow::GraphDef& net); +void simplifySubgraphs(tensorflow::GraphDef& net, bool newEngine); Mat getTensorContent(const tensorflow::TensorProto& tensor, bool forceCopy = true); diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index f431a77d83..d8f9150128 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -10,9 +10,11 @@ Implementation of Tensorflow models parser */ #include "../precomp.hpp" +#include "../net_impl.hpp" #include +#include #include #include #undef CV_LOG_STRIP_LEVEL @@ -25,6 +27,8 @@ Implementation of Tensorflow models parser #include #include #include +#include +#include #include #include #include "tf_graph_simplifier.hpp" @@ -466,6 +470,19 @@ Pin parsePin(const std::string &name) return pin; } +// Removes zero index from name suffix, keeps other indices +std::string formatTensorName(std::string name) +{ + size_t delimiter_pos = name.find_first_of(':'); + if (delimiter_pos != std::string::npos) + { + std::string index = name.substr(delimiter_pos + 1); + if (index == "0") + name = name.substr(0, delimiter_pos); + } + return name; +} + StrIntVector getNextLayers(const tensorflow::GraphDef& net, const String& layer_name, const String& type = "") { StrIntVector layers; @@ -512,9 +529,13 @@ class TFImporter { FPDenormalsIgnoreHintScope fp_denormals_ignore_scope; public: - TFImporter(Net& net, const char *model, const char *config = NULL); + TFImporter(Net& net, const char *model, const char *config = NULL, + bool newEngine = true, + const std::vector& extraOutputs=std::vector()); TFImporter(Net& net, const char *dataModel, size_t lenModel, - const char *dataConfig = NULL, size_t lenConfig = 0); + const char *dataConfig = NULL, size_t lenConfig = 0, + bool newEngine = true, + const std::vector& extraOutputs=std::vector()); protected: std::unique_ptr layerHandler; Net& dstNet; @@ -528,8 +549,6 @@ protected: void connect(const std::map& layers_name_id_map, Net& network, const Pin& outPin, const int input_layer_id, const int input_blob_id); - void connectToAllBlobs(const std::map& layer_id, Net& network, const Pin& outPin, - const int input_layer_id, const int input_blobs_count); const tensorflow::TensorProto& getConstBlob(const tensorflow::NodeDef &layer, std::map const_layers, int input_blob_index = -1, int* actual_inp_blob_idx = 0); @@ -543,6 +562,7 @@ protected: std::vector netInputsNames; std::vector netInputShapes; + std::vector extraOutputs; std::set layers_to_ignore; std::map data_layouts; @@ -554,8 +574,18 @@ protected: std::map layer_id; + bool newEngine; + std::vector> curProg; + std::vector> layersOutputs; + std::vector modelInputs; + std::unordered_map tensorsShape; + private: - void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId, int orderSize = 4); + void addLayer(const std::string& name, const std::string& type, LayerParams& layerParams, std::vector inputs, int numOutputs = 1, int repeatInputs = 0); + // Calculates and saves layer's output shapes if input shapes are available + void calcLayerOutputShapes(const Layer& layer, const std::vector& inputs, const std::vector& outputs); + + void addPermuteLayer(const int* order, const std::string& permName, const std::string& inputName, int orderSize = 4); void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value = 0.); friend class TFLayerHandler; @@ -603,6 +633,80 @@ private: void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; +void TFImporter::addLayer(const std::string& name, const std::string& type, LayerParams& layerParams, std::vector inputs, int numOutputs, int repeatInputs) +{ + if (repeatInputs != 0) + { + CV_CheckEQ(inputs.size(), 1u, ""); + for (int i = 0; i < repeatInputs; i++) + inputs.push_back(inputs[0]); + } + + if (newEngine) + { + layerParams.type = type; + layerParams.name = name; + + Ptr layer = LayerFactory::createLayerInstance(type, layerParams); + if (!layer) { + CV_Error(Error::StsError, "Can't create layer " + name + " with type " + type); + } + layer_id[name] = -1; + + for (size_t i = 0; i < inputs.size(); i++) + { + inputs[i] = formatTensorName(inputs[i]); + layer->inputs.push_back(dstNet.getArg(inputs[i])); + } + + layersOutputs.push_back({name}); + for (int i = 1; i < numOutputs; i++) + layersOutputs.back().push_back(name + ":" + std::to_string(i)); + layer->netimpl = dstNet.getImpl(); + curProg.push_back(layer); + + calcLayerOutputShapes(*layer.get(), inputs, layersOutputs.back()); + } + else + { + int id = dstNet.addLayer(name, type, layerParams); + if (id < 0) { + CV_Error(Error::StsError, "Can't create layer " + name + " with type " + type); + } + layer_id[name] = id; + for (size_t i = 0; i < inputs.size(); ++i) + { + Pin inp = parsePin(inputs[i]); + if (layer_id.find(inp.name) == layer_id.end()) { + for (const auto& [key, value] : layer_id) + std::cout << '[' << key << "] = " << value << "; "; + CV_Error(Error::StsError, "Input layer not found: " + inp.name); + } + connect(layer_id, dstNet, inp, id, i); + } + } +} + +void TFImporter::calcLayerOutputShapes(const Layer& layer, const std::vector& inputs, const std::vector& outputs) +{ + std::vector inputsShape; + for (const std::string& inputName : inputs) { + const auto& it = tensorsShape.find(inputName); + if (it == tensorsShape.end()) + return; + inputsShape.push_back(it->second); + } + + std::vector outputsShape; + std::vector intermediatesShape; + + layer.getMemoryShapes(inputsShape, outputs.size(), outputsShape, intermediatesShape); + + CV_Assert(outputsShape.size() == outputs.size()); + for(size_t i = 0; i < outputs.size(); i++) + tensorsShape[outputs[i]] = outputsShape[i]; +} + void TFImporter::setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value) { setPadMode(layerParams, layer); @@ -619,10 +723,7 @@ void TFImporter::setPadding(LayerParams &layerParams, const tensorflow::NodeDef padLp.set("paddings", DictValue::arrayInt(pads, sizeof(pads) / sizeof(pads[0]))); padLp.set("value", (double)value); - int id = dstNet.addLayer(padLp.name, padLp.type, padLp); - layer_id[padLp.name] = id; - - connect(layer_id, dstNet, parsePin(inputName), id, 0); + addLayer(padLp.name, padLp.type, padLp, {inputName}); inputName = padLp.name; layerParams.set("pad_mode", "VALID"); @@ -762,10 +863,7 @@ void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::N if (type == "MirrorPad") layerParams.set("type", "reflect"); - int id = dstNet.addLayer(name, "Padding", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(input), id, 0); + addLayer(name, "Padding", layerParams, {input}); return; } else @@ -893,11 +991,7 @@ void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::N layers_to_ignore.insert(next_layers[0].first); } - int id = dstNet.addLayer(name, "Convolution", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(input), id, 0); + addLayer(name, "Convolution", layerParams, {input}); if (getDataLayout(name, data_layouts) == DNN_LAYOUT_UNKNOWN) @@ -927,26 +1021,24 @@ void TFImporter::parseBias(tensorflow::GraphDef& net, const tensorflow::NodeDef& if (type == "Sub") values *= -1.0f; - int id; + std::string inputName; + Pin inp0 = parsePin(layer.input(0)); + if (layer_id.find(inp0.name) != layer_id.end()) + // First operand is a constant. + inputName = layer.input(0); + else + inputName = layer.input(1); + if (values.total() == 1) // is a scalar. { layerParams.set("shift", values.at(0)); - id = dstNet.addLayer(name, "Power", layerParams); + addLayer(name, "Power", layerParams, {inputName}); } else // is a vector { layerParams.blobs.resize(1, values); - id = dstNet.addLayer(name, "Shift", layerParams); + addLayer(name, "Shift", layerParams, {inputName}); } - layer_id[name] = id; - - // one input only - Pin inp0 = parsePin(layer.input(0)); - if (layer_id.find(inp0.name) != layer_id.end()) - // First operand is a constant. - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - else - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); } else { @@ -957,16 +1049,11 @@ void TFImporter::parseBias(tensorflow::GraphDef& net, const tensorflow::NodeDef& layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); } - int id = dstNet.addLayer(name, "Eltwise", layerParams); - layer_id[name] = id; - + std::vector inputs; for (int ii = 0; ii < num_inputs; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii); - } + inputs.push_back(layer.input(ii)); + + addLayer(name, "Eltwise", layerParams, inputs); } } @@ -995,13 +1082,11 @@ void TFImporter::parseMatMul(tensorflow::GraphDef& net, const tensorflow::NodeDe if (!hasConstBlob) { layerParams.blobs.clear(); - int id = dstNet.addLayer(name, "InnerProduct", layerParams); - layer_id[name] = id; + std::vector inputs; + for (int ii = 0; ii < layer.input_size(); ii++) + inputs.push_back(layer.input(ii)); - // two inputs - for(int ii=0; ii= 2 || newShape.at(1) == 1) { int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - addPermuteLayer(order, name + "/nhwc", inpId); + addPermuteLayer(order, name + "/nhwc", inputName); + inputName = name + "/nhwc"; if (newShapeSize < 4) { inpLayout = DNN_LAYOUT_NCHW; @@ -1114,16 +1196,13 @@ void TFImporter::parseReshape(tensorflow::GraphDef& net, const tensorflow::NodeD } } } - layerParams.set("dim", DictValue::arrayInt(newShape.ptr(), newShapeSize)); - std::string setName = changedType ? name + "/realReshape" : name; + layerParams.set(newEngine ? "shape" : "dim", DictValue::arrayInt(newShape.ptr(), newShapeSize)); - int id = dstNet.addLayer(setName, "Reshape", layerParams); - layer_id[setName] = id; + std::string setName = changedType ? name + "/realReshape" : name; - // one input only - connect(layer_id, dstNet, inpId, id, 0); - inpId = Pin(setName); + addLayer(setName, newEngine ? "Reshape2" : "Reshape", layerParams, {inputName}); + inputName = setName; if ((inpLayout == DNN_LAYOUT_NHWC || inpLayout == DNN_LAYOUT_UNKNOWN || inpLayout == DNN_LAYOUT_PLANAR) && newShapeSize == 4 && !hasSwap) @@ -1131,7 +1210,7 @@ void TFImporter::parseReshape(tensorflow::GraphDef& net, const tensorflow::NodeD int order[] = {0, 3, 1, 2}; // Transform back to OpenCV's NCHW. setName = changedType ? name : name + "/nchw"; - addPermuteLayer(order, setName, inpId); + addPermuteLayer(order, setName, inputName); inpLayout = DNN_LAYOUT_NCHW; } @@ -1139,10 +1218,7 @@ void TFImporter::parseReshape(tensorflow::GraphDef& net, const tensorflow::NodeD } else { - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); + addLayer(name, newEngine ? "Reshape2" : "Reshape", layerParams, {layer.input(0), layer.input(1)}); data_layouts[name] = inpLayout; } } @@ -1155,15 +1231,25 @@ void TFImporter::parseExpandDims(tensorflow::GraphDef& net, const tensorflow::No CV_Assert(!netInputShapes.empty()); CV_CheckGT(num_inputs, 0, ""); - Pin inpId = parsePin(layer.input(0)); + std::string inputName = layer.input(0); DataLayout inpLayout = getDataLayout(layer.input(0), data_layouts); // Get input shape - std::vector inShape_, outShape_; - int inpIdindex = layer_id.find(inpId.name)->second; - std::vector netInputTypes(netInputShapes.size(), CV_32F); - dstNet.getLayerShapes(netInputShapes, netInputTypes, inpIdindex, inShape_, outShape_); - MatShape inpShape = outShape_[0]; + MatShape inpShape; + if (newEngine) + { + const auto& it = tensorsShape.find(formatTensorName(inputName)); + CV_Assert(it != tensorsShape.end()); + inpShape = it->second; + } + else + { + std::vector inShape_, outShape_; + int inpIdindex = layer_id.find(inputName)->second; + std::vector netInputTypes(netInputShapes.size(), CV_32F); + dstNet.getLayerShapes(netInputShapes, netInputTypes, inpIdindex, inShape_, outShape_); + inpShape = outShape_[0]; + } MatShape outShape = inpShape; int outShapeSize = (int)outShape.size(); @@ -1188,7 +1274,8 @@ void TFImporter::parseExpandDims(tensorflow::GraphDef& net, const tensorflow::No if(axis != outShapeSize) { int order[] = {0, 2, 1}; // From OpenCV's NHC to NCH. - addPermuteLayer(order, name + "/nch", inpId, 3); + addPermuteLayer(order, name + "/nch", inputName, 3); + inputName = name + "/nch"; std::swap(outShape[1], outShape[2]); } @@ -1200,7 +1287,8 @@ void TFImporter::parseExpandDims(tensorflow::GraphDef& net, const tensorflow::No if(axis == inpShape.size()) { int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - addPermuteLayer(order, name + "/nhwc", inpId); + addPermuteLayer(order, name + "/nhwc", inputName); + inputName = name + "/nhwc"; // Convert shape From OpenCV's NCHW to NHWC. if(inpLayout == DNN_LAYOUT_NHWC) @@ -1221,7 +1309,8 @@ void TFImporter::parseExpandDims(tensorflow::GraphDef& net, const tensorflow::No if (inpShape.size() == 5 && (inpLayout == DNN_LAYOUT_NDHWC || inpLayout == DNN_LAYOUT_UNKNOWN)) { int order[] = {0, 2, 3, 4, 1}; // From OpenCV's NCDHW to NDHWC. - addPermuteLayer(order, name + "/ndhwc", inpId, 5); + addPermuteLayer(order, name + "/ndhwc", inputName, 5); + inputName = name + "/ndhwc"; // Convert shape From OpenCV's NCDHW to NDHWC. if(inpLayout == DNN_LAYOUT_NDHWC) @@ -1245,10 +1334,7 @@ void TFImporter::parseExpandDims(tensorflow::GraphDef& net, const tensorflow::No } layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, inpId, id, 0); + addLayer(name, "Reshape", layerParams, {inputName}); if(outShapeSize == 5) { @@ -1272,15 +1358,8 @@ void TFImporter::parseSquare(tensorflow::GraphDef& net, const tensorflow::NodeDe CV_CheckEQ(num_inputs, 1, ""); - int id; layerParams.set("operation", "prod"); - id = dstNet.addLayer(name, "Eltwise", layerParams); - - layer_id[name] = id; - - Pin inp = parsePin(layer.input(0)); - connect(layer_id, dstNet, inp, id, 0); - connect(layer_id, dstNet, inp, id, 1); + addLayer(name, "Eltwise", layerParams, {layer.input(0), layer.input(0)}); } // "Flatten" "Squeeze" @@ -1291,7 +1370,7 @@ void TFImporter::parseFlatten(tensorflow::GraphDef& net, const tensorflow::NodeD const int num_inputs = layer.input_size(); CV_CheckGT(num_inputs, 0, ""); - Pin inpId = parsePin(layer.input(0)); + std::string inputName = layer.input(0); int inpLayout = getDataLayout(layer.input(0), data_layouts); if (type == "Squeeze") { @@ -1325,14 +1404,10 @@ void TFImporter::parseFlatten(tensorflow::GraphDef& net, const tensorflow::NodeD std::string permName = name + "/nchw"; CV_Assert(layer_id.find(permName) == layer_id.end()); - int permId = dstNet.addLayer(permName, "Permute", permLP); - layer_id[permName] = permId; - connect(layer_id, dstNet, inpId, permId, 0); - inpId = Pin(permName); - } - int id = dstNet.addLayer(name, "Flatten", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); + addLayer(permName, "Permute", permLP, {inputName}); + inputName = permName; + } + addLayer(name, "Flatten", layerParams, {inputName}); data_layouts[name] = DNN_LAYOUT_PLANAR; } @@ -1394,19 +1469,13 @@ void TFImporter::parseTranspose(tensorflow::GraphDef& net, const tensorflow::Nod else CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); } - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, type, layerParams, {layer.input(0)}); } else { layerParams.set("order", DictValue::arrayInt(permData, perm.total())); - int id = dstNet.addLayer(name, "Permute", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, "Permute", layerParams, {layer.input(0)}); data_layouts[name] = DNN_LAYOUT_UNKNOWN; } } @@ -1436,10 +1505,7 @@ void TFImporter::parseLrn(tensorflow::GraphDef& net, const tensorflow::NodeDef& } layerParams.set("norm_by_size", false); - int id = dstNet.addLayer(name, "LRN", layerParams); - layer_id[name] = id; - - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); + addLayer(name, "LRN", layerParams, {layer.input(0)}, 1, num_inputs - 1); } // "Concat" "ConcatV2" @@ -1476,21 +1542,15 @@ void TFImporter::parseConcat(tensorflow::GraphDef& net, const tensorflow::NodeDe blobFromTensor(getConstBlob(layer, value_id, ii), lp.blobs.back()); CV_Assert_N(!lp.blobs[0].empty(), lp.blobs[0].type() == CV_32F); - int constInpId = dstNet.addLayer(lp.name, lp.type, lp); - layer_id[lp.name] = constInpId; + addLayer(lp.name, lp.type, lp, {}); } } - int id = dstNet.addLayer(name, "Concat", layerParams); - layer_id[name] = id; + std::vector inputs; + for (int ii = from; ii < to; ii++) + inputs.push_back(layer.input(ii)); - for (int ii = from; ii < to; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii - from); - } + addLayer(name, "Concat", layerParams, inputs); } // "MaxPool" "MaxPool3D" @@ -1509,10 +1569,7 @@ void TFImporter::parseMaxPool(tensorflow::GraphDef& net, const tensorflow::NodeD // Test_TensorFlow_nets.EAST_text_detection/1, NGRAPH/CPU layerParams.set("ceil_mode", false); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - - connectToAllBlobs(layer_id, dstNet, parsePin(inputName), id, num_inputs); + addLayer(name, "Pooling", layerParams, {inputName}, 2, num_inputs - 1); } // "AvgPool" "AvgPool3D" @@ -1528,10 +1585,7 @@ void TFImporter::parseAvgPool(tensorflow::GraphDef& net, const tensorflow::NodeD setStrides(layerParams, layer); setPadMode(layerParams, layer); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); + addLayer(name, "Pooling", layerParams, {layer.input(0)}, 1, num_inputs - 1); } void TFImporter::parseMaxPoolGrad(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -1548,12 +1602,7 @@ void TFImporter::parseMaxPoolGrad(tensorflow::GraphDef& net, const tensorflow::N layerParams.set("pool_pad_h", 0); layerParams.set("pool_pad_w", 0); - int id = dstNet.addLayer(name, "MaxUnpool", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1) + ":1"), id, 1); - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 2); + addLayer(name, "MaxUnpool", layerParams, {layer.input(2), layer.input(1) + ":1", layer.input(0)}); } void TFImporter::parsePlaceholder(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -1565,6 +1614,11 @@ void TFImporter::parsePlaceholder(tensorflow::GraphDef& net, const tensorflow::N if (!hasLayerAttr(layer, "dtype") || getLayerAttr(layer, "dtype").type() != tensorflow::DT_BOOL) // If input is not a train/test flag. { + if (newEngine) + { + modelInputs.push_back(dstNet.getImpl()->newArg(name, DNN_ARG_INPUT)); + dstNet.getImpl()->args.at(modelInputs.back().idx).type = CV_32F; + } netInputsNames.push_back(name); layer_id[name] = 0; } @@ -1604,7 +1658,12 @@ void TFImporter::parsePlaceholder(tensorflow::GraphDef& net, const tensorflow::N hasNeg = dims[i] < 0; } if (!hasNeg) + { + if (newEngine) + dstNet.getImpl()->args.at(modelInputs.back().idx).shape = dims; netInputShapes.push_back(dims); + tensorsShape[formatTensorName(name)] = dims; + } } } @@ -1627,12 +1686,10 @@ void TFImporter::parseSplit(tensorflow::GraphDef& net, const tensorflow::NodeDef if (hasLayerAttr(layer, "num_split")) layerParams.set("num_split", getLayerAttr(layer, "num_split").i()); + else + CV_Error(Error::StsNotImplemented, "Can't parse split layer. num_split attribut is missing"); - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); + addLayer(name, "Slice", layerParams, {layer.input(1)}, getLayerAttr(layer, "num_split").i()); } void TFImporter::parseSlice(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -1662,10 +1719,7 @@ void TFImporter::parseSlice(tensorflow::GraphDef& net, const tensorflow::NodeDef layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); layerParams.set("size", DictValue::arrayInt((int*)sizes.data, sizes.total())); - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, "Slice", layerParams, {layer.input(0)}); } void TFImporter::parseStridedSlice(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -1712,14 +1766,10 @@ void TFImporter::parseStridedSlice(tensorflow::GraphDef& net, const tensorflow:: lp.type = "Const"; lp.blobs.push_back(getTensorContent(getConstBlob(layer, value_id, 0))); - int constInpId = dstNet.addLayer(lp.name, lp.type, lp); - layer_id[lp.name] = constInpId; + addLayer(lp.name, lp.type, lp, {}); } - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, "Slice", layerParams, {layer.input(0)}); } // "Mul" "RealDiv" @@ -1755,7 +1805,14 @@ void TFImporter::parseMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& scaleMat = 1.0f / scaleMat; } - int id; + Pin inp0 = parsePin(layer.input(0)); + std::string inputName; + if (layer_id.find(inp0.name) != layer_id.end()) + // First operand is a constant. + inputName = layer.input(0); + else + inputName = layer.input(1); + if (scaleMat.total() == 1) // is a scalar. { // Try to match with a LeakyRelu: @@ -1785,13 +1842,13 @@ void TFImporter::parseMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& layers_to_ignore.insert(next_layers[0].first); layerParams.set("negative_slope", scaleMat.at(0)); - id = dstNet.addLayer(name, "ReLU", layerParams); + addLayer(name, "ReLU", layerParams, {inputName}); } else { // Just a multiplication. layerParams.set("scale", scaleMat.at(0)); - id = dstNet.addLayer(name, "Power", layerParams); + addLayer(name, "Power", layerParams, {inputName}); } } else // is a vector @@ -1813,71 +1870,14 @@ void TFImporter::parseMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& if (hasLayerAttr(layer, "axis")) layerParams.set("axis", getLayerAttr(layer, "axis").i()); - id = dstNet.addLayer(name, "Scale", layerParams); + addLayer(name, "Scale", layerParams, {inputName}); } - layer_id[name] = id; - - Pin inp0 = parsePin(layer.input(0)); - if (layer_id.find(inp0.name) != layer_id.end()) - // First operand is a constant. - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - else - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); } else { - // Check if all the inputs have the same shape. - bool equalInpShapes = true; - bool isShapeOnes = false; - MatShape outShape0; - for (int ii = 0; ii < num_inputs && !netInputShapes.empty(); ii++) - { - Pin pin = parsePin(layer.input(ii)); - int inpId = layer_id.find(pin.name)->second; - - // Get input shape - MatShape outShape; - std::vector inpShapes, outShapes; - std::vector netInputTypes(netInputShapes.size(), CV_32F); - dstNet.getLayerShapes(netInputShapes, netInputTypes, inpId, inpShapes, outShapes); - CV_CheckGT(static_cast(outShapes.size()), pin.blobIndex, ""); - outShape = outShapes[pin.blobIndex]; - - if (ii == 0) - { - outShape0 = outShape; - } - else if (outShape != outShape0) - { - equalInpShapes = false; - isShapeOnes = isAllOnes(outShape, 2, outShape.size()) || - isAllOnes(outShape0, 2, outShape0.size()); - break; - } - } - - int id; - if (equalInpShapes || netInputShapes.empty() || (!equalInpShapes && isShapeOnes)) - { - layerParams.set("operation", type == "RealDiv" ? "div" : "prod"); - id = dstNet.addLayer(name, "Eltwise", layerParams); - } - else - { - if (type == "RealDiv") - CV_Error(Error::StsNotImplemented, "Division of non equal tensors"); - id = dstNet.addLayer(name, "Scale", layerParams); - } - - layer_id[name] = id; - - for (int ii = 0; ii < num_inputs; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii); - } + CV_CheckEQ(num_inputs, 2, ""); + layerParams.set("operation", type == "RealDiv" ? "div" : "mul"); + addLayer(name, "NaryEltwise", layerParams, {layer.input(0), layer.input(1)}); } } @@ -1895,7 +1895,7 @@ void TFImporter::parseFusedBatchNorm(tensorflow::GraphDef& net, const tensorflow const int num_inputs = layer.input_size(); CV_CheckEQ(num_inputs, 5, "Expected gamma, beta, mean and std"); - Pin inpId = parsePin(layer.input(0)); + std::string inputName = layer.input(0); bool isTraining = hasLayerAttr(layer, "is_training") && getLayerAttr(layer, "is_training").b(); @@ -1934,10 +1934,8 @@ void TFImporter::parseFusedBatchNorm(tensorflow::GraphDef& net, const tensorflow LayerParams mvnParams; std::string mvnName = name + "/MVN"; CV_Assert(layer_id.find(mvnName) == layer_id.end()); - int mvnId = dstNet.addLayer(mvnName, "MVN", mvnParams); - layer_id[mvnName] = mvnId; - connect(layer_id, dstNet, inpId, mvnId, 0); - inpId = Pin(mvnName); + addLayer(mvnName, "MVN", mvnParams, {inputName}); + inputName = mvnName; } else { @@ -1950,11 +1948,7 @@ void TFImporter::parseFusedBatchNorm(tensorflow::GraphDef& net, const tensorflow if (hasLayerAttr(layer, "epsilon")) layerParams.set("eps", getLayerAttr(layer, "epsilon").f()); - int id = dstNet.addLayer(name, "BatchNorm", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, inpId, id, 0); + addLayer(name, "BatchNorm", layerParams, {inputName}); } void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -2040,20 +2034,13 @@ void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tenso layerParams.set("adj_w", (outW - kernelW) % strideX); layerParams.set("adj_h", (outH - kernelH) % strideY); } - int id = dstNet.addLayer(name, "Deconvolution", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); + addLayer(name, "Deconvolution", layerParams, {layer.input(2)}); if (explicit_pads) // If we have explicit paddings, remove extra data { layerParams.set("begin", DictValue::arrayInt(begs, sizeof(begs) / sizeof(begs[0]))); layerParams.set("end", DictValue::arrayInt(ends, sizeof(ends) / sizeof(ends[0]))); - int id = dstNet.addLayer(layer.name(), "Slice", layerParams); - layer_id[layer.name()] = id; - - connect(layer_id, dstNet, parsePin(name), id, 0); + addLayer(layer.name(), "Slice", layerParams, {name}); } } @@ -2134,11 +2121,7 @@ void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::Nod } } - int id = dstNet.addLayer(name, "LSTM", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); + addLayer(name, "LSTM", layerParams, {layer.input(1)}); data_layouts[name] = DNN_LAYOUT_UNKNOWN; } @@ -2206,10 +2189,7 @@ void TFImporter::parseResize(tensorflow::GraphDef& net, const tensorflow::NodeDe if (hasLayerAttr(layer, "half_pixel_centers")) layerParams.set("half_pixel_centers", getLayerAttr(layer, "half_pixel_centers").b()); - int id = dstNet.addLayer(name, "Resize", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, "Resize", layerParams, {layer.input(0)}); // Step back to add convolution if (type == "FusedResizeAndPadConv2D") @@ -2251,9 +2231,7 @@ void TFImporter::parseL2Normalize(tensorflow::GraphDef& net, const tensorflow::N layerParams.set("start_axis", reductionIndices.at(0)); layerParams.set("end_axis", reductionIndices.at(numAxes - 1)); - int id = dstNet.addLayer(name, "Normalize", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, "Normalize", layerParams, {layer.input(0)}); } void TFImporter::parsePriorBox(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -2286,10 +2264,7 @@ void TFImporter::parsePriorBox(tensorflow::GraphDef& net, const tensorflow::Node DictValue::arrayReal((float*)values.data, values.total())); } } - int id = dstNet.addLayer(name, "PriorBox", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); + addLayer(name, "PriorBox", layerParams, {layer.input(0), layer.input(1)}); data_layouts[name] = DNN_LAYOUT_UNKNOWN; } @@ -2308,9 +2283,7 @@ void TFImporter::parseSoftmax(tensorflow::GraphDef& net, const tensorflow::NodeD else layerParams.set("axis", 1); - int id = dstNet.addLayer(name, "Softmax", layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); + addLayer(name, "Softmax", layerParams, {layer.input(0)}, 1, num_inputs - 1); } void TFImporter::parseCropAndResize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -2330,11 +2303,7 @@ void TFImporter::parseCropAndResize(tensorflow::GraphDef& net, const tensorflow: layerParams.set("height", cropSize.at(0)); layerParams.set("width", cropSize.at(1)); - int id = dstNet.addLayer(name, "CropAndResize", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); + addLayer(name, "CropAndResize", layerParams, {layer.input(0), layer.input(1)}); } // "Mean" "Sum" "Max" @@ -2380,9 +2349,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& LayerParams flattenLp; std::string flattenName = name + "/flatten"; CV_Assert(layer_id.find(flattenName) == layer_id.end()); - int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); - layer_id[flattenName] = flattenId; - connect(layer_id, dstNet, parsePin(layer.input(0)), flattenId, 0); + addLayer(flattenName, "Flatten", flattenLp, {layer.input(0)}); LayerParams reshapeLp; std::string reshapeName = name + "/reshape"; @@ -2392,9 +2359,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& int newShape[] = {1, 1, -1}; reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 3)); - int reshapeId = dstNet.addLayer(reshapeName, "Reshape", reshapeLp); - layer_id[reshapeName] = reshapeId; - connect(layer_id, dstNet, Pin(flattenName), reshapeId, 0); + addLayer(reshapeName, "Reshape", reshapeLp, {flattenName}); LayerParams avgLp; std::string avgName = name + "/avg"; @@ -2403,9 +2368,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& // pooling kernel H x 1 avgLp.set("global_pooling_h", true); avgLp.set("kernel_w", 1); - int avgId = dstNet.addLayer(avgName, "Pooling", avgLp); - layer_id[avgName] = avgId; - connect(layer_id, dstNet, Pin(reshapeName), avgId, 0); + addLayer(avgName, "Pooling", avgLp, {reshapeName}); LayerParams sliceLp; std::string layerShapeName = name + "/slice"; @@ -2415,9 +2378,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& int size[] = {1}; sliceLp.set("begin", DictValue::arrayInt(&begin[0], 1)); sliceLp.set("size", DictValue::arrayInt(&size[0], 1)); - int sliceId = dstNet.addLayer(layerShapeName, "Slice", sliceLp); - layer_id[layerShapeName] = sliceId; - connect(layer_id, dstNet, Pin(layer.input(0)), sliceId, 0); + addLayer(layerShapeName, "Slice", sliceLp, {layer.input(0)}); if (!keepDims) { @@ -2426,8 +2387,7 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& LayerParams permLP; int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. std::string permName = name + "/nhwc"; - Pin inpId = Pin(layerShapeName); - addPermuteLayer(order, permName, inpId); + addPermuteLayer(order, permName, layerShapeName); layerShapeName = permName; } @@ -2436,16 +2396,11 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& CV_Assert(layer_id.find(squeezeName) == layer_id.end()); squeezeLp.set("axis", 0); squeezeLp.set("end_axis", 1); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(layerShapeName), squeezeId, 0); + addLayer(squeezeName, "Flatten", squeezeLp, {layerShapeName}); layerShapeName = squeezeName; } - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, Pin(avgName), id, 0); - connect(layer_id, dstNet, Pin(layerShapeName), id, 1); + addLayer(name, "Reshape", layerParams, {avgName, layerShapeName}); } else if (indices.total() == 1) { int axis = toNCHW(indices.at(0)); if (axis == 2 || axis == 3) @@ -2456,49 +2411,39 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& if (keepDims) { - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, "Pooling", layerParams, {layer.input(0)}); } else { // To keep correct order after squeeze dims we first need to change layout from NCHW to NHWC std::string poolingName = name + "/Pooling"; CV_Assert(layer_id.find(poolingName) == layer_id.end()); - int id = dstNet.addLayer(poolingName, "Pooling", layerParams); - layer_id[poolingName] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(poolingName, "Pooling", layerParams, {layer.input(0)}); LayerParams permLP; int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. std::string permName = name + "/nhwc"; - Pin inpId = Pin(poolingName); - addPermuteLayer(order, permName, inpId); + addPermuteLayer(order, permName, poolingName); LayerParams squeezeLp; const std::string& squeezeName = name; squeezeLp.set("axis", indices.at(0)); squeezeLp.set("end_axis", indices.at(0) + 1); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(permName), squeezeId, 0); + addLayer(squeezeName, "Flatten", squeezeLp, {permName}); } } else if (axis == 1) { int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - Pin inpId = parsePin(layer.input(0)); std::string permName = name + "/nhwc"; - addPermuteLayer(order, permName, inpId); + addPermuteLayer(order, permName, layer.input(0)); layerParams.set("pool", pool_type); layerParams.set("kernel_h", 1); layerParams.set("global_pooling_w", true); std::string poolingName = name + "/Pooling"; CV_Assert(layer_id.find(poolingName) == layer_id.end()); - int id = dstNet.addLayer(poolingName, "Pooling", layerParams); - layer_id[poolingName] = id; - connect(layer_id, dstNet, Pin(permName), id, 0); + addLayer(poolingName, "Pooling", layerParams, {permName}); if (!keepDims) { @@ -2507,15 +2452,12 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& int channel_id = 3; // TF NHWC layout squeezeLp.set("axis", channel_id - 1); squeezeLp.set("end_axis", channel_id); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(poolingName), squeezeId, 0); + addLayer(squeezeName, "Flatten", squeezeLp, {poolingName}); } else { int order[] = {0, 3, 1, 2}; // From NHWC to OpenCV's NCHW. - Pin inpId = parsePin(poolingName); - addPermuteLayer(order, name, inpId); + addPermuteLayer(order, name, poolingName); } } } else { @@ -2527,22 +2469,16 @@ void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& if (keepDims) { - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, "Pooling", layerParams, {layer.input(0)}); } else { std::string poolingName = name + "/Pooling"; CV_Assert(layer_id.find(poolingName) == layer_id.end()); - int id = dstNet.addLayer(poolingName, "Pooling", layerParams); - layer_id[poolingName] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(poolingName, "Pooling", layerParams, {layer.input(0)}); LayerParams flattenLp; const std::string& flattenName = name; - int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); - layer_id[flattenName] = flattenId; - connect(layer_id, dstNet, Pin(poolingName), flattenId, 0); + addLayer(flattenName, "Flatten", flattenLp, {poolingName}); data_layouts[name] = DNN_LAYOUT_PLANAR; } } @@ -2572,7 +2508,7 @@ void TFImporter::parsePack(tensorflow::GraphDef& net, const tensorflow::NodeDef& int num = (int)getLayerAttr(layer, "N").i(); CV_CheckEQ(num_inputs, num, ""); std::string base_name = name + "/reshape_"; - std::vector reshape_ids; + std::vector inputs; for (int i = 0; i < num; i++) { std::ostringstream ss; ss << i; @@ -2582,18 +2518,12 @@ void TFImporter::parsePack(tensorflow::GraphDef& net, const tensorflow::NodeDef& reshapeLP.set("num_axes", 1); int outShape[] = {1, -1}; reshapeLP.set("dim", DictValue::arrayInt(&outShape[0], 2)); - int id = dstNet.addLayer(reshape_name, "Reshape", reshapeLP); - layer_id[reshape_name] = id; - reshape_ids.push_back(id); - connect(layer_id, dstNet, parsePin(layer.input(i)), id, 0); + addLayer(reshape_name, "Reshape", reshapeLP, {layer.input(i)}); + inputs.push_back(reshape_name); } layerParams.set("axis", dim); - int id = dstNet.addLayer(name, "Concat", layerParams); - layer_id[name] = id; - - for (int li = 0; li < num; li++) - dstNet.connect(reshape_ids[li], 0, id, li); + addLayer(name, "Concat", layerParams, inputs); } void TFImporter::parseClipByValue(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -2616,10 +2546,7 @@ void TFImporter::parseClipByValue(tensorflow::GraphDef& net, const tensorflow::N layerParams.set("min_value", minValue.at(0)); layerParams.set("max_value", maxValue.at(0)); - int id = dstNet.addLayer(name, "ReLU6", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, "ReLU6", layerParams, {layer.input(0)}); } void TFImporter::parseLeakyRelu(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -2631,9 +2558,7 @@ void TFImporter::parseLeakyRelu(tensorflow::GraphDef& net, const tensorflow::Nod CV_Assert(hasLayerAttr(layer, "alpha")); layerParams.set("negative_slope", getLayerAttr(layer, "alpha").f()); - int id = dstNet.addLayer(name, "ReLU", layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); + addLayer(name, "ReLU", layerParams, {layer.input(0)}, 1, num_inputs - 1); } void TFImporter::parsePReLU(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -2652,9 +2577,7 @@ void TFImporter::parsePReLU(tensorflow::GraphDef& net, const tensorflow::NodeDef layerParams.blobs[0] = scales; } - int id = dstNet.addLayer(name, "PReLU", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, "PReLU", layerParams, {layer.input(0)}); } // "Abs" "Tanh" "Sigmoid" "Relu" "Elu" "Exp" "Identity" "Relu6" @@ -2672,9 +2595,7 @@ void TFImporter::parseActivation(tensorflow::GraphDef& net, const tensorflow::No else if (type == "Relu6") dnnType = "ReLU6"; else if (type == "Elu") dnnType = "ELU"; - int id = dstNet.addLayer(name, dnnType, layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); + addLayer(name, dnnType, layerParams, {layer.input(0)}, 1, num_inputs - 1); } // ArgMin or ArgMax node @@ -2694,9 +2615,7 @@ void TFImporter::parseArg(tensorflow::GraphDef& net, const tensorflow::NodeDef& layerParams.set("op", type == "ArgMax" ? "max" : "min"); layerParams.set("keepdims", false); //tensorflow doesn't have this atrr, the output's dims minus one(default); - int id = dstNet.addLayer(name, "Arg", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + addLayer(name, "Arg", layerParams, {layer.input(0)}); } void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) @@ -2736,18 +2655,13 @@ void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::N else inputsNames.push_back(layer.input(i)); } - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; - - for (int i = 0; i < inputsNames.size(); ++i) - { - connect(layer_id, dstNet, parsePin(inputsNames[i]), id, i); - } + addLayer(name, type, layerParams, inputsNames); } -TFImporter::TFImporter(Net& net, const char *model, const char *config) +TFImporter::TFImporter(Net& net, const char *model, const char *config, bool newEngine, + const std::vector& extraOutputs_) : layerHandler(DNN_DIAGNOSTICS_RUN ? new TFLayerHandler(this) : nullptr), - dstNet(net), dispatch(buildDispatchMap()) + dstNet(net), newEngine(newEngine), dispatch(buildDispatchMap()) { if (model && model[0]) { @@ -2760,16 +2674,18 @@ TFImporter::TFImporter(Net& net, const char *model, const char *config) ReadTFNetParamsFromTextFileOrDie(config, &netTxt); } + extraOutputs = extraOutputs_; populateNet(); } TFImporter::TFImporter( Net& net, const char *dataModel, size_t lenModel, - const char *dataConfig, size_t lenConfig + const char *dataConfig, size_t lenConfig, + bool newEngine, const std::vector& extraOutputs_ ) : layerHandler(DNN_DIAGNOSTICS_RUN ? new TFLayerHandler(this) : nullptr), - dstNet(net), dispatch(buildDispatchMap()) + dstNet(net), newEngine(newEngine), dispatch(buildDispatchMap()) { if (dataModel != NULL && lenModel > 0) { @@ -2781,6 +2697,8 @@ TFImporter::TFImporter( CV_LOG_DEBUG(NULL, "DNN/TF: processing TensorFlow config from memory (" << lenConfig << " bytes)"); ReadTFNetParamsFromTextBufferOrDie(dataConfig, lenConfig, &netTxt); } + + extraOutputs = extraOutputs_; populateNet(); } @@ -2861,13 +2779,6 @@ void TFImporter::connect(const std::map& layers_name_id_map, Net& n network.connect(it->second, blobIndex, input_layer_id, input_blob_id); } -void TFImporter::connectToAllBlobs(const std::map& layer_id, Net& network, const Pin& outPin, - const int input_layer_id, const int input_blobs_count) -{ - for (int input_blob_id = 0; input_blob_id < input_blobs_count; input_blob_id++) - connect(layer_id, network, outPin, input_layer_id, input_blob_id); -} - const tensorflow::TensorProto& TFImporter::getConstBlob(const tensorflow::NodeDef &layer, std::map const_layers, int input_blob_index, int* actual_inp_blob_idx) { if (input_blob_index == -1) { @@ -3072,7 +2983,7 @@ void TFImporter::populateNet() RemoveIdentityOps(netBin); CV_LOG_DEBUG(NULL, "DNN/TF: RemoveIdentityOps(model) => " << netBin.node_size() << " nodes"); - simplifySubgraphs(netBin); + simplifySubgraphs(netBin, newEngine); CV_LOG_DEBUG(NULL, "DNN/TF: simplifySubgraphs(model) => " << netBin.node_size() << " nodes"); sortByExecutionOrder(netBin); CV_LOG_DEBUG(NULL, "DNN/TF: sortByExecutionOrder(model) => " << netBin.node_size() << " nodes"); @@ -3147,6 +3058,8 @@ void TFImporter::populateNet() layerHandler->fillRegistry(net); } + std::vector modelOutputs; + for (int li = 0; li < layersSize; li++) { const tensorflow::NodeDef& layer = net.node(li); @@ -3162,19 +3075,84 @@ void TFImporter::populateNet() CV_LOG_DEBUG(NULL, "DNN/TF: Model input: " << i << " - '" << netInputsNames[i] << "'"); CV_Assert(!netInputsNames[i].empty()); } - dstNet.setInputsNames(netInputsNames); + + if (newEngine) + { + Net::Impl* netimpl = dstNet.getImpl(); + for (const std::string& outputName : layersOutputs.back()) + netimpl->newArg(outputName, DNN_ARG_OUTPUT); + for (size_t layerId = 0; layerId < layersOutputs.size(); layerId++) + { + for (const std::string& outputName : layersOutputs[layerId]) + curProg[layerId]->outputs.push_back(dstNet.getArg(outputName)); + } + + std::set extraOutSet; + for (const String& extraOutName: extraOutputs) { + if (!dstNet.haveArg(extraOutName)) { + CV_LOG_WARNING(NULL, "DNN/TF: the model extra output '" << extraOutName << "' is not found"); + } + Arg arg = dstNet.getArg(extraOutName); + extraOutSet.insert(arg.idx); + } + std::map modelOutputsMap; + std::map extraOutputsMap; + int layer_idx = 0; + for (const auto& layer: curProg) { + layer_idx++; + const std::vector& layer_inputs = layer->inputs; + const std::vector& layer_outputs = layer->outputs; + for (Arg inp: layer_inputs) { + modelOutputsMap.erase(inp.idx); + } + size_t noutputs = layer_outputs.size(); + if (noutputs == 2 && layer->type == "Pooling") + noutputs = 1; + for (size_t i = 0; i < noutputs; i++) { + Arg out = layer_outputs[i]; + auto p = std::make_pair(out.idx, layer_idx); + modelOutputsMap.insert(p); + if (extraOutSet.find(out.idx) != extraOutSet.end()) + extraOutputsMap.insert(p); + } + } + + for (auto p: extraOutputsMap) + modelOutputsMap.insert(p); + + std::vector > modelOutputs_pairs; + for (auto p: modelOutputsMap) { + modelOutputs_pairs.push_back(p); + } + std::sort(modelOutputs_pairs.begin(), modelOutputs_pairs.end()); + for (auto p: modelOutputs_pairs) { + ArgData& data = netimpl->args[p.first]; + data.kind = DNN_ARG_OUTPUT; + modelOutputs.push_back(Arg(p.first)); + } + + Ptr curr_graph = netimpl->newGraph("", modelInputs, true); + curr_graph->setOutputs(modelOutputs); + curr_graph->setProg(curProg); + + netimpl->mainGraph = curr_graph; + netimpl->modelFormat = DNN_MODEL_TF; + netimpl->originalLayout = DATA_LAYOUT_NCHW; // TODO Should we set NHWC? + netimpl->prepareForInference(); + } + else + { + dstNet.setInputsNames(netInputsNames); + } CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN? "DNN/TF: diagnostic run completed!" : "DNN/TF: import completed!")); } -void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId, int orderSize) +void TFImporter::addPermuteLayer(const int* order, const std::string& permName, const std::string& inputName, int orderSize) { LayerParams permLP; permLP.set("order", DictValue::arrayInt(order, orderSize)); CV_Assert(layer_id.find(permName) == layer_id.end()); - int permId = dstNet.addLayer(permName, "Permute", permLP); - layer_id[permName] = permId; - connect(layer_id, dstNet, inpId, permId, 0); - inpId = Pin(permName); + addLayer(permName, "Permute", permLP, {inputName}); } void TFImporter::parseNode(const tensorflow::NodeDef& layer) @@ -3262,33 +3240,90 @@ void TFLayerHandler::handleFailed(const tensorflow::NodeDef& layer) LayerParams lp = getNotImplementedParams(layer.name(), layer.op()); // the layer will be created or its params and type will be replaced - int id = importer->dstNet.addLayer(lp.name, lp.type, lp); - if (id != -1) // internal layer failure before the call to addLayer() + if (importer->newEngine) + { + Ptr layer = LayerFactory::createLayerInstance(lp.type, lp); + if (layer) { + layer->inputs.push_back(importer->dstNet.getArg("NotImplementedInput")); + importer->layer_id[lp.name] = -1; + importer->curProg.push_back(layer); + } + } + else { - importer->layer_id[lp.name] = id; + int id = importer->dstNet.addLayer(lp.name, lp.type, lp); + if (id != -1) // internal layer failure before the call to addLayer() + { + importer->layer_id[lp.name] = id; + } } } } // namespace -Net readNetFromTensorflow(const String &model, const String &config) +Net readNetFromTensorflow(const String &model, const String &config, int engine, + const std::vector& extraOutputs) { - return detail::readNetDiagnostic(model.c_str(), config.c_str()); + static const int engine_forced = utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO); + if(engine_forced != ENGINE_AUTO) + engine = engine_forced; + + if (engine == ENGINE_AUTO) + { + try + { + return detail::readNetDiagnostic(model.c_str(), config.c_str(), + true, extraOutputs); + } + catch(const std::exception& e) + { + CV_LOG_WARNING(NULL, "Can't parse model with the new dnn engine, trying to parse with the old dnn engine: " << e.what()); + return detail::readNetDiagnostic(model.c_str(), config.c_str(), + false, extraOutputs); + } + } + else + { + return detail::readNetDiagnostic(model.c_str(), config.c_str(), engine == ENGINE_NEW || engine == ENGINE_AUTO, extraOutputs); + } } Net readNetFromTensorflow(const char* bufferModel, size_t lenModel, - const char* bufferConfig, size_t lenConfig) + const char* bufferConfig, size_t lenConfig, + int engine, + const std::vector& extraOutputs) { - return detail::readNetDiagnostic(bufferModel, lenModel, bufferConfig, lenConfig); + static const int engine_forced = utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO); + if(engine_forced != ENGINE_AUTO) + engine = engine_forced; + + if (engine == ENGINE_AUTO) + { + try + { + return detail::readNetDiagnostic(bufferModel, lenModel, bufferConfig, lenConfig, true, extraOutputs); + } + catch(const std::exception& e) + { + CV_LOG_WARNING(NULL, "Can't parse model with the new dnn engine, trying to parse with the old dnn engine: " << e.what()); + return detail::readNetDiagnostic(bufferModel, lenModel, bufferConfig, lenConfig, false, extraOutputs); + } + } + else + { + return detail::readNetDiagnostic(bufferModel, lenModel, bufferConfig, lenConfig, engine == ENGINE_NEW || engine == ENGINE_AUTO, extraOutputs); + } } -Net readNetFromTensorflow(const std::vector& bufferModel, const std::vector& bufferConfig) +Net readNetFromTensorflow(const std::vector& bufferModel, const std::vector& bufferConfig, int engine, + const std::vector& extraOutputs) { const char* bufferModelPtr = reinterpret_cast(&bufferModel[0]); const char* bufferConfigPtr = bufferConfig.empty() ? NULL : reinterpret_cast(&bufferConfig[0]); return readNetFromTensorflow(bufferModelPtr, bufferModel.size(), - bufferConfigPtr, bufferConfig.size()); + bufferConfigPtr, bufferConfig.size(), + engine, extraOutputs); } void writeTextGraph(const String& _model, const String& output) diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 964fcbbbe8..4e0e5bd734 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -46,7 +46,7 @@ TEST(Test_TensorFlow, read_inception) Mat inputBlob = blobFromImage(input); net.setInput(inputBlob, "input"); - Mat out = net.forward("softmax2"); + Mat out = net.forward(); std::cout << out.dims << std::endl; } @@ -66,7 +66,7 @@ TEST(Test_TensorFlow, inception_accuracy) Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true); net.setInput(inputBlob, "input"); - Mat out = net.forward("softmax2"); + Mat out = net.forward(); Mat ref = blobFromNPY(_tf("tf_inception_prob.npy")); @@ -1812,7 +1812,12 @@ TEST_P(Test_TensorFlow_nets, Mask_RCNN) std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt"); std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false); - Net net = readNetFromTensorflow(model, proto); + // Mask-RCNN predicts bounding boxes and segmentation masks. + std::vector outNames(2); + outNames[0] = "detection_out_final"; + outNames[1] = "detection_masks"; + + Net net = readNetFromTensorflow(model, proto, ENGINE_AUTO, outNames); Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy")); Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy")); Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false); @@ -1824,14 +1829,8 @@ TEST_P(Test_TensorFlow_nets, Mask_RCNN) net.setInput(blob); - // Mask-RCNN predicts bounding boxes and segmentation masks. - std::vector outNames(2); - outNames[0] = "detection_out_final"; - outNames[1] = "detection_masks"; - std::vector outs; net.forward(outs, outNames); - Mat outDetections = outs[0]; Mat outMasks = outs[1]; @@ -1940,4 +1939,24 @@ TEST(Test_TensorFlow_Importer, tf_graph_simplifier_buffer_overflow_21947) EXPECT_ANY_THROW(readNetFromTensorflow(reinterpret_cast(payload), sizeof(payload) / sizeof(payload[0]))); } +TEST(Test_TF_Model, Inception_GetLayer) +{ + const std::string model = findDataFile("dnn/tensorflow_inception_graph.pb", false); + auto net = readNetFromTensorflow(model); + + auto layernames = net.getLayerNames(); + + ASSERT_FALSE(layernames.empty()); + + // this is empirical initialization: + // * in the case of new engine the very first layer always has id == 0 for any model. + // * in the case of the old engine at least for this model the very first layer has id == 1 + int layer_id = net.getLayer(0)->name != layernames[0]; + for (auto name: layernames) { + auto layer = net.getLayer(layer_id); + EXPECT_EQ(layer->name, name); + layer_id++; + } +} + }