From 9e332dc5fbfabca99fc98a4e663500bea0df2cc6 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Thu, 5 Mar 2020 23:53:50 +0300 Subject: [PATCH] Broadcasting from ONNX --- modules/dnn/src/layers/scale_layer.cpp | 51 ++++++++++------ modules/dnn/src/onnx/onnx_importer.cpp | 81 +++++++++++++++++++------ modules/dnn/test/test_onnx_importer.cpp | 41 ++++++------- 3 files changed, 111 insertions(+), 62 deletions(-) diff --git a/modules/dnn/src/layers/scale_layer.cpp b/modules/dnn/src/layers/scale_layer.cpp index ea2d117901..7453c38a1c 100644 --- a/modules/dnn/src/layers/scale_layer.cpp +++ b/modules/dnn/src/layers/scale_layer.cpp @@ -46,14 +46,14 @@ public: { std::vector inputs; inputs_arr.getMatVector(inputs); - hasWeights = blobs.size() == 2 || (blobs.size() == 1 && !hasBias); + hasWeights = blobs.size() == 2 || (blobs.size() <= 1 && !hasBias); CV_Assert((inputs.size() == 2 && blobs.empty()) || blobs.size() == (int)hasWeights + (int)hasBias); } virtual bool supportBackend(int backendId) CV_OVERRIDE { return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE || - (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && axis == 1) || + (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && axis == 1 && !blobs.empty()) || (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && axis > 0); } @@ -78,10 +78,9 @@ public: Mat &outBlob = outputs[0]; // There is a mode when we multiply a first blob by a second one // instead of trainable weights. - Mat weights = blobs.empty() ? inputs[1] : (hasWeights ? blobs[0] : Mat()); - Mat bias = hasBias ? blobs.back().reshape(1, 1) : Mat(); - if (!weights.empty()) - weights = weights.reshape(1, 1); + Mat weights = hasWeights ? (blobs.empty() ? inputs[1] : blobs[0]).reshape(1, 1) : Mat();; + Mat bias = hasBias ? (blobs.empty() ? inputs[1] : blobs.back()).reshape(1, 1) : Mat(); + MatShape inpShape = shape(inpBlob); const int numWeights = !weights.empty() ? weights.total() : bias.total(); CV_Assert(numWeights != 0); @@ -229,28 +228,40 @@ public: #ifdef HAVE_DNN_NGRAPH virtual Ptr initNgraph(const std::vector >& inputs, const std::vector >& nodes) CV_OVERRIDE { - CV_Assert(!blobs.empty()); - const size_t numChannels = blobs[0].total(); - auto ieInpNode = nodes[0].dynamicCast()->node; + auto ieInpNode0 = nodes[0].dynamicCast()->node; + auto ieInpNode1 = nodes.size() > 1 ? nodes[1].dynamicCast()->node : nullptr; + + size_t numChannels = 1; + if (blobs.empty()) + for (const size_t& dim : ieInpNode1->get_shape()) + numChannels *= dim; + else + numChannels = blobs[0].total(); - std::vector shape(ieInpNode->get_shape().size(), 1); + std::vector shape(ieInpNode0->get_shape().size(), 1); int cAxis = clamp(axis, shape.size()); shape[cAxis] = numChannels; - auto node = ieInpNode; + auto node = ieInpNode0; if (hasWeights) { - auto weight = std::make_shared(ngraph::element::f32, - ngraph::Shape(shape), blobs[0].data); + auto weight = blobs.empty() ? ieInpNode1 : + std::make_shared(ngraph::element::f32, ngraph::Shape(shape), blobs[0].data); + node = std::make_shared(node, weight, ngraph::op::AutoBroadcastType::NUMPY); } if (hasBias || !hasWeights) { - auto bias = hasBias ? - std::make_shared(ngraph::element::f32, - ngraph::Shape(shape), blobs.back().data) : - std::make_shared(ngraph::element::f32, - ngraph::Shape(shape), std::vector(numChannels, 0).data()); + std::shared_ptr bias; + if (hasBias) + { + bias = blobs.empty() ? ieInpNode1 : + std::make_shared(ngraph::element::f32, + ngraph::Shape(shape), blobs.back().data); + } + else + bias = std::make_shared(ngraph::element::f32, + ngraph::Shape(shape), std::vector(numChannels, 0).data()); node = std::make_shared(node, bias, ngraph::op::AutoBroadcastType::NUMPY); } return Ptr(new InfEngineNgraphNode(node)); @@ -259,8 +270,8 @@ public: void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE { - scale = hasWeights ? blobs[0] : Mat(); - shift = hasBias ? blobs.back() : Mat(); + scale = (hasWeights && !blobs.empty()) ? blobs[0] : Mat(); + shift = (hasBias && !blobs.empty()) ? blobs.back() : Mat(); } virtual int64 getFLOPS(const std::vector &inputs, diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 3d7e33a37f..0ca909597e 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -427,24 +427,57 @@ void ONNXImporter::populateNet(Net dstNet) } layerParams.type = "Slice"; } - else if (layer_type == "Add" || layer_type == "Sum") + else if (layer_type == "Add" || layer_type == "Sum" || layer_type == "Sub") { + bool isSub = layer_type == "Sub"; + CV_CheckEQ(node_proto.input_size(), 2, ""); if (layer_id.find(node_proto.input(1)) == layer_id.end()) { Mat blob = getBlob(node_proto, constBlobs, 1); blob = blob.reshape(1, 1); if (blob.total() == 1) { layerParams.type = "Power"; - layerParams.set("shift", blob.at(0)); + layerParams.set("shift", (isSub ? -1 : 1) * blob.at(0)); } else { layerParams.type = "Scale"; layerParams.set("bias_term", true); - layerParams.blobs.push_back(blob); + layerParams.blobs.push_back((isSub ? -1 : 1) * blob); } } - else { + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { layerParams.type = "Eltwise"; + if (isSub) + { + static float subCoeffs[] = {1.f, -1.f}; + layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); + } + } + else + { + if (isSub) + { + LayerParams powerParams; + powerParams.name = layerParams.name + "/neg"; + powerParams.type = "Power"; + powerParams.set("scale", -1); + + //Create Power layer + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + layerId = layer_id.find(node_proto.input(1)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; + + //Replace input to Power + node_proto.set_input(1, powerParams.name); + } + layerParams.type = "Scale"; + layerParams.set("bias_term", true); } } else if (layer_type == "Max") @@ -452,19 +485,6 @@ void ONNXImporter::populateNet(Net dstNet) layerParams.type = "Eltwise"; layerParams.set("operation", "max"); } - else if (layer_type == "Sub") - { - Mat blob = getBlob(node_proto, constBlobs, 1); - if (blob.total() == 1) { - layerParams.type = "Power"; - layerParams.set("shift", -blob.at(0)); - } - else { - layerParams.type = "Scale"; - layerParams.set("has_bias", true); - layerParams.blobs.push_back(-1.0f * blob.reshape(1, 1)); - } - } else if (layer_type == "Neg") { layerParams.type = "Power"; @@ -643,10 +663,35 @@ void ONNXImporter::populateNet(Net dstNet) layerParams.type = "Scale"; } } - else { + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { layerParams.type = "Eltwise"; layerParams.set("operation", isDiv ? "div" : "prod"); } + else + { + if (isDiv) + { + LayerParams powerParams; + powerParams.name = layerParams.name + "/inv"; + powerParams.type = "Power"; + powerParams.set("power", -1); + + //Create Power layer + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + layerId = layer_id.find(node_proto.input(1)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; + + //Replace input to Power + node_proto.set_input(1, powerParams.name); + } + layerParams.type = "Scale"; + } if (!haveVariables) { diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 2838a72ea7..f284eed45b 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -32,29 +32,33 @@ public: void testONNXModels(const String& basename, const Extension ext = npy, const double l1 = 0, const float lInf = 0, const bool useSoftmax = false, - bool checkNoFallbacks = true) + bool checkNoFallbacks = true, int numInps = 1) { String onnxmodel = _tf("models/" + basename + ".onnx", required); - Mat inp, ref; + std::vector inps(numInps); + Mat ref; if (ext == npy) { - inp = blobFromNPY(_tf("data/input_" + basename + ".npy")); + for (int i = 0; i < numInps; ++i) + inps[i] = blobFromNPY(_tf("data/input_" + basename + (numInps > 1 ? format("_%d", i) : "") + ".npy")); ref = blobFromNPY(_tf("data/output_" + basename + ".npy")); } else if (ext == pb) { - inp = readTensorFromONNX(_tf("data/input_" + basename + ".pb")); + for (int i = 0; i < numInps; ++i) + inps[i] = readTensorFromONNX(_tf("data/input_" + basename + (numInps > 1 ? format("_%d", i) : "") + ".pb")); ref = readTensorFromONNX(_tf("data/output_" + basename + ".pb")); } else CV_Error(Error::StsUnsupportedFormat, "Unsupported extension"); - checkBackend(&inp, &ref); + checkBackend(&inps[0], &ref); Net net = readNetFromONNX(onnxmodel); ASSERT_FALSE(net.empty()); net.setPreferableBackend(backend); net.setPreferableTarget(target); - net.setInput(inp); + for (int i = 0; i < numInps; ++i) + net.setInput(inps[i], numInps > 1 ? format("%d", i) : ""); Mat out = net.forward(""); if (useSoftmax) @@ -328,25 +332,14 @@ TEST_P(Test_ONNX_layers, ResizeUnfused) TEST_P(Test_ONNX_layers, MultyInputs) { - const String model = _tf("models/multy_inputs.onnx"); - - Net net = readNetFromONNX(model); - ASSERT_FALSE(net.empty()); - - net.setPreferableBackend(backend); - net.setPreferableTarget(target); - - Mat inp1 = blobFromNPY(_tf("data/input_multy_inputs_0.npy")); - Mat inp2 = blobFromNPY(_tf("data/input_multy_inputs_1.npy")); - Mat ref = blobFromNPY(_tf("data/output_multy_inputs.npy")); - checkBackend(&inp1, &ref); - - net.setInput(inp1, "0"); - net.setInput(inp2, "1"); - Mat out = net.forward(); + testONNXModels("multy_inputs", npy, 0, 0, false, true, 2); +} - normAssert(ref, out, "", default_l1, default_lInf); - expectNoFallbacksFromIE(net); +TEST_P(Test_ONNX_layers, Broadcast) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + testONNXModels("channel_broadcast", npy, 0, 0, false, true, 2); } TEST_P(Test_ONNX_layers, Div)